Commit 65ef8181 authored by Balasankar C's avatar Balasankar C

Merge tag 'upstream/0.7.3+dfsg'

Upstream version 0.7.3+dfsg
parents f5dca9c5 ae5a9582
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @author: starenka
# @email: 'moc]tod[liamg].T.E[0aknerats'[::-1]
import warnings, hashlib, simplejson, string
from os.path import dirname, abspath
from flask import Flask, render_template, request
from flaskext.sqlalchemy import SQLAlchemy
from sqlalchemy.exceptions import IntegrityError
except ImportError:
from sqlalchemy.exc import IntegrityError
#Hey monkey patcher! NLTK's NgramModel is not serializable w/ pickle.HIGHEST_PROTOCOL (2)
from werkzeug.contrib import cache
from werkzeug.contrib.cache import SimpleCache
from BeautifulSoup import BeautifulSoup
import nltk
PUNCT = list(unicode(string.punctuation))
app = Flask(__name__)
cache = SimpleCache()
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///%s/db.sqlite3'%abspath(dirname(__file__))
db = SQLAlchemy(app)
class Sample(db.Model):
id = db.Column(db.Integer, primary_key=True)
url = db.Column(db.String(80), unique=True)
text = db.Column(db.String())
enabled = db.Column(db.Boolean())
def __unicode__(self):
str = unicode(BeautifulSoup(self.text,convertEntities=BeautifulSoup.HTML_ENTITIES))
return nltk.clean_html(str)
def get_all(self):
cached = cache.get('samples')
if cached is None:
cached = self.query.filter_by(enabled=True).all()
cache.set('samples', cached, timeout=app.config['CACHE_MINUTES'] * 60)
return cached
class Output(db.Model):
id = db.Column(db.Integer, primary_key=True)
hash = db.Column(db.String(128),unique=True)
text = db.Column(db.String())
params = db.Column(db.String(100))
def __init__(self,text,**params):
self.hash = hashlib.sha512(text.encode('utf8')).hexdigest()
self.text = text
self.params = simplejson.dumps(params)
def base_context():
return dict(settings=app.config,
hits = Output.query.count() + app.config['INIT_HITS']
def page_not_found(error):
return render_template('404.html',title=u"To tady nemáme!"), 404
def faq():
return render_template('faq.html',title=u"Často kladené dotazy",samples=Sample.get_all())
def permalink(hash):
one = Output.query.filter_by(hash=hash).first_or_404()
return render_template('generator.html', title=u"Henrykuj!",
text=one.text, hash=one.hash,
def index():
bigrams = request.args.get('bigrams',False)
words = int(request.args.get('words',app.config['WORDS']))
if words > app.config['MAX_WORDS']:
words = app.config['MAX_WORDS']
except ValueError:
words = app.config['WORDS']
out = _generate(words,bigrams)
output = Output(out,words=words,bigrams=bool(bigrams))
except IntegrityError:
return render_template('generator.html', title=u"Henrykuj!",
text=out, hash=output.hash,
words=words, bigrams=bigrams
def _get_ngram_model(bigrams):
#NLTK produces a LOT of warnings - don't mess with my error log
cached = cache.get('ngram_model')
if cached is None:
samples = Sample.get_all()
if samples:
text = [unicode(s) for s in samples]
tokenizer = nltk.tokenize.WordPunctTokenizer()
tokenized = tokenizer.tokenize(' '.join(text))
cached = nltk.NgramModel(3-int(bool(bigrams)), tokenized)
cache.set('ngram_model', cached, timeout=app.config['CACHE_MINUTES'] * 60)
return cached
def _generate(words,bigrams):
model = _get_ngram_model(bigrams)
starts = model.generate(100)[-4:]
starts = filter(lambda a: a not in PUNCT,starts)
generated = model.generate(words, starts)
out = ' '.join(generated).replace(' , ',', ').replace(' . ','. ')
return '%s%s&hellip;'%(out[0].upper(),out[1:])
if __name__ == '__main__':
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment