# # # delete "web" # # delete "web/__init__.py" # # delete "web/cheetah.py" # # delete "web/db.py" # # delete "web/debugerror.py" # # delete "web/form.py" # # delete "web/http.py" # # delete "web/httpserver.py" # # delete "web/net.py" # # delete "web/request.py" # # delete "web/template.py" # # delete "web/utils.py" # # delete "web/webapi.py" # # delete "web/wsgi.py" # # add_dir "web" # # add_dir "web/wsgiserver" # # add_dir "webpy" # # add_file "web/__init__.py" # content [4ad6a0c17ac8571d6099b2b771b32b3607197d0e] # # add_file "web/cheetah.py" # content [264c19ddc664f70dd676c87f7c0987e2aec3d037] # # add_file "web/db.py" # content [c2c7c92981bbe09f198800977980c642cc09a018] # # add_file "web/debugerror.py" # content [dbd19a03c29519c7f45301222c73fc3b36053362] # # add_file "web/form.py" # content [5d23f4c195cb1d0b51d23bc6088630c0cfb9fa0d] # # add_file "web/http.py" # content [0344eb823380e8fd6b9aa3996ceb73cedb42a983] # # add_file "web/httpserver.py" # content [ae3d4631644bf59c612f4aca85feb43252fa784e] # # add_file "web/net.py" # content [ac59560505e4c2998eb9b579b98e63cecada37d8] # # add_file "web/request.py" # content [d84d3ee98b7786434cdbda3403860489376b1b7c] # # add_file "web/template.py" # content [acc5f5166520ac922aeb052cc63cd0bd6cc4964f] # # add_file "web/utils.py" # content [ffbc0498151371a2a099bb8233f7d2bd0676f1d2] # # add_file "web/webapi.py" # content [adba8f74b58b03514c8ccb47c926676139d4044b] # # add_file "web/wsgi.py" # content [d7214b2a4c8693a87db5d17eacd245d441ad7113] # # add_file "web/wsgiserver/__init__.py" # content [7459212c431b1a3561d948c6852c41fa765e9b9d] # # patch "ChangeLog" # from [356aa12709506db7f10ce9cbe43b7a69da776e2f] # to [b019f88c92309d5cd7ba45061bffe75186eb4e2a] # # patch "config.py.example" # from [782031278d2d977661b7e566495c264626557f3a] # to [0a0614a487e5a055da10046a6d6a701211abce19] # # patch "mk2.py" # from [160b4053f2d72fa2807c10cc1b72df4d11828ebf] # to [db6cdd07694eb91b5828985c8eaf5fcfc4835602] # # patch "release.py" # from [b530a2e112abc57d740dc931d41a16ac265af00f] # to [7039ccac8caa4f0a7f22b9e6ca05f406e26044cf] # # patch "viewmtn.py" # from [5de4dc55065621a80377580d13ff4fada54f5029] # to [c0d63918a4f6f3ea8cfa923166fdd243b55a8e46] # ============================================================ --- web/__init__.py 4ad6a0c17ac8571d6099b2b771b32b3607197d0e +++ web/__init__.py 4ad6a0c17ac8571d6099b2b771b32b3607197d0e @@ -0,0 +1,62 @@ +#!/usr/bin/env python +from __future__ import generators + +"""web.py: makes web apps (http://webpy.org)""" +__version__ = "0.22" +__revision__ = "$Rev: 183 $" +__author__ = "Aaron Swartz " +__license__ = "public domain" +__contributors__ = "see http://webpy.org/changes" + +# todo: +# - some sort of accounts system + +import utils, db, net, wsgi, http, webapi, request, httpserver, debugerror +import template, form + +from utils import * +from db import * +from net import * +from wsgi import * +from http import * +from webapi import * +from request import * +from httpserver import * +from debugerror import * + +try: + import cheetah + from cheetah import * +except ImportError: + pass + +def main(): + import doctest + + doctest.testmod(utils) + doctest.testmod(db) + doctest.testmod(net) + doctest.testmod(wsgi) + doctest.testmod(http) + doctest.testmod(webapi) + doctest.testmod(request) + + try: + doctest.testmod(cheetah) + except NameError: + pass + + template.test() + + import sys + urls = ('/web.py', 'source') + class source: + def GET(self): + header('Content-Type', 'text/python') + print open(sys.argv[0]).read() + + if listget(sys.argv, 1) != 'test': + run(urls, locals()) + +if __name__ == "__main__": main() + ============================================================ --- web/cheetah.py 264c19ddc664f70dd676c87f7c0987e2aec3d037 +++ web/cheetah.py 264c19ddc664f70dd676c87f7c0987e2aec3d037 @@ -0,0 +1,98 @@ +""" +Cheetah API +(from web.py) +""" + +__all__ = ["render"] + +import re, urlparse, pprint, traceback, sys +from Cheetah.Compiler import Compiler +from Cheetah.Filters import Filter +from utils import re_compile, memoize, dictadd +from net import htmlquote, websafe +from webapi import ctx, header, output, input, cookies, loadhooks + +def upvars(level=2): + """Guido van Rossum sez: don't use this function.""" + return dictadd( + sys._getframe(level).f_globals, + sys._getframe(level).f_locals) + +r_include = re_compile(r'(?!\\)#include \"(.*?)\"($|#)', re.M) +def __compiletemplate(template, base=None, isString=False): + if isString: + text = template + else: + text = open('templates/'+template).read() + # implement #include at compile-time + def do_include(match): + text = open('templates/'+match.groups()[0]).read() + return text + while r_include.findall(text): + text = r_include.sub(do_include, text) + + execspace = _compiletemplate.bases.copy() + tmpl_compiler = Compiler(source=text, mainClassName='GenTemplate') + tmpl_compiler.addImportedVarNames(execspace.keys()) + exec str(tmpl_compiler) in execspace + if base: + _compiletemplate.bases[base] = execspace['GenTemplate'] + + return execspace['GenTemplate'] + +_compiletemplate = memoize(__compiletemplate) +_compiletemplate.bases = {} + +def render(template, terms=None, asTemplate=False, base=None, + isString=False): + """ + Renders a template, caching where it can. + + `template` is the name of a file containing the a template in + the `templates/` folder, unless `isString`, in which case it's the + template itself. + + `terms` is a dictionary used to fill the template. If it's None, then + the caller's local variables are used instead, plus context, if it's not + already set, is set to `context`. + + If asTemplate is False, it `output`s the template directly. Otherwise, + it returns the template object. + + If the template is a potential base template (that is, something other templates) + can extend, then base should be a string with the name of the template. The + template will be cached and made available for future calls to `render`. + + Requires [Cheetah](http://cheetahtemplate.org/). + """ + # terms=['var1', 'var2'] means grab those variables + if isinstance(terms, list): + new = {} + old = upvars() + for k in terms: + new[k] = old[k] + terms = new + # default: grab all locals + elif terms is None: + terms = {'context': ctx, 'ctx':ctx} + terms.update(sys._getframe(1).f_locals) + # terms=d means use d as the searchList + if not isinstance(terms, tuple): + terms = (terms,) + + if 'headers' in ctx and not isString and template.endswith('.html'): + header('Content-Type','text/html; charset=utf-8', unique=True) + + if loadhooks.has_key('reloader'): + compiled_tmpl = __compiletemplate(template, base=base, isString=isString) + else: + compiled_tmpl = _compiletemplate(template, base=base, isString=isString) + compiled_tmpl = compiled_tmpl(searchList=terms, filter=WebSafe) + if asTemplate: + return compiled_tmpl + else: + return output(str(compiled_tmpl)) + +class WebSafe(Filter): + def filter(self, val, **keywords): + return websafe(val) ============================================================ --- web/db.py c2c7c92981bbe09f198800977980c642cc09a018 +++ web/db.py c2c7c92981bbe09f198800977980c642cc09a018 @@ -0,0 +1,703 @@ +""" +Database API +(part of web.py) +""" + +# todo: +# - test with sqlite +# - a store function? + +__all__ = [ + "UnknownParamstyle", "UnknownDB", + "sqllist", "sqlors", "aparam", "reparam", + "SQLQuery", "sqlquote", + "SQLLiteral", "sqlliteral", + "connect", + "TransactionError", "transaction", "transact", "commit", "rollback", + "query", + "select", "insert", "update", "delete" +] + +import time +try: import datetime +except ImportError: datetime = None + +from utils import storage, iters, iterbetter +import webapi as web + +try: + from DBUtils import PooledDB + web.config._hasPooling = True +except ImportError: + web.config._hasPooling = False + +class _ItplError(ValueError): + def __init__(self, text, pos): + ValueError.__init__(self) + self.text = text + self.pos = pos + def __str__(self): + return "unfinished expression in %s at char %d" % ( + repr(self.text), self.pos) + +def _interpolate(format): + """ + Takes a format string and returns a list of 2-tuples of the form + (boolean, string) where boolean says whether string should be evaled + or not. + + from (public domain, Ka-Ping Yee) + """ + from tokenize import tokenprog + + def matchorfail(text, pos): + match = tokenprog.match(text, pos) + if match is None: + raise _ItplError(text, pos) + return match, match.end() + + namechars = "abcdefghijklmnopqrstuvwxyz" \ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + chunks = [] + pos = 0 + + while 1: + dollar = format.find("$", pos) + if dollar < 0: + break + nextchar = format[dollar + 1] + + if nextchar == "{": + chunks.append((0, format[pos:dollar])) + pos, level = dollar + 2, 1 + while level: + match, pos = matchorfail(format, pos) + tstart, tend = match.regs[3] + token = format[tstart:tend] + if token == "{": + level = level + 1 + elif token == "}": + level = level - 1 + chunks.append((1, format[dollar + 2:pos - 1])) + + elif nextchar in namechars: + chunks.append((0, format[pos:dollar])) + match, pos = matchorfail(format, dollar + 1) + while pos < len(format): + if format[pos] == "." and \ + pos + 1 < len(format) and format[pos + 1] in namechars: + match, pos = matchorfail(format, pos + 1) + elif format[pos] in "([": + pos, level = pos + 1, 1 + while level: + match, pos = matchorfail(format, pos) + tstart, tend = match.regs[3] + token = format[tstart:tend] + if token[0] in "([": + level = level + 1 + elif token[0] in ")]": + level = level - 1 + else: + break + chunks.append((1, format[dollar + 1:pos])) + + else: + chunks.append((0, format[pos:dollar + 1])) + pos = dollar + 1 + (nextchar == "$") + + if pos < len(format): + chunks.append((0, format[pos:])) + return chunks + +class UnknownParamstyle(Exception): + """ + raised for unsupported db paramstyles + + (currently supported: qmark, numeric, format, pyformat) + """ + pass + +def aparam(): + """ + Returns the appropriate string to be used to interpolate + a value with the current `web.ctx.db_module` or simply %s + if there isn't one. + + >>> aparam() + '%s' + """ + if hasattr(web.ctx, 'db_module'): + style = web.ctx.db_module.paramstyle + else: + style = 'pyformat' + + if style == 'qmark': + return '?' + elif style == 'numeric': + return ':1' + elif style in ['format', 'pyformat']: + return '%s' + raise UnknownParamstyle, style + +def reparam(string_, dictionary): + """ + Takes a string and a dictionary and interpolates the string + using values from the dictionary. Returns an `SQLQuery` for the result. + + >>> reparam("s = $s", dict(s=True)) + + """ + vals = [] + result = [] + for live, chunk in _interpolate(string_): + if live: + result.append(aparam()) + vals.append(eval(chunk, dictionary)) + else: result.append(chunk) + return SQLQuery(''.join(result), vals) + +def sqlify(obj): + """ + converts `obj` to its proper SQL version + + >>> sqlify(None) + 'NULL' + >>> sqlify(True) + "'t'" + >>> sqlify(3) + '3' + """ + + # because `1 == True and hash(1) == hash(True)` + # we have to do this the hard way... + + if obj is None: + return 'NULL' + elif obj is True: + return "'t'" + elif obj is False: + return "'f'" + elif datetime and isinstance(obj, datetime.datetime): + return repr(obj.isoformat()) + else: + return repr(obj) + +class SQLQuery: + """ + You can pass this sort of thing as a clause in any db function. + Otherwise, you can pass a dictionary to the keyword argument `vars` + and the function will call reparam for you. + """ + # tested in sqlquote's docstring + def __init__(self, s='', v=()): + self.s, self.v = str(s), tuple(v) + + def __getitem__(self, key): # for backwards-compatibility + return [self.s, self.v][key] + + def __add__(self, other): + if isinstance(other, str): + self.s += other + elif isinstance(other, SQLQuery): + self.s += other.s + self.v += other.v + return self + + def __radd__(self, other): + if isinstance(other, str): + self.s = other + self.s + return self + else: + return NotImplemented + + def __str__(self): + try: + return self.s % tuple([sqlify(x) for x in self.v]) + except (ValueError, TypeError): + return self.s + + def __repr__(self): + return '' % repr(str(self)) + +class SQLLiteral: + """ + Protects a string from `sqlquote`. + + >>> insert('foo', time=SQLLiteral('NOW()'), _test=True) + + """ + def __init__(self, v): + self.v = v + + def __repr__(self): + return self.v + +sqlliteral = SQLLiteral + +def sqlquote(a): + """ + Ensures `a` is quoted properly for use in a SQL query. + + >>> 'WHERE x = ' + sqlquote(True) + ' AND y = ' + sqlquote(3) + + """ + return SQLQuery(aparam(), (a,)) + +class UnknownDB(Exception): + """raised for unsupported dbms""" + pass + +def connect(dbn, **keywords): + """ + Connects to the specified database. + + `dbn` currently must be "postgres", "mysql", or "sqlite". + + If DBUtils is installed, connection pooling will be used. + """ + if dbn == "postgres": + try: + import psycopg2 as db + except ImportError: + try: + import psycopg as db + except ImportError: + import pgdb as db + if 'pw' in keywords: + keywords['password'] = keywords['pw'] + del keywords['pw'] + keywords['database'] = keywords['db'] + del keywords['db'] + + elif dbn == "mysql": + import MySQLdb as db + if 'pw' in keywords: + keywords['passwd'] = keywords['pw'] + del keywords['pw'] + db.paramstyle = 'pyformat' # it's both, like psycopg + + elif dbn == "sqlite": + try: + import sqlite3 as db + db.paramstyle = 'qmark' + except ImportError: + try: + from pysqlite2 import dbapi2 as db + db.paramstyle = 'qmark' + except ImportError: + import sqlite as db + web.config._hasPooling = False + keywords['database'] = keywords['db'] + del keywords['db'] + + elif dbn == "firebird": + import kinterbasdb as db + if 'pw' in keywords: + keywords['passwd'] = keywords['pw'] + del keywords['pw'] + keywords['database'] = keywords['db'] + del keywords['db'] + + else: + raise UnknownDB, dbn + + web.ctx.db_name = dbn + web.ctx.db_module = db + web.ctx.db_transaction = 0 + web.ctx.db = keywords + + def _PooledDB(db, keywords): + # In DBUtils 0.9.3, `dbapi` argument is renamed as `creator` + # see Bug#122112 + if PooledDB.__version__.split('.') < '0.9.3'.split('.'): + return PooledDB.PooledDB(dbapi=db, **keywords) + else: + return PooledDB.PooledDB(creator=db, **keywords) + + def db_cursor(): + if isinstance(web.ctx.db, dict): + keywords = web.ctx.db + if web.config._hasPooling: + if 'db' not in globals(): + globals()['db'] = _PooledDB(db, keywords) + web.ctx.db = globals()['db'].connection() + else: + web.ctx.db = db.connect(**keywords) + return web.ctx.db.cursor() + web.ctx.db_cursor = db_cursor + + web.ctx.dbq_count = 0 + + def db_execute(cur, sql_query, dorollback=True): + """executes an sql query""" + + web.ctx.dbq_count += 1 + + try: + a = time.time() + out = cur.execute(sql_query.s, sql_query.v) + b = time.time() + except: + if web.config.get('db_printing'): + print >> web.debug, 'ERR:', str(sql_query) + if dorollback: rollback(care=False) + raise + + if web.config.get('db_printing'): + print >> web.debug, '%s (%s): %s' % (round(b-a, 2), web.ctx.dbq_count, str(sql_query)) + + return out + web.ctx.db_execute = db_execute + return web.ctx.db + +class TransactionError(Exception): pass + +class transaction: + """ + A context that can be used in conjunction with "with" statements + to implement SQL transactions. Starts a transaction on enter, + rolls it back if there's an error; otherwise it commits it at the + end. + """ + def __enter__(self): + transact() + + def __exit__(self, exctype, excvalue, traceback): + if exctype is not None: + rollback() + else: + commit() + +def transact(): + """Start a transaction.""" + if not web.ctx.db_transaction: + # commit everything up to now, so we don't rollback it later + if hasattr(web.ctx.db, 'commit'): + web.ctx.db.commit() + else: + db_cursor = web.ctx.db_cursor() + web.ctx.db_execute(db_cursor, + SQLQuery("SAVEPOINT webpy_sp_%s" % web.ctx.db_transaction)) + web.ctx.db_transaction += 1 + +def commit(): + """Commits a transaction.""" + web.ctx.db_transaction -= 1 + if web.ctx.db_transaction < 0: + raise TransactionError, "not in a transaction" + + if not web.ctx.db_transaction: + if hasattr(web.ctx.db, 'commit'): + web.ctx.db.commit() + else: + db_cursor = web.ctx.db_cursor() + web.ctx.db_execute(db_cursor, + SQLQuery("RELEASE SAVEPOINT webpy_sp_%s" % web.ctx.db_transaction)) + +def rollback(care=True): + """Rolls back a transaction.""" + web.ctx.db_transaction -= 1 + if web.ctx.db_transaction < 0: + web.db_transaction = 0 + if care: + raise TransactionError, "not in a transaction" + else: + return + + if not web.ctx.db_transaction: + if hasattr(web.ctx.db, 'rollback'): + web.ctx.db.rollback() + else: + db_cursor = web.ctx.db_cursor() + web.ctx.db_execute(db_cursor, + SQLQuery("ROLLBACK TO SAVEPOINT webpy_sp_%s" % web.ctx.db_transaction), + dorollback=False) + +def query(sql_query, vars=None, processed=False, _test=False): + """ + Execute SQL query `sql_query` using dictionary `vars` to interpolate it. + If `processed=True`, `vars` is a `reparam`-style list to use + instead of interpolating. + + >>> query("SELECT * FROM foo", _test=True) + + >>> query("SELECT * FROM foo WHERE x = $x", vars=dict(x='f'), _test=True) + + >>> query("SELECT * FROM foo WHERE x = " + sqlquote('f'), _test=True) + + """ + if vars is None: vars = {} + + if not processed and not isinstance(sql_query, SQLQuery): + sql_query = reparam(sql_query, vars) + + if _test: return sql_query + + db_cursor = web.ctx.db_cursor() + web.ctx.db_execute(db_cursor, sql_query) + + if db_cursor.description: + names = [x[0] for x in db_cursor.description] + def iterwrapper(): + row = db_cursor.fetchone() + while row: + yield storage(dict(zip(names, row))) + row = db_cursor.fetchone() + out = iterbetter(iterwrapper()) + if web.ctx.db_name != "sqlite": + out.__len__ = lambda: int(db_cursor.rowcount) + out.list = lambda: [storage(dict(zip(names, x))) \ + for x in db_cursor.fetchall()] + else: + out = db_cursor.rowcount + + if not web.ctx.db_transaction: web.ctx.db.commit() + return out + +def sqllist(lst): + """ + Converts the arguments for use in something like a WHERE clause. + + >>> sqllist(['a', 'b']) + 'a, b' + >>> sqllist('a') + 'a' + + """ + if isinstance(lst, str): + return lst + else: + return ', '.join(lst) + +def sqlors(left, lst): + """ + `left is a SQL clause like `tablename.arg = ` + and `lst` is a list of values. Returns a reparam-style + pair featuring the SQL that ORs together the clause + for each item in the lst. + + >>> sqlors('foo = ', []) + + >>> sqlors('foo = ', [1]) + + >>> sqlors('foo = ', 1) + + >>> sqlors('foo = ', [1,2,3]) + + """ + if isinstance(lst, iters): + lst = list(lst) + ln = len(lst) + if ln == 0: + return SQLQuery("2+2=5", []) + if ln == 1: + lst = lst[0] + + if isinstance(lst, iters): + return SQLQuery('(' + left + + (' OR ' + left).join([aparam() for param in lst]) + ")", lst) + else: + return SQLQuery(left + aparam(), [lst]) + +def sqlwhere(dictionary, grouping=' AND '): + """ + Converts a `dictionary` to an SQL WHERE clause `SQLQuery`. + + >>> sqlwhere({'cust_id': 2, 'order_id':3}) + + >>> sqlwhere({'cust_id': 2, 'order_id':3}, grouping=', ') + + """ + + return SQLQuery(grouping.join([ + '%s = %s' % (k, aparam()) for k in dictionary.keys() + ]), dictionary.values()) + +def select(tables, vars=None, what='*', where=None, order=None, group=None, + limit=None, offset=None, _test=False): + """ + Selects `what` from `tables` with clauses `where`, `order`, + `group`, `limit`, and `offset`. Uses vars to interpolate. + Otherwise, each clause can be a SQLQuery. + + >>> select('foo', _test=True) + + >>> select(['foo', 'bar'], where="foo.bar_id = bar.id", limit=5, _test=True) + + """ + if vars is None: vars = {} + qout = "" + + def gen_clause(sql, val): + if isinstance(val, (int, long)): + if sql == 'WHERE': + nout = 'id = ' + sqlquote(val) + else: + nout = SQLQuery(val) + elif isinstance(val, (list, tuple)) and len(val) == 2: + nout = SQLQuery(val[0], val[1]) # backwards-compatibility + elif isinstance(val, SQLQuery): + nout = val + elif val: + nout = reparam(val, vars) + else: + return "" + + out = "" + if qout: out += " " + out += sql + " " + nout + return out + + if web.ctx.get('db_name') == "firebird": + for (sql, val) in ( + ('FIRST', limit), + ('SKIP', offset) + ): + qout += gen_clause(sql, val) + if qout: + SELECT = 'SELECT ' + qout + else: + SELECT = 'SELECT' + qout = "" + sql_clauses = ( + (SELECT, what), + ('FROM', sqllist(tables)), + ('WHERE', where), + ('GROUP BY', group), + ('ORDER BY', order) + ) + else: + sql_clauses = ( + ('SELECT', what), + ('FROM', sqllist(tables)), + ('WHERE', where), + ('GROUP BY', group), + ('ORDER BY', order), + ('LIMIT', limit), + ('OFFSET', offset) + ) + + for (sql, val) in sql_clauses: + qout += gen_clause(sql, val) + + if _test: return qout + return query(qout, processed=True) + +def insert(tablename, seqname=None, _test=False, **values): + """ + Inserts `values` into `tablename`. Returns current sequence ID. + Set `seqname` to the ID if it's not the default, or to `False` + if there isn't one. + + >>> insert('foo', joe='bob', a=2, _test=True) + + """ + + if values: + sql_query = SQLQuery("INSERT INTO %s (%s) VALUES (%s)" % ( + tablename, + ", ".join(values.keys()), + ', '.join([aparam() for x in values]) + ), values.values()) + else: + sql_query = SQLQuery("INSERT INTO %s DEFAULT VALUES" % tablename) + + if _test: return sql_query + + db_cursor = web.ctx.db_cursor() + if seqname is False: + pass + elif web.ctx.db_name == "postgres": + if seqname is None: + seqname = tablename + "_id_seq" + sql_query += "; SELECT currval('%s')" % seqname + elif web.ctx.db_name == "mysql": + web.ctx.db_execute(db_cursor, sql_query) + sql_query = SQLQuery("SELECT last_insert_id()") + elif web.ctx.db_name == "sqlite": + web.ctx.db_execute(db_cursor, sql_query) + # not really the same... + sql_query = SQLQuery("SELECT last_insert_rowid()") + + web.ctx.db_execute(db_cursor, sql_query) + try: + out = db_cursor.fetchone()[0] + except Exception: + out = None + + if not web.ctx.db_transaction: web.ctx.db.commit() + + return out + +def update(tables, where, vars=None, _test=False, **values): + """ + Update `tables` with clause `where` (interpolated using `vars`) + and setting `values`. + + >>> joe = 'Joseph' + >>> update('foo', where='name = $joe', name='bob', age=5, + ... vars=locals(), _test=True) + + """ + if vars is None: vars = {} + + if isinstance(where, (int, long)): + where = "id = " + sqlquote(where) + elif isinstance(where, (list, tuple)) and len(where) == 2: + where = SQLQuery(where[0], where[1]) + elif isinstance(where, SQLQuery): + pass + else: + where = reparam(where, vars) + + query = ( + "UPDATE " + sqllist(tables) + + " SET " + sqlwhere(values, ', ') + + " WHERE " + where) + + if _test: return query + + db_cursor = web.ctx.db_cursor() + web.ctx.db_execute(db_cursor, query) + + if not web.ctx.db_transaction: web.ctx.db.commit() + return db_cursor.rowcount + +def delete(table, where=None, using=None, vars=None, _test=False): + """ + Deletes from `table` with clauses `where` and `using`. + + >>> name = 'Joe' + >>> delete('foo', where='name = $name', vars=locals(), _test=True) + + """ + if vars is None: vars = {} + + if isinstance(where, (int, long)): + where = "id = " + sqlquote(where) + elif isinstance(where, (list, tuple)) and len(where) == 2: + where = SQLQuery(where[0], where[1]) + elif isinstance(where, SQLQuery): + pass + elif where is None: + pass + else: + where = reparam(where, vars) + + q = 'DELETE FROM ' + table + if where: + q += ' WHERE ' + where + if using and web.ctx.get('db_name') != "firebird": + q += ' USING ' + sqllist(using) + + if _test: return q + + db_cursor = web.ctx.db_cursor() + web.ctx.db_execute(db_cursor, q) + + if not web.ctx.db_transaction: web.ctx.db.commit() + return db_cursor.rowcount + +if __name__ == "__main__": + import doctest + doctest.testmod() ============================================================ --- web/debugerror.py dbd19a03c29519c7f45301222c73fc3b36053362 +++ web/debugerror.py dbd19a03c29519c7f45301222c73fc3b36053362 @@ -0,0 +1,316 @@ +""" +pretty debug errors +(part of web.py) + +adapted from Django +Copyright (c) 2005, the Lawrence Journal-World +Used under the modified BSD license: +http://www.xfree86.org/3.3.6/COPYRIGHT2.html#5 +""" + +__all__ = ["debugerror", "djangoerror"] + +import sys, urlparse, pprint +from net import websafe +from template import Template +import webapi as web + +import os, os.path +whereami = os.path.join(os.getcwd(), __file__) +whereami = os.path.sep.join(whereami.split(os.path.sep)[:-1]) +djangoerror_t = """\ +$def with (exception_type, exception_value, frames) + + + + + + $exception_type at $ctx.path + + + + + +
+

$exception_type at $ctx.path

+

$exception_value

+ + + + + + +
Python$frames[0].filename in $frames[0].function, line $frames[0].lineno
Web$ctx.method $ctx.home$ctx.path
+
+
+

Traceback (innermost first)

+
    +$for frame in frames: +
  • + $frame.filename in $frame.function + $if frame.context_line: +
    + $if frame.pre_context: +
      + $for line in frame.pre_context: +
    1. $line
      2. +
    +
    1. $frame.context_line ...
    + $if frame.post_context: +
      + $for line in frame.post_context: +
    1. $line
      2. +
    +
    + + $if frame.vars: +
    + ▶ Local vars + $# $inspect.formatargvalues(*inspect.getargvalues(frame['tb'].tb_frame)) +
    + $:dicttable(frame.vars, kls='vars', id=('v' + str(frame.id))) +
    • +
+
+ +
+$if ctx.output or ctx.headers: +

Response so far

+

HEADERS

+

+ $for kv in ctx.headers: + $kv[0]: $kv[1]
+ $else: + [no headers] +

+ +

BODY

+

+ $ctx.output +

+ +

Request information

+ +

INPUT

+$:dicttable(web.input()) + + +$:dicttable(web.cookies()) + +

META

+$ newctx = [] +$# ) and (k not in ['env', 'output', 'headers', 'environ', 'status', 'db_execute']): +$for k, v in ctx.iteritems(): + $if not k.startswith('_') and (k in x): + $newctx.append(kv) +$:dicttable(dict(newctx)) + +

ENVIRONMENT

+$:dicttable(ctx.env) +
+ +
+

+ You're seeing this error because you have web.internalerror + set to web.debugerror. Change that if you want a different one. +

+
+ + + +""" + +dicttable_t = r"""$def with (d, kls='req', id=None) +$if d: + + + $ temp = d.items() + $temp.sort() + $for kv in temp: + + +
VariableValue
$kv[0]
$prettify(kv[1])
+$else: +

No data.

+""" + +dicttable_r = Template(dicttable_t, filter=websafe) +djangoerror_r = Template(djangoerror_t, filter=websafe) + +def djangoerror(): + def _get_lines_from_file(filename, lineno, context_lines): + """ + Returns context_lines before and after lineno from file. + Returns (pre_context_lineno, pre_context, context_line, post_context). + """ + try: + source = open(filename).readlines() + lower_bound = max(0, lineno - context_lines) + upper_bound = lineno + context_lines + + pre_context = \ + [line.strip('\n') for line in source[lower_bound:lineno]] + context_line = source[lineno].strip('\n') + post_context = \ + [line.strip('\n') for line in source[lineno + 1:upper_bound]] + + return lower_bound, pre_context, context_line, post_context + except (OSError, IOError): + return None, [], None, [] + + exception_type, exception_value, tback = sys.exc_info() + frames = [] + while tback is not None: + filename = tback.tb_frame.f_code.co_filename + function = tback.tb_frame.f_code.co_name + lineno = tback.tb_lineno - 1 + pre_context_lineno, pre_context, context_line, post_context = \ + _get_lines_from_file(filename, lineno, 7) + frames.append(web.storage({ + 'tback': tback, + 'filename': filename, + 'function': function, + 'lineno': lineno, + 'vars': tback.tb_frame.f_locals, + 'id': id(tback), + 'pre_context': pre_context, + 'context_line': context_line, + 'post_context': post_context, + 'pre_context_lineno': pre_context_lineno, + })) + tback = tback.tb_next + frames.reverse() + urljoin = urlparse.urljoin + def prettify(x): + try: + out = pprint.pformat(x) + except Exception, e: + out = '[could not display: <' + e.__class__.__name__ + \ + ': '+str(e)+'>]' + return out + dt = dicttable_r + dt.globals = {'prettify': prettify} + t = djangoerror_r + t.globals = {'ctx': web.ctx, 'web':web, 'dicttable':dt, 'dict':dict, 'str':str} + return t(exception_type, exception_value, frames) + +def debugerror(): + """ + A replacement for `internalerror` that presents a nice page with lots + of debug information for the programmer. + + (Based on the beautiful 500 page from [Django](http://djangoproject.com/), + designed by [Wilson Miner](http://wilsonminer.com/).) + """ + + web.ctx.headers = [('Content-Type', 'text/html')] + web.ctx.output = djangoerror() + +if __name__ == "__main__": + urls = ( + '/', 'index' + ) + + class index: + def GET(self): + thisdoesnotexist + + web.internalerror = web.debugerror + web.run(urls) \ No newline at end of file ============================================================ --- web/form.py 5d23f4c195cb1d0b51d23bc6088630c0cfb9fa0d +++ web/form.py 5d23f4c195cb1d0b51d23bc6088630c0cfb9fa0d @@ -0,0 +1,215 @@ +""" +HTML forms +(part of web.py) +""" + +import copy, re +import webapi as web +import utils, net + +def attrget(obj, attr, value=None): + if hasattr(obj, 'has_key') and obj.has_key(attr): return obj[attr] + if hasattr(obj, attr): return getattr(obj, attr) + return value + +class Form: + def __init__(self, *inputs, **kw): + self.inputs = inputs + self.valid = True + self.note = None + self.validators = kw.pop('validators', []) + + def __call__(self, x=None): + o = copy.deepcopy(self) + if x: o.validates(x) + return o + + def render(self): + out = '' + out += self.rendernote(self.note) + out += '\n' + for i in self.inputs: + out += ' ' % (i.id, i.description) + out += "" + out += '\n' % (i.id, self.rendernote(i.note)) + out += "
"+i.pre+i.render()+i.post+"%s
" + return out + + def rendernote(self, note): + if note: return '%s' % note + else: return "" + + def validates(self, source=None, _validate=True, **kw): + source = source or kw or web.input() + out = True + for i in self.inputs: + v = attrget(source, i.name) + if _validate: + out = i.validate(v) and out + else: + i.value = v + if _validate: + out = out and self._validate(source) + self.valid = out + return out + + def _validate(self, value): + self.value = value + for v in self.validators: + if not v.valid(value): + self.note = v.msg + return False + return True + + def fill(self, source=None, **kw): + return self.validates(source, _validate=False, **kw) + + def __getitem__(self, i): + for x in self.inputs: + if x.name == i: return x + raise KeyError, i + + def _get_d(self): #@@ should really be form.attr, no? + return utils.storage([(i.name, i.value) for i in self.inputs]) + d = property(_get_d) + +class Input(object): + def __init__(self, name, *validators, **attrs): + self.description = attrs.pop('description', name) + self.value = attrs.pop('value', None) + self.pre = attrs.pop('pre', "") + self.post = attrs.pop('post', "") + self.id = attrs.setdefault('id', name) + if 'class_' in attrs: + attrs['class'] = attrs['class_'] + del attrs['class_'] + self.name, self.validators, self.attrs, self.note = name, validators, attrs, None + + def validate(self, value): + self.value = value + for v in self.validators: + if not v.valid(value): + self.note = v.msg + return False + return True + + def render(self): raise NotImplementedError + + def addatts(self): + str = "" + for (n, v) in self.attrs.items(): + str += ' %s="%s"' % (n, net.websafe(v)) + return str + +#@@ quoting + +class Textbox(Input): + def render(self): + x = '' + +class Checkbox(Input): + def render(self): + x = 'moved permanently') + +def found(url): + """A `302 Found` redirect.""" + return redirect(url, '302 Found') + +def seeother(url): + """A `303 See Other` redirect.""" + return redirect(url, '303 See Other') + +def tempredirect(url): + """A `307 Temporary Redirect` redirect.""" + return redirect(url, '307 Temporary Redirect') + +def write(cgi_response): + """ + Converts a standard CGI-style string response into `header` and + `output` calls. + """ + cgi_response = str(cgi_response) + cgi_response.replace('\r\n', '\n') + head, body = cgi_response.split('\n\n', 1) + lines = head.split('\n') + + for line in lines: + if line.isspace(): + continue + hdr, value = line.split(":", 1) + value = value.strip() + if hdr.lower() == "status": + web.ctx.status = value + else: + web.header(hdr, value) + + web.output(body) + +def urlencode(query): + """ + Same as urllib.urlencode, but supports unicode strings. + + >>> urlencode({'text':'foo bar'}) + 'text=foo+bar' + """ + query = dict([(k, utils.utf8(v)) for k, v in query.items()]) + return urllib.urlencode(query) + +def changequery(query=None, **kw): + """ + Imagine you're at `/foo?a=1&b=2`. Then `changequery(a=3)` will return + `/foo?a=3&b=2` -- the same URL but with the arguments you requested + changed. + """ + if query is None: + query = web.input(_method='get') + for k, v in kw.iteritems(): + if v is None: + query.pop(k, None) + else: + query[k] = v + out = web.ctx.path + if query: + out += '?' + urlencode(query) + return out + +def url(path=None, **kw): + """ + Makes url by concatinating web.ctx.homepath and path and the + query string created using the arguments. + """ + if path is None: + path = web.ctx.path + if path.startswith("/"): + out = web.ctx.homepath + path + else: + out = path + + if kw: + out += '?' + urlencode(kw) + + return out + +def background(func): + """A function decorator to run a long-running function as a background thread.""" + def internal(*a, **kw): + web.data() # cache it + + tmpctx = web._context[threading.currentThread()] + web._context[threading.currentThread()] = utils.storage(web.ctx.copy()) + + def newfunc(): + web._context[threading.currentThread()] = tmpctx + func(*a, **kw) + myctx = web._context[threading.currentThread()] + for k in myctx.keys(): + if k not in ['status', 'headers', 'output']: + try: del myctx[k] + except KeyError: pass + + t = threading.Thread(target=newfunc) + background.threaddb[id(t)] = t + t.start() + web.ctx.headers = [] + return seeother(changequery(_t=id(t))) + return internal +background.threaddb = {} + +def backgrounder(func): + def internal(*a, **kw): + i = web.input(_method='get') + if '_t' in i: + try: + t = background.threaddb[int(i._t)] + except KeyError: + return web.notfound() + web._context[threading.currentThread()] = web._context[t] + return + else: + return func(*a, **kw) + return internal + +class Reloader: + """ + Before every request, checks to see if any loaded modules have changed on + disk and, if so, reloads them. + """ + def __init__(self, func): + self.func = func + self.mtimes = {} + # cheetah: + # b = _compiletemplate.bases + # _compiletemplate = globals()['__compiletemplate'] + # _compiletemplate.bases = b + + web.loadhooks['reloader'] = self.check + # todo: + # - replace relrcheck with a loadhook + #if reloader in middleware: + # relr = reloader(None) + # relrcheck = relr.check + # middleware.remove(reloader) + #else: + # relr = None + # relrcheck = lambda: None + # if relr: + # relr.func = wsgifunc + # return wsgifunc + # + + + def check(self): + for mod in sys.modules.values(): + try: + mtime = os.stat(mod.__file__).st_mtime + except (AttributeError, OSError, IOError): + continue + if mod.__file__.endswith('.pyc') and \ + os.path.exists(mod.__file__[:-1]): + mtime = max(os.stat(mod.__file__[:-1]).st_mtime, mtime) + if mod not in self.mtimes: + self.mtimes[mod] = mtime + elif self.mtimes[mod] < mtime: + try: + reload(mod) + self.mtimes[mod] = mtime + except ImportError: + pass + return True + + def __call__(self, e, o): + self.check() + return self.func(e, o) + +reloader = Reloader + +def profiler(app): + """Outputs basic profiling information at the bottom of each response.""" + from utils import profile + def profile_internal(e, o): + out, result = profile(app)(e, o) + return out + ['
' + net.websafe(result) + '
'] + return profile_internal + +if __name__ == "__main__": + import doctest + doctest.testmod() ============================================================ --- web/httpserver.py ae3d4631644bf59c612f4aca85feb43252fa784e +++ web/httpserver.py ae3d4631644bf59c612f4aca85feb43252fa784e @@ -0,0 +1,224 @@ +__all__ = ["runsimple"] + +import sys, os +import webapi as web +import net + +def runbasic(func, server_address=("0.0.0.0", 8080)): + """ + Runs a simple HTTP server hosting WSGI app `func`. The directory `static/` + is hosted statically. + + Based on [WsgiServer][ws] from [Colin Stewart][cs]. + + [ws]: http://www.owlfish.com/software/wsgiutils/documentation/wsgi-server-api.html + [cs]: http://www.owlfish.com/ + """ + # Copyright (c) 2004 Colin Stewart (http://www.owlfish.com/) + # Modified somewhat for simplicity + # Used under the modified BSD license: + # http://www.xfree86.org/3.3.6/COPYRIGHT2.html#5 + + import SimpleHTTPServer, SocketServer, BaseHTTPServer, urlparse + import socket, errno + import traceback + + class WSGIHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def run_wsgi_app(self): + protocol, host, path, parameters, query, fragment = \ + urlparse.urlparse('http://dummyhost%s' % self.path) + + # we only use path, query + env = {'wsgi.version': (1, 0) + ,'wsgi.url_scheme': 'http' + ,'wsgi.input': self.rfile + ,'wsgi.errors': sys.stderr + ,'wsgi.multithread': 1 + ,'wsgi.multiprocess': 0 + ,'wsgi.run_once': 0 + ,'REQUEST_METHOD': self.command + ,'REQUEST_URI': self.path + ,'PATH_INFO': path + ,'QUERY_STRING': query + ,'CONTENT_TYPE': self.headers.get('Content-Type', '') + ,'CONTENT_LENGTH': self.headers.get('Content-Length', '') + ,'REMOTE_ADDR': self.client_address[0] + ,'SERVER_NAME': self.server.server_address[0] + ,'SERVER_PORT': str(self.server.server_address[1]) + ,'SERVER_PROTOCOL': self.request_version + } + + for http_header, http_value in self.headers.items(): + env ['HTTP_%s' % http_header.replace('-', '_').upper()] = \ + http_value + + # Setup the state + self.wsgi_sent_headers = 0 + self.wsgi_headers = [] + + try: + # We have there environment, now invoke the application + result = self.server.app(env, self.wsgi_start_response) + try: + try: + for data in result: + if data: + self.wsgi_write_data(data) + finally: + if hasattr(result, 'close'): + result.close() + except socket.error, socket_err: + # Catch common network errors and suppress them + if (socket_err.args[0] in \ + (errno.ECONNABORTED, errno.EPIPE)): + return + except socket.timeout, socket_timeout: + return + except: + print >> web.debug, traceback.format_exc(), + + if (not self.wsgi_sent_headers): + # We must write out something! + self.wsgi_write_data(" ") + return + + do_POST = run_wsgi_app + do_PUT = run_wsgi_app + do_DELETE = run_wsgi_app + + def do_GET(self): + if self.path.startswith('/static/'): + SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + else: + self.run_wsgi_app() + + def wsgi_start_response(self, response_status, response_headers, + exc_info=None): + if (self.wsgi_sent_headers): + raise Exception \ + ("Headers already sent and start_response called again!") + # Should really take a copy to avoid changes in the application.... + self.wsgi_headers = (response_status, response_headers) + return self.wsgi_write_data + + def wsgi_write_data(self, data): + if (not self.wsgi_sent_headers): + status, headers = self.wsgi_headers + # Need to send header prior to data + status_code = status[:status.find(' ')] + status_msg = status[status.find(' ') + 1:] + self.send_response(int(status_code), status_msg) + for header, value in headers: + self.send_header(header, value) + self.end_headers() + self.wsgi_sent_headers = 1 + # Send the data + self.wfile.write(data) + + class WSGIServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): + def __init__(self, func, server_address): + BaseHTTPServer.HTTPServer.__init__(self, + server_address, + WSGIHandler) + self.app = func + self.serverShuttingDown = 0 + + print "http://%s:%d/" % server_address + WSGIServer(func, server_address).serve_forever() + +def runsimple(func, server_address=("0.0.0.0", 8080)): + """ + Runs [CherryPy][cp] WSGI server hosting WSGI app `func`. + The directory `static/` is hosted statically. + + [cp]: http://www.cherrypy.org + """ + from wsgiserver import CherryPyWSGIServer + from SimpleHTTPServer import SimpleHTTPRequestHandler + from BaseHTTPServer import BaseHTTPRequestHandler + + class StaticApp(SimpleHTTPRequestHandler): + """WSGI application for serving static files.""" + def __init__(self, environ, start_response): + self.headers = [] + self.environ = environ + self.start_response = start_response + + def send_response(self, status, msg=""): + self.status = str(status) + " " + msg + + def send_header(self, name, value): + self.headers.append((name, value)) + + def end_headers(self): + pass + + def log_message(*a): pass + + def __iter__(self): + environ = self.environ + + self.path = environ.get('PATH_INFO', '') + self.client_address = environ.get('REMOTE_ADDR','-'), \ + environ.get('REMOTE_PORT','-') + self.command = environ.get('REQUEST_METHOD', '-') + + from cStringIO import StringIO + self.wfile = StringIO() # for capturing error + + f = self.send_head() + self.start_response(self.status, self.headers) + + if f: + block_size = 16 * 1024 + while True: + buf = f.read(block_size) + if not buf: + break + yield buf + f.close() + else: + value = self.wfile.getvalue() + yield value + + class WSGIWrapper(BaseHTTPRequestHandler): + """WSGI wrapper for logging the status and serving static files.""" + def __init__(self, app): + self.app = app + self.format = '%s - - [%s] "%s %s %s" - %s' + + def __call__(self, environ, start_response): + def xstart_response(status, response_headers, *args): + write = start_response(status, response_headers, *args) + self.log(status, environ) + return write + + path = environ.get('PATH_INFO', '') + if path.startswith('/static/'): + return StaticApp(environ, xstart_response) + else: + return self.app(environ, xstart_response) + + def log(self, status, environ): + outfile = environ.get('wsgi.errors', web.debug) + req = environ.get('PATH_INFO', '_') + protocol = environ.get('ACTUAL_SERVER_PROTOCOL', '-') + method = environ.get('REQUEST_METHOD', '-') + host = "%s:%s" % (environ.get('REMOTE_ADDR','-'), + environ.get('REMOTE_PORT','-')) + + #@@ It is really bad to extend from + #@@ BaseHTTPRequestHandler just for this method + time = self.log_date_time_string() + + print >> outfile, self.format % (host, time, protocol, + method, req, status) + + func = WSGIWrapper(func) + server = CherryPyWSGIServer(server_address, func, server_name="localhost") + + print "http://%s:%d/" % server_address + try: + server.start() + except KeyboardInterrupt: + server.stop() ============================================================ --- web/net.py ac59560505e4c2998eb9b579b98e63cecada37d8 +++ web/net.py ac59560505e4c2998eb9b579b98e63cecada37d8 @@ -0,0 +1,155 @@ +""" +Network Utilities +(from web.py) +""" + +__all__ = [ + "validipaddr", "validipport", "validip", "validaddr", + "urlquote", + "httpdate", "parsehttpdate", + "htmlquote", "websafe", +] + +import urllib, time +try: import datetime +except ImportError: pass + +def validipaddr(address): + """returns True if `address` is a valid IPv4 address""" + try: + octets = address.split('.') + assert len(octets) == 4 + for x in octets: + assert 0 <= int(x) <= 255 + except (AssertionError, ValueError): + return False + return True + +def validipport(port): + """returns True if `port` is a valid IPv4 port""" + try: + assert 0 <= int(port) <= 65535 + except (AssertionError, ValueError): + return False + return True + +def validip(ip, defaultaddr="0.0.0.0", defaultport=8080): + """returns `(ip_address, port)` from string `ip_addr_port`""" + addr = defaultaddr + port = defaultport + + ip = ip.split(":", 1) + if len(ip) == 1: + if not ip[0]: + pass + elif validipaddr(ip[0]): + addr = ip[0] + elif validipport(ip[0]): + port = int(ip[0]) + else: + raise ValueError, ':'.join(ip) + ' is not a valid IP address/port' + elif len(ip) == 2: + addr, port = ip + if not validipaddr(addr) and validipport(port): + raise ValueError, ':'.join(ip) + ' is not a valid IP address/port' + port = int(port) + else: + raise ValueError, ':'.join(ip) + ' is not a valid IP address/port' + return (addr, port) + +def validaddr(string_): + """ + returns either (ip_address, port) or "/path/to/socket" from string_ + + >>> validaddr('/path/to/socket') + '/path/to/socket' + >>> validaddr('8000') + ('0.0.0.0', 8000) + >>> validaddr('127.0.0.1') + ('127.0.0.1', 8080) + >>> validaddr('127.0.0.1:8000') + ('127.0.0.1', 8000) + >>> validaddr('fff') + Traceback (most recent call last): + ... + ValueError: fff is not a valid IP address/port + """ + if '/' in string_: + return string_ + else: + return validip(string_) + +def urlquote(val): + """ + Quotes a string for use in a URL. + + >>> urlquote('://?f=1&j=1') + '%3A//%3Ff%3D1%26j%3D1' + >>> urlquote(None) + '' + >>> urlquote(u'\u203d') + '%E2%80%BD' + """ + if val is None: return '' + if not isinstance(val, unicode): val = str(val) + else: val = val.encode('utf-8') + return urllib.quote(val) + +def httpdate(date_obj): + """ + Formats a datetime object for use in HTTP headers. + + >>> import datetime + >>> httpdate(datetime.datetime(1970, 1, 1, 1, 1, 1)) + 'Thu, 01 Jan 1970 01:01:01 GMT' + """ + return date_obj.strftime("%a, %d %b %Y %H:%M:%S GMT") + +def parsehttpdate(string_): + """ + Parses an HTTP date into a datetime object. + + >>> parsehttpdate('Thu, 01 Jan 1970 01:01:01 GMT') + datetime.datetime(1970, 1, 1, 1, 1, 1) + """ + try: + t = time.strptime(string_, "%a, %d %b %Y %H:%M:%S %Z") + except ValueError: + return None + return datetime.datetime(*t[:6]) + +def htmlquote(text): + """ + Encodes `text` for raw use in HTML. + + >>> htmlquote("<'&\\">") + '<'&">' + """ + text = text.replace("&", "&") # Must be done first! + text = text.replace("<", "<") + text = text.replace(">", ">") + text = text.replace("'", "'") + text = text.replace('"', """) + return text + +def websafe(val): + """ + Converts `val` so that it's safe for use in UTF-8 HTML. + + >>> websafe("<'&\\">") + '<'&">' + >>> websafe(None) + '' + >>> websafe(u'\u203d') + '\\xe2\\x80\\xbd' + """ + if val is None: + return '' + if isinstance(val, unicode): + val = val.encode('utf-8') + val = str(val) + return htmlquote(val) + +if __name__ == "__main__": + import doctest + doctest.testmod() ============================================================ --- web/request.py d84d3ee98b7786434cdbda3403860489376b1b7c +++ web/request.py d84d3ee98b7786434cdbda3403860489376b1b7c @@ -0,0 +1,153 @@ +""" +Request Delegation +(from web.py) +""" + +__all__ = ["handle", "nomethod", "autodelegate", "webpyfunc", "run"] + +import sys, re, types, os.path, urllib + +import http, wsgi, utils, webapi +import webapi as web + +def handle(mapping, fvars=None): + """ + Call the appropriate function based on the url to function mapping in `mapping`. + If no module for the function is specified, look up the function in `fvars`. If + `fvars` is empty, using the caller's context. + + `mapping` should be a tuple of paired regular expressions with function name + substitutions. `handle` will import modules as necessary. + """ + for url, ofno in utils.group(mapping, 2): + if isinstance(ofno, tuple): + ofn, fna = ofno[0], list(ofno[1:]) + else: + ofn, fna = ofno, [] + fn, result = utils.re_subm('^' + url + '$', ofn, web.ctx.path) + if result: # it's a match + if fn.split(' ', 1)[0] == "redirect": + url = fn.split(' ', 1)[1] + if web.ctx.method == "GET": + x = web.ctx.env.get('QUERY_STRING', '') + if x: + url += '?' + x + return http.redirect(url) + elif '.' in fn: + x = fn.split('.') + mod, cls = '.'.join(x[:-1]), x[-1] + mod = __import__(mod, globals(), locals(), [""]) + cls = getattr(mod, cls) + else: + cls = fn + mod = fvars + if isinstance(mod, types.ModuleType): + mod = vars(mod) + try: + cls = mod[cls] + except KeyError: + return web.notfound() + + meth = web.ctx.method + if meth == "HEAD": + if not hasattr(cls, meth): + meth = "GET" + if not hasattr(cls, meth): + return nomethod(cls) + tocall = getattr(cls(), meth) + args = list(result.groups()) + for d in re.findall(r'\\(\d+)', ofn): + args.pop(int(d) - 1) + return tocall(*([x and urllib.unquote(x) for x in args] + fna)) + + return web.notfound() + +def nomethod(cls): + """Returns a `405 Method Not Allowed` error for `cls`.""" + web.ctx.status = '405 Method Not Allowed' + web.header('Content-Type', 'text/html') + web.header('Allow', \ + ', '.join([method for method in \ + ['GET', 'HEAD', 'POST', 'PUT', 'DELETE'] \ + if hasattr(cls, method)])) + + # commented out for the same reason redirect is + # return output('method not allowed') + +def autodelegate(prefix=''): + """ + Returns a method that takes one argument and calls the method named prefix+arg, + calling `notfound()` if there isn't one. Example: + + urls = ('/prefs/(.*)', 'prefs') + + class prefs: + GET = autodelegate('GET_') + def GET_password(self): pass + def GET_privacy(self): pass + + `GET_password` would get called for `/prefs/password` while `GET_privacy` for + `GET_privacy` gets called for `/prefs/privacy`. + + If a user visits `/prefs/password/change` then `GET_password(self, '/change')` + is called. + """ + def internal(self, arg): + if '/' in arg: + first, rest = arg.split('/', 1) + func = prefix + first + args = ['/' + rest] + else: + func = prefix + arg + args = [] + + if hasattr(self, func): + try: + return getattr(self, func)(*args) + except TypeError: + return web.notfound() + else: + return web.notfound() + return internal + +def webpyfunc(inp, fvars, autoreload=False): + """If `inp` is a url mapping, returns a function that calls handle.""" + if not hasattr(inp, '__call__'): + if autoreload: + def modname(): + """find name of the module name from fvars.""" + file, name = fvars['__file__'], fvars['__name__'] + if name == '__main__': + # Since the __main__ module can't be reloaded, the module has + # to be imported using its file name. + name = os.path.splitext(os.path.basename(file))[0] + return name + + mod = __import__(modname(), None, None, [""]) + #@@probably should replace this with some inspect magic + name = utils.dictfind(fvars, inp) + func = lambda: handle(getattr(mod, name), mod) + else: + func = lambda: handle(inp, fvars) + else: + func = inp + return func + +def run(inp, fvars, *middleware): + """ + Starts handling requests. If called in a CGI or FastCGI context, it will follow + that protocol. If called from the command line, it will start an HTTP + server on the port named in the first command line argument, or, if there + is no argument, on port 8080. + + `input` is a callable, then it's called with no arguments. + Otherwise, it's a `mapping` object to be passed to `handle(...)`. + + **Caveat:** So that `reloader` will work correctly, input has to be a variable, + it can't be a tuple passed in directly. + + `middleware` is a list of WSGI middleware which is applied to the resulting WSGI + function. + """ + autoreload = http.reloader in middleware + return wsgi.runwsgi(webapi.wsgifunc(webpyfunc(inp, fvars, autoreload), *middleware)) ============================================================ --- web/template.py acc5f5166520ac922aeb052cc63cd0bd6cc4964f +++ web/template.py acc5f5166520ac922aeb052cc63cd0bd6cc4964f @@ -0,0 +1,878 @@ +""" +simple, elegant templating +(part of web.py) +""" + +import re, glob, os, os.path +from types import FunctionType as function +from utils import storage, group, utf8 +from net import websafe + +# differences from python: +# - for: has an optional else: that gets called if the loop never runs +# differences to add: +# - you can use the expression inside if, while blocks +# - special for loop attributes, like django? +# - you can check to see if a variable is defined (perhaps w/ get func?) +# all these are probably good ideas for python... + +# todo: +# inline tuple +# relax constraints on spacing +# continue, break, etc. +# tracebacks + +global_globals = {'None':None, 'False':False, 'True': True} +MAX_ITERS = 100000 + +WHAT = 0 +ARGS = 4 +KWARGS = 6 +NAME = 2 +BODY = 4 +CLAUSE = 2 +ELIF = 6 +ELSE = 8 +IN = 6 +NAME = 2 +EXPR = 4 +FILTER = 4 +THING = 2 +ATTR = 4 +ITEM = 4 +NEGATE = 4 +X = 2 +OP = 4 +Y = 6 +LINENO = -1 + +# http://docs.python.org/ref/identifiers.html +r_var = '[a-zA-Z_][a-zA-Z0-9_]*' + +class ParseError(Exception): pass +class Parser: + def __init__(self, text, name=""): + self.t = text + self.p = 0 + self._lock = [False] + self.name = name + + def lock(self): + self._lock[-1] = True + + def curline(self): + return self.t[:self.p].count('\n')+1 + + def csome(self): + return repr(self.t[self.p:self.p+5]+'...') + + def Error(self, x, y=None): + if y is None: y = self.csome() + raise ParseError, "%s: expected %s, got %s (line %s)" % (self.name, x, y, self.curline()) + + def q(self, f): + def internal(*a, **kw): + checkp = self.p + self._lock.append(False) + try: + q = f(*a, **kw) + except ParseError: + if self._lock[-1]: + raise + self.p = checkp + self._lock.pop() + return False + self._lock.pop() + return q or True + return internal + + def tokr(self, t): + text = self.c(len(t)) + if text != t: + self.Error(repr(t), repr(text)) + return t + + def ltokr(self, *l): + for x in l: + o = self.tokq(x) + if o: return o + self.Error('one of '+repr(l)) + + def rer(self, r): + x = re.match(r, self.t[self.p:]) #@@re_compile + if not x: + self.Error('r'+repr(r)) + return self.tokr(x.group()) + + def endr(self): + if self.p != len(self.t): + self.Error('EOF') + + def c(self, n=1): + out = self.t[self.p:self.p+n] + if out == '' and n != 0: + self.Error('character', 'EOF') + self.p += n + return out + + def lookbehind(self, t): + return self.t[self.p-len(t):self.p] == t + + def __getattr__(self, a): + if a.endswith('q'): + return self.q(getattr(self, a[:-1]+'r')) + raise AttributeError, a + +class TemplateParser(Parser): + def __init__(self, *a, **kw): + Parser.__init__(self, *a, **kw) + self.curws = '' + self.curind = '' + + def o(self, *a): + return a+('lineno', self.curline()) + + def go(self): + # maybe try to do some traceback parsing/hacking + return self.gor() + + def gor(self): + header = self.defwithq() + results = self.lines(start=True) + self.endr() + return header, results + + def ws(self): + n = 0 + while self.tokq(" "): n += 1 + return " " * n + + def defwithr(self): + self.tokr('$def with ') + self.lock() + self.tokr('(') + args = [] + kw = [] + x = self.req(r_var) + while x: + if self.tokq('='): + v = self.exprr() + kw.append((x, v)) + else: + args.append(x) + x = self.tokq(', ') and self.req(r_var) + self.tokr(')\n') + return self.o('defwith', 'null', None, 'args', args, 'kwargs', kw) + + def literalr(self): + o = ( + self.req('"[^"]*"') or #@@ no support for escapes + self.req("'[^']*'") + ) + if o is False: + o = self.req('\-?[0-9]+(\.[0-9]*)?') + if o is not False: + if '.' in o: o = float(o) + else: o = int(o) + + if o is False: self.Error('literal') + return self.o('literal', 'thing', o) + + def listr(self): + self.tokr('[') + self.lock() + x = [] + if not self.tokq(']'): + while True: + t = self.exprr() + x.append(t) + if not self.tokq(', '): break + self.tokr(']') + return self.o('list', 'thing', x) + + def dictr(self): + self.tokr('{') + self.lock() + x = {} + if not self.tokq('}'): + while True: + k = self.exprr() + self.tokr(': ') + v = self.exprr() + x[k] = v + if not self.tokq(', '): break + self.tokr('}') + return self.o('dict', 'thing', x) + + def parenr(self): + self.tokr('(') + self.lock() + o = self.exprr() # todo: allow list + self.tokr(')') + return self.o('paren', 'thing', o) + + def atomr(self): + """returns var, literal, paren, dict, or list""" + o = ( + self.varq() or + self.parenq() or + self.dictq() or + self.listq() or + self.literalq() + ) + if o is False: self.Error('atom') + return o + + def primaryr(self): + """returns getattr, call, or getitem""" + n = self.atomr() + while 1: + if self.tokq('.'): + v = self.req(r_var) + if not v: + self.p -= 1 # get rid of the '.' + break + else: + n = self.o('getattr', 'thing', n, 'attr', v) + elif self.tokq('('): + args = [] + kw = [] + + while 1: + # need to see if we're doing a keyword argument + checkp = self.p + k = self.req(r_var) + if k and self.tokq('='): # yup + v = self.exprr() + kw.append((k, v)) + else: + self.p = checkp + x = self.exprq() + if x: # at least it's something + args.append(x) + else: + break + + if not self.tokq(', '): break + self.tokr(')') + n = self.o('call', 'thing', n, 'args', args, 'kwargs', kw) + elif self.tokq('['): + v = self.exprr() + self.tokr(']') + n = self.o('getitem', 'thing', n, 'item', v) + else: + break + + return n + + def exprr(self): + negate = self.tokq('not ') + x = self.primaryr() + if self.tokq(' '): + operator = self.ltokr('not in', 'in', 'is not', 'is', '==', '!=', '>=', '<=', '<', '>', 'and', 'or', '*', '+', '-', '/', '%') + self.tokr(' ') + y = self.exprr() + x = self.o('test', 'x', x, 'op', operator, 'y', y) + + return self.o('expr', 'thing', x, 'negate', negate) + + def varr(self): + return self.o('var', 'name', self.rer(r_var)) + + def liner(self): + out = [] + o = self.curws + while 1: + c = self.c() + self.lock() + if c == '\n': + self.p -= 1 + break + if c == '$': + if self.lookbehind('\\$'): + o = o[:-1] + c + else: + filter = not bool(self.tokq(':')) + + if self.tokq('{'): + out.append(o) + out.append(self.o('itpl', 'name', self.exprr(), 'filter', filter)) + self.tokr('}') + o = '' + else: + g = self.primaryq() + if g: + out.append(o) + out.append(self.o('itpl', 'name', g, 'filter', filter)) + o = '' + else: + o += c + else: + o += c + self.tokr('\n') + if not self.lookbehind('\\\n'): + o += '\n' + else: + o = o[:-1] + out.append(o) + return self.o('line', 'thing', out) + + def varsetr(self): + self.tokr('$var ') + self.lock() + what = self.rer(r_var) + self.tokr(':') + body = self.lines() + return self.o('varset', 'name', what, 'body', body) + + def ifr(self): + self.tokr("$if ") + self.lock() + expr = self.exprr() + self.tokr(":") + ifc = self.lines() + + elifs = [] + while self.tokq(self.curws + self.curind + '$elif '): + v = self.exprr() + self.tokr(':') + c = self.lines() + elifs.append(self.o('elif', 'clause', v, 'body', c)) + + if self.tokq(self.curws + self.curind + "$else:"): + elsec = self.lines() + else: + elsec = None + + return self.o('if', 'clause', expr, 'then', ifc, 'elif', elifs, 'else', elsec) + + def forr(self): + self.tokr("$for ") + self.lock() + v = self.setabler() + self.tokr(" in ") + g = self.exprr() + self.tokr(":") + l = self.lines() + + if self.tokq(self.curws + self.curind + '$else:'): + elsec = self.lines() + else: + elsec = None + + return self.o('for', 'name', v, 'body', l, 'in', g, 'else', elsec) + + def whiler(self): + self.tokr('$while ') + self.lock() + v = self.exprr() + self.tokr(":") + l = self.lines() + + if self.tokq(self.curws + self.curind + '$else:'): + elsec = self.lines() + else: + elsec = None + + return self.o('while', 'clause', v, 'body', l, 'null', None, 'else', elsec) + + def assignr(self): + self.tokr('$ ') + assign = self.rer(r_var) # NOTE: setable + self.tokr(' = ') + expr = self.exprr() + self.tokr('\n') + + return self.o('assign', 'name', assign, 'expr', expr) + + def commentr(self): + self.tokr('$#') + self.lock() + while self.c() != '\n': pass + return self.o('comment') + + def setabler(self): + out = [self.varr()] #@@ not quite right + while self.tokq(', '): + out.append(self.varr()) + return out + + def lines(self, start=False): + """ + This function gets called from two places: + 1. at the start, where it's matching the document itself + 2. after any command, where it matches one line or an indented block + """ + o = [] + if not start: # try to match just one line + singleline = self.tokq(' ') and self.lineq() + if singleline: + return [singleline] + else: + self.rer(' *') #@@slurp space? + self.tokr('\n') + oldind = self.curind + self.curind += ' ' + while 1: + oldws = self.curws + t = self.tokq(oldws + self.curind) + if not t: break + + self.curws += self.ws() + x = t and ( + self.varsetq() or + self.ifq() or + self.forq() or + self.whileq() or + self.assignq() or + self.commentq() or + self.lineq()) + self.curws = oldws + if not x: + break + elif x[WHAT] == 'comment': + pass + else: + o.append(x) + + if not start: self.curind = oldind + return o + +class Stowage(storage): + def __str__(self): return self.get('_str') + #@@ edits in place + def __add__(self, other): + if isinstance(other, (unicode, str)): + self._str += other + return self + else: + raise TypeError, 'cannot add' + def __radd__(self, other): + if isinstance(other, (unicode, str)): + self._str = other + self._str + return self + else: + raise TypeError, 'cannot add' + +class WTF(AssertionError): pass +class SecurityError(Exception): + """The template seems to be trying to do something naughty.""" + pass + + + + +Required = object() +class Template: + globals = {} + content_types = { + '.html' : 'text/html; charset=utf-8', + '.txt' : 'text/plain', + } + + def __init__(self, text, filter=None, filename=""): + self.filter = filter + self.filename = filename + # universal newlines: + text = text.replace('\r\n', '\n').replace('\r', '\n').expandtabs() + if not text.endswith('\n'): text += '\n' + header, tree = TemplateParser(text, filename).go() + self.tree = tree + if header: + self.h_defwith(header) + else: + self.args, self.kwargs = (), {} + + def __call__(self, *a, **kw): + d = self.globals.copy() + d.update(self._parseargs(a, kw)) + f = Fill(self.tree, d=d) + if self.filter: f.filter = self.filter + + import webapi as web + if 'headers' in web.ctx and self.filename: + content_type = self.find_content_type() + if content_type: + web.header('Content-Type', content_type, unique=True) + + return f.go() + + def find_content_type(self): + for ext, content_type in self.content_types.iteritems(): + if self.filename.endswith(ext): + return content_type + + def _parseargs(self, inargs, inkwargs): + # difference from Python: + # no error on setting a keyword arg twice + d = {} + for arg in self.args: + d[arg] = Required + for kw, val in self.kwargs: + d[kw] = val + + for n, val in enumerate(inargs): + if n < len(self.args): + d[self.args[n]] = val + elif n < len(self.args)+len(self.kwargs): + kw = self.kwargs[n - len(self.args)][0] + d[kw] = val + + for kw, val in inkwargs.iteritems(): + d[kw] = val + + unset = [] + for k, v in d.iteritems(): + if v is Required: + unset.append(k) + if unset: + raise TypeError, 'values for %s are required' % unset + + return d + + def h_defwith(self, header): + assert header[WHAT] == 'defwith' + f = Fill(self.tree, d={}) + + self.args = header[ARGS] + self.kwargs = [] + for var, valexpr in header[KWARGS]: + self.kwargs.append((var, f.h(valexpr))) + + def __repr__(self): + return "" % self.filename + +class Handle: + def __init__(self, parsetree, **kw): + self._funccache = {} + self.parsetree = parsetree + for (k, v) in kw.iteritems(): setattr(self, k, v) + + def h(self, item): + return getattr(self, 'h_' + item[WHAT])(item) + +class Fill(Handle): + builtins = global_globals + def filter(self, text): + if text is None: return '' + else: return utf8(text) + # often replaced with stuff like net.websafe + + def h_literal(self, i): + item = i[THING] + if isinstance(item, (unicode, str)) and item[0] in ['"', "'"]: + item = item[1:-1] + elif isinstance(item, (float, int)): + pass + return item + + def h_list(self, i): + x = i[THING] + out = [] + for item in x: + out.append(self.h(item)) + return out + + def h_dict(self, i): + x = i[THING] + out = {} + for k, v in x.iteritems(): + out[self.h(k)] = self.h(v) + return out + + def h_paren(self, i): + item = i[THING] + if isinstance(item, list): + raise NotImplementedError, 'tuples' + return self.h(item) + + def h_getattr(self, i): + thing, attr = i[THING], i[ATTR] + thing = self.h(thing) + if attr.startswith('_') or attr.startswith('func_') or attr.startswith('im_'): + raise SecurityError, 'tried to get ' + attr + try: + if thing in self.builtins: + raise SecurityError, 'tried to getattr on ' + repr(thing) + except TypeError: + pass # raised when testing an unhashable object + try: + return getattr(thing, attr) + except AttributeError: + if isinstance(thing, list) and attr == 'join': + return lambda s: s.join(thing) + else: + raise + + def h_call(self, i): + call = self.h(i[THING]) + args = [self.h(x) for x in i[ARGS]] + kw = dict([(x, self.h(y)) for (x, y) in i[KWARGS]]) + return call(*args, **kw) + + def h_getitem(self, i): + thing, item = i[THING], i[ITEM] + thing = self.h(thing) + item = self.h(item) + return thing[item] + + def h_expr(self, i): + item = self.h(i[THING]) + if i[NEGATE]: + item = not item + return item + + def h_test(self, item): + ox, op, oy = item[X], item[OP], item[Y] + # for short-circuiting to work, we can't eval these here + e = self.h + if op == 'is': + return e(ox) is e(oy) + elif op == 'is not': + return e(ox) is not e(oy) + elif op == 'in': + return e(ox) in e(oy) + elif op == 'not in': + return e(ox) not in e(oy) + elif op == '==': + return e(ox) == e(oy) + elif op == '!=': + return e(ox) != e(oy) + elif op == '>': + return e(ox) > e(oy) + elif op == '<': + return e(ox) < e(oy) + elif op == '<=': + return e(ox) <= e(oy) + elif op == '>=': + return e(ox) >= e(oy) + elif op == 'and': + return e(ox) and e(oy) + elif op == 'or': + return e(ox) or e(oy) + elif op == '+': + return e(ox) + e(oy) + elif op == '-': + return e(ox) - e(oy) + elif op == '*': + return e(ox) * e(oy) + elif op == '/': + return e(ox) / e(oy) + elif op == '%': + return e(ox) % e(oy) + else: + raise WTF, 'op ' + op + + def h_var(self, i): + v = i[NAME] + if v in self.d: + return self.d[v] + elif v in self.builtins: + return self.builtins[v] + elif v == 'self': + return self.output + else: + raise NameError, 'could not find %s (line %s)' % (repr(i[NAME]), i[LINENO]) + + def h_line(self, i): + out = [] + for x in i[THING]: + #@@ what if x is unicode + if isinstance(x, str): + out.append(x) + elif x[WHAT] == 'itpl': + o = self.h(x[NAME]) + if x[FILTER]: + o = self.filter(o) + else: + o = (o is not None and utf8(o)) or "" + out.append(o) + else: + raise WTF, x + return ''.join(out) + + def h_varset(self, i): + self.output[i[NAME]] = ''.join(self.h_lines(i[BODY])) + return '' + + def h_if(self, i): + expr = self.h(i[CLAUSE]) + if expr: + do = i[BODY] + else: + for e in i[ELIF]: + expr = self.h(e[CLAUSE]) + if expr: + do = e[BODY] + break + else: + do = i[ELSE] + return ''.join(self.h_lines(do)) + + def h_for(self, i): + out = [] + assert i[IN][WHAT] == 'expr' + invar = self.h(i[IN]) + forvar = i[NAME] + if invar: + for nv in invar: + if len(forvar) == 1: + fv = forvar[0] + assert fv[WHAT] == 'var' + self.d[fv[NAME]] = nv # same (lack of) scoping as Python + else: + for x, y in zip(forvar, nv): + assert x[WHAT] == 'var' + self.d[x[NAME]] = y + + out.extend(self.h_lines(i[BODY])) + else: + if i[ELSE]: + out.extend(self.h_lines(i[ELSE])) + return ''.join(out) + + def h_while(self, i): + out = [] + expr = self.h(i[CLAUSE]) + if not expr: + return ''.join(self.h_lines(i[ELSE])) + c = 0 + while expr: + c += 1 + if c >= MAX_ITERS: + raise RuntimeError, 'too many while-loop iterations (line %s)' % i[LINENO] + out.extend(self.h_lines(i[BODY])) + expr = self.h(i[CLAUSE]) + return ''.join(out) + + def h_assign(self, i): + self.d[i[NAME]] = self.h(i[EXPR]) + return '' + + def h_comment(self, i): pass + + def h_lines(self, lines): + if lines is None: return [] + return map(self.h, lines) + + def go(self): + self.output = Stowage() + self.output._str = ''.join(map(self.h, self.parsetree)) + if self.output.keys() == ['_str']: + self.output = self.output['_str'] + return self.output + +class render: + def __init__(self, loc='templates/', cache=True): + self.loc = loc + if cache: + self.cache = {} + else: + self.cache = False + + def _do(self, name, filter=None): + if self.cache is False or name not in self.cache: + + tmplpath = os.path.join(self.loc, name) + p = [f for f in glob.glob(tmplpath + '.*') if not f.endswith('~')] # skip backup files + if not p and os.path.isdir(tmplpath): + return render(tmplpath, cache=self.cache) + elif not p: + raise AttributeError, 'no template named ' + name + + p = p[0] + c = Template(open(p).read(), filename=p) + if self.cache is not False: self.cache[name] = (p, c) + + if self.cache is not False: p, c = self.cache[name] + + if p.endswith('.html') or p.endswith('.xml'): + if not filter: c.filter = websafe + return c + + def __getattr__(self, p): + return self._do(p) + +def frender(fn, *a, **kw): + return Template(open(fn).read(), *a, **kw) + +def test(): + import sys + verbose = '-v' in sys.argv + def assertEqual(a, b): + if a == b: + if verbose: + sys.stderr.write('.') + sys.stderr.flush() + else: + assert a == b, "\nexpected: %s\ngot: %s" % (repr(b), repr(a)) + + from utils import storage, group + + class t: + def __init__(self, text): + self.text = text + + def __call__(self, *a, **kw): + return TestResult(self.text, Template(self.text)(*a, **kw)) + + class TestResult: + def __init__(self, source, value): + self.source = source + self.value = value + + def __eq__(self, other): + if self.value == other: + if verbose: + sys.stderr.write('.') + else: + print >> sys.stderr, 'FAIL:', repr(self.source), 'expected', repr(other), ', got', repr(self.value) + sys.stderr.flush() + + t('1')() == '1\n' + t('$def with ()\n1')() == '1\n' + t('$def with (a)\n$a')(1) == '1\n' + t('$def with (a=0)\n$a')(1) == '1\n' + t('$def with (a=0)\n$a')(a=1) == '1\n' + t('$if 1: 1')() == '1\n' + t('$if 1:\n 1')() == '1\n' + t('$if 0: 0\n$elif 1: 1')() == '1\n' + t('$if 0: 0\n$elif None: 0\n$else: 1')() == '1\n' + t('$if (0 < 1) and (1 < 2): 1')() == '1\n' + t('$for x in [1, 2, 3]: $x')() == '1\n2\n3\n' + t('$for x in []: 0\n$else: 1')() == '1\n' + t('$def with (a)\n$while a and a.pop(): 1')([1, 2, 3]) == '1\n1\n1\n' + t('$while 0: 0\n$else: 1')() == '1\n' + t('$ a = 1\n$a')() == '1\n' + t('$# 0')() == '' + t('$def with (d)\n$for k, v in d.iteritems(): $k')({1: 1}) == '1\n' + t('$def with (a)\n$(a)')(1) == '1\n' + t('$def with (a)\n$a')(1) == '1\n' + t('$def with (a)\n$a.b')(storage(b=1)) == '1\n' + t('$def with (a)\n$a[0]')([1]) == '1\n' + t('${0 or 1}')() == '1\n' + t('$ a = [1]\n$a[0]')() == '1\n' + t('$ a = {1: 1}\n$a.keys()[0]')() == '1\n' + t('$ a = []\n$if not a: 1')() == '1\n' + t('$ a = {}\n$if not a: 1')() == '1\n' + t('$ a = -1\n$a')() == '-1\n' + t('$ a = "1"\n$a')() == '1\n' + t('$if 1 is 1: 1')() == '1\n' + t('$if not 0: 1')() == '1\n' + t('$if 1:\n $if 1: 1')() == '1\n' + t('$ a = 1\n$a')() == '1\n' + t('$ a = 1.\n$a')() == '1.0\n' + t('$({1: 1}.keys()[0])')() == '1\n' + t('$for x in [1, 2, 3]:\n\t$x')() == ' 1\n 2\n 3\n' + t('$def with (a)\n$:a')(1) == '1\n' + t('$def with (a)\n$a')(u'\u203d') == '\xe2\x80\xbd\n' + t(u'$def with (f)\n$:f("x")')(lambda x: x) == 'x\n' + + j = Template("$var foo: bar")() + assertEqual(str(j), '') + assertEqual(j.foo, 'bar\n') + if verbose: sys.stderr.write('\n') + + +if __name__ == "__main__": + test() ============================================================ --- web/utils.py ffbc0498151371a2a099bb8233f7d2bd0676f1d2 +++ web/utils.py ffbc0498151371a2a099bb8233f7d2bd0676f1d2 @@ -0,0 +1,796 @@ +""" +General Utilities +(part of web.py) +""" + +__all__ = [ + "Storage", "storage", "storify", + "iters", + "rstrips", "lstrips", "strips", "utf8", + "TimeoutError", "timelimit", + "Memoize", "memoize", + "re_compile", "re_subm", + "group", + "IterBetter", "iterbetter", + "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd", + "listget", "intget", "datestr", + "numify", "denumify", "dateify", + "CaptureStdout", "capturestdout", "Profile", "profile", + "tryall", + "ThreadedDict", + "autoassign", + "to36", + "safemarkdown" +] + +import re, sys, time, threading +try: import datetime +except ImportError: pass + +class Storage(dict): + """ + A Storage object is like a dictionary except `obj.foo` can be used + in addition to `obj['foo']`. + + >>> o = storage(a=1) + >>> o.a + 1 + >>> o['a'] + 1 + >>> o.a = 2 + >>> o['a'] + 2 + >>> del o.a + >>> o.a + Traceback (most recent call last): + ... + AttributeError: 'a' + + """ + def __getattr__(self, key): + try: + return self[key] + except KeyError, k: + raise AttributeError, k + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + try: + del self[key] + except KeyError, k: + raise AttributeError, k + + def __repr__(self): + return '' + +storage = Storage + +def storify(mapping, *requireds, **defaults): + """ + Creates a `storage` object from dictionary `mapping`, raising `KeyError` if + d doesn't have all of the keys in `requireds` and using the default + values for keys found in `defaults`. + + For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of + `storage({'a':1, 'b':2, 'c':3})`. + + If a `storify` value is a list (e.g. multiple values in a form submission), + `storify` returns the last element of the list, unless the key appears in + `defaults` as a list. Thus: + + >>> storify({'a':[1, 2]}).a + 2 + >>> storify({'a':[1, 2]}, a=[]).a + [1, 2] + >>> storify({'a':1}, a=[]).a + [1] + >>> storify({}, a=[]).a + [] + + Similarly, if the value has a `value` attribute, `storify will return _its_ + value, unless the key appears in `defaults` as a dictionary. + + >>> storify({'a':storage(value=1)}).a + 1 + >>> storify({'a':storage(value=1)}, a={}).a + + >>> storify({}, a={}).a + {} + + """ + def getvalue(x): + if hasattr(x, 'value'): + return x.value + else: + return x + + stor = Storage() + for key in requireds + tuple(mapping.keys()): + value = mapping[key] + if isinstance(value, list): + if isinstance(defaults.get(key), list): + value = [getvalue(x) for x in value] + else: + value = value[-1] + if not isinstance(defaults.get(key), dict): + value = getvalue(value) + if isinstance(defaults.get(key), list) and not isinstance(value, list): + value = [value] + setattr(stor, key, value) + + for (key, value) in defaults.iteritems(): + result = value + if hasattr(stor, key): + result = stor[key] + if value == () and not isinstance(result, tuple): + result = (result,) + setattr(stor, key, result) + + return stor + +iters = [list, tuple] +import __builtin__ +if hasattr(__builtin__, 'set'): + iters.append(set) +try: + from sets import Set + iters.append(Set) +except ImportError: + pass + +class _hack(tuple): pass +iters = _hack(iters) +iters.__doc__ = """ +A list of iterable items (like lists, but not strings). Includes whichever +of lists, tuples, sets, and Sets are available in this version of Python. +""" + +def _strips(direction, text, remove): + if direction == 'l': + if text.startswith(remove): + return text[len(remove):] + elif direction == 'r': + if text.endswith(remove): + return text[:-len(remove)] + else: + raise ValueError, "Direction needs to be r or l." + return text + +def rstrips(text, remove): + """ + removes the string `remove` from the right of `text` + + >>> rstrips("foobar", "bar") + 'foo' + + """ + return _strips('r', text, remove) + +def lstrips(text, remove): + """ + removes the string `remove` from the left of `text` + + >>> lstrips("foobar", "foo") + 'bar' + + """ + return _strips('l', text, remove) + +def strips(text, remove): + """removes the string `remove` from the both sides of `text` + + >>> strips("foobarfoo", "foo") + 'bar' + + """ + return rstrips(lstrips(text, remove), remove) + +def utf8(text): + """Encodes text in utf-8. + + >> utf8(u'\u1234') # doctest doesn't seem to like utf-8 + '\xe1\x88\xb4' + + >>> utf8('hello') + 'hello' + >>> utf8(42) + '42' + """ + if isinstance(text, unicode): + return text.encode('utf-8') + elif isinstance(text, str): + return text + else: + return str(text) + +class TimeoutError(Exception): pass +def timelimit(timeout): + """ + A decorator to limit a function to `timeout` seconds, raising `TimeoutError` + if it takes longer. + + >>> import time + >>> def meaningoflife(): + ... time.sleep(.2) + ... return 42 + >>> + >>> timelimit(.1)(meaningoflife)() + Traceback (most recent call last): + ... + TimeoutError: took too long + >>> timelimit(1)(meaningoflife)() + 42 + + _Caveat:_ The function isn't stopped after `timeout` seconds but continues + executing in a separate thread. (There seems to be no way to kill a thread.) + + inspired by + """ + def _1(function): + def _2(*args, **kw): + class Dispatch(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.result = None + self.error = None + + self.setDaemon(True) + self.start() + + def run(self): + try: + self.result = function(*args, **kw) + except: + self.error = sys.exc_info() + + c = Dispatch() + c.join(timeout) + if c.isAlive(): + raise TimeoutError, 'took too long' + if c.error: + raise c.error[0], c.error[1] + return c.result + return _2 + return _1 + +class Memoize: + """ + 'Memoizes' a function, caching its return values for each input. + + >>> import time + >>> def meaningoflife(): + ... time.sleep(.2) + ... return 42 + >>> fastlife = memoize(meaningoflife) + >>> meaningoflife() + 42 + >>> timelimit(.1)(meaningoflife)() + Traceback (most recent call last): + ... + TimeoutError: took too long + >>> fastlife() + 42 + >>> timelimit(.1)(fastlife)() + 42 + + """ + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args, **keywords): + key = (args, tuple(keywords.items())) + if key not in self.cache: + self.cache[key] = self.func(*args, **keywords) + return self.cache[key] + +memoize = Memoize + +re_compile = memoize(re.compile) #@@ threadsafe? +re_compile.__doc__ = """ +A memoized version of re.compile. +""" + +class _re_subm_proxy: + def __init__(self): + self.match = None + def __call__(self, match): + self.match = match + return '' + +def re_subm(pat, repl, string): + """ + Like re.sub, but returns the replacement _and_ the match object. + + >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball') + >>> t + 'foooooolish' + >>> m.groups() + ('oooooo',) + """ + compiled_pat = re_compile(pat) + proxy = _re_subm_proxy() + compiled_pat.sub(proxy.__call__, string) + return compiled_pat.sub(repl, string), proxy.match + +def group(seq, size): + """ + Returns an iterator over a series of lists of length size from iterable. + + >>> list(group([1,2,3,4], 2)) + [[1, 2], [3, 4]] + """ + if not hasattr(seq, 'next'): + seq = iter(seq) + while True: + yield [seq.next() for i in xrange(size)] + +class IterBetter: + """ + Returns an object that can be used as an iterator + but can also be used via __getitem__ (although it + cannot go backwards -- that is, you cannot request + `iterbetter[0]` after requesting `iterbetter[1]`). + + >>> import itertools + >>> c = iterbetter(itertools.count()) + >>> c[1] + 1 + >>> c[5] + 5 + >>> c[3] + Traceback (most recent call last): + ... + IndexError: already passed 3 + """ + def __init__(self, iterator): + self.i, self.c = iterator, 0 + def __iter__(self): + while 1: + yield self.i.next() + self.c += 1 + def __getitem__(self, i): + #todo: slices + if i < self.c: + raise IndexError, "already passed "+str(i) + try: + while i > self.c: + self.i.next() + self.c += 1 + # now self.c == i + self.c += 1 + return self.i.next() + except StopIteration: + raise IndexError, str(i) +iterbetter = IterBetter + +def dictreverse(mapping): + """ + >>> dictreverse({1: 2, 3: 4}) + {2: 1, 4: 3} + """ + return dict([(value, key) for (key, value) in mapping.iteritems()]) + +def dictfind(dictionary, element): + """ + Returns a key whose value in `dictionary` is `element` + or, if none exists, None. + + >>> d = {1:2, 3:4} + >>> dictfind(d, 4) + 3 + >>> dictfind(d, 5) + """ + for (key, value) in dictionary.iteritems(): + if element is value: + return key + +def dictfindall(dictionary, element): + """ + Returns the keys whose values in `dictionary` are `element` + or, if none exists, []. + + >>> d = {1:4, 3:4} + >>> dictfindall(d, 4) + [1, 3] + >>> dictfindall(d, 5) + [] + """ + res = [] + for (key, value) in dictionary.iteritems(): + if element is value: + res.append(key) + return res + +def dictincr(dictionary, element): + """ + Increments `element` in `dictionary`, + setting it to one if it doesn't exist. + + >>> d = {1:2, 3:4} + >>> dictincr(d, 1) + 3 + >>> d[1] + 3 + >>> dictincr(d, 5) + 1 + >>> d[5] + 1 + """ + dictionary.setdefault(element, 0) + dictionary[element] += 1 + return dictionary[element] + +def dictadd(*dicts): + """ + Returns a dictionary consisting of the keys in the argument dictionaries. + If they share a key, the value from the last argument is used. + + >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1}) + {1: 0, 2: 1, 3: 1} + """ + result = {} + for dct in dicts: + result.update(dct) + return result + +def listget(lst, ind, default=None): + """ + Returns `lst[ind]` if it exists, `default` otherwise. + + >>> listget(['a'], 0) + 'a' + >>> listget(['a'], 1) + >>> listget(['a'], 1, 'b') + 'b' + """ + if len(lst)-1 < ind: + return default + return lst[ind] + +def intget(integer, default=None): + """ + Returns `integer` as an int or `default` if it can't. + + >>> intget('3') + 3 + >>> intget('3a') + >>> intget('3a', 0) + 0 + """ + try: + return int(integer) + except (TypeError, ValueError): + return default + +def datestr(then, now=None): + """ + Converts a (UTC) datetime object to a nice string representation. + + >>> from datetime import datetime, timedelta + >>> d = datetime(1970, 5, 1) + >>> datestr(d, now=d) + '0 microseconds ago' + >>> for t, v in { + ... timedelta(microseconds=1): '1 microsecond ago', + ... timedelta(microseconds=2): '2 microseconds ago', + ... -timedelta(microseconds=1): '1 microsecond from now', + ... -timedelta(microseconds=2): '2 microseconds from now', + ... timedelta(microseconds=2000): '2 milliseconds ago', + ... timedelta(seconds=2): '2 seconds ago', + ... timedelta(seconds=2*60): '2 minutes ago', + ... timedelta(seconds=2*60*60): '2 hours ago', + ... timedelta(days=2): '2 days ago', + ... }.iteritems(): + ... assert datestr(d, now=d+t) == v + >>> datestr(datetime(1970, 1, 1), now=d) + 'January 1' + >>> datestr(datetime(1969, 1, 1), now=d) + 'January 1, 1969' + >>> datestr(datetime(1970, 6, 1), now=d) + 'June 1, 1970' + """ + def agohence(n, what, divisor=None): + if divisor: n = n // divisor + + out = str(abs(n)) + ' ' + what # '2 day' + if abs(n) != 1: out += 's' # '2 days' + out += ' ' # '2 days ' + if n < 0: + out += 'from now' + else: + out += 'ago' + return out # '2 days ago' + + oneday = 24 * 60 * 60 + + if not now: now = datetime.datetime.utcnow() + if type(now).__name__ == "DateTime": + now = datetime.datetime.fromtimestamp(now) + if type(then).__name__ == "DateTime": + then = datetime.datetime.fromtimestamp(then) + delta = now - then + deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06) + deltadays = abs(deltaseconds) // oneday + if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor + + if deltadays: + if abs(deltadays) < 4: + return agohence(deltadays, 'day') + + out = then.strftime('%B %e') # e.g. 'June 13' + if then.year != now.year or deltadays < 0: + out += ', %s' % then.year + return out + + if int(deltaseconds): + if abs(deltaseconds) > (60 * 60): + return agohence(deltaseconds, 'hour', 60 * 60) + elif abs(deltaseconds) > 60: + return agohence(deltaseconds, 'minute', 60) + else: + return agohence(deltaseconds, 'second') + + deltamicroseconds = delta.microseconds + if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity + if abs(deltamicroseconds) > 1000: + return agohence(deltamicroseconds, 'millisecond', 1000) + + return agohence(deltamicroseconds, 'microsecond') + +def numify(string): + """ + Removes all non-digit characters from `string`. + + >>> numify('800-555-1212') + '8005551212' + >>> numify('800.555.1212') + '8005551212' + + """ + return ''.join([c for c in str(string) if c.isdigit()]) + +def denumify(string, pattern): + """ + Formats `string` according to `pattern`, where the letter X gets replaced + by characters from `string`. + + >>> denumify("8005551212", "(XXX) XXX-XXXX") + '(800) 555-1212' + + """ + out = [] + for c in pattern: + if c == "X": + out.append(string[0]) + string = string[1:] + else: + out.append(c) + return ''.join(out) + +def dateify(datestring): + """ + Formats a numified `datestring` properly. + """ + return denumify(datestring, "XXXX-XX-XX XX:XX:XX") + +class CaptureStdout: + """ + Captures everything `func` prints to stdout and returns it instead. + + >>> def idiot(): + ... print "foo" + >>> capturestdout(idiot)() + 'foo\\n' + + **WARNING:** Not threadsafe! + """ + def __init__(self, func): + self.func = func + def __call__(self, *args, **keywords): + from cStringIO import StringIO + # Not threadsafe! + out = StringIO() + oldstdout = sys.stdout + sys.stdout = out + try: + self.func(*args, **keywords) + finally: + sys.stdout = oldstdout + return out.getvalue() + +capturestdout = CaptureStdout + +class Profile: + """ + Profiles `func` and returns a tuple containing its output + and a string with human-readable profiling information. + + >>> import time + >>> out, inf = profile(time.sleep)(.001) + >>> out + >>> inf[:10].strip() + 'took 0.0' + """ + def __init__(self, func): + self.func = func + def __call__(self, *args): ##, **kw): kw unused + import hotshot, hotshot.stats, tempfile ##, time already imported + temp = tempfile.NamedTemporaryFile() + prof = hotshot.Profile(temp.name) + + stime = time.time() + result = prof.runcall(self.func, *args) + stime = time.time() - stime + + prof.close() + stats = hotshot.stats.load(temp.name) + stats.strip_dirs() + stats.sort_stats('time', 'calls') + x = '\n\ntook '+ str(stime) + ' seconds\n' + x += capturestdout(stats.print_stats)(40) + x += capturestdout(stats.print_callers)() + return result, x + +profile = Profile + + +import traceback +# hack for compatibility with Python 2.3: +if not hasattr(traceback, 'format_exc'): + from cStringIO import StringIO + def format_exc(limit=None): + strbuf = StringIO() + traceback.print_exc(limit, strbuf) + return strbuf.getvalue() + traceback.format_exc = format_exc + +def tryall(context, prefix=None): + """ + Tries a series of functions and prints their results. + `context` is a dictionary mapping names to values; + the value will only be tried if it's callable. + + >>> tryall(dict(j=lambda: True)) + j: True + ---------------------------------------- + results: + True: 1 + + For example, you might have a file `test/stuff.py` + with a series of functions testing various things in it. + At the bottom, have a line: + + if __name__ == "__main__": tryall(globals()) + + Then you can run `python test/stuff.py` and get the results of + all the tests. + """ + context = context.copy() # vars() would update + results = {} + for (key, value) in context.iteritems(): + if not hasattr(value, '__call__'): + continue + if prefix and not key.startswith(prefix): + continue + print key + ':', + try: + r = value() + dictincr(results, r) + print r + except: + print 'ERROR' + dictincr(results, 'ERROR') + print ' ' + '\n '.join(traceback.format_exc().split('\n')) + + print '-'*40 + print 'results:' + for (key, value) in results.iteritems(): + print ' '*2, str(key)+':', value + +class ThreadedDict: + """ + Takes a dictionary that maps threads to objects. + When a thread tries to get or set an attribute or item + of the threadeddict, it passes it on to the object + for that thread in dictionary. + """ + def __init__(self, dictionary): + self.__dict__['_ThreadedDict__d'] = dictionary + + def __getattr__(self, attr): + return getattr(self.__d[threading.currentThread()], attr) + + def __getitem__(self, item): + return self.__d[threading.currentThread()][item] + + def __setattr__(self, attr, value): + if attr == '__doc__': + self.__dict__[attr] = value + else: + return setattr(self.__d[threading.currentThread()], attr, value) + + def __delattr__(self, item): + try: + del self.__d[threading.currentThread()][item] + except KeyError, k: + raise AttributeError, k + + def __delitem__(self, item): + del self.__d[threading.currentThread()][item] + + def __setitem__(self, item, value): + self.__d[threading.currentThread()][item] = value + + def __hash__(self): + return hash(self.__d[threading.currentThread()]) + +threadeddict = ThreadedDict + +def autoassign(self, locals): + """ + Automatically assigns local variables to `self`. + + >>> self = storage() + >>> autoassign(self, dict(a=1, b=2)) + >>> self + + + Generally used in `__init__` methods, as in: + + def __init__(self, foo, bar, baz=1): autoassign(self, locals()) + """ + for (key, value) in locals.iteritems(): + if key == 'self': + continue + setattr(self, key, value) + +def to36(q): + """ + Converts an integer to base 36 (a useful scheme for human-sayable IDs). + + >>> to36(35) + 'z' + >>> to36(119292) + '2k1o' + >>> int(to36(939387374), 36) + 939387374 + >>> to36(0) + '0' + >>> to36(-393) + Traceback (most recent call last): + ... + ValueError: must supply a positive integer + + """ + if q < 0: raise ValueError, "must supply a positive integer" + letters = "0123456789abcdefghijklmnopqrstuvwxyz" + converted = [] + while q != 0: + q, r = divmod(q, 36) + converted.insert(0, letters[r]) + return "".join(converted) or '0' + + +r_url = re_compile('(?', text) + text = markdown(text) + return text + + +if __name__ == "__main__": + import doctest + doctest.testmod() ============================================================ --- web/webapi.py adba8f74b58b03514c8ccb47c926676139d4044b +++ web/webapi.py adba8f74b58b03514c8ccb47c926676139d4044b @@ -0,0 +1,369 @@ +""" +Web API (wrapper around WSGI) +(from web.py) +""" + +__all__ = [ + "config", + "badrequest", "notfound", "gone", "internalerror", + "header", "output", "flush", "debug", + "input", "data", + "setcookie", "cookies", + "ctx", + "loadhooks", "load", "unloadhooks", "unload", "_loadhooks", + "wsgifunc" +] + +import sys, os, cgi, threading, Cookie, pprint, traceback +try: import itertools +except ImportError: pass +from utils import storage, storify, threadeddict, dictadd, intget, lstrips, utf8 + +config = storage() +config.__doc__ = """ +A configuration object for various aspects of web.py. + +`db_parameters` + : A dictionary containing the parameters to be passed to `connect` + when `load()` is called. +`db_printing` + : Set to `True` if you would like SQL queries and timings to be + printed to the debug output. + +""" + +def badrequest(): + """Return a `400 Bad Request` error.""" + ctx.status = '400 Bad Request' + header('Content-Type', 'text/html') + return output('bad request') + +def notfound(): + """Returns a `404 Not Found` error.""" + ctx.status = '404 Not Found' + header('Content-Type', 'text/html') + return output('not found') + +def gone(): + """Returns a `410 Gone` error.""" + ctx.status = '410 Gone' + header('Content-Type', 'text/html') + return output("gone") + +def internalerror(): + """Returns a `500 Internal Server` error.""" + ctx.status = "500 Internal Server Error" + ctx.headers = [('Content-Type', 'text/html')] + ctx.output = "internal server error" + +def header(hdr, value, unique=False): + """ + Adds the header `hdr: value` with the response. + + If `unique` is True and a header with that name already exists, + it doesn't add a new one. + """ + hdr, value = utf8(hdr), utf8(value) + # protection against HTTP response splitting attack + if '\n' in hdr or '\r' in hdr or '\n' in value or '\r' in value: + raise ValueError, 'invalid characters in header' + + if unique is True: + for h, v in ctx.headers: + if h.lower() == hdr.lower(): return + + ctx.headers.append((hdr, value)) + +def output(string_): + """Appends `string_` to the response.""" + if isinstance(string_, unicode): string_ = string_.encode('utf8') + if ctx.get('flush'): + ctx._write(string_) + else: + ctx.output += str(string_) + +def flush(): + ctx.flush = True + return flush + +def input(*requireds, **defaults): + """ + Returns a `storage` object with the GET and POST arguments. + See `storify` for how `requireds` and `defaults` work. + """ + from cStringIO import StringIO + def dictify(fs): return dict([(k, fs[k]) for k in fs.keys()]) + + _method = defaults.pop('_method', 'both') + + e = ctx.env.copy() + a = b = {} + + if _method.lower() in ['both', 'post']: + if e['REQUEST_METHOD'] == 'POST': + a = cgi.FieldStorage(fp = StringIO(data()), environ=e, + keep_blank_values=1) + a = dictify(a) + + if _method.lower() in ['both', 'get']: + e['REQUEST_METHOD'] = 'GET' + b = dictify(cgi.FieldStorage(environ=e, keep_blank_values=1)) + + out = dictadd(b, a) + try: + return storify(out, *requireds, **defaults) + except KeyError: + badrequest() + raise StopIteration + +def data(): + """Returns the data sent with the request.""" + if 'data' not in ctx: + cl = intget(ctx.env.get('CONTENT_LENGTH'), 0) + ctx.data = ctx.env['wsgi.input'].read(cl) + return ctx.data + +def setcookie(name, value, expires="", domain=None): + """Sets a cookie.""" + if expires < 0: + expires = -1000000000 + kargs = {'expires': expires, 'path':'/'} + if domain: + kargs['domain'] = domain + # @@ should we limit cookies to a different path? + cookie = Cookie.SimpleCookie() + cookie[name] = value + for key, val in kargs.iteritems(): + cookie[name][key] = val + header('Set-Cookie', cookie.items()[0][1].OutputString()) + +def cookies(*requireds, **defaults): + """ + Returns a `storage` object with all the cookies in it. + See `storify` for how `requireds` and `defaults` work. + """ + cookie = Cookie.SimpleCookie() + cookie.load(ctx.env.get('HTTP_COOKIE', '')) + try: + return storify(cookie, *requireds, **defaults) + except KeyError: + badrequest() + raise StopIteration + +def debug(*args): + """ + Prints a prettyprinted version of `args` to stderr. + """ + try: + out = ctx.environ['wsgi.errors'] + except: + out = sys.stderr + for arg in args: + print >> out, pprint.pformat(arg) + return '' + +def _debugwrite(x): + try: + out = ctx.environ['wsgi.errors'] + except: + out = sys.stderr + out.write(x) +debug.write = _debugwrite + +class _outputter: + """Wraps `sys.stdout` so that print statements go into the response.""" + def __init__(self, file): self.file = file + def write(self, string_): + if hasattr(ctx, 'output'): + return output(string_) + else: + self.file.write(string_) + def __getattr__(self, attr): return getattr(self.file, attr) + def __getitem__(self, item): return self.file[item] + +def _capturedstdout(): + sysstd = sys.stdout + while hasattr(sysstd, 'file'): + if isinstance(sys.stdout, _outputter): return True + sysstd = sysstd.file + if isinstance(sys.stdout, _outputter): return True + return False + +if not _capturedstdout(): + sys.stdout = _outputter(sys.stdout) + +_context = {threading.currentThread(): storage()} +ctx = context = threadeddict(_context) + +ctx.__doc__ = """ +A `storage` object containing various information about the request: + +`environ` (aka `env`) + : A dictionary containing the standard WSGI environment variables. + +`host` + : The domain (`Host` header) requested by the user. + +`home` + : The base path for the application. + +`ip` + : The IP address of the requester. + +`method` + : The HTTP method used. + +`path` + : The path request. + +`query` + : If there are no query arguments, the empty string. Otherwise, a `?` followed + by the query string. + +`fullpath` + : The full path requested, including query arguments (`== path + query`). + +### Response Data + +`status` (default: "200 OK") + : The status code to be used in the response. + +`headers` + : A list of 2-tuples to be used in the response. + +`output` + : A string to be used as the response. +""" + +loadhooks = {} +_loadhooks = {} + +def load(): + """ + Loads a new context for the thread. + + You can ask for a function to be run at loadtime by + adding it to the dictionary `loadhooks`. + """ + _context[threading.currentThread()] = storage() + ctx.status = '200 OK' + ctx.headers = [] + if config.get('db_parameters'): + import db + db.connect(**config.db_parameters) + + for x in loadhooks.values(): x() + +def _load(env): + load() + ctx.output = '' + ctx.environ = ctx.env = env + ctx.host = env.get('HTTP_HOST') + ctx.homedomain = 'http://' + env.get('HTTP_HOST', '[unknown]') + ctx.homepath = os.environ.get('REAL_SCRIPT_NAME', env.get('SCRIPT_NAME', '')) + ctx.home = ctx.homedomain + ctx.homepath + ctx.ip = env.get('REMOTE_ADDR') + ctx.method = env.get('REQUEST_METHOD') + ctx.path = env.get('PATH_INFO') + # http://trac.lighttpd.net/trac/ticket/406 requires: + if env.get('SERVER_SOFTWARE', '').startswith('lighttpd/'): + ctx.path = lstrips(env.get('REQUEST_URI').split('?')[0], + os.environ.get('REAL_SCRIPT_NAME', env.get('SCRIPT_NAME', ''))) + + if env.get('QUERY_STRING'): + ctx.query = '?' + env.get('QUERY_STRING', '') + else: + ctx.query = '' + + ctx.fullpath = ctx.path + ctx.query + for x in _loadhooks.values(): x() + +unloadhooks = {} + +def unload(): + """ + Unloads the context for the thread. + + You can ask for a function to be run at loadtime by + adding it ot the dictionary `unloadhooks`. + """ + for x in unloadhooks.values(): x() + # ensures db cursors and such are GCed promptly + del _context[threading.currentThread()] + +def _unload(): + unload() + +def wsgifunc(func, *middleware): + """Returns a WSGI-compatible function from a webpy-function.""" + middleware = list(middleware) + + def wsgifunc(env, start_resp): + _load(env) + try: + result = func() + except StopIteration: + result = None + except: + print >> debug, traceback.format_exc() + result = internalerror() + + is_generator = result and hasattr(result, 'next') + if is_generator: + # wsgi requires the headers first + # so we need to do an iteration + # and save the result for later + try: + firstchunk = result.next() + except StopIteration: + firstchunk = '' + + status, headers, output = ctx.status, ctx.headers, ctx.output + ctx._write = start_resp(status, headers) + + # and now, the fun: + + def cleanup(): + # we insert this little generator + # at the end of our itertools.chain + # so that it unloads the request + # when everything else is done + + yield '' # force it to be a generator + _unload() + + # result is the output of calling the webpy function + # it could be a generator... + + if is_generator: + if firstchunk is flush: + # oh, it's just our special flush mode + # ctx._write is set up, so just continue execution + try: + result.next() + except StopIteration: + pass + + _unload() + return [] + else: + return itertools.chain([firstchunk], result, cleanup()) + + # ... but it's usually just None + # + # output is the stuff in ctx.output + # it's usually a string... + if isinstance(output, str): #@@ other stringlikes? + _unload() + return [output] + # it could be a generator... + elif hasattr(output, 'next'): + return itertools.chain(output, cleanup()) + else: + _unload() + raise Exception, "Invalid ctx.output" + + for mw_func in middleware: + wsgifunc = mw_func(wsgifunc) + + return wsgifunc ============================================================ --- web/wsgi.py d7214b2a4c8693a87db5d17eacd245d441ad7113 +++ web/wsgi.py d7214b2a4c8693a87db5d17eacd245d441ad7113 @@ -0,0 +1,54 @@ +""" +WSGI Utilities +(from web.py) +""" + +import os, sys + +import http +import webapi as web +from utils import listget +from net import validaddr, validip +import httpserver + +def runfcgi(func, addr=('localhost', 8000)): + """Runs a WSGI function as a FastCGI server.""" + import flup.server.fcgi as flups + return flups.WSGIServer(func, multiplexed=True, bindAddress=addr).run() + +def runscgi(func, addr=('localhost', 4000)): + """Runs a WSGI function as an SCGI server.""" + import flup.server.scgi as flups + return flups.WSGIServer(func, bindAddress=addr).run() + +def runwsgi(func): + """ + Runs a WSGI-compatible `func` using FCGI, SCGI, or a simple web server, + as appropriate based on context and `sys.argv`. + """ + + if os.environ.has_key('SERVER_SOFTWARE'): # cgi + os.environ['FCGI_FORCE_CGI'] = 'Y' + + if (os.environ.has_key('PHP_FCGI_CHILDREN') #lighttpd fastcgi + or os.environ.has_key('SERVER_SOFTWARE')): + return runfcgi(func, None) + + if 'fcgi' in sys.argv or 'fastcgi' in sys.argv: + args = sys.argv[1:] + if 'fastcgi' in args: args.remove('fastcgi') + elif 'fcgi' in args: args.remove('fcgi') + if args: + return runfcgi(func, validaddr(args[0])) + else: + return runfcgi(func, None) + + if 'scgi' in sys.argv: + args = sys.argv[1:] + args.remove('scgi') + if args: + return runscgi(func, validaddr(args[0])) + else: + return runscgi(func) + + return httpserver.runsimple(func, validip(listget(sys.argv, 1, ''))) ============================================================ --- web/wsgiserver/__init__.py 7459212c431b1a3561d948c6852c41fa765e9b9d +++ web/wsgiserver/__init__.py 7459212c431b1a3561d948c6852c41fa765e9b9d @@ -0,0 +1,1019 @@ +"""A high-speed, production ready, thread pooled, generic WSGI server. + +Simplest example on how to use this module directly +(without using CherryPy's application machinery): + + from cherrypy import wsgiserver + + def my_crazy_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type','text/plain')] + start_response(status, response_headers) + return ['Hello world!\n'] + + # Here we set our application to the script_name '/' + wsgi_apps = [('/', my_crazy_app)] + + server = wsgiserver.CherryPyWSGIServer(('localhost', 8070), wsgi_apps, + server_name='localhost') + + # Want SSL support? Just set these attributes + # server.ssl_certificate = + # server.ssl_private_key = + + if __name__ == '__main__': + try: + server.start() + except KeyboardInterrupt: + server.stop() + +This won't call the CherryPy engine (application side) at all, only the +WSGI server, which is independant from the rest of CherryPy. Don't +let the name "CherryPyWSGIServer" throw you; the name merely reflects +its origin, not it's coupling. + +The CherryPy WSGI server can serve as many WSGI applications +as you want in one instance: + + wsgi_apps = [('/', my_crazy_app), ('/blog', my_blog_app)] + +""" + +import base64 +import Queue +import os +import re +quoted_slash = re.compile("(?i)%2F") +import rfc822 +import socket +try: + import cStringIO as StringIO +except ImportError: + import StringIO +import sys +import threading +import time +import traceback +from urllib import unquote +from urlparse import urlparse + +try: + from OpenSSL import SSL + from OpenSSL import crypto +except ImportError: + SSL = None + +import errno +socket_errors_to_ignore = [] +# Not all of these names will be defined for every platform. +for _ in ("EPIPE", "ETIMEDOUT", "ECONNREFUSED", "ECONNRESET", + "EHOSTDOWN", "EHOSTUNREACH", + "WSAECONNABORTED", "WSAECONNREFUSED", "WSAECONNRESET", + "WSAENETRESET", "WSAETIMEDOUT"): + if _ in dir(errno): + socket_errors_to_ignore.append(getattr(errno, _)) +# de-dupe the list +socket_errors_to_ignore = dict.fromkeys(socket_errors_to_ignore).keys() +socket_errors_to_ignore.append("timed out") + + +comma_separated_headers = ['ACCEPT', 'ACCEPT-CHARSET', 'ACCEPT-ENCODING', + 'ACCEPT-LANGUAGE', 'ACCEPT-RANGES', 'ALLOW', 'CACHE-CONTROL', + 'CONNECTION', 'CONTENT-ENCODING', 'CONTENT-LANGUAGE', 'EXPECT', + 'IF-MATCH', 'IF-NONE-MATCH', 'PRAGMA', 'PROXY-AUTHENTICATE', 'TE', + 'TRAILER', 'TRANSFER-ENCODING', 'UPGRADE', 'VARY', 'VIA', 'WARNING', + 'WWW-AUTHENTICATE'] + +class HTTPRequest(object): + """An HTTP Request (and response). + + A single HTTP connection may consist of multiple request/response pairs. + + connection: the HTTP Connection object which spawned this request. + rfile: the 'read' fileobject from the connection's socket + ready: when True, the request has been parsed and is ready to begin + generating the response. When False, signals the calling Connection + that the response should not be generated and the connection should + close. + close_connection: signals the calling Connection that the request + should close. This does not imply an error! The client and/or + server may each request that the connection be closed. + chunked_write: if True, output will be encoded with the "chunked" + transfer-coding. This value is set automatically inside + send_headers. + """ + + def __init__(self, connection): + self.connection = connection + self.rfile = self.connection.rfile + self.sendall = self.connection.sendall + self.environ = connection.environ.copy() + + self.ready = False + self.started_response = False + self.status = "" + self.outheaders = [] + self.sent_headers = False + self.close_connection = False + self.chunked_write = False + + def parse_request(self): + """Parse the next HTTP request start-line and message-headers.""" + # HTTP/1.1 connections are persistent by default. If a client + # requests a page, then idles (leaves the connection open), + # then rfile.readline() will raise socket.error("timed out"). + # Note that it does this based on the value given to settimeout(), + # and doesn't need the client to request or acknowledge the close + # (although your TCP stack might suffer for it: cf Apache's history + # with FIN_WAIT_2). + request_line = self.rfile.readline() + if not request_line: + # Force self.ready = False so the connection will close. + self.ready = False + return + + if request_line == "\r\n": + # RFC 2616 sec 4.1: "...if the server is reading the protocol + # stream at the beginning of a message and receives a CRLF + # first, it should ignore the CRLF." + # But only ignore one leading line! else we enable a DoS. + request_line = self.rfile.readline() + if not request_line: + self.ready = False + return + + server = self.connection.server + environ = self.environ + environ["SERVER_SOFTWARE"] = "%s WSGI Server" % server.version + + method, path, req_protocol = request_line.strip().split(" ", 2) + environ["REQUEST_METHOD"] = method + + # path may be an abs_path (including "http://host.domain.tld"); + scheme, location, path, params, qs, frag = urlparse(path) + + if frag: + self.simple_response("400 Bad Request", + "Illegal #fragment in Request-URI.") + return + + if scheme: + environ["wsgi.url_scheme"] = scheme + if params: + path = path + ";" + params + + # Unquote the path+params (e.g. "/this%20path" -> "this path"). + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 + # + # But note that "...a URI must be separated into its components + # before the escaped characters within those components can be + # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 + atoms = [unquote(x) for x in quoted_slash.split(path)] + path = "%2F".join(atoms) + + if path == "*": + # This means, of course, that the last wsgi_app (shortest path) + # will always handle a URI of "*". + environ["SCRIPT_NAME"] = "" + environ["PATH_INFO"] = "*" + self.wsgi_app = server.mount_points[-1][1] + else: + for mount_point, wsgi_app in server.mount_points: + # The mount_points list should be sorted by length, descending. + if path.startswith(mount_point + "/") or path == mount_point: + environ["SCRIPT_NAME"] = mount_point + environ["PATH_INFO"] = path[len(mount_point):] + self.wsgi_app = wsgi_app + break + else: + self.simple_response("404 Not Found") + return + + # Note that, like wsgiref and most other WSGI servers, + # we unquote the path but not the query string. + environ["QUERY_STRING"] = qs + + # Compare request and server HTTP protocol versions, in case our + # server does not support the requested protocol. Limit our output + # to min(req, server). We want the following output: + # request server actual written supported response + # protocol protocol response protocol feature set + # a 1.0 1.0 1.0 1.0 + # b 1.0 1.1 1.1 1.0 + # c 1.1 1.0 1.0 1.0 + # d 1.1 1.1 1.1 1.1 + # Notice that, in (b), the response will be "HTTP/1.1" even though + # the client only understands 1.0. RFC 2616 10.5.6 says we should + # only return 505 if the _major_ version is different. + rp = int(req_protocol[5]), int(req_protocol[7]) + sp = int(server.protocol[5]), int(server.protocol[7]) + if sp[0] != rp[0]: + self.simple_response("505 HTTP Version Not Supported") + return + # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. + environ["SERVER_PROTOCOL"] = req_protocol + # set a non-standard environ entry so the WSGI app can know what + # the *real* server protocol is (and what features to support). + # See http://www.faqs.org/rfcs/rfc2145.html. + environ["ACTUAL_SERVER_PROTOCOL"] = server.protocol + self.response_protocol = "HTTP/%s.%s" % min(rp, sp) + + # If the Request-URI was an absoluteURI, use its location atom. + if location: + environ["SERVER_NAME"] = location + + # then all the http headers + try: + self.read_headers() + except ValueError, ex: + self.simple_response("400 Bad Request", repr(ex.args)) + return + + creds = environ.get("HTTP_AUTHORIZATION", "").split(" ", 1) + environ["AUTH_TYPE"] = creds[0] + if creds[0].lower() == 'basic': + user, pw = base64.decodestring(creds[1]).split(":", 1) + environ["REMOTE_USER"] = user + + # Persistent connection support + if self.response_protocol == "HTTP/1.1": + if environ.get("HTTP_CONNECTION", "") == "close": + self.close_connection = True + else: + # HTTP/1.0 + if environ.get("HTTP_CONNECTION", "") != "Keep-Alive": + self.close_connection = True + + # Transfer-Encoding support + te = None + if self.response_protocol == "HTTP/1.1": + te = environ.get("HTTP_TRANSFER_ENCODING") + if te: + te = [x.strip().lower() for x in te.split(",") if x.strip()] + + read_chunked = False + + if te: + for enc in te: + if enc == "chunked": + read_chunked = True + else: + # Note that, even if we see "chunked", we must reject + # if there is an extension we don't recognize. + self.simple_response("501 Unimplemented") + self.close_connection = True + return + + if read_chunked: + if not self.decode_chunked(): + return + + # From PEP 333: + # "Servers and gateways that implement HTTP 1.1 must provide + # transparent support for HTTP 1.1's "expect/continue" mechanism. + # This may be done in any of several ways: + # 1. Respond to requests containing an Expect: 100-continue request + # with an immediate "100 Continue" response, and proceed normally. + # 2. Proceed with the request normally, but provide the application + # with a wsgi.input stream that will send the "100 Continue" + # response if/when the application first attempts to read from + # the input stream. The read request must then remain blocked + # until the client responds. + # 3. Wait until the client decides that the server does not support + # expect/continue, and sends the request body on its own. + # (This is suboptimal, and is not recommended.) + # + # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, + # but it seems like it would be a big slowdown for such a rare case. + if environ.get("HTTP_EXPECT", "") == "100-continue": + self.simple_response(100) + + self.ready = True + + def read_headers(self): + """Read header lines from the incoming stream.""" + environ = self.environ + + while True: + line = self.rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + if line == '\r\n': + # Normal end of headers + break + + if line[0] in ' \t': + # It's a continuation line. + v = line.strip() + else: + k, v = line.split(":", 1) + k, v = k.strip().upper(), v.strip() + envname = "HTTP_" + k.replace("-", "_") + + if k in comma_separated_headers: + existing = environ.get(envname) + if existing: + v = ", ".join((existing, v)) + environ[envname] = v + + ct = environ.pop("HTTP_CONTENT_TYPE", None) + if ct: + environ["CONTENT_TYPE"] = ct + cl = environ.pop("HTTP_CONTENT_LENGTH", None) + if cl: + environ["CONTENT_LENGTH"] = cl + + def decode_chunked(self): + """Decode the 'chunked' transfer coding.""" + cl = 0 + data = StringIO.StringIO() + while True: + line = self.rfile.readline().strip().split(";", 1) + chunk_size = int(line.pop(0), 16) + if chunk_size <= 0: + break +## if line: chunk_extension = line[0] + cl += chunk_size + data.write(self.rfile.read(chunk_size)) + crlf = self.rfile.read(2) + if crlf != "\r\n": + self.simple_response("400 Bad Request", + "Bad chunked transfer coding " + "(expected '\\r\\n', got %r)" % crlf) + return + + # Grab any trailer headers + self.read_headers() + + data.seek(0) + self.environ["wsgi.input"] = data + self.environ["CONTENT_LENGTH"] = str(cl) or "" + return True + + def respond(self): + """Call the appropriate WSGI app and write its iterable output.""" + response = self.wsgi_app(self.environ, self.start_response) + try: + for chunk in response: + # "The start_response callable must not actually transmit + # the response headers. Instead, it must store them for the + # server or gateway to transmit only after the first + # iteration of the application return value that yields + # a NON-EMPTY string, or upon the application's first + # invocation of the write() callable." (PEP 333) + if chunk: + self.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + if (self.ready and not self.sent_headers + and not self.connection.server.interrupt): + self.sent_headers = True + self.send_headers() + if self.chunked_write: + self.sendall("0\r\n\r\n") + + def simple_response(self, status, msg=""): + """Write a simple response back to the client.""" + status = str(status) + buf = ["%s %s\r\n" % (self.connection.server.protocol, status), + "Content-Length: %s\r\n" % len(msg)] + + if status[:3] == "413" and self.response_protocol == 'HTTP/1.1': + # Request Entity Too Large + self.close_connection = True + buf.append("Connection: close\r\n") + + buf.append("\r\n") + if msg: + buf.append(msg) + self.sendall("".join(buf)) + + def start_response(self, status, headers, exc_info = None): + """WSGI callable to begin the HTTP response.""" + if self.started_response: + if not exc_info: + raise AssertionError("WSGI start_response called a second " + "time with no exc_info.") + else: + try: + raise exc_info[0], exc_info[1], exc_info[2] + finally: + exc_info = None + self.started_response = True + self.status = status + self.outheaders.extend(headers) + return self.write + + def write(self, chunk): + """WSGI callable to write unbuffered data to the client. + + This method is also used internally by start_response (to write + data from the iterable returned by the WSGI application). + """ + if not self.started_response: + raise AssertionError("WSGI write called before start_response.") + + if not self.sent_headers: + self.sent_headers = True + self.send_headers() + + if self.chunked_write and chunk: + buf = [hex(len(chunk))[2:], "\r\n", chunk, "\r\n"] + self.sendall("".join(buf)) + else: + self.sendall(chunk) + + def send_headers(self): + """Assert, process, and send the HTTP response message-headers.""" + hkeys = [key.lower() for key, value in self.outheaders] + status = int(self.status[:3]) + + if status == 413: + # Request Entity Too Large. Close conn to avoid garbage. + self.close_connection = True + elif "content-length" not in hkeys: + # "All 1xx (informational), 204 (no content), + # and 304 (not modified) responses MUST NOT + # include a message-body." So no point chunking. + if status < 200 or status in (204, 205, 304): + pass + else: + if self.response_protocol == 'HTTP/1.1': + # Use the chunked transfer-coding + self.chunked_write = True + self.outheaders.append(("Transfer-Encoding", "chunked")) + else: + # Closing the conn is the only way to determine len. + self.close_connection = True + + if "connection" not in hkeys: + if self.response_protocol == 'HTTP/1.1': + if self.close_connection: + self.outheaders.append(("Connection", "close")) + else: + if not self.close_connection: + self.outheaders.append(("Connection", "Keep-Alive")) + + if "date" not in hkeys: + self.outheaders.append(("Date", rfc822.formatdate())) + + server = self.connection.server + + if "server" not in hkeys: + self.outheaders.append(("Server", server.version)) + + buf = [server.protocol, " ", self.status, "\r\n"] + try: + buf += [k + ": " + v + "\r\n" for k, v in self.outheaders] + except TypeError: + if not isinstance(k, str): + raise TypeError("WSGI response header key %r is not a string.") + if not isinstance(v, str): + raise TypeError("WSGI response header value %r is not a string.") + else: + raise + buf.append("\r\n") + self.sendall("".join(buf)) + + +class NoSSLError(Exception): + """Exception raised when a client speaks HTTP to an HTTPS socket.""" + pass + + +def _ssl_wrap_method(method, is_reader=False): + """Wrap the given method with SSL error-trapping. + + is_reader: if False (the default), EOF errors will be raised. + If True, EOF errors will return "" (to emulate normal sockets). + """ + def ssl_method_wrapper(self, *args, **kwargs): +## print (id(self), method, args, kwargs) + start = time.time() + while True: + try: + return method(self, *args, **kwargs) + except (SSL.WantReadError, SSL.WantWriteError): + # Sleep and try again. This is dangerous, because it means + # the rest of the stack has no way of differentiating + # between a "new handshake" error and "client dropped". + # Note this isn't an endless loop: there's a timeout below. + time.sleep(self.ssl_retry) + except SSL.SysCallError, e: + if is_reader and e.args == (-1, 'Unexpected EOF'): + return "" + + errno = e.args[0] + if is_reader and errno in socket_errors_to_ignore: + return "" + raise socket.error(errno) + except SSL.Error, e: + if is_reader and e.args == (-1, 'Unexpected EOF'): + return "" + + thirdarg = None + try: + thirdarg = e.args[0][0][2] + except IndexError: + pass + + if is_reader and thirdarg == 'ssl handshake failure': + return "" + if thirdarg == 'http request': + # The client is talking HTTP to an HTTPS server. + raise NoSSLError() + raise + if time.time() - start > self.ssl_timeout: + raise socket.timeout("timed out") + return ssl_method_wrapper + +class SSL_fileobject(socket._fileobject): + """Faux file object attached to a socket object.""" + + ssl_timeout = 3 + ssl_retry = .01 + + close = _ssl_wrap_method(socket._fileobject.close) + flush = _ssl_wrap_method(socket._fileobject.flush) + write = _ssl_wrap_method(socket._fileobject.write) + writelines = _ssl_wrap_method(socket._fileobject.writelines) + read = _ssl_wrap_method(socket._fileobject.read, is_reader=True) + readline = _ssl_wrap_method(socket._fileobject.readline, is_reader=True) + readlines = _ssl_wrap_method(socket._fileobject.readlines, is_reader=True) + + +class HTTPConnection(object): + """An HTTP connection (active socket). + + socket: the raw socket object (usually TCP) for this connection. + addr: the "bind address" for the remote end of the socket. + For IP sockets, this is a tuple of (REMOTE_ADDR, REMOTE_PORT). + For UNIX domain sockets, this will be a string. + server: the HTTP Server for this Connection. Usually, the server + object possesses a passive (server) socket which spawns multiple, + active (client) sockets, one for each connection. + + environ: a WSGI environ template. This will be copied for each request. + rfile: a fileobject for reading from the socket. + sendall: a function for writing (+ flush) to the socket. + """ + + rbufsize = -1 + RequestHandlerClass = HTTPRequest + environ = {"wsgi.version": (1, 0), + "wsgi.url_scheme": "http", + "wsgi.multithread": True, + "wsgi.multiprocess": False, + "wsgi.run_once": False, + "wsgi.errors": sys.stderr, + } + + def __init__(self, sock, addr, server): + self.socket = sock + self.addr = addr + self.server = server + + # Copy the class environ into self. + self.environ = self.environ.copy() + + if SSL and isinstance(sock, SSL.ConnectionType): + timeout = sock.gettimeout() + self.rfile = SSL_fileobject(sock, "r", self.rbufsize) + self.rfile.ssl_timeout = timeout + self.sendall = _ssl_wrap_method(sock.sendall) + self.environ["wsgi.url_scheme"] = "https" + self.environ["HTTPS"] = "on" + sslenv = getattr(server, "ssl_environ", None) + if sslenv: + self.environ.update(sslenv) + else: + self.rfile = sock.makefile("r", self.rbufsize) + self.sendall = sock.sendall + + self.environ.update({"wsgi.input": self.rfile, + "SERVER_NAME": self.server.server_name, + }) + + if isinstance(self.server.bind_addr, basestring): + # AF_UNIX. This isn't really allowed by WSGI, which doesn't + # address unix domain sockets. But it's better than nothing. + self.environ["SERVER_PORT"] = "" + else: + self.environ["SERVER_PORT"] = str(self.server.bind_addr[1]) + # optional values + # Until we do DNS lookups, omit REMOTE_HOST + self.environ["REMOTE_ADDR"] = self.addr[0] + self.environ["REMOTE_PORT"] = str(self.addr[1]) + + def communicate(self): + """Read each request and respond appropriately.""" + try: + while True: + # (re)set req to None so that if something goes wrong in + # the RequestHandlerClass constructor, the error doesn't + # get written to the previous request. + req = None + req = self.RequestHandlerClass(self) + # This order of operations should guarantee correct pipelining. + req.parse_request() + if not req.ready: + return + req.respond() + if req.close_connection: + return + except socket.error, e: + errno = e.args[0] + if errno not in socket_errors_to_ignore: + if req: + req.simple_response("500 Internal Server Error", + format_exc()) + return + except (KeyboardInterrupt, SystemExit): + raise + except NoSSLError: + # Unwrap our sendall + req.sendall = self.socket._sock.sendall + req.simple_response("400 Bad Request", + "The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + except: + if req: + req.simple_response("500 Internal Server Error", format_exc()) + + def close(self): + """Close the socket underlying this connection.""" + self.rfile.close() + self.socket.close() + + +def format_exc(limit=None): + """Like print_exc() but return a string. Backport for Python 2.3.""" + try: + etype, value, tb = sys.exc_info() + return ''.join(traceback.format_exception(etype, value, tb, limit)) + finally: + etype = value = tb = None + + +_SHUTDOWNREQUEST = None + +class WorkerThread(threading.Thread): + """Thread which continuously polls a Queue for Connection objects. + + server: the HTTP Server which spawned this thread, and which owns the + Queue and is placing active connections into it. + ready: a simple flag for the calling server to know when this thread + has begun polling the Queue. + + Due to the timing issues of polling a Queue, a WorkerThread does not + check its own 'ready' flag after it has started. To stop the thread, + it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue + (one for each running WorkerThread). + """ + + def __init__(self, server): + self.ready = False + self.server = server + threading.Thread.__init__(self) + + def run(self): + try: + self.ready = True + while True: + conn = self.server.requests.get() + if conn is _SHUTDOWNREQUEST: + return + + try: + conn.communicate() + finally: + conn.close() + except (KeyboardInterrupt, SystemExit), exc: + self.server.interrupt = exc + + +class SSLConnection: + """A thread-safe wrapper for an SSL.Connection. + + *args: the arguments to create the wrapped SSL.Connection(*args). + """ + + def __init__(self, *args): + self._ssl_conn = SSL.Connection(*args) + self._lock = threading.RLock() + + for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read', + 'renegotiate', 'bind', 'listen', 'connect', 'accept', + 'setblocking', 'fileno', 'shutdown', 'close', 'get_cipher_list', + 'getpeername', 'getsockname', 'getsockopt', 'setsockopt', + 'makefile', 'get_app_data', 'set_app_data', 'state_string', + 'sock_shutdown', 'get_peer_certificate', 'want_read', + 'want_write', 'set_connect_state', 'set_accept_state', + 'connect_ex', 'sendall', 'settimeout'): + exec """def %s(self, *args): + self._lock.acquire() + try: + return self._ssl_conn.%s(*args) + finally: + self._lock.release() +""" % (f, f) + + +class CherryPyWSGIServer(object): + """An HTTP server for WSGI. + + bind_addr: a (host, port) tuple if TCP sockets are desired; + for UNIX sockets, supply the filename as a string. + wsgi_app: the WSGI 'application callable'; multiple WSGI applications + may be passed as (script_name, callable) pairs. + numthreads: the number of worker threads to create (default 10). + server_name: the string to set for WSGI's SERVER_NAME environ entry. + Defaults to socket.gethostname(). + max: the maximum number of queued requests (defaults to -1 = no limit). + request_queue_size: the 'backlog' argument to socket.listen(); + specifies the maximum number of queued connections (default 5). + timeout: the timeout in seconds for accepted connections (default 10). + + protocol: the version string to write in the Status-Line of all + HTTP responses. For example, "HTTP/1.1" (the default). This + also limits the supported features used in the response. + + + SSL/HTTPS + --------- + The OpenSSL module must be importable for SSL functionality. + You can obtain it from http://pyopenssl.sourceforge.net/ + + ssl_certificate: the filename of the server SSL certificate. + ssl_privatekey: the filename of the server's private key file. + + If either of these is None (both are None by default), this server + will not use SSL. If both are given and are valid, they will be read + on server start and used in the SSL context for the listening socket. + """ + + protocol = "HTTP/1.1" + version = "CherryPy/3.0.1" + ready = False + _interrupt = None + ConnectionClass = HTTPConnection + + # Paths to certificate and private key files + ssl_certificate = None + ssl_private_key = None + + def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, + max=-1, request_queue_size=5, timeout=10): + self.requests = Queue.Queue(max) + + if callable(wsgi_app): + # We've been handed a single wsgi_app, in CP-2.1 style. + # Assume it's mounted at "". + self.mount_points = [("", wsgi_app)] + else: + # We've been handed a list of (mount_point, wsgi_app) tuples, + # so that the server can call different wsgi_apps, and also + # correctly set SCRIPT_NAME. + self.mount_points = wsgi_app + self.mount_points.sort() + self.mount_points.reverse() + + self.bind_addr = bind_addr + self.numthreads = numthreads or 1 + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.request_queue_size = request_queue_size + self._workerThreads = [] + + self.timeout = timeout + + def start(self): + """Run the server forever.""" + # We don't have to trap KeyboardInterrupt or SystemExit here, + # because cherrpy.server already does so, calling self.stop() for us. + # If you're using this server with another framework, you should + # trap those exceptions in whatever code block calls start(). + self._interrupt = None + + # Select the appropriate socket + if isinstance(self.bind_addr, basestring): + # AF_UNIX socket + + # So we can reuse the socket... + try: os.unlink(self.bind_addr) + except: pass + + # So everyone can access the socket... + try: os.chmod(self.bind_addr, 0777) + except: pass + + info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] + else: + # AF_INET or AF_INET6 socket + # Get the correct address family for our host (allows IPv6 addresses) + host, port = self.bind_addr + flags = 0 + if host == '': + # Despite the socket module docs, using '' does not + # allow AI_PASSIVE to work. Passing None instead + # returns '0.0.0.0' like we want. In other words: + # host AI_PASSIVE result + # '' Y 192.168.x.y + # '' N 192.168.x.y + # None Y 0.0.0.0 + # None N 127.0.0.1 + host = None + flags = socket.AI_PASSIVE + try: + info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, 0, flags) + except socket.gaierror: + # Probably a DNS issue. Assume IPv4. + info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", self.bind_addr)] + + self.socket = None + msg = "No socket could be created" + for res in info: + af, socktype, proto, canonname, sa = res + try: + self.bind(af, socktype, proto) + except socket.error, msg: + if self.socket: + self.socket.close() + self.socket = None + continue + break + if not self.socket: + raise socket.error, msg + + # Timeout so KeyboardInterrupt can be caught on Win32 + self.socket.settimeout(1) + self.socket.listen(self.request_queue_size) + + # Create worker threads + for i in xrange(self.numthreads): + self._workerThreads.append(WorkerThread(self)) + for worker in self._workerThreads: + worker.setName("CP WSGIServer " + worker.getName()) + worker.start() + for worker in self._workerThreads: + while not worker.ready: + time.sleep(.1) + + self.ready = True + while self.ready: + self.tick() + if self.interrupt: + while self.interrupt is True: + # Wait for self.stop() to complete. See _set_interrupt. + time.sleep(0.1) + raise self.interrupt + + def bind(self, family, type, proto=0): + """Create (or recreate) the actual socket object.""" + self.socket = socket.socket(family, type, proto) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +## self.socket.setsockopt(socket.SOL_SOCKET, socket.TCP_NODELAY, 1) + if self.ssl_certificate and self.ssl_private_key: + if SSL is None: + raise ImportError("You must install pyOpenSSL to use HTTPS.") + + # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473 + ctx = SSL.Context(SSL.SSLv23_METHOD) + ctx.use_privatekey_file(self.ssl_private_key) + ctx.use_certificate_file(self.ssl_certificate) + self.socket = SSLConnection(ctx, self.socket) + self.populate_ssl_environ() + self.socket.bind(self.bind_addr) + + def tick(self): + """Accept a new connection and put it on the Queue.""" + try: + s, addr = self.socket.accept() + if not self.ready: + return + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + conn = self.ConnectionClass(s, addr, self) + self.requests.put(conn) + except socket.timeout: + # The only reason for the timeout in start() is so we can + # notice keyboard interrupts on Win32, which don't interrupt + # accept() by default + return + except socket.error, x: + msg = x.args[1] + if msg in ("Bad file descriptor", "Socket operation on non-socket"): + # Our socket was closed. + return + if msg == "Resource temporarily unavailable": + # Just try again. See http://www.cherrypy.org/ticket/479. + return + raise + + def _get_interrupt(self): + return self._interrupt + def _set_interrupt(self, interrupt): + self._interrupt = True + self.stop() + self._interrupt = interrupt + interrupt = property(_get_interrupt, _set_interrupt, + doc="Set this to an Exception instance to " + "interrupt the server.") + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + self.ready = False + + sock = getattr(self, "socket", None) + if sock: + if not isinstance(self.bind_addr, basestring): + # Touch our own socket to make accept() return immediately. + try: + host, port = sock.getsockname()[:2] + except socket.error, x: + if x.args[1] != "Bad file descriptor": + raise + else: + # Note that we're explicitly NOT using AI_PASSIVE, + # here, because we want an actual IP to touch. + # localhost won't work if we've bound to a public IP, + # but it would if we bound to INADDR_ANY via host = ''. + for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(1.0) + s.connect((host, port)) + s.close() + except socket.error: + if s: + s.close() + if hasattr(sock, "close"): + sock.close() + self.socket = None + + # Must shut down threads here so the code that calls + # this method can know when all threads are stopped. + for worker in self._workerThreads: + self.requests.put(_SHUTDOWNREQUEST) + + # Don't join currentThread (when stop is called inside a request). + current = threading.currentThread() + while self._workerThreads: + worker = self._workerThreads.pop() + if worker is not current and worker.isAlive: + try: + worker.join() + except AssertionError: + pass + + def populate_ssl_environ(self): + """Create WSGI environ entries to be merged into each request.""" + cert = open(self.ssl_certificate).read() + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) + self.ssl_environ = { + # pyOpenSSL doesn't provide access to any of these AFAICT +## 'SSL_PROTOCOL': 'SSLv2', +## SSL_CIPHER string The cipher specification name +## SSL_VERSION_INTERFACE string The mod_ssl program version +## SSL_VERSION_LIBRARY string The OpenSSL program version + } + + # Server certificate attributes + self.ssl_environ.update({ + 'SSL_SERVER_M_VERSION': cert.get_version(), + 'SSL_SERVER_M_SERIAL': cert.get_serial_number(), +## 'SSL_SERVER_V_START': Validity of server's certificate (start time), +## 'SSL_SERVER_V_END': Validity of server's certificate (end time), + }) + + for prefix, dn in [("I", cert.get_issuer()), + ("S", cert.get_subject())]: + # X509Name objects don't seem to have a way to get the + # complete DN string. Use str() and slice it instead, + # because str(dn) == "" + dnstr = str(dn)[18:-2] + + wsgikey = 'SSL_SERVER_%s_DN' % prefix + self.ssl_environ[wsgikey] = dnstr + + # The DN should be of the form: /k1=v1/k2=v2, but we must allow + # for any value to contain slashes itself (in a URL). + while dnstr: + pos = dnstr.rfind("=") + dnstr, value = dnstr[:pos], dnstr[pos + 1:] + pos = dnstr.rfind("/") + dnstr, key = dnstr[:pos], dnstr[pos + 1:] + if key and value: + wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key) + self.ssl_environ[wsgikey] = value + ============================================================ --- ChangeLog 356aa12709506db7f10ce9cbe43b7a69da776e2f +++ ChangeLog b019f88c92309d5cd7ba45061bffe75186eb4e2a @@ -1,3 +1,12 @@ +2008-01-13 Grahame Bowland + + * merge patch from Roland McGrath, making + template directory a configurable option (some + servers require that it be an absolute path) + * upgrade to latest web.py (0.22) + * patches tuning automatic branch grouping code + for the branch listing web page. + 2007-07-05 Grahame Bowland * support remapping MIME types, to allow ============================================================ --- config.py.example 782031278d2d977661b7e566495c264626557f3a +++ config.py.example 0a0614a487e5a055da10046a6d6a701211abce19 @@ -27,6 +27,10 @@ static_uri_path = 'http://localhost:8080 dynamic_uri_path = 'http://localhost:8080/' static_uri_path = 'http://localhost:8080/static/' +# Directory in which to find the templates/ files. +# Depending on the web server setup, this might need to be absolute. +templates_directory = 'templates/' + # if you are running under Apache2, set this. # don't set it otherwise, it breaks any other configuration # including running standalone. ============================================================ --- mk2.py 160b4053f2d72fa2807c10cc1b72df4d11828ebf +++ mk2.py db6cdd07694eb91b5828985c8eaf5fcfc4835602 @@ -131,7 +131,7 @@ class MarkovChain(object): max_h = h candidate = state cutoff = self.cutoff_func (self, entropies) - print >>sys.stderr, "best entropy vs. cutoff is: %s :: %.2f vs. cutoff %.2f" % (candidate.state, candidate.entropy(), cutoff) +# print >>sys.stderr, "best entropy vs. cutoff is: %s :: %.2f vs. cutoff %.2f" % (candidate.state, candidate.entropy(), cutoff) if candidate.entropy() < cutoff: return None, None else: ============================================================ --- release.py b530a2e112abc57d740dc931d41a16ac265af00f +++ release.py 7039ccac8caa4f0a7f22b9e6ca05f406e26044cf @@ -1,4 +1,4 @@ -version='0.07' +version='0.08' authors=''' Authors: Grahame Bowland ============================================================ --- viewmtn.py 5de4dc55065621a80377580d13ff4fada54f5029 +++ viewmtn.py c0d63918a4f6f3ea8cfa923166fdd243b55a8e46 @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.4 +#!/usr/bin/env python # Copyright (C) 2005 Grahame Bowland #