# # # add_file "www/viewmtn.diff" # content [f068709d25426720218355b05afc835627bf45f2] # # patch "www/viewmtn/config.py.example" # from [b076986c7d2ba90eacbbccdb772ab4cd38574e36] # to [1372c7ff5e516d80f25ac66c939bd66e3fcb3a9f] # # patch "www/viewmtn/mtn.py" # from [146041b3f010749063b798fca752e9a687447513] # to [c09dd76e71aadf61e8cf8af280c6ef65f115a762] # # patch "www/viewmtn/viewmtn.py" # from [843122d52484c7d36c7b12a911e844aa3828d4ef] # to [7d191aa63ba24ea0427246c4e395645ec07fe46f] # ============================================================ --- www/viewmtn.diff f068709d25426720218355b05afc835627bf45f2 +++ www/viewmtn.diff f068709d25426720218355b05afc835627bf45f2 @@ -0,0 +1,328 @@ +diff -ru viewmtn-0.10/config.py.example webhost/www/viewmtn/config.py.example +--- viewmtn-0.10/config.py.example 2010-02-28 16:32:01.093397729 -0600 ++++ webhost/www/viewmtn/config.py.example 2010-02-27 23:48:09.321400937 -0600 +@@ -20,12 +20,43 @@ + # + + import sys ++import os ++import re ++ ++name = 'NAME' ++confdir = 'CONFDIR' ++conffile = confdir + '/hostconfig' ++ ++# if you are running under Apache2, set this. ++# don't set it otherwise, it breaks any other configuration ++# including running standalone. ++running_under_apache2 = True ++def file_tokens(fn): ++ f = open(fn, 'r') ++ data = f.read() ++ f.close() ++ tok_str = r'"(?:[^"]|\\")*"' ++ tok_key = r'[^"\s[]\S*' ++ tok_hex = r'\[[[:xdigit:]]*\]' ++ anytok = '(%s|%s|%s)' % (tok_key, tok_str, tok_hex) ++ return re.split(anytok, data) ++ ++config_data = {} ++ ++for s in file_tokens(conffile): ++ if not re.match(r'^\s', s): ++ if re.match(r'^["[]', s): ++ config_data[lastkey].append(s[1:-1]) ++ else: ++ lastkey = s ++ config_data[lastkey] = [] + + # default addresses should work without modification + # if running viewmtn standalone. You must change these + # if you run viewmtn with a web server, see INSTALL. +-dynamic_uri_path = 'http://localhost:8080/' +-static_uri_path = 'http://localhost:8080/static/' ++dynamic_uri_path = config_data['base_url'][0] + 'viewmtn/' ++static_uri_path = config_data['base_url'][0] + 'viewmtn/static/' ++ + + # Directory in which to find the templates/ files. + # Depending on the web server setup, this might need to be absolute. +@@ -34,10 +65,10 @@ + # if you are running under Apache2, set this. + # don't set it otherwise, it breaks any other configuration + # including running standalone. +-running_under_apache2 = False ++running_under_apache2 = True + + # the path to the 'mtn' binary +-monotone = '/usr/bin/mtn' ++monotone = config_data['monotone'][0] + + # + # Database files: +@@ -71,6 +102,9 @@ + # "database2", "/path/to/junk.db", "my other stuff") + # defaultdb = "database1" + ++dbfiles = () ++automate_addr = config_data['automateaddr'][0].split(':') ++ + # Style (2b) + # WARNING: this will publish _all_ the databases in the directory + # matched by the call to glob, if they have a corresponding .descr +@@ -84,6 +118,8 @@ + # ) for t in glob.glob("/Users/grahame/mtn/db/*.db") + # if os.access(t+'.descr', os.R_OK)]) + ++project_dir = config_data['project_dir'][0] ++ + # highlight from http://andre-simon.de/ + # if you don't have this available, just comment + # the "highlight_command" line out +Only in viewmtn-0.10: _MTN +diff -ru viewmtn-0.10/mtn.py webhost/www/viewmtn/mtn.py +--- viewmtn-0.10/mtn.py 2010-02-28 16:32:01.097407097 -0600 ++++ webhost/www/viewmtn/mtn.py 2010-02-27 23:48:09.325401327 -0600 +@@ -13,6 +13,7 @@ + import fcntl + import pipes + import select ++import socket + import threading + import popen2 + from common import set_nonblocking, terminate_popen3 +@@ -55,12 +56,11 @@ + self.obj_type = "author" + + class Runner(object): +- def __init__(self, monotone, database): ++ def __init__(self, stdio_addr, database): + self.database = database +- self.base_command = [monotone, "--db=%s" % pipes.quote(database)] ++ self.stdio_addr = stdio_addr + + packet_header_re = re.compile(r'^(\d+):(\d+):([lm]):(\d+):') +-import web + + class Automate(Runner): + """Runs commands via a particular monotone process. This +@@ -74,32 +74,26 @@ + def __init__(self, *args, **kwargs): + Runner.__init__(*[self] + list(args), **kwargs) + self.lock = threading.Lock() +- self.process = None +- self.running_mtime = None ++ self.sock = None + + def stop(self): +- if not self.process: ++ if not self.sock: + return +- terminate_popen3(self.process) +- self.process = None ++ self.sock.close() ++ self.sock = None + + def database_mtime(self): + return os.stat(self.database).st_mtime + + def check_current(self): +- if self.process != None and self.database_mtime() > self.running_mtime: +- debug("stopped process, database has changed.") +- self.stop() ++ pass + + def __process_required(self): +- if self.process != None: ++ if self.sock != None: + return +- to_run = self.base_command + ['automate', 'stdio'] +- self.running_mtime = self.database_mtime() +- self.process = popen2.Popen3(to_run, capturestderr=True) +- map (set_nonblocking, [ self.process.fromchild, +- self.process.tochild, +- self.process.childerr ]) ++ self.sock = socket.socket() ++ self.sock.connect((self.stdio_addr[0], int(self.stdio_addr[1]))) ++ self.sock.send(self.database + "\n") + + def run(self, *args, **kwargs): + #debug(("automate is running:", args, kwargs)) +@@ -140,19 +134,16 @@ + + return CleanRequest(self.__run(*args, **kwargs)) + +- def __run(self, command, args, **kwargs): ++ def __run(self, command, args, opts = []): + def str_with_len(l, s): + l.append(str(len(s))) + l.append(":") + l.append(s) + parts = [] +- for k in kwargs: +- # can't use a '-' as a named function argument +- kd = k.replace('_', '-') +- parts.append("o") +- str_with_len(parts, kd) +- str_with_len(parts, kwargs[k]) +- parts.append("e") ++ parts.append("o") ++ for x in opts: # I don't think **kwargs allows duplicates? ++ str_with_len(parts, x) ++ parts.append("e") + parts.append("l") + str_with_len(parts, command) + for x in args: +@@ -164,23 +155,22 @@ + for i in xrange(2): + self.__process_required() + try: +- self.process.tochild.write(enc) +- self.process.tochild.flush() ++ self.sock.send(enc) + break + except: + # mtn has died underneath the automate; restart it +- debug("exception writing to child process; attempting restart: %s" % format_exc()) ++ debug("exception writing to automation server; attempting restart: %s" % format_exc()) + self.stop() + + def read_result_packets(): + buffer = "" + while True: +- r_stdin, r_stdout, r_stderr = select.select([self.process.fromchild], [], [], None) ++ r_stdin, r_stdout, r_stderr = select.select([self.sock], [], [], None) + if not r_stdin and not r_stdout and not r_stderr: + break + +- if self.process.fromchild in r_stdin: +- data = self.process.fromchild.read() ++ if self.sock in r_stdin: ++ data = self.sock.recv(4096) + if data == "": + break + buffer += data +@@ -230,25 +220,6 @@ + if code_max > 0: + raise MonotoneException("error code %d in automate packet." % (code_max)) + +-class Standalone(Runner): +- """Runs commands by running monotone. One monotone process +- per command""" +- +- def run(self, command, args): +- # as we pass popen3 as sequence, it executes monotone with these +- # arguments - and does not pass them through the shell according +- # to help(os.popen3) +-# debug(("standalone is running:", command, args)) +- to_run = self.base_command + [command] + args +- process = popen2.Popen3(to_run, capturestderr=True) +- for line in process.fromchild: +- yield line +- stderr_data = process.childerr.read() +- if len(stderr_data) > 0: +- raise MonotoneException("data on stderr for command '%s': %s" % (command, +- stderr_data)) +- terminate_popen3(process) +- + class MtnObject(object): + def __init__(self, obj_type): + self.obj_type = obj_type +@@ -370,11 +341,10 @@ + + class Operations(object): + def __init__(self, runner_args): +- self.standalone = apply(Standalone, runner_args) + self.automate = apply(Automate, runner_args) + + def per_request(self): +- """"Call this method every distinct request, to allow Operations to do any ++ """Call this method every distinct request, to allow Operations to do any + cleanup operations. + """ + self.automate.check_current() +@@ -392,7 +362,7 @@ + yield Revision(revision_id) + + def branches(self): +- for line in (t.strip() for t in self.automate.run('branches', [], ignore_suspend_certs="")): ++ for line in (t.strip() for t in self.automate.run('branches', [], ['ignore-suspend-certs', ""])): + if not line: + continue + yield apply(Branch, (line,)) +@@ -452,8 +422,12 @@ + yield stanza + + def diff(self, revision_from, revision_to, files=[]): +- args = ['-r', revision_from, '-r', revision_to] + files +- for line in self.standalone.run('diff', args): ++ for line in self.automate.run('content_diff', ++ files, ++ ['with-header', '', ++ 'revision', revision_from, ++ 'revision', revision_to]): ++ + yield line + + ### +diff -ru viewmtn-0.10/viewmtn.py webhost/www/viewmtn/viewmtn.py +--- viewmtn-0.10/viewmtn.py 2010-02-28 16:32:01.106065059 -0600 ++++ webhost/www/viewmtn/viewmtn.py 2010-02-27 23:48:09.337400710 -0600 +@@ -70,7 +70,7 @@ + self.default = config.defaultdb + self.have_multidb = True + else: +- self.add_to_store(None, ops=mtn.Operations([config.monotone, config.dbfile]), ++ self.add_to_store(None, ops=mtn.Operations([config.automate_addr, config.dbfile]), + branchdivs=branchdiv.BranchDivisions(), + dbdescr="") + self.have_multidb = False +@@ -84,22 +84,25 @@ + return self.dbstore[k].get(name, default) + + def __getitem__(self, name): +- if name is None: +- name = self.default ++ if not os.path.isdir("%s/%s" % (config.project_dir, name)): ++ print >>sys.stderr, "Cannot find project %s in directory %s\n" % (name, config.project_dir) ++ return None + ops = self.get_from_store("ops", name) + if ops is None: +- return None +- else: +- dbdescr = self.get_from_store("dbdescr", name) +- branchdivs = self.get_from_store("branchdivs", name) +- return RequestContext(dbname=name, +- ops=ops, +- dbdescr=dbdescr, +- nodb_join=self.nodb_join, +- static_join=self.static_join, +- branchdivs=branchdivs, +- renderer=self.renderer, +- db_summary=self.db_summary) ++ self.add_to_store(name, ops=mtn.Operations([config.automate_addr, name]), ++ branchdivs=branchdiv.BranchDivisions(), ++ dbdescr="") ++ ops = self.get_from_store("ops", name) ++ dbdescr = self.get_from_store("dbdescr", name) ++ branchdivs = self.get_from_store("branchdivs", name) ++ return RequestContext(dbname=name, ++ ops=ops, ++ dbdescr=dbdescr, ++ nodb_join=self.nodb_join, ++ static_join=self.static_join, ++ branchdivs=branchdivs, ++ renderer=self.renderer, ++ db_summary=self.db_summary) + + def runfcgi_apache(func): + web.wsgi.runfcgi(func, None) +@@ -156,7 +159,7 @@ + lambda f: nodefault_closure(f())) + urls += assemble('d', + perdb_urls, +- lambda u: r'^/([A-Za-z]+/)?' + u, ++ lambda u: r'^/([-A-Za-z0-9]+/)?' + u, + lambda f: per_db_closure(f())) + + # import pprint ============================================================ --- www/viewmtn/config.py.example b076986c7d2ba90eacbbccdb772ab4cd38574e36 +++ www/viewmtn/config.py.example 1372c7ff5e516d80f25ac66c939bd66e3fcb3a9f @@ -20,44 +20,13 @@ import sys # import sys -import os -import re -name = 'NAME' -confdir = 'CONFDIR' -conffile = confdir + '/hostconfig' - -# if you are running under Apache2, set this. -# don't set it otherwise, it breaks any other configuration -# including running standalone. -running_under_apache2 = True -def file_tokens(fn): - f = open(fn, 'r') - data = f.read() - f.close() - tok_str = r'"(?:[^"]|\\")*"' - tok_key = r'[^"\s[]\S*' - tok_hex = r'\[[[:xdigit:]]*\]' - anytok = '(%s|%s|%s)' % (tok_key, tok_str, tok_hex) - return re.split(anytok, data) - -config_data = {} - -for s in file_tokens(conffile): - if not re.match(r'^\s', s): - if re.match(r'^["[]', s): - config_data[lastkey].append(s[1:-1]) - else: - lastkey = s - config_data[lastkey] = [] - # default addresses should work without modification # if running viewmtn standalone. You must change these # if you run viewmtn with a web server, see INSTALL. -dynamic_uri_path = config_data['base_url'][0] + 'viewmtn/' -static_uri_path = config_data['base_url'][0] + 'viewmtn/static/' +dynamic_uri_path = 'http://localhost:8080/' +static_uri_path = 'http://localhost:8080/static/' - # Directory in which to find the templates/ files. # Depending on the web server setup, this might need to be absolute. templates_directory = 'templates/' @@ -65,10 +34,10 @@ templates_directory = 'templates/' # if you are running under Apache2, set this. # don't set it otherwise, it breaks any other configuration # including running standalone. -running_under_apache2 = True +running_under_apache2 = False # the path to the 'mtn' binary -monotone = config_data['monotone'][0] +monotone = '/usr/bin/mtn' # # Database files: @@ -102,9 +71,6 @@ monotone = config_data['monotone'][0] # "database2", "/path/to/junk.db", "my other stuff") # defaultdb = "database1" -dbfiles = () -automate_addr = config_data['automateaddr'][0].split(':') - # Style (2b) # WARNING: this will publish _all_ the databases in the directory # matched by the call to glob, if they have a corresponding .descr @@ -118,8 +84,6 @@ automate_addr = config_data['automateadd # ) for t in glob.glob("/Users/grahame/mtn/db/*.db") # if os.access(t+'.descr', os.R_OK)]) -project_dir = config_data['project_dir'][0] - # highlight from http://andre-simon.de/ # if you don't have this available, just comment # the "highlight_command" line out ============================================================ --- www/viewmtn/mtn.py 146041b3f010749063b798fca752e9a687447513 +++ www/viewmtn/mtn.py c09dd76e71aadf61e8cf8af280c6ef65f115a762 @@ -13,7 +13,6 @@ import select import fcntl import pipes import select -import socket import threading import popen2 from common import set_nonblocking, terminate_popen3 @@ -56,11 +55,12 @@ class Runner(object): self.obj_type = "author" class Runner(object): - def __init__(self, stdio_addr, database): + def __init__(self, monotone, database): self.database = database - self.stdio_addr = stdio_addr + self.base_command = [monotone, "--db=%s" % pipes.quote(database)] packet_header_re = re.compile(r'^(\d+):(\d+):([lm]):(\d+):') +import web class Automate(Runner): """Runs commands via a particular monotone process. This @@ -74,26 +74,32 @@ class Automate(Runner): def __init__(self, *args, **kwargs): Runner.__init__(*[self] + list(args), **kwargs) self.lock = threading.Lock() - self.sock = None + self.process = None + self.running_mtime = None def stop(self): - if not self.sock: + if not self.process: return - self.sock.close() - self.sock = None + terminate_popen3(self.process) + self.process = None def database_mtime(self): return os.stat(self.database).st_mtime def check_current(self): - pass + if self.process != None and self.database_mtime() > self.running_mtime: + debug("stopped process, database has changed.") + self.stop() def __process_required(self): - if self.sock != None: + if self.process != None: return - self.sock = socket.socket() - self.sock.connect((self.stdio_addr[0], int(self.stdio_addr[1]))) - self.sock.send(self.database + "\n") + to_run = self.base_command + ['automate', 'stdio'] + self.running_mtime = self.database_mtime() + self.process = popen2.Popen3(to_run, capturestderr=True) + map (set_nonblocking, [ self.process.fromchild, + self.process.tochild, + self.process.childerr ]) def run(self, *args, **kwargs): #debug(("automate is running:", args, kwargs)) @@ -134,16 +140,19 @@ class Automate(Runner): return CleanRequest(self.__run(*args, **kwargs)) - def __run(self, command, args, opts = []): + def __run(self, command, args, **kwargs): def str_with_len(l, s): l.append(str(len(s))) l.append(":") l.append(s) parts = [] - parts.append("o") - for x in opts: # I don't think **kwargs allows duplicates? - str_with_len(parts, x) - parts.append("e") + for k in kwargs: + # can't use a '-' as a named function argument + kd = k.replace('_', '-') + parts.append("o") + str_with_len(parts, kd) + str_with_len(parts, kwargs[k]) + parts.append("e") parts.append("l") str_with_len(parts, command) for x in args: @@ -155,22 +164,23 @@ class Automate(Runner): for i in xrange(2): self.__process_required() try: - self.sock.send(enc) + self.process.tochild.write(enc) + self.process.tochild.flush() break except: # mtn has died underneath the automate; restart it - debug("exception writing to automation server; attempting restart: %s" % format_exc()) + debug("exception writing to child process; attempting restart: %s" % format_exc()) self.stop() def read_result_packets(): buffer = "" while True: - r_stdin, r_stdout, r_stderr = select.select([self.sock], [], [], None) + r_stdin, r_stdout, r_stderr = select.select([self.process.fromchild], [], [], None) if not r_stdin and not r_stdout and not r_stderr: break - if self.sock in r_stdin: - data = self.sock.recv(4096) + if self.process.fromchild in r_stdin: + data = self.process.fromchild.read() if data == "": break buffer += data @@ -220,6 +230,25 @@ class Automate(Runner): if code_max > 0: raise MonotoneException("error code %d in automate packet." % (code_max)) +class Standalone(Runner): + """Runs commands by running monotone. One monotone process + per command""" + + def run(self, command, args): + # as we pass popen3 as sequence, it executes monotone with these + # arguments - and does not pass them through the shell according + # to help(os.popen3) +# debug(("standalone is running:", command, args)) + to_run = self.base_command + [command] + args + process = popen2.Popen3(to_run, capturestderr=True) + for line in process.fromchild: + yield line + stderr_data = process.childerr.read() + if len(stderr_data) > 0: + raise MonotoneException("data on stderr for command '%s': %s" % (command, + stderr_data)) + terminate_popen3(process) + class MtnObject(object): def __init__(self, obj_type): self.obj_type = obj_type @@ -341,10 +370,11 @@ class Operations(object): class Operations(object): def __init__(self, runner_args): + self.standalone = apply(Standalone, runner_args) self.automate = apply(Automate, runner_args) def per_request(self): - """Call this method every distinct request, to allow Operations to do any + """"Call this method every distinct request, to allow Operations to do any cleanup operations. """ self.automate.check_current() @@ -362,7 +392,7 @@ class Operations(object): yield Revision(revision_id) def branches(self): - for line in (t.strip() for t in self.automate.run('branches', [], ['ignore-suspend-certs', ""])): + for line in (t.strip() for t in self.automate.run('branches', [], ignore_suspend_certs="")): if not line: continue yield apply(Branch, (line,)) @@ -422,12 +452,8 @@ class Operations(object): yield stanza def diff(self, revision_from, revision_to, files=[]): - for line in self.automate.run('content_diff', - files, - ['with-header', '', - 'revision', revision_from, - 'revision', revision_to]): - + args = ['-r', revision_from, '-r', revision_to] + files + for line in self.standalone.run('diff', args): yield line ### ============================================================ --- www/viewmtn/viewmtn.py 843122d52484c7d36c7b12a911e844aa3828d4ef +++ www/viewmtn/viewmtn.py 7d191aa63ba24ea0427246c4e395645ec07fe46f @@ -70,7 +70,7 @@ class RequestContextFactory(object): self.default = config.defaultdb self.have_multidb = True else: - self.add_to_store(None, ops=mtn.Operations([config.automate_addr, config.dbfile]), + self.add_to_store(None, ops=mtn.Operations([config.monotone, config.dbfile]), branchdivs=branchdiv.BranchDivisions(), dbdescr="") self.have_multidb = False @@ -84,25 +84,22 @@ class RequestContextFactory(object): return self.dbstore[k].get(name, default) def __getitem__(self, name): - if not os.path.isdir("%s/%s" % (config.project_dir, name)): - print >>sys.stderr, "Cannot find project %s in directory %s\n" % (name, config.project_dir) - return None + if name is None: + name = self.default ops = self.get_from_store("ops", name) if ops is None: - self.add_to_store(name, ops=mtn.Operations([config.automate_addr, name]), - branchdivs=branchdiv.BranchDivisions(), - dbdescr="") - ops = self.get_from_store("ops", name) - dbdescr = self.get_from_store("dbdescr", name) - branchdivs = self.get_from_store("branchdivs", name) - return RequestContext(dbname=name, - ops=ops, - dbdescr=dbdescr, - nodb_join=self.nodb_join, - static_join=self.static_join, - branchdivs=branchdivs, - renderer=self.renderer, - db_summary=self.db_summary) + return None + else: + dbdescr = self.get_from_store("dbdescr", name) + branchdivs = self.get_from_store("branchdivs", name) + return RequestContext(dbname=name, + ops=ops, + dbdescr=dbdescr, + nodb_join=self.nodb_join, + static_join=self.static_join, + branchdivs=branchdivs, + renderer=self.renderer, + db_summary=self.db_summary) def runfcgi_apache(func): web.wsgi.runfcgi(func, None) @@ -159,7 +156,7 @@ def assemble_urls(): lambda f: nodefault_closure(f())) urls += assemble('d', perdb_urls, - lambda u: r'^/([-A-Za-z0-9]+/)?' + u, + lambda u: r'^/([A-Za-z]+/)?' + u, lambda f: per_db_closure(f())) # import pprint