# # # patch "tracvc/mtn/automate.py" # from [6787b45e149070d179346ead8fea47a0e6186b0a] # to [26b9c724d86bb12747b9dde784c10b38b3675583] # # patch "tracvc/mtn/basic_io.py" # from [695987e1e39467a82c95bb1027dcbc8dc2457e5c] # to [d4749eacd44c037360f74b8f1405eef48a357993] # # patch "tracvc/mtn/util.py" # from [e073d76fa3050b8fd42874838ba78ed3734ed291] # to [e65c638466ededc3aef8e7eb96bb4e414c68a3a0] # ============================================================ --- tracvc/mtn/automate.py 6787b45e149070d179346ead8fea47a0e6186b0a +++ tracvc/mtn/automate.py 26b9c724d86bb12747b9dde784c10b38b3675583 @@ -41,7 +41,7 @@ class Automate: def __init__(self, db, binary): (self.to_child, self.from_child) = os.popen2( - "%s --norc --automate-stdio-size=1048576 --db=%s automate stdio" % + "%s --norc --automate-stdio-size=1048576 --db=%s --root=. automate stdio" % (binary, db), "b") self.lock = _threading.Lock() @@ -190,26 +190,23 @@ class MTN: status, result = self.automate.command("get_manifest_of", rev) manifest = {} if status != 0: return manifest - for stanza in basic_io.get_stanzas(result): - path, kind, content, attrs = ('', '', '', {}) - for (key, value) in basic_io.get_pairs(stanza): - if key == 'content': content = value - elif key == 'attr': - attrs[value[0]] = value[1] - elif key == 'file' or key == 'dir': - path = '/' + value - kind = key - if path: manifest[path] = (kind, content, attrs) + + # stanzas have variable length, trigger on next 'path' or eof + path = None + for key, values in basic_io.items(result): + if key == 'dir' or key == 'file': + if path: + manifest[path] = (kind, content, attrs) + path = util.add_slash(util.u(values[0])) + kind, content, attrs = key, None, {} + elif key == 'content': + content = values[0] + elif key == 'attrs': + attrs[util.u(values[0])] = util.u(values[1]) + if path: + manifest[path] = (kind, content, attrs) return manifest - @staticmethod - def _u(x): - """ - Convert x from utf-8 to unicode, thereby replacing unknown - characters with the unicode replace char. - """ - return unicode(x, "utf-8", "replace") - @memoize(get_cachespec) def certs(self, rev): """ @@ -220,14 +217,13 @@ class MTN: status, result = self.automate.command("certs", rev) certs = {} if status != 0: return certs - for stanza in basic_io.get_stanzas(result): - cert = basic_io.get_hash_from_stanza(stanza) - name = self._u(cert['name']) - value = self._u(cert['value']) - if not name in certs: - certs[name] = [value] - else: - certs[name].append(value) + + for key, values in basic_io.items(result): + if key == 'name': + name = util.u(values[0]) + elif key == 'value': + value = util.u(values[0]) + certs.setdefault(name, []).append(value) return certs def file(self, id): @@ -254,28 +250,28 @@ class MTN: status, result = self.automate.command("get_revision", rev) if status != 0: return {} changesets = [] - - for stanza in basic_io.get_stanzas(result): - entry = basic_io.get_hash_from_stanza(stanza) - if entry.has_key('old_revision'): + + for key, values in basic_io.items(result): + if key == 'old_revision': # start a new changeset - cs = Changeset(entry['old_revision']) + cs = Changeset(values[0]) changesets.append(cs) - elif entry.has_key('add_dir'): - path = util.add_slash(entry['add_dir']) + elif key == 'delete': + path = util.add_slash(util.u(values[0])) + cs.deleted.append(path) + elif key == 'to': + newpath = util.add_slash(util.u(values[0])) + elif key == 'rename': + oldpath = util.add_slash(util.u(values[0])) + cs.renamed[newpath] = oldpath + elif key == 'add_dir': + path = util.add_slash(util.u(values[0])) cs.added[path] = 'dir' - elif entry.has_key('add_file'): - path = util.add_slash(entry['add_file']) + elif key == 'add_file': + path = util.add_slash(util.u(values[0])) cs.added[path] = 'file' - elif entry.has_key('delete'): - path = util.add_slash(entry['delete']) - cs.deleted.append(path) - elif entry.has_key('rename'): - oldpath = util.add_slash(entry['rename']) - newpath = util.add_slash(entry['to']) - cs.renamed[newpath] = oldpath - elif entry.has_key('patch'): - path = util.add_slash(entry['patch']) + elif key == 'patch': + path = util.add_slash(util.u(values[0])) cs.patched.append(path) # fixme: what about 'set' and 'clear'? These are edits, # but not if applied to new files. @@ -284,7 +280,7 @@ class MTN: def branchnames(self): """Returns a list of branch names.""" status, result = self.automate.command("branches") - if status == 0: return result.splitlines() + if status == 0: return map(util.u, result.splitlines()) else: return [] def branches(self): @@ -312,10 +308,13 @@ class MTN: status, result = self.automate.command("tags") if status != 0: return [] tags = [] - for stanza in basic_io.get_stanzas(result): - entry = basic_io.get_hash_from_stanza(stanza) - if 'tag' in entry: - tags.append((entry['tag'], entry['revision'])) + + for key, values in basic_io.items(result): + if key == 'tag': + tag = util.u(values[0]) + elif key == 'revision': + revision = values[0] + tags.append((tag, revision)) tags.sort(key=lambda i: util.natsort_key(i[0])) return tags @@ -327,10 +326,10 @@ class MTN: status, result = self.automate.command("get_content_changed", rev, file[1:]) if status != 0: return [] revs = [] - for stanza in basic_io.get_stanzas(result): - entry = basic_io.get_hash_from_stanza(stanza) - if 'content_mark' in entry: - revs.append(entry['content_mark']) + + for key, values in basic_io.items(result): + if key == 'content_mark': + revs.append(values[0]) return revs ============================================================ --- tracvc/mtn/basic_io.py 695987e1e39467a82c95bb1027dcbc8dc2457e5c +++ tracvc/mtn/basic_io.py d4749eacd44c037360f74b8f1405eef48a357993 @@ -25,59 +25,32 @@ import re """ import re -STR = r'"(?P(\\\\|\\"|[^"])*)" *' -ID = r'\[(?P[a-f0-9]{40}|)\] *' -VAL = STR + r'|' + ID -LINE = r' *(?P\w+) *(?P(' + VAL + r')+)' + '\n' -STANZA = r'(?P(' + LINE + r')+)(\n|$)' +TOKEN = re.compile(r''' + "(?P(\\\\|\\"|[^"])*)" + |\[(?P[a-f0-9]{40}|)\] + |(?P\w+) + |(?P\s+) +''', re.VERBOSE) -STANZA_RULE = re.compile(STANZA, re.DOTALL) -LINE_RULE = re.compile(LINE, re.DOTALL) -VAL_RULE = re.compile(VAL, re.DOTALL) +def items(raw): + # we yield when we find a new key or hit end of input + key = None + for m in TOKEN.finditer(raw): + if m.lastgroup == 'key': + if key: + yield key, values + key = m.group('key') + values = [] + elif m.lastgroup == 'id': + values.append(m.group('id')) + elif m.lastgroup == 'str': + value = m.group('str') + # dequote: replace \" with " + value = re.sub(r'\\"', '"', value) + # dequote: replace \\ with \ + value = re.sub(r'\\\\', r'\\', value) + values.append(value) + if key: + yield key, values -def get_stanzas(raw): - """Generator, splits a basic_io blob into stanzas.""" - for match in STANZA_RULE.finditer(raw): - yield match.group('stanza') - -def get_pairs(stanza): - """ - Generator, returns (key, value) pairs in one stanza. - - Empty string values ("") are passed through, while empty - revision ids ([]) are converted to None. - - If key has more than one value, the values are collected in a list. - """ - for match in LINE_RULE.finditer(stanza): - key = match.group('key') - - values = match.group('values') - valarray = [] - for match in VAL_RULE.finditer(values): - value = match.group('strval') - if value != None: - # dequote: replace \" with " - value = re.sub(r'\\"', '"', value) - # dequote: replace \\ with \ - value = re.sub(r'\\\\', r'\\', value) - else: - value = match.group('idval') or None - valarray.append(value) - - if len(valarray) == 1: - yield key, valarray[0] - else: - yield key, valarray - -def get_hash_from_stanza(stanza): - """Return a dictionary of the key, value elements in one stanza""" - return dict([(k,v) for k,v in get_pairs(stanza)]) - -def process(raw): - """Parse any basic io output.""" - result = [] - for stanza in get_stanzas(raw): - result.append(get_hash_from_stanza(stanza)) - return result ============================================================ --- tracvc/mtn/util.py e073d76fa3050b8fd42874838ba78ed3734ed291 +++ tracvc/mtn/util.py e65c638466ededc3aef8e7eb96bb4e414c68a3a0 @@ -26,7 +26,9 @@ import re import re +NATSORT_BIT = re.compile(r'(\d+|\D+)') + def add_slash(path): """Prepend a slash. Throughout the this plugin, all paths including the root dir, start with a slash.""" @@ -71,9 +73,13 @@ def try_int(s): return s -NATSORT_BIT = re.compile(r'(\d+|\D+)') - def natsort_key(s): """Generate the key for sorting strings naturally.""" return [try_int(w) for w in re.findall(NATSORT_BIT, s)] + + +def u(s): + """Convert s from utf-8 to unicode, thereby replacing unknown + characters with the unicode replace char.""" + return unicode(s, "utf-8", "replace")