Abstract manifest block parsing.

mercurial/manifest.py

@@ -1,199 +1,209 b''
 # manifest.py - manifest revision class for mercurial
 #
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
 from revlog import *
 from i18n import gettext as _
 from demandload import *
 demandload(globals(), "array bisect struct")
+demandload(globals(), "mdiff")
 
 class manifestdict(dict):
     def __init__(self, mapping=None, flags=None):
         if mapping is None: mapping = {}
         if flags is None: flags = {}
         dict.__init__(self, mapping)
         self._flags = flags
     def flags(self, f):
         return self._flags.get(f, "")
     def execf(self, f):
         "test for executable in manifest flags"
         return "x" in self.flags(f)
     def linkf(self, f):
         "test for symlink in manifest flags"
         return "l" in self.flags(f)
     def rawset(self, f, entry):
         self[f] = bin(entry[:40])
         fl = entry[40:-1]
         if fl: self._flags[f] = fl
     def set(self, f, execf=False, linkf=False):
         if linkf: self._flags[f] = "l"
         elif execf: self._flags[f] = "x"
         else: self._flags[f] = ""
     def copy(self):
         return manifestdict(dict.copy(self), dict.copy(self._flags))
 
 class manifest(revlog):
     def __init__(self, opener, defversion=REVLOGV0):
         self.mapcache = None
         self.listcache = None
         revlog.__init__(self, opener, "00manifest.i", "00manifest.d",
                         defversion)
 
+    def parselines(self, lines):
+        for l in lines.splitlines(1):
+            yield l.split('\0')
+
+    def readdelta(self, node):
+        delta = mdiff.patchtext(self.delta(node))
+        deltamap = manifestdict()
+        for f, n in self.parselines(delta):
+            deltamap.rawset(f, n)
+        return deltamap
+            
     def read(self, node):
         if node == nullid: return manifestdict() # don't upset local cache
         if self.mapcache and self.mapcache[0] == node:
             return self.mapcache[1]
         text = self.revision(node)
         self.listcache = array.array('c', text)
-        lines = text.splitlines(1)
         mapping = manifestdict()
-        for l in lines:
-            (f, n) = l.split('\0')
+        for f, n in self.parselines(text):
             mapping.rawset(f, n)
         self.mapcache = (node, mapping)
         return mapping
 
     def _search(self, m, s, lo=0, hi=None):
         '''return a tuple (start, end) that says where to find s within m.
 
         If the string is found m[start:end] are the line containing
         that string.  If start == end the string was not found and
         they indicate the proper sorted insertion point.  This was
         taken from bisect_left, and modified to find line start/end as
         it goes along.
 
         m should be a buffer or a string
         s is a string'''
         def advance(i, c):
             while i < lenm and m[i] != c:
                 i += 1
             return i
         lenm = len(m)
         if not hi:
             hi = lenm
         while lo < hi:
             mid = (lo + hi) // 2
             start = mid
             while start > 0 and m[start-1] != '\n':
                 start -= 1
             end = advance(start, '\0')
             if m[start:end] < s:
                 # we know that after the null there are 40 bytes of sha1
                 # this translates to the bisect lo = mid + 1
                 lo = advance(end + 40, '\n') + 1
             else:
                 # this translates to the bisect hi = mid
                 hi = start
         end = advance(lo, '\0')
         found = m[lo:end]
         if cmp(s, found) == 0:
             # we know that after the null there are 40 bytes of sha1
             end = advance(end + 40, '\n')
             return (lo, end+1)
         else:
             return (lo, lo)
 
     def find(self, node, f):
         '''look up entry for a single file efficiently.
         return (node, flag) pair if found, (None, None) if not.'''
         if self.mapcache and node == self.mapcache[0]:
             return self.mapcache[1].get(f), self.mapcache[1].flags(f)
         text = self.revision(node)
         start, end = self._search(text, f)
         if start == end:
             return None, None
         l = text[start:end]
         f, n = l.split('\0')
         return bin(n[:40]), n[40:-1] == 'x'
 
     def add(self, map, transaction, link, p1=None, p2=None,
             changed=None):
         # apply the changes collected during the bisect loop to our addlist
         # return a delta suitable for addrevision
         def addlistdelta(addlist, x):
             # start from the bottom up
             # so changes to the offsets don't mess things up.
             i = len(x)
             while i > 0:
                 i -= 1
                 start = x[i][0]
                 end = x[i][1]
                 if x[i][2]:
                     addlist[start:end] = array.array('c', x[i][2])
                 else:
                     del addlist[start:end]
             return "".join([struct.pack(">lll", d[0], d[1], len(d[2])) + d[2] \
                             for d in x ])
 
         # if we're using the listcache, make sure it is valid and
         # parented by the same node we're diffing against
         if not changed or not self.listcache or not p1 or \
                self.mapcache[0] != p1:
             files = map.keys()
             files.sort()
 
             # if this is changed to support newlines in filenames,
             # be sure to check the templates/ dir again (especially *-raw.tmpl)
             text = ["%s\000%s%s\n" % (f, hex(map[f]), map.flags(f)) for f in files]
             self.listcache = array.array('c', "".join(text))
             cachedelta = None
         else:
             addlist = self.listcache
 
             # combine the changed lists into one list for sorting
             work = [[x, 0] for x in changed[0]]
             work[len(work):] = [[x, 1] for x in changed[1]]
             work.sort()
 
             delta = []
             dstart = None
             dend = None
             dline = [""]
             start = 0
             # zero copy representation of addlist as a buffer
             addbuf = buffer(addlist)
 
             # start with a readonly loop that finds the offset of
             # each line and creates the deltas
             for w in work:
                 f = w[0]
                 # bs will either be the index of the item or the insert point
                 start, end = self._search(addbuf, f, start)
                 if w[1] == 0:
                     l = "%s\000%s%s\n" % (f, hex(map[f]), map.flags(f))
                 else:
                     l = ""
                 if start == end and w[1] == 1:
                     # item we want to delete was not found, error out
                     raise AssertionError(
                             _("failed to remove %s from manifest") % f)
                 if dstart != None and dstart <= start and dend >= start:
                     if dend < end:
                         dend = end
                     if l:
                         dline.append(l)
                 else:
                     if dstart != None:
                         delta.append([dstart, dend, "".join(dline)])
                     dstart = start
                     dend = end
                     dline = [l]
 
             if dstart != None:
                 delta.append([dstart, dend, "".join(dline)])
             # apply the delta to the addlist, and get a delta for addrevision
             cachedelta = addlistdelta(addlist, delta)
 
             # the delta is only valid if we've been processing the tip revision
             if self.mapcache[0] != self.tip():
                 cachedelta = None
             self.listcache = addlist
 
         n = self.addrevision(buffer(self.listcache), transaction, link, p1,  \
                              p2, cachedelta)
         self.mapcache = (n, map)
 
         return n

mercurial/verify.py

@@ -1,200 +1,194 b''
 # verify.py - repository integrity checking for Mercurial
 #
 # Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
 from node import *
 from i18n import gettext as _
 import revlog, mdiff
 
 def verify(repo):
     filelinkrevs = {}
     filenodes = {}
     changesets = revisions = files = 0
     errors = [0]
     warnings = [0]
     neededmanifests = {}
 
     def err(msg):
         repo.ui.warn(msg + "\n")
         errors[0] += 1
 
     def warn(msg):
         repo.ui.warn(msg + "\n")
         warnings[0] += 1
 
     def checksize(obj, name):
         d = obj.checksize()
         if d[0]:
             err(_("%s data length off by %d bytes") % (name, d[0]))
         if d[1]:
             err(_("%s index contains %d extra bytes") % (name, d[1]))
 
     def checkversion(obj, name):
         if obj.version != revlog.REVLOGV0:
             if not revlogv1:
                 warn(_("warning: `%s' uses revlog format 1") % name)
         elif revlogv1:
             warn(_("warning: `%s' uses revlog format 0") % name)
 
     revlogv1 = repo.revlogversion != revlog.REVLOGV0
     if repo.ui.verbose or revlogv1 != repo.revlogv1:
         repo.ui.status(_("repository uses revlog format %d\n") %
                        (revlogv1 and 1 or 0))
 
     seen = {}
     repo.ui.status(_("checking changesets\n"))
     checksize(repo.changelog, "changelog")
 
     for i in range(repo.changelog.count()):
         changesets += 1
         n = repo.changelog.node(i)
         l = repo.changelog.linkrev(n)
         if l != i:
             err(_("incorrect link (%d) for changeset revision %d") %(l, i))
         if n in seen:
             err(_("duplicate changeset at revision %d") % i)
         seen[n] = 1
 
         for p in repo.changelog.parents(n):
             if p not in repo.changelog.nodemap:
                 err(_("changeset %s has unknown parent %s") %
                              (short(n), short(p)))
         try:
             changes = repo.changelog.read(n)
         except KeyboardInterrupt:
             repo.ui.warn(_("interrupted"))
             raise
         except Exception, inst:
             err(_("unpacking changeset %s: %s") % (short(n), inst))
             continue
 
         neededmanifests[changes[0]] = n
 
         for f in changes[3]:
             filelinkrevs.setdefault(f, []).append(i)
 
     seen = {}
     repo.ui.status(_("checking manifests\n"))
     checkversion(repo.manifest, "manifest")
     checksize(repo.manifest, "manifest")
 
     for i in range(repo.manifest.count()):
         n = repo.manifest.node(i)
         l = repo.manifest.linkrev(n)
 
         if l < 0 or l >= repo.changelog.count():
             err(_("bad manifest link (%d) at revision %d") % (l, i))
 
         if n in neededmanifests:
             del neededmanifests[n]
 
         if n in seen:
             err(_("duplicate manifest at revision %d") % i)
 
         seen[n] = 1
 
         for p in repo.manifest.parents(n):
             if p not in repo.manifest.nodemap:
                 err(_("manifest %s has unknown parent %s") %
                     (short(n), short(p)))
 
         try:
-            delta = mdiff.patchtext(repo.manifest.delta(n))
+            for f, fn in repo.manifest.readdelta(n).iteritems():
+                filenodes.setdefault(f, {})[fn] = 1
         except KeyboardInterrupt:
             repo.ui.warn(_("interrupted"))
             raise
         except Exception, inst:
-            err(_("unpacking manifest %s: %s") % (short(n), inst))
+            err(_("reading delta for manifest %s: %s") % (short(n), inst))
             continue
 
-        try:
-            ff = [ l.split('\0') for l in delta.splitlines() ]
-            for f, fn in ff:
-                filenodes.setdefault(f, {})[bin(fn[:40])] = 1
-        except (ValueError, TypeError), inst:
-            err(_("broken delta in manifest %s: %s") % (short(n), inst))
-
     repo.ui.status(_("crosschecking files in changesets and manifests\n"))
 
     for m, c in neededmanifests.items():
         err(_("Changeset %s refers to unknown manifest %s") %
             (short(m), short(c)))
     del neededmanifests
 
     for f in filenodes:
         if f not in filelinkrevs:
             err(_("file %s in manifest but not in changesets") % f)
 
     for f in filelinkrevs:
         if f not in filenodes:
             err(_("file %s in changeset but not in manifest") % f)
 
     repo.ui.status(_("checking files\n"))
     ff = filenodes.keys()
     ff.sort()
     for f in ff:
         if f == "/dev/null":
             continue
         files += 1
         if not f:
             err(_("file without name in manifest %s") % short(n))
             continue
         fl = repo.file(f)
         checkversion(fl, f)
         checksize(fl, f)
 
         nodes = {nullid: 1}
         seen = {}
         for i in range(fl.count()):
             revisions += 1
             n = fl.node(i)
 
             if n in seen:
                 err(_("%s: duplicate revision %d") % (f, i))
             if n not in filenodes[f]:
                 err(_("%s: %d:%s not in manifests") % (f, i, short(n)))
             else:
                 del filenodes[f][n]
 
             flr = fl.linkrev(n)
             if flr not in filelinkrevs.get(f, []):
                 err(_("%s:%s points to unexpected changeset %d")
                         % (f, short(n), flr))
             else:
                 filelinkrevs[f].remove(flr)
 
             # verify contents
             try:
                 t = fl.read(n)
             except KeyboardInterrupt:
                 repo.ui.warn(_("interrupted"))
                 raise
             except Exception, inst:
                 err(_("unpacking file %s %s: %s") % (f, short(n), inst))
 
             # verify parents
             (p1, p2) = fl.parents(n)
             if p1 not in nodes:
                 err(_("file %s:%s unknown parent 1 %s") %
                     (f, short(n), short(p1)))
             if p2 not in nodes:
                 err(_("file %s:%s unknown parent 2 %s") %
                         (f, short(n), short(p1)))
             nodes[n] = 1
 
         # cross-check
         for node in filenodes[f]:
             err(_("node %s in manifests not in %s") % (hex(node), f))
 
     repo.ui.status(_("%d files, %d changesets, %d total revisions\n") %
                    (files, changesets, revisions))
 
     if warnings[0]:
         repo.ui.warn(_("%d warnings encountered!\n") % warnings[0])
     if errors[0]:
         repo.ui.warn(_("%d integrity errors encountered!\n") % errors[0])
         return 1