ancestor: add lazy membership testing to lazyancestors

This also makes the perfancestorset command use lazy membership testing. In a
linear repository with over 400,000 commits, without this patch, hg
perfancestorset takes 0.80 seconds no matter how far behind we're looking.
With this patch, hg perfancestorset -- X takes:

    Rev X       Time
       -1      0.00s
    -4000      0.01s
   -20000      0.04s
   -80000      0.17s
  -200000      0.43s
  -300000      0.69s
        0      0.88s

Thus, for revisions close to tip, we're up to several orders of magnitude
faster. At 0 we're around 10% slower.

contrib/perf.py

@@ -1,277 +1,277
 # perf.py - performance test routines
 '''helper extension to measure performance'''
 
 from mercurial import cmdutil, scmutil, util, match, commands
 import time, os, sys
 
 def timer(func, title=None):
     results = []
     begin = time.time()
     count = 0
     while True:
         ostart = os.times()
         cstart = time.time()
         r = func()
         cstop = time.time()
         ostop = os.times()
         count += 1
         a, b = ostart, ostop
         results.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
         if cstop - begin > 3 and count >= 100:
             break
         if cstop - begin > 10 and count >= 3:
             break
     if title:
         sys.stderr.write("! %s\n" % title)
     if r:
         sys.stderr.write("! result: %s\n" % r)
     m = min(results)
     sys.stderr.write("! wall %f comb %f user %f sys %f (best of %d)\n"
                      % (m[0], m[1] + m[2], m[1], m[2], count))
 
 def perfwalk(ui, repo, *pats):
     try:
         m = scmutil.match(repo[None], pats, {})
         timer(lambda: len(list(repo.dirstate.walk(m, [], True, False))))
     except Exception:
         try:
             m = scmutil.match(repo[None], pats, {})
             timer(lambda: len([b for a, b, c in repo.dirstate.statwalk([], m)]))
         except Exception:
             timer(lambda: len(list(cmdutil.walk(repo, pats, {}))))
 
 def perfstatus(ui, repo, **opts):
     #m = match.always(repo.root, repo.getcwd())
     #timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer(lambda: sum(map(len, repo.status(**opts))))
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, 'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, '_nodecache'):
         from mercurial.node import nullid, nullrev
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 def perfheads(ui, repo):
     cl = repo.changelog
     def d():
         len(cl.headrevs())
         clearcaches(cl)
     timer(d)
 
 def perftags(ui, repo):
     import mercurial.changelog, mercurial.manifest
     def t():
         repo.changelog = mercurial.changelog.changelog(repo.sopener)
         repo.manifest = mercurial.manifest.manifest(repo.sopener)
         repo._tags = None
         return len(repo.tags())
     timer(t)
 
 def perfancestors(ui, repo):
     heads = repo.changelog.headrevs()
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
     timer(d)
 
 def perfancestorset(ui, repo, revset):
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
     def d():
-        s = set(repo.changelog.ancestors(heads))
+        s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
     timer(d)
 
 def perfdirstate(ui, repo):
     "a" in repo.dirstate
     def d():
         repo.dirstate.invalidate()
         "a" in repo.dirstate
     timer(d)
 
 def perfdirstatedirs(ui, repo):
     "a" in repo.dirstate
     def d():
         "a" in repo.dirstate._dirs
         del repo.dirstate._dirs
     timer(d)
 
 def perfdirstatewrite(ui, repo):
     ds = repo.dirstate
     "a" in ds
     def d():
         ds._dirty = True
         ds.write()
     timer(d)
 
 def perfmanifest(ui, repo):
     def d():
         t = repo.manifest.tip()
         m = repo.manifest.read(t)
         repo.manifest.mapcache = None
         repo.manifest._cache = None
     timer(d)
 
 def perfchangeset(ui, repo, rev):
     n = repo[rev].node()
     def d():
         c = repo.changelog.read(n)
         #repo.changelog._cache = None
     timer(d)
 
 def perfindex(ui, repo):
     import mercurial.revlog
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
     n = repo["tip"].node()
     def d():
         cl = mercurial.revlog.revlog(repo.sopener, "00changelog.i")
         cl.rev(n)
     timer(d)
 
 def perfstartup(ui, repo):
     cmd = sys.argv[0]
     def d():
         os.system("HGRCPATH= %s version -q > /dev/null" % cmd)
     timer(d)
 
 def perfparents(ui, repo):
     nl = [repo.changelog.node(i) for i in xrange(1000)]
     def d():
         for n in nl:
             repo.changelog.parents(n)
     timer(d)
 
 def perflookup(ui, repo, rev):
     timer(lambda: len(repo.lookup(rev)))
 
 def perfrevrange(ui, repo, *specs):
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
 
 def perfnodelookup(ui, repo, rev):
     import mercurial.revlog
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
     n = repo[rev].node()
     def d():
         cl = mercurial.revlog.revlog(repo.sopener, "00changelog.i")
         cl.rev(n)
     timer(d)
 
 def perfnodelookup(ui, repo, rev):
     import mercurial.revlog
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
     n = repo[rev].node()
     cl = mercurial.revlog.revlog(repo.sopener, "00changelog.i")
     def d():
         cl.rev(n)
         clearcaches(cl)
     timer(d)
 
 def perflog(ui, repo, **opts):
     ui.pushbuffer()
     timer(lambda: commands.log(ui, repo, rev=[], date='', user='',
                                copies=opts.get('rename')))
     ui.popbuffer()
 
 def perftemplating(ui, repo):
     ui.pushbuffer()
     timer(lambda: commands.log(ui, repo, rev=[], date='', user='',
                                template='{date|shortdate} [{rev}:{node|short}]'
                                ' {author|person}: {desc|firstline}\n'))
     ui.popbuffer()
 
 def perfcca(ui, repo):
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
 
 def perffncacheload(ui, repo):
     s = repo.store
     def d():
         s.fncache._load()
     timer(d)
 
 def perffncachewrite(ui, repo):
     s = repo.store
     s.fncache._load()
     def d():
         s.fncache._dirty = True
         s.fncache.write()
     timer(d)
 
 def perffncacheencode(ui, repo):
     s = repo.store
     s.fncache._load()
     def d():
         for p in s.fncache.entries:
             s.encode(p)
     timer(d)
 
 def perfdiffwd(ui, repo):
     """Profile diff of working directory changes"""
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
         }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
         opts = dict((options[c], '1') for c in diffopt)
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
         title = 'diffopts: %s' % (diffopt and ('-' + diffopt) or 'none')
         timer(d, title)
 
 def perfrevlog(ui, repo, file_, **opts):
     from mercurial import revlog
     dist = opts['dist']
     def d():
         r = revlog.revlog(lambda fn: open(fn, 'rb'), file_)
         for x in xrange(0, len(r), dist):
             r.revision(r.node(x))
 
     timer(d)
 
 def perfrevset(ui, repo, expr):
     def d():
         repo.revs(expr)
     timer(d)
 
 cmdtable = {
     'perfcca': (perfcca, []),
     'perffncacheload': (perffncacheload, []),
     'perffncachewrite': (perffncachewrite, []),
     'perffncacheencode': (perffncacheencode, []),
     'perflookup': (perflookup, []),
     'perfrevrange': (perfrevrange, []),
     'perfnodelookup': (perfnodelookup, []),
     'perfparents': (perfparents, []),
     'perfstartup': (perfstartup, []),
     'perfstatus': (perfstatus,
                    [('u', 'unknown', False,
                      'ask status to look for unknown files')]),
     'perfwalk': (perfwalk, []),
     'perfmanifest': (perfmanifest, []),
     'perfchangeset': (perfchangeset, []),
     'perfindex': (perfindex, []),
     'perfheads': (perfheads, []),
     'perftags': (perftags, []),
     'perfancestors': (perfancestors, []),
     'perfancestorset': (perfancestorset, [], "REVSET"),
     'perfdirstate': (perfdirstate, []),
     'perfdirstatedirs': (perfdirstate, []),
     'perfdirstatewrite': (perfdirstatewrite, []),
     'perflog': (perflog,
                 [('', 'rename', False, 'ask log to follow renames')]),
     'perftemplating': (perftemplating, []),
     'perfdiffwd': (perfdiffwd, []),
     'perfrevlog': (perfrevlog,
                    [('d', 'dist', 100, 'distance between the revisions')],
                    "[INDEXFILE]"),
     'perfrevset': (perfrevset, [], "REVSET")
 }

mercurial/ancestor.py

@@ -1,221 +1,264
 # ancestor.py - generic DAG ancestor algorithm for mercurial
 #
 # Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import heapq, util
 from node import nullrev
 
 def ancestor(a, b, pfunc):
     """
     Returns the common ancestor of a and b that is furthest from a
     root (as measured by longest path) or None if no ancestor is
     found. If there are multiple common ancestors at the same
     distance, the first one found is returned.
 
     pfunc must return a list of parent vertices for a given vertex
     """
 
     if a == b:
         return a
 
     a, b = sorted([a, b])
 
     # find depth from root of all ancestors
     # depth is stored as a negative for heapq
     parentcache = {}
     visit = [a, b]
     depth = {}
     while visit:
         vertex = visit[-1]
         pl = pfunc(vertex)
         parentcache[vertex] = pl
         if not pl:
             depth[vertex] = 0
             visit.pop()
         else:
             for p in pl:
                 if p == a or p == b: # did we find a or b as a parent?
                     return p # we're done
                 if p not in depth:
                     visit.append(p)
             if visit[-1] == vertex:
                 # -(maximum distance of parents + 1)
                 depth[vertex] = min([depth[p] for p in pl]) - 1
                 visit.pop()
 
     # traverse ancestors in order of decreasing distance from root
     def ancestors(vertex):
         h = [(depth[vertex], vertex)]
         seen = set()
         while h:
             d, n = heapq.heappop(h)
             if n not in seen:
                 seen.add(n)
                 yield (d, n)
                 for p in parentcache[n]:
                     heapq.heappush(h, (depth[p], p))
 
     def generations(vertex):
         sg, s = None, set()
         for g, v in ancestors(vertex):
             if g != sg:
                 if sg:
                     yield sg, s
                 sg, s = g, set((v,))
             else:
                 s.add(v)
         yield sg, s
 
     x = generations(a)
     y = generations(b)
     gx = x.next()
     gy = y.next()
 
     # increment each ancestor list until it is closer to root than
     # the other, or they match
     try:
         while True:
             if gx[0] == gy[0]:
                 for v in gx[1]:
                     if v in gy[1]:
                         return v
                 gy = y.next()
                 gx = x.next()
             elif gx[0] > gy[0]:
                 gy = y.next()
             else:
                 gx = x.next()
     except StopIteration:
         return None
 
 def missingancestors(revs, bases, pfunc):
     """Return all the ancestors of revs that are not ancestors of bases.
 
     This may include elements from revs.
 
     Equivalent to the revset (::revs - ::bases). Revs are returned in
     revision number order, which is a topological order.
 
     revs and bases should both be iterables. pfunc must return a list of
     parent revs for a given revs.
     """
 
     revsvisit = set(revs)
     basesvisit = set(bases)
     if not revsvisit:
         return []
     if not basesvisit:
         basesvisit.add(nullrev)
     start = max(max(revsvisit), max(basesvisit))
     bothvisit = revsvisit.intersection(basesvisit)
     revsvisit.difference_update(bothvisit)
     basesvisit.difference_update(bothvisit)
     # At this point, we hold the invariants that:
     # - revsvisit is the set of nodes we know are an ancestor of at least one
     #   of the nodes in revs
     # - basesvisit is the same for bases
     # - bothvisit is the set of nodes we know are ancestors of at least one of
     #   the nodes in revs and one of the nodes in bases
     # - a node may be in none or one, but not more, of revsvisit, basesvisit
     #   and bothvisit at any given time
     # Now we walk down in reverse topo order, adding parents of nodes already
     # visited to the sets while maintaining the invariants. When a node is
     # found in both revsvisit and basesvisit, it is removed from them and
     # added to bothvisit instead. When revsvisit becomes empty, there are no
     # more ancestors of revs that aren't also ancestors of bases, so exit.
 
     missing = []
     for curr in xrange(start, nullrev, -1):
         if not revsvisit:
             break
 
         if curr in bothvisit:
             bothvisit.remove(curr)
             # curr's parents might have made it into revsvisit or basesvisit
             # through another path
             for p in pfunc(curr):
                 revsvisit.discard(p)
                 basesvisit.discard(p)
                 bothvisit.add(p)
             continue
 
         # curr will never be in both revsvisit and basesvisit, since if it
         # were it'd have been pushed to bothvisit
         if curr in revsvisit:
             missing.append(curr)
             thisvisit = revsvisit
             othervisit = basesvisit
         elif curr in basesvisit:
             thisvisit = basesvisit
             othervisit = revsvisit
         else:
             # not an ancestor of revs or bases: ignore
             continue
 
         thisvisit.remove(curr)
         for p in pfunc(curr):
             if p == nullrev:
                 pass
             elif p in othervisit or p in bothvisit:
                 # p is implicitly in thisvisit. This means p is or should be
                 # in bothvisit
                 revsvisit.discard(p)
                 basesvisit.discard(p)
                 bothvisit.add(p)
             else:
                 # visit later
                 thisvisit.add(p)
 
     missing.reverse()
     return missing
 
 class lazyancestors(object):
     def __init__(self, cl, revs, stoprev=0, inclusive=False):
         """Create a new object generating ancestors for the given revs. Does
         not generate revs lower than stoprev.
 
-        This is computed lazily starting from revs. The object only supports
-        iteration.
+        This is computed lazily starting from revs. The object supports
+        iteration and membership.
 
         cl should be a changelog and revs should be an iterable. inclusive is
         a boolean that indicates whether revs should be included. Revs lower
         than stoprev will not be generated.
 
         Result does not include the null revision."""
         self._parentrevs = cl.parentrevs
         self._initrevs = revs
         self._stoprev = stoprev
         self._inclusive = inclusive
 
+        # Initialize data structures for __contains__.
+        # For __contains__, we use a heap rather than a deque because
+        # (a) it minimizes the number of parentrevs calls made
+        # (b) it makes the loop termination condition obvious
+        # Python's heap is a min-heap. Multiply all values by -1 to convert it
+        # into a max-heap.
+        self._containsvisit = [-rev for rev in revs]
+        heapq.heapify(self._containsvisit)
+        if inclusive:
+            self._containsseen = set(revs)
+        else:
+            self._containsseen = set()
+
     def __iter__(self):
         """Generate the ancestors of _initrevs in reverse topological order.
 
         If inclusive is False, yield a sequence of revision numbers starting
         with the parents of each revision in revs, i.e., each revision is *not*
         considered an ancestor of itself.  Results are in breadth-first order:
         parents of each rev in revs, then parents of those, etc.
 
         If inclusive is True, yield all the revs first (ignoring stoprev),
         then yield all the ancestors of revs as when inclusive is False.
         If an element in revs is an ancestor of a different rev it is not
         yielded again."""
         seen = set()
         revs = self._initrevs
         if self._inclusive:
             for rev in revs:
                 yield rev
             seen.update(revs)
 
         parentrevs = self._parentrevs
         stoprev = self._stoprev
         visit = util.deque(revs)
 
         while visit:
             for parent in parentrevs(visit.popleft()):
                 if parent >= stoprev and parent not in seen:
                     visit.append(parent)
                     seen.add(parent)
                     yield parent
+
+    def __contains__(self, target):
+        """Test whether target is an ancestor of self._initrevs."""
+        # Trying to do both __iter__ and __contains__ using the same visit
+        # heap and seen set is complex enough that it slows down both. Keep
+        # them separate.
+        seen = self._containsseen
+        if target in seen:
+            return True
+
+        parentrevs = self._parentrevs
+        visit = self._containsvisit
+        stoprev = self._stoprev
+        heappop = heapq.heappop
+        heappush = heapq.heappush
+
+        targetseen = False
+
+        while visit and -visit[0] > target and not targetseen:
+            for parent in parentrevs(-heappop(visit)):
+                if parent < stoprev or parent in seen:
+                    continue
+                # We need to make sure we push all parents into the heap so
+                # that we leave it in a consistent state for future calls.
+                heappush(visit, -parent)
+                seen.add(parent)
+                if parent == target:
+                    targetseen = True
+
+        return targetseen

tests/test-ancestor.py

@@ -1,74 +1,106
 from mercurial import ancestor
 
 # graph is a dict of child->parent adjacency lists for this graph:
 # o  13
 # |
 # | o  12
 # | |
 # | | o    11
 # | | |\
 # | | | | o  10
 # | | | | |
 # | o---+ |  9
 # | | | | |
 # o | | | |  8
 #  / / / /
 # | | o |  7
 # | | | |
 # o---+ |  6
 #  / / /
 # | | o  5
 # | |/
 # | o  4
 # | |
 # o |  3
 # | |
 # | o  2
 # |/
 # o  1
 # |
 # o  0
 
 graph = {0: [-1], 1: [0], 2: [1], 3: [1], 4: [2], 5: [4], 6: [4],
          7: [4], 8: [-1], 9: [6, 7], 10: [5], 11: [3, 7], 12: [9],
          13: [8]}
 pfunc = graph.get
 
+class mockchangelog(object):
+    parentrevs = graph.get
+
 def runmissingancestors(revs, bases):
     print "%% ancestors of %s and not of %s" % (revs, bases)
     print ancestor.missingancestors(revs, bases, pfunc)
 
 def test_missingancestors():
     # Empty revs
     runmissingancestors([], [1])
     runmissingancestors([], [])
 
     # If bases is empty, it's the same as if it were [nullrev]
     runmissingancestors([12], [])
 
     # Trivial case: revs == bases
     runmissingancestors([0], [0])
     runmissingancestors([4, 5, 6], [6, 5, 4])
 
     # With nullrev
     runmissingancestors([-1], [12])
     runmissingancestors([12], [-1])
 
     # 9 is a parent of 12. 7 is a parent of 9, so an ancestor of 12. 6 is an
     # ancestor of 12 but not of 7.
     runmissingancestors([12], [9])
     runmissingancestors([9], [12])
     runmissingancestors([12, 9], [7])
     runmissingancestors([7, 6], [12])
 
     # More complex cases
     runmissingancestors([10], [11, 12])
     runmissingancestors([11], [10])
     runmissingancestors([11], [10, 12])
     runmissingancestors([12], [10])
     runmissingancestors([12], [11])
     runmissingancestors([10, 11, 12], [13])
     runmissingancestors([13], [10, 11, 12])
 
+def genlazyancestors(revs, stoprev=0, inclusive=False):
+    print ("%% lazy ancestor set for %s, stoprev = %s, inclusive = %s" %
+           (revs, stoprev, inclusive))
+    return ancestor.lazyancestors(mockchangelog, revs, stoprev=stoprev,
+                                  inclusive=inclusive)
+
+def printlazyancestors(s, l):
+    print [n for n in l if n in s]
+
+def test_lazyancestors():
+    # Empty revs
+    s = genlazyancestors([])
+    printlazyancestors(s, [3, 0, -1])
+
+    # Standard example
+    s = genlazyancestors([11, 13])
+    printlazyancestors(s, [11, 13, 7, 9, 8, 3, 6, 4, 1, -1, 0])
+
+    # Including revs
+    s = genlazyancestors([11, 13], inclusive=True)
+    printlazyancestors(s, [11, 13, 7, 9, 8, 3, 6, 4, 1, -1, 0])
+
+    # Test with stoprev
+    s = genlazyancestors([11, 13], stoprev=6)
+    printlazyancestors(s, [11, 13, 7, 9, 8, 3, 6, 4, 1, -1, 0])
+    s = genlazyancestors([11, 13], stoprev=6, inclusive=True)
+    printlazyancestors(s, [11, 13, 7, 9, 8, 3, 6, 4, 1, -1, 0])
+
 if __name__ == '__main__':
     test_missingancestors()
+    test_lazyancestors()

tests/test-ancestor.py.out

@@ -1,36 +1,46
 % ancestors of [] and not of [1]
 []
 % ancestors of [] and not of []
 []
 % ancestors of [12] and not of []
 [0, 1, 2, 4, 6, 7, 9, 12]
 % ancestors of [0] and not of [0]
 []
 % ancestors of [4, 5, 6] and not of [6, 5, 4]
 []
 % ancestors of [-1] and not of [12]
 []
 % ancestors of [12] and not of [-1]
 [0, 1, 2, 4, 6, 7, 9, 12]
 % ancestors of [12] and not of [9]
 [12]
 % ancestors of [9] and not of [12]
 []
 % ancestors of [12, 9] and not of [7]
 [6, 9, 12]
 % ancestors of [7, 6] and not of [12]
 []
 % ancestors of [10] and not of [11, 12]
 [5, 10]
 % ancestors of [11] and not of [10]
 [3, 7, 11]
 % ancestors of [11] and not of [10, 12]
 [3, 11]
 % ancestors of [12] and not of [10]
 [6, 7, 9, 12]
 % ancestors of [12] and not of [11]
 [6, 9, 12]
 % ancestors of [10, 11, 12] and not of [13]
 [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12]
 % ancestors of [13] and not of [10, 11, 12]
 [8, 13]
+% lazy ancestor set for [], stoprev = 0, inclusive = False
+[]
+% lazy ancestor set for [11, 13], stoprev = 0, inclusive = False
+[7, 8, 3, 4, 1, 0]
+% lazy ancestor set for [11, 13], stoprev = 0, inclusive = True
+[11, 13, 7, 8, 3, 4, 1, 0]
+% lazy ancestor set for [11, 13], stoprev = 6, inclusive = False
+[7, 8]
+% lazy ancestor set for [11, 13], stoprev = 6, inclusive = True
+[11, 13, 7, 8]