Fix up a bunch of bugs in the new merge code

Move getchangegroup/addchangegroup to generators

hg

@@ -282,7 +282,7 b' elif cmd == "debugindex":'
 elif cmd == "merge":
     if args:
         other = hg.repository(ui, args[0])
-        print "retrieving changegroup"
+        print "requesting changegroup"
         cg = repo.getchangegroup(other)
         repo.addchangegroup(cg)
     else:

mercurial/hg.py

@@ -613,22 +613,23 b' class localrepository:'
 
     def getchangegroup(self, remote):
         tip = remote.branches([])[0]
-        cl = self.changelog
+        m = self.changelog.nodemap
         unknown = [tip]
         search = []
         fetch = []
 
-        if tip[0] == self.changelog.tip():
+        if tip[0] in m:
             return None
 
         while unknown:
             n = unknown.pop(0)
             if n == nullid: break
-            if n[1] and cl.nodemap.has_key(n[1]): # do we know the base?
+            if n[1] and n[1] in m: # do we know the base?
                 search.append(n) # schedule branch range for scanning
             else:
                 for b in remote.branches([n[2], n[3]]):
-                    if cl.nodemap.has_key(b[0]):
+                    if b[0] in m:
+                        if n[1] not in fetch:
                         fetch.append(n[1]) # earliest unknown
                     else:
                         unknown.append(b)
@@ -639,13 +640,18 b' class localrepository:'
             p = n[0]
             f = 1
             for i in l + [n[1]]:
-                if self.changelog.nodemap.has_key(i):
+                if i in m:
                     if f <= 4:
                         fetch.append(p)
                     else:
                         search.append((p, i))
+                    break
                 p, f = i, f * 2
 
+        for f in fetch:
+            if f in m:
+                raise "already have", hex(f[:4])
+
         return remote.changegroup(fetch)
     
     def changegroup(self, basenodes):
@@ -677,55 +683,63 b' class localrepository:'
             l = struct.pack(">l", len(f))
             yield "".join([l, f, g])
 
-    def addchangegroup(self, data):
-        def getlen(data, pos):
-            return struct.unpack(">l", data[pos:pos + 4])[0]
+    def addchangegroup(self, generator):
+        class genread:
+            def __init__(self, generator):
+                self.g = generator
+                self.buf = ""
+            def read(self, l):
+                while l > len(self.buf):
+                    try:
+                        self.buf += self.g.next()
+                    except StopIteration:
+                        break
+                d, self.buf = self.buf[:l], self.buf[l:]
+                return d
 
-        if not data: return
+        if not generator: return
+        source = genread(generator)
+
+        def getchunk(add = 0):
+            d = source.read(4)
+            if not d: return ""
+            l = struct.unpack(">l", d)[0]
+            return source.read(l - 4 + add)
         
         tr = self.transaction()
         simple = True
 
         print "merging changesets"
         # pull off the changeset group
-        l = getlen(data, 0)
-        csg = data[0:l]
-        pos = l
+        csg = getchunk()
         co = self.changelog.tip()
         cn = self.changelog.addgroup(csg, lambda x: self.changelog.count(), tr)
 
         print "merging manifests"
         # pull off the manifest group
-        l = getlen(data, pos)
-        mfg = data[pos: pos + l]
-        pos += l
+        mfg = getchunk()
         mo = self.manifest.tip()
-        mn = self.manifest.addgroup(mfg, lambda x: self.changelog.rev(x), tr)
+        mm = self.manifest.addgroup(mfg, lambda x: self.changelog.rev(x), tr)
 
         # do we need a resolve?
         if self.changelog.ancestor(co, cn) != co:
-            print "NEED RESOLVE"
             simple = False
             resolverev = self.changelog.count()
 
         # process the files
         print "merging files"
         new = {}
-        while pos < len(data):
-            l = getlen(data, pos)
-            pos += 4
-            f = data[pos:pos + l]
-            pos += l
-
-            l = getlen(data, pos)
-            fg = data[pos: pos + l]
-            pos += l
+        while 1:
+            f = getchunk(4)
+            if not f: break
+            fg = getchunk()
 
             fl = self.file(f)
             o = fl.tip()
             n = fl.addgroup(fg, lambda x: self.changelog.rev(x), tr)
             if not simple:
-                new[fl] = fl.resolvedag(o, n, tr, resolverev)
+                nn = fl.resolvedag(o, n, tr, resolverev)
+                if nn: new[f] = nn
 
         # For simple merges, we don't need to resolve manifests or changesets
         if simple:
@@ -794,24 +808,30 b' class remoterepository:'
         q.update(args)
         qs = urllib.urlencode(q)
         cu = "%s?%s" % (self.url, qs)
-        return urllib.urlopen(cu).read()
+        return urllib.urlopen(cu)
 
     def branches(self, nodes):
         n = " ".join(map(hex, nodes))
-        d = self.do_cmd("branches", nodes=n)
+        d = self.do_cmd("branches", nodes=n).read()
         br = [ map(bin, b.split(" ")) for b in d.splitlines() ]
         return br
 
     def between(self, pairs):
         n = "\n".join(["-".join(map(hex, p)) for p in pairs])
-        d = self.do_cmd("between", pairs=n)
+        d = self.do_cmd("between", pairs=n).read()
         p = [ map(bin, l.split(" ")) for l in d.splitlines() ]
         return p
 
     def changegroup(self, nodes):
         n = " ".join(map(hex, nodes))
-        d = self.do_cmd("changegroup", roots=n)
-        return zlib.decompress(d)
+        zd = zlib.decompressobj()
+        f = self.do_cmd("changegroup", roots=n)
+        while 1:
+            d = f.read(4096)
+            if not d:
+                yield zd.flush()
+                break
+            yield zd.decompress(d)
 
 def repository(ui, path=None, create=0):
     if path and path[:5] == "hg://":

mercurial/revlog.py

@@ -345,11 +345,10 b' class revlog:'
         # first delta is against its parent, which should be in our
         # log, the rest are against the previous delta.
 
-        if len(data) <= 4: return
+        if not data: return self.tip()
 
         # retrieve the parent revision of the delta chain
-        chain = data[28:48]
-        text = self.revision(chain)
+        chain = data[24:44]
 
         # track the base of the current delta log
         r = self.count()
@@ -370,7 +369,7 b' class revlog:'
         ifh = self.opener(self.indexfile, "a")
 
         # loop through our set of deltas
-        pos = 4
+        pos = 0
         while pos < len(data):
             l, node, p1, p2, cs = struct.unpack(">l20s20s20s20s",
                                                 data[pos:pos+84])
@@ -391,7 +390,7 b' class revlog:'
                 # flush our writes here so we can read it in revision
                 dfh.flush()
                 ifh.flush()
-                text = self.revision(self.node(t))
+                text = self.revision(chain)
                 text = self.patch(text, delta)
                 chk = self.addrevision(text, transaction, link, p1, p2)
                 if chk != node:
@@ -404,8 +403,7 b' class revlog:'
                 dfh.write(cdelta)
                 ifh.write(struct.pack(indexformat, *e))
 
-            t, r = r, r + 1
-            chain = prev
+            t, r, chain, prev = r, r + 1, node, node
             start = self.start(self.base(t))
             end = self.end(t)