bundlerepo: remove duplication of bundle decompressors

mercurial/bundlerepo.py

@@ -1,305 +1,289 b''
 # bundlerepo.py - repository class for viewing uncompressed bundles
 #
 # Copyright 2006, 2007 Benoit Boissinot <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Repository class for viewing uncompressed bundles.
 
 This provides a read-only repository interface to bundles as if they
 were part of the actual repository.
 """
 
 from node import nullid
 from i18n import _
-import os, struct, bz2, zlib, tempfile, shutil
+import os, struct, tempfile, shutil
 import changegroup, util, mdiff
 import localrepo, changelog, manifest, filelog, revlog, error
 
 class bundlerevlog(revlog.revlog):
     def __init__(self, opener, indexfile, bundlefile,
                  linkmapper=None):
         # How it works:
         # to retrieve a revision, we need to know the offset of
         # the revision in the bundlefile (an opened file).
         #
         # We store this offset in the index (start), to differentiate a
         # rev in the bundle and from a rev in the revlog, we check
         # len(index[r]). If the tuple is bigger than 7, it is a bundle
         # (it is bigger since we store the node to which the delta is)
         #
         revlog.revlog.__init__(self, opener, indexfile)
         self.bundlefile = bundlefile
         self.basemap = {}
         def chunkpositer():
             for chunk in changegroup.chunkiter(bundlefile):
                 pos = bundlefile.tell()
                 yield chunk, pos - len(chunk)
         n = len(self)
         prev = None
         for chunk, start in chunkpositer():
             size = len(chunk)
             if size < 80:
                 raise util.Abort(_("invalid changegroup"))
             start += 80
             size -= 80
             node, p1, p2, cs = struct.unpack("20s20s20s20s", chunk[:80])
             if node in self.nodemap:
                 prev = node
                 continue
             for p in (p1, p2):
                 if not p in self.nodemap:
                     raise error.LookupError(p, self.indexfile,
                                             _("unknown parent"))
             if linkmapper is None:
                 link = n
             else:
                 link = linkmapper(cs)
 
             if not prev:
                 prev = p1
             # start, size, full unc. size, base (unused), link, p1, p2, node
             e = (revlog.offset_type(start, 0), size, -1, -1, link,
                  self.rev(p1), self.rev(p2), node)
             self.basemap[n] = prev
             self.index.insert(-1, e)
             self.nodemap[node] = n
             prev = node
             n += 1
 
     def bundle(self, rev):
         """is rev from the bundle"""
         if rev < 0:
             return False
         return rev in self.basemap
     def bundlebase(self, rev):
         return self.basemap[rev]
     def _chunk(self, rev):
         # Warning: in case of bundle, the diff is against bundlebase,
         # not against rev - 1
         # XXX: could use some caching
         if not self.bundle(rev):
             return revlog.revlog._chunk(self, rev)
         self.bundlefile.seek(self.start(rev))
         return self.bundlefile.read(self.length(rev))
 
     def revdiff(self, rev1, rev2):
         """return or calculate a delta between two revisions"""
         if self.bundle(rev1) and self.bundle(rev2):
             # hot path for bundle
             revb = self.rev(self.bundlebase(rev2))
             if revb == rev1:
                 return self._chunk(rev2)
         elif not self.bundle(rev1) and not self.bundle(rev2):
             return revlog.revlog.revdiff(self, rev1, rev2)
 
         return mdiff.textdiff(self.revision(self.node(rev1)),
                          self.revision(self.node(rev2)))
 
     def revision(self, node):
         """return an uncompressed revision of a given"""
         if node == nullid:
             return ""
 
         text = None
         chain = []
         iter_node = node
         rev = self.rev(iter_node)
         # reconstruct the revision if it is from a changegroup
         while self.bundle(rev):
             if self._cache and self._cache[0] == iter_node:
                 text = self._cache[2]
                 break
             chain.append(rev)
             iter_node = self.bundlebase(rev)
             rev = self.rev(iter_node)
         if text is None:
             text = revlog.revlog.revision(self, iter_node)
 
         while chain:
             delta = self._chunk(chain.pop())
             text = mdiff.patches(text, [delta])
 
         p1, p2 = self.parents(node)
         if node != revlog.hash(text, p1, p2):
             raise error.RevlogError(_("integrity check failed on %s:%d")
                                      % (self.datafile, self.rev(node)))
 
         self._cache = (node, self.rev(node), text)
         return text
 
     def addrevision(self, text, transaction, link, p1=None, p2=None, d=None):
         raise NotImplementedError
     def addgroup(self, revs, linkmapper, transaction):
         raise NotImplementedError
     def strip(self, rev, minlink):
         raise NotImplementedError
     def checksize(self):
         raise NotImplementedError
 
 class bundlechangelog(bundlerevlog, changelog.changelog):
     def __init__(self, opener, bundlefile):
         changelog.changelog.__init__(self, opener)
         bundlerevlog.__init__(self, opener, self.indexfile, bundlefile)
 
 class bundlemanifest(bundlerevlog, manifest.manifest):
     def __init__(self, opener, bundlefile, linkmapper):
         manifest.manifest.__init__(self, opener)
         bundlerevlog.__init__(self, opener, self.indexfile, bundlefile,
                               linkmapper)
 
 class bundlefilelog(bundlerevlog, filelog.filelog):
     def __init__(self, opener, path, bundlefile, linkmapper):
         filelog.filelog.__init__(self, opener, path)
         bundlerevlog.__init__(self, opener, self.indexfile, bundlefile,
                               linkmapper)
 
 class bundlerepository(localrepo.localrepository):
     def __init__(self, ui, path, bundlename):
         self._tempparent = None
         try:
             localrepo.localrepository.__init__(self, ui, path)
         except error.RepoError:
             self._tempparent = tempfile.mkdtemp()
             localrepo.instance(ui, self._tempparent, 1)
             localrepo.localrepository.__init__(self, ui, self._tempparent)
 
         if path:
             self._url = 'bundle:' + util.expandpath(path) + '+' + bundlename
         else:
             self._url = 'bundle:' + bundlename
 
         self.tempfile = None
         self.bundlefile = open(bundlename, "rb")
-        header = self.bundlefile.read(6)
-        if not header.startswith("HG"):
-            raise util.Abort(_("%s: not a Mercurial bundle file") % bundlename)
-        elif not header.startswith("HG10"):
-            raise util.Abort(_("%s: unknown bundle version") % bundlename)
-        elif (header == "HG10BZ") or (header == "HG10GZ"):
+        b = changegroup.readbundle(self.bundlefile, bundlename)
+        if b.compressed():
             fdtemp, temp = tempfile.mkstemp(prefix="hg-bundle-",
                                             suffix=".hg10un", dir=self.path)
             self.tempfile = temp
             fptemp = os.fdopen(fdtemp, 'wb')
-            def generator(f):
-                if header == "HG10BZ":
-                    zd = bz2.BZ2Decompressor()
-                    zd.decompress("BZ")
-                elif header == "HG10GZ":
-                    zd = zlib.decompressobj()
-                for chunk in f:
-                    yield zd.decompress(chunk)
-            gen = generator(util.filechunkiter(self.bundlefile, 4096))
 
             try:
                 fptemp.write("HG10UN")
-                for chunk in gen:
+                while 1:
+                    chunk = b.read(2**18)
+                    if not chunk:
+                        break
                     fptemp.write(chunk)
             finally:
                 fptemp.close()
                 self.bundlefile.close()
 
             self.bundlefile = open(self.tempfile, "rb")
-            # seek right after the header
             self.bundlefile.seek(6)
-        elif header == "HG10UN":
-            # nothing to do
-            pass
-        else:
-            raise util.Abort(_("%s: unknown bundle compression type")
-                             % bundlename)
+
         # dict with the mapping 'filename' -> position in the bundle
         self.bundlefilespos = {}
 
     @util.propertycache
     def changelog(self):
         c = bundlechangelog(self.sopener, self.bundlefile)
         self.manstart = self.bundlefile.tell()
         return c
 
     @util.propertycache
     def manifest(self):
         self.bundlefile.seek(self.manstart)
         m = bundlemanifest(self.sopener, self.bundlefile, self.changelog.rev)
         self.filestart = self.bundlefile.tell()
         return m
 
     @util.propertycache
     def manstart(self):
         self.changelog
         return self.manstart
 
     @util.propertycache
     def filestart(self):
         self.manifest
         return self.filestart
 
     def url(self):
         return self._url
 
     def file(self, f):
         if not self.bundlefilespos:
             self.bundlefile.seek(self.filestart)
             while 1:
                 chunk = changegroup.getchunk(self.bundlefile)
                 if not chunk:
                     break
                 self.bundlefilespos[chunk] = self.bundlefile.tell()
                 for c in changegroup.chunkiter(self.bundlefile):
                     pass
 
         if f[0] == '/':
             f = f[1:]
         if f in self.bundlefilespos:
             self.bundlefile.seek(self.bundlefilespos[f])
             return bundlefilelog(self.sopener, f, self.bundlefile,
                                  self.changelog.rev)
         else:
             return filelog.filelog(self.sopener, f)
 
     def close(self):
         """Close assigned bundle file immediately."""
         self.bundlefile.close()
 
     def __del__(self):
         bundlefile = getattr(self, 'bundlefile', None)
         if bundlefile and not bundlefile.closed:
             bundlefile.close()
         tempfile = getattr(self, 'tempfile', None)
         if tempfile is not None:
             os.unlink(tempfile)
         if self._tempparent:
             shutil.rmtree(self._tempparent, True)
 
     def cancopy(self):
         return False
 
     def getcwd(self):
         return os.getcwd() # always outside the repo
 
 def instance(ui, path, create):
     if create:
         raise util.Abort(_('cannot create new bundle repository'))
     parentpath = ui.config("bundle", "mainreporoot", "")
     if parentpath:
         # Try to make the full path relative so we get a nice, short URL.
         # In particular, we don't want temp dir names in test outputs.
         cwd = os.getcwd()
         if parentpath == cwd:
             parentpath = ''
         else:
             cwd = os.path.join(cwd,'')
             if parentpath.startswith(cwd):
                 parentpath = parentpath[len(cwd):]
     path = util.drop_scheme('file', path)
     if path.startswith('bundle:'):
         path = util.drop_scheme('bundle', path)
         s = path.split("+", 1)
         if len(s) == 1:
             repopath, bundlename = parentpath, s[0]
         else:
             repopath, bundlename = s
     else:
         repopath, bundlename = parentpath, path
     return bundlerepository(ui, repopath, bundlename)

mercurial/changegroup.py

@@ -1,167 +1,170 b''
 # changegroup.py - Mercurial changegroup manipulation functions
 #
 #  Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from i18n import _
 import util
 import struct, os, bz2, zlib, tempfile
 
 def getchunk(source):
     """return the next chunk from changegroup 'source' as a string"""
     d = source.read(4)
     if not d:
         return ""
     l = struct.unpack(">l", d)[0]
     if l <= 4:
         return ""
     d = source.read(l - 4)
     if len(d) < l - 4:
         raise util.Abort(_("premature EOF reading chunk"
                            " (got %d bytes, expected %d)")
                           % (len(d), l - 4))
     return d
 
 def chunkiter(source, progress=None):
     """iterate through the chunks in source, yielding a sequence of chunks
     (strings)"""
     while 1:
         c = getchunk(source)
         if not c:
             break
         elif progress is not None:
             progress()
         yield c
 
 def chunkheader(length):
     """return a changegroup chunk header (string)"""
     return struct.pack(">l", length + 4)
 
 def closechunk():
     """return a changegroup chunk header (string) for a zero-length chunk"""
     return struct.pack(">l", 0)
 
 class nocompress(object):
     def compress(self, x):
         return x
     def flush(self):
         return ""
 
 bundletypes = {
     "": ("", nocompress),
     "HG10UN": ("HG10UN", nocompress),
     "HG10BZ": ("HG10", lambda: bz2.BZ2Compressor()),
     "HG10GZ": ("HG10GZ", lambda: zlib.compressobj()),
 }
 
 def collector(cl, mmfs, files):
     # Gather information about changeset nodes going out in a bundle.
     # We want to gather manifests needed and filelogs affected.
     def collect(node):
         c = cl.read(node)
         files.update(c[3])
         mmfs.setdefault(c[0], node)
     return collect
 
 # hgweb uses this list to communicate its preferred type
 bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN']
 
 def writebundle(cg, filename, bundletype):
     """Write a bundle file and return its filename.
 
     Existing files will not be overwritten.
     If no filename is specified, a temporary file is created.
     bz2 compression can be turned off.
     The bundle file will be deleted in case of errors.
     """
 
     fh = None
     cleanup = None
     try:
         if filename:
             fh = open(filename, "wb")
         else:
             fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
             fh = os.fdopen(fd, "wb")
         cleanup = filename
 
         header, compressor = bundletypes[bundletype]
         fh.write(header)
         z = compressor()
 
         # parse the changegroup data, otherwise we will block
         # in case of sshrepo because we don't know the end of the stream
 
         # an empty chunkiter is the end of the changegroup
         # a changegroup has at least 2 chunkiters (changelog and manifest).
         # after that, an empty chunkiter is the end of the changegroup
         empty = False
         count = 0
         while not empty or count <= 2:
             empty = True
             count += 1
             for chunk in chunkiter(cg):
                 empty = False
                 fh.write(z.compress(chunkheader(len(chunk))))
                 pos = 0
                 while pos < len(chunk):
                     next = pos + 2**20
                     fh.write(z.compress(chunk[pos:next]))
                     pos = next
             fh.write(z.compress(closechunk()))
         fh.write(z.flush())
         cleanup = None
         return filename
     finally:
         if fh is not None:
             fh.close()
         if cleanup is not None:
             os.unlink(cleanup)
 
 def decompressor(fh, alg):
     if alg == 'UN':
         return fh
     elif alg == 'GZ':
         def generator(f):
             zd = zlib.decompressobj()
             for chunk in f:
                 yield zd.decompress(chunk)
     elif alg == 'BZ':
         def generator(f):
             zd = bz2.BZ2Decompressor()
             zd.decompress("BZ")
             for chunk in util.filechunkiter(f, 4096):
                 yield zd.decompress(chunk)
     else:
         raise util.Abort("unknown bundle compression '%s'" % alg)
     return generator(fh)
 
 class unbundle10(object):
     def __init__(self, fh, alg):
         self._stream = util.chunkbuffer(decompressor(fh, alg))
+        self._type = alg
+    def compressed(self):
+        return self._type != 'UN'
     def read(self, l):
         return self._stream.read(l)
 
 def readbundle(fh, fname):
     header = fh.read(6)
 
     if not fname:
         fname = "stream"
         if not header.startswith('HG') and header.startswith('\0'):
             # headerless bundle, clean things up
             def fixup(f, h):
                 yield h
                 for x in f:
                     yield x
             fh = fixup(fh, header)
             header = "HG10UN"
 
     magic, version, alg = header[0:2], header[2:4], header[4:6]
 
     if magic != 'HG':
         raise util.Abort(_('%s: not a Mercurial bundle') % fname)
     if version != '10':
         raise util.Abort(_('%s: unknown bundle version %s') % (fname, version))
     return unbundle10(fh, alg)

tests/test-bundle-type.t

@@ -1,99 +1,99 b''
 bundle w/o type option
 
   $ hg init t1
   $ hg init t2
   $ cd t1
   $ echo blablablablabla > file.txt
   $ hg ci -Ama
   adding file.txt
   $ hg log | grep summary
   summary:     a
   $ hg bundle ../b1 ../t2
   searching for changes
   1 changesets found
 
   $ cd ../t2
   $ hg pull ../b1
   pulling from ../b1
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
   $ hg up
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg log | grep summary
   summary:     a
   $ cd ..
 
 test bundle types
 
   $ for t in "None" "bzip2" "gzip"; do
   >   echo % test bundle type $t
   >   hg init t$t
   >   cd t1
   >   hg bundle -t $t ../b$t ../t$t
   >   cut -b 1-6 ../b$t | head -n 1
   >   cd ../t$t
   >   hg pull ../b$t
   >   hg up
   >   hg log | grep summary
   >   cd ..
   > done
   % test bundle type None
   searching for changes
   1 changesets found
   HG10UN
   pulling from ../bNone
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   summary:     a
   % test bundle type bzip2
   searching for changes
   1 changesets found
   HG10BZ
   pulling from ../bbzip2
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   summary:     a
   % test bundle type gzip
   searching for changes
   1 changesets found
   HG10GZ
   pulling from ../bgzip
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   summary:     a
 
 test garbage file
 
   $ echo garbage > bgarbage
   $ hg init tgarbage
   $ cd tgarbage
   $ hg pull ../bgarbage
-  abort: ../bgarbage: not a Mercurial bundle file
+  abort: ../bgarbage: not a Mercurial bundle
   $ cd ..
 
 test invalid bundle type
 
   $ cd t1
   $ hg bundle -a -t garbage ../bgarbage
   1 changesets found
   abort: unknown bundle type specified with --type
   $ cd ..