archive: add support for progress extension

mercurial/archival.py

@@ -1,274 +1,279 b''
 # archival.py - revision archival for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from i18n import _
 from node import hex
 import cmdutil
 import util, encoding
 import cStringIO, os, stat, tarfile, time, zipfile
 import zlib, gzip
 
 def tidyprefix(dest, kind, prefix):
     '''choose prefix to use for names in archive.  make sure prefix is
     safe for consumers.'''
 
     if prefix:
         prefix = util.normpath(prefix)
     else:
         if not isinstance(dest, str):
             raise ValueError('dest must be string if no prefix')
         prefix = os.path.basename(dest)
         lower = prefix.lower()
         for sfx in exts.get(kind, []):
             if lower.endswith(sfx):
                 prefix = prefix[:-len(sfx)]
                 break
     lpfx = os.path.normpath(util.localpath(prefix))
     prefix = util.pconvert(lpfx)
     if not prefix.endswith('/'):
         prefix += '/'
     if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
         raise util.Abort(_('archive prefix contains illegal components'))
     return prefix
 
 exts = {
     'tar': ['.tar'],
     'tbz2': ['.tbz2', '.tar.bz2'],
     'tgz': ['.tgz', '.tar.gz'],
     'zip': ['.zip'],
     }
 
 def guesskind(dest):
     for kind, extensions in exts.iteritems():
         if util.any(dest.endswith(ext) for ext in extensions):
             return kind
     return None
 
 
 class tarit(object):
     '''write archive to tar file or stream.  can write uncompressed,
     or compress with gzip or bzip2.'''
 
     class GzipFileWithTime(gzip.GzipFile):
 
         def __init__(self, *args, **kw):
             timestamp = None
             if 'timestamp' in kw:
                 timestamp = kw.pop('timestamp')
             if timestamp is None:
                 self.timestamp = time.time()
             else:
                 self.timestamp = timestamp
             gzip.GzipFile.__init__(self, *args, **kw)
 
         def _write_gzip_header(self):
             self.fileobj.write('\037\213')             # magic header
             self.fileobj.write('\010')                 # compression method
             # Python 2.6 deprecates self.filename
             fname = getattr(self, 'name', None) or self.filename
             if fname and fname.endswith('.gz'):
                 fname = fname[:-3]
             flags = 0
             if fname:
                 flags = gzip.FNAME
             self.fileobj.write(chr(flags))
             gzip.write32u(self.fileobj, long(self.timestamp))
             self.fileobj.write('\002')
             self.fileobj.write('\377')
             if fname:
                 self.fileobj.write(fname + '\000')
 
     def __init__(self, dest, mtime, kind=''):
         self.mtime = mtime
 
         def taropen(name, mode, fileobj=None):
             if kind == 'gz':
                 mode = mode[0]
                 if not fileobj:
                     fileobj = open(name, mode + 'b')
                 gzfileobj = self.GzipFileWithTime(name, mode + 'b',
                                                   zlib.Z_BEST_COMPRESSION,
                                                   fileobj, timestamp=mtime)
                 return tarfile.TarFile.taropen(name, mode, gzfileobj)
             else:
                 return tarfile.open(name, mode + kind, fileobj)
 
         if isinstance(dest, str):
             self.z = taropen(dest, mode='w:')
         else:
             # Python 2.5-2.5.1 have a regression that requires a name arg
             self.z = taropen(name='', mode='w|', fileobj=dest)
 
     def addfile(self, name, mode, islink, data):
         i = tarfile.TarInfo(name)
         i.mtime = self.mtime
         i.size = len(data)
         if islink:
             i.type = tarfile.SYMTYPE
             i.mode = 0777
             i.linkname = data
             data = None
             i.size = 0
         else:
             i.mode = mode
             data = cStringIO.StringIO(data)
         self.z.addfile(i, data)
 
     def done(self):
         self.z.close()
 
 class tellable(object):
     '''provide tell method for zipfile.ZipFile when writing to http
     response file object.'''
 
     def __init__(self, fp):
         self.fp = fp
         self.offset = 0
 
     def __getattr__(self, key):
         return getattr(self.fp, key)
 
     def write(self, s):
         self.fp.write(s)
         self.offset += len(s)
 
     def tell(self):
         return self.offset
 
 class zipit(object):
     '''write archive to zip file or stream.  can write uncompressed,
     or compressed with deflate.'''
 
     def __init__(self, dest, mtime, compress=True):
         if not isinstance(dest, str):
             try:
                 dest.tell()
             except (AttributeError, IOError):
                 dest = tellable(dest)
         self.z = zipfile.ZipFile(dest, 'w',
                                  compress and zipfile.ZIP_DEFLATED or
                                  zipfile.ZIP_STORED)
 
         # Python's zipfile module emits deprecation warnings if we try
         # to store files with a date before 1980.
         epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
         if mtime < epoch:
             mtime = epoch
 
         self.date_time = time.gmtime(mtime)[:6]
 
     def addfile(self, name, mode, islink, data):
         i = zipfile.ZipInfo(name, self.date_time)
         i.compress_type = self.z.compression
         # unzip will not honor unix file modes unless file creator is
         # set to unix (id 3).
         i.create_system = 3
         ftype = stat.S_IFREG
         if islink:
             mode = 0777
             ftype = stat.S_IFLNK
         i.external_attr = (mode | ftype) << 16L
         self.z.writestr(i, data)
 
     def done(self):
         self.z.close()
 
 class fileit(object):
     '''write archive as files in directory.'''
 
     def __init__(self, name, mtime):
         self.basedir = name
         self.opener = util.opener(self.basedir)
 
     def addfile(self, name, mode, islink, data):
         if islink:
             self.opener.symlink(data, name)
             return
         f = self.opener(name, "w", atomictemp=True)
         f.write(data)
         f.rename()
         destfile = os.path.join(self.basedir, name)
         os.chmod(destfile, mode)
 
     def done(self):
         pass
 
 archivers = {
     'files': fileit,
     'tar': tarit,
     'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
     'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
     'uzip': lambda name, mtime: zipit(name, mtime, False),
     'zip': zipit,
     }
 
 def archive(repo, dest, node, kind, decode=True, matchfn=None,
             prefix=None, mtime=None, subrepos=False):
     '''create archive of repo as it was at node.
 
     dest can be name of directory, name of archive file, or file
     object to write archive to.
 
     kind is type of archive to create.
 
     decode tells whether to put files through decode filters from
     hgrc.
 
     matchfn is function to filter names of files to write to archive.
 
     prefix is name of path to put before every archive member.'''
 
     if kind == 'files':
         if prefix:
             raise util.Abort(_('cannot give prefix when archiving to files'))
     else:
         prefix = tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         if matchfn and not matchfn(name):
             return
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     if kind not in archivers:
         raise util.Abort(_("unknown archive type '%s'") % kind)
 
     ctx = repo[node]
     archiver = archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool("ui", "archivemeta", True):
         def metadata():
             base = 'repo: %s\nnode: %s\nbranch: %s\n' % (
                 repo[0].hex(), hex(node), encoding.fromlocal(ctx.branch()))
 
             tags = ''.join('tag: %s\n' % t for t in ctx.tags()
                            if repo.tagtype(t) == 'global')
             if not tags:
                 repo.ui.pushbuffer()
                 opts = {'template': '{latesttag}\n{latesttagdistance}',
                         'style': '', 'patch': None, 'git': None}
                 cmdutil.show_changeset(repo.ui, repo, opts).show(ctx)
                 ltags, dist = repo.ui.popbuffer().split('\n')
                 tags = ''.join('latesttag: %s\n' % t for t in ltags.split(':'))
                 tags += 'latesttagdistance: %s\n' % dist
 
             return base + tags
 
         write('.hg_archival.txt', 0644, False, metadata)
 
-    for f in ctx:
+    total = len(ctx.manifest())
+    repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
+    for i, f in enumerate(ctx):
         ff = ctx.flags(f)
         write(f, 'x' in ff and 0755 or 0644, 'l' in ff, ctx[f].data)
+        repo.ui.progress(_('archiving'), i + 1, item=f,
+                         unit=_('files'), total=total)
+    repo.ui.progress(_('archiving'), None)
 
     if subrepos:
         for subpath in ctx.substate:
             sub = ctx.sub(subpath)
             sub.archive(archiver, prefix)
 
     archiver.done()

tests/test-archive.t

@@ -1,234 +1,265 b''
   $ mkdir test
   $ cd test
   $ hg init
   $ echo foo>foo
   $ hg commit -Am 1 -d '1 0'
   adding foo
   $ echo bar>bar
   $ hg commit -Am 2 -d '2 0'
   adding bar
   $ mkdir baz
   $ echo bletch>baz/bletch
   $ hg commit -Am 3 -d '1000000000 0'
   adding baz/bletch
   $ echo "[web]" >> .hg/hgrc
   $ echo "name = test-archive" >> .hg/hgrc
   $ cp .hg/hgrc .hg/hgrc-base
   > test_archtype() {
   >     echo "allow_archive = $1" >> .hg/hgrc
   >     hg serve -p $HGPORT -d --pid-file=hg.pid -E errors.log
   >     cat hg.pid >> $DAEMON_PIDS
   >     echo % $1 allowed should give 200
   >     "$TESTDIR/get-with-headers.py" localhost:$HGPORT "/archive/tip.$2" | head -n 1
   >     echo % $3 and $4 disallowed should both give 403
   >     "$TESTDIR/get-with-headers.py" localhost:$HGPORT "/archive/tip.$3" | head -n 1
   >     "$TESTDIR/get-with-headers.py" localhost:$HGPORT "/archive/tip.$4" | head -n 1
   >     "$TESTDIR/killdaemons.py"
   >     cat errors.log
   >     cp .hg/hgrc-base .hg/hgrc
   > }
 
 check http return codes
 
   $ test_archtype gz tar.gz tar.bz2 zip
   % gz allowed should give 200
   200 Script output follows
   % tar.bz2 and zip disallowed should both give 403
   403 Archive type not allowed: bz2
   403 Archive type not allowed: zip
   $ test_archtype bz2 tar.bz2 zip tar.gz
   % bz2 allowed should give 200
   200 Script output follows
   % zip and tar.gz disallowed should both give 403
   403 Archive type not allowed: zip
   403 Archive type not allowed: gz
   $ test_archtype zip zip tar.gz tar.bz2
   % zip allowed should give 200
   200 Script output follows
   % tar.gz and tar.bz2 disallowed should both give 403
   403 Archive type not allowed: gz
   403 Archive type not allowed: bz2
 
   $ echo "allow_archive = gz bz2 zip" >> .hg/hgrc
   $ hg serve -p $HGPORT -d --pid-file=hg.pid -E errors.log
   $ cat hg.pid >> $DAEMON_PIDS
 
 invalid arch type should give 404
 
   $ "$TESTDIR/get-with-headers.py" localhost:$HGPORT "/archive/tip.invalid" | head -n 1
   404 Unsupported archive type: None
 
   $ TIP=`hg id -v | cut -f1 -d' '`
   $ QTIP=`hg id -q`
   $ cat > getarchive.py <<EOF
   > import os, sys, urllib2
   > try:
   >     # Set stdout to binary mode for win32 platforms
   >     import msvcrt
   >     msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
   > except ImportError:
   >     pass
   > node, archive = sys.argv[1:]
   > f = urllib2.urlopen('http://127.0.0.1:%s/?cmd=archive;node=%s;type=%s'
   >                     % (os.environ['HGPORT'], node, archive))
   > sys.stdout.write(f.read())
   > EOF
   $ python getarchive.py "$TIP" gz | gunzip | tar tf - 2>/dev/null
   test-archive-2c0277f05ed4/.hg_archival.txt
   test-archive-2c0277f05ed4/bar
   test-archive-2c0277f05ed4/baz/bletch
   test-archive-2c0277f05ed4/foo
   $ python getarchive.py "$TIP" bz2 | bunzip2 | tar tf - 2>/dev/null
   test-archive-2c0277f05ed4/.hg_archival.txt
   test-archive-2c0277f05ed4/bar
   test-archive-2c0277f05ed4/baz/bletch
   test-archive-2c0277f05ed4/foo
   $ python getarchive.py "$TIP" zip > archive.zip
   $ unzip -t archive.zip
   Archive:  archive.zip
       testing: test-archive-2c0277f05ed4/.hg_archival.txt   OK
       testing: test-archive-2c0277f05ed4/bar   OK
       testing: test-archive-2c0277f05ed4/baz/bletch   OK
       testing: test-archive-2c0277f05ed4/foo   OK
   No errors detected in compressed data of archive.zip.
 
   $ "$TESTDIR/killdaemons.py"
 
   $ hg archive -t tar test.tar
   $ tar tf test.tar
   test/.hg_archival.txt
   test/bar
   test/baz/bletch
   test/foo
 
   $ hg archive -t tbz2 -X baz test.tar.bz2
   $ bunzip2 -dc test.tar.bz2 | tar tf - 2>/dev/null
   test/.hg_archival.txt
   test/bar
   test/foo
 
   $ hg archive -t tgz -p %b-%h test-%h.tar.gz
   $ gzip -dc test-$QTIP.tar.gz | tar tf - 2>/dev/null
   test-2c0277f05ed4/.hg_archival.txt
   test-2c0277f05ed4/bar
   test-2c0277f05ed4/baz/bletch
   test-2c0277f05ed4/foo
 
   $ hg archive autodetected_test.tar
   $ tar tf autodetected_test.tar
   autodetected_test/.hg_archival.txt
   autodetected_test/bar
   autodetected_test/baz/bletch
   autodetected_test/foo
 
 The '-t' should override autodetection
 
   $ hg archive -t tar autodetect_override_test.zip
   $ tar tf autodetect_override_test.zip
   autodetect_override_test.zip/.hg_archival.txt
   autodetect_override_test.zip/bar
   autodetect_override_test.zip/baz/bletch
   autodetect_override_test.zip/foo
 
   $ for ext in tar tar.gz tgz tar.bz2 tbz2 zip; do
   >     hg archive auto_test.$ext
   >     if [ -d auto_test.$ext ]; then
   >         echo "extension $ext was not autodetected."
   >     fi
   > done
 
   $ cat > md5comp.py <<EOF
   > try:
   >     from hashlib import md5
   > except ImportError:
   >     from md5 import md5
   > import sys
   > f1, f2 = sys.argv[1:3]
   > h1 = md5(file(f1, 'rb').read()).hexdigest()
   > h2 = md5(file(f2, 'rb').read()).hexdigest()
   > print h1 == h2 or "md5 differ: " + repr((h1, h2))
   > EOF
 
 archive name is stored in the archive, so create similar archives and
 rename them afterwards.
 
   $ hg archive -t tgz tip.tar.gz
   $ mv tip.tar.gz tip1.tar.gz
   $ sleep 1
   $ hg archive -t tgz tip.tar.gz
   $ mv tip.tar.gz tip2.tar.gz
   $ python md5comp.py tip1.tar.gz tip2.tar.gz
   True
 
   $ hg archive -t zip -p /illegal test.zip
   abort: archive prefix contains illegal components
   [255]
   $ hg archive -t zip -p very/../bad test.zip
 
   $ hg archive --config ui.archivemeta=false -t zip -r 2 test.zip
   $ unzip -t test.zip
   Archive:  test.zip
       testing: test/bar                 OK
       testing: test/baz/bletch          OK
       testing: test/foo                 OK
   No errors detected in compressed data of test.zip.
 
   $ hg archive -t tar - | tar tf - 2>/dev/null
   test-2c0277f05ed4/.hg_archival.txt
   test-2c0277f05ed4/bar
   test-2c0277f05ed4/baz/bletch
   test-2c0277f05ed4/foo
 
   $ hg archive -r 0 -t tar rev-%r.tar
   $ if [ -f rev-0.tar ]; then
   $ fi
 
 test .hg_archival.txt
 
   $ hg archive ../test-tags
   $ cat ../test-tags/.hg_archival.txt
   repo: daa7f7c60e0a224faa4ff77ca41b2760562af264
   node: 2c0277f05ed49d1c8328fb9ba92fba7a5ebcb33e
   branch: default
   latesttag: null
   latesttagdistance: 3
   $ hg tag -r 2 mytag
   $ hg tag -r 2 anothertag
   $ hg archive -r 2 ../test-lasttag
   $ cat ../test-lasttag/.hg_archival.txt
   repo: daa7f7c60e0a224faa4ff77ca41b2760562af264
   node: 2c0277f05ed49d1c8328fb9ba92fba7a5ebcb33e
   branch: default
   tag: anothertag
   tag: mytag
 
   $ hg archive -t bogus test.bogus
   abort: unknown archive type 'bogus'
   [255]
 
+enable progress extension:
+
+  $ cp $HGRCPATH $HGRCPATH.no-progress
+  $ cat >> $HGRCPATH <<EOF
+  > [extensions]
+  > progress =
+  > [progress]
+  > assume-tty = 1
+  > delay = 0
+  > refresh = 0
+  > width = 60
+  > EOF
+
+  $ hg archive ../with-progress 2>&1 | $TESTDIR/filtercr.py
+  
+  archiving [                                           ] 0/4
+  archiving [                                           ] 0/4
+  archiving [=========>                                 ] 1/4
+  archiving [=========>                                 ] 1/4
+  archiving [====================>                      ] 2/4
+  archiving [====================>                      ] 2/4
+  archiving [===============================>           ] 3/4
+  archiving [===============================>           ] 3/4
+  archiving [==========================================>] 4/4
+  archiving [==========================================>] 4/4
+                                                              \r (esc)
+
+cleanup after progress extension test:
+
+  $ cp $HGRCPATH.no-progress $HGRCPATH
+
 server errors
 
   $ cat errors.log
 
 empty repo
 
   $ hg init ../empty
   $ cd ../empty
   $ hg archive ../test-empty
   abort: no working directory: please specify a revision
   [255]
 
 old file -- date clamped to 1980
 
   $ touch -t 197501010000 old
   $ hg add old
   $ hg commit -m old
   $ hg archive ../old.zip
   $ unzip -l ../old.zip
   Archive:  ../old.zip
   \s*Length.* (re)
   *-----* (glob)
   *147*80*00:00*old/.hg_archival.txt (glob)
   *0*80*00:00*old/old (glob)
   *-----* (glob)
   \s*147\s+2 files (re)