archival.py
344 lines
| 10.8 KiB
| text/x-python
|
PythonLexer
/ mercurial / archival.py
|
r2112 | # archival.py - revision archival for mercurial | ||
# | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r2112 | |||
|
r25916 | from __future__ import absolute_import | ||
import gzip | ||||
import os | ||||
|
r17628 | import struct | ||
|
r25916 | import tarfile | ||
import time | ||||
import zipfile | ||||
import zlib | ||||
from .i18n import _ | ||||
from . import ( | ||||
error, | ||||
|
r33544 | formatter, | ||
|
r25916 | match as matchmod, | ||
|
r36724 | pycompat, | ||
|
r36156 | scmutil, | ||
|
r25916 | util, | ||
|
r31235 | vfs as vfsmod, | ||
|
r25916 | ) | ||
|
r28861 | stringio = util.stringio | ||
|
r2112 | |||
|
r17429 | # from unzip source code: | ||
_UNX_IFREG = 0x8000 | ||||
_UNX_IFLNK = 0xa000 | ||||
|
r11558 | def tidyprefix(dest, kind, prefix): | ||
|
r2112 | '''choose prefix to use for names in archive. make sure prefix is | ||
safe for consumers.''' | ||||
if prefix: | ||||
|
r5842 | prefix = util.normpath(prefix) | ||
|
r2112 | else: | ||
|
r36455 | if not isinstance(dest, bytes): | ||
|
r2112 | raise ValueError('dest must be string if no prefix') | ||
prefix = os.path.basename(dest) | ||||
lower = prefix.lower() | ||||
|
r11558 | for sfx in exts.get(kind, []): | ||
|
r2112 | if lower.endswith(sfx): | ||
prefix = prefix[:-len(sfx)] | ||||
break | ||||
lpfx = os.path.normpath(util.localpath(prefix)) | ||||
prefix = util.pconvert(lpfx) | ||||
if not prefix.endswith('/'): | ||||
prefix += '/' | ||||
|
r24953 | # Drop the leading '.' path component if present, so Windows can read the | ||
# zip files (issue4634) | ||||
if prefix.startswith('./'): | ||||
prefix = prefix[2:] | ||||
|
r2112 | if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix: | ||
|
r26587 | raise error.Abort(_('archive prefix contains illegal components')) | ||
|
r2112 | return prefix | ||
|
r11557 | exts = { | ||
'tar': ['.tar'], | ||||
'tbz2': ['.tbz2', '.tar.bz2'], | ||||
'tgz': ['.tgz', '.tar.gz'], | ||||
'zip': ['.zip'], | ||||
} | ||||
def guesskind(dest): | ||||
for kind, extensions in exts.iteritems(): | ||||
|
r25149 | if any(dest.endswith(ext) for ext in extensions): | ||
|
r11557 | return kind | ||
return None | ||||
|
r24681 | def _rootctx(repo): | ||
# repo[0] may be hidden | ||||
for rev in repo: | ||||
return repo[rev] | ||||
return repo['null'] | ||||
|
r35923 | # {tags} on ctx includes local tags and 'tip', with no current way to limit | ||
# that to global tags. Therefore, use {latesttag} as a substitute when | ||||
# the distance is 0, since that will be the list of global tags on ctx. | ||||
_defaultmetatemplate = br''' | ||||
repo: {root} | ||||
node: {ifcontains(rev, revset("wdir()"), "{p1node}{dirty}", "{node}")} | ||||
branch: {branch|utf8} | ||||
{ifeq(latesttagdistance, 0, join(latesttag % "tag: {tag}", "\n"), | ||||
separate("\n", | ||||
join(latesttag % "latesttag: {tag}", "\n"), | ||||
"latesttagdistance: {latesttagdistance}", | ||||
"changessincelatesttag: {changessincelatesttag}"))} | ||||
'''[1:] # drop leading '\n' | ||||
|
r24678 | def buildmetadata(ctx): | ||
'''build content of .hg_archival.txt''' | ||||
repo = ctx.repo() | ||||
|
r33544 | |||
opts = { | ||||
|
r33545 | 'template': repo.ui.config('experimental', 'archivemetatemplate', | ||
|
r35923 | _defaultmetatemplate) | ||
|
r33544 | } | ||
out = util.stringio() | ||||
|
r24678 | |||
|
r33544 | fm = formatter.formatter(repo.ui, out, 'archive', opts) | ||
fm.startitem() | ||||
fm.context(ctx=ctx) | ||||
fm.data(root=_rootctx(repo).hex()) | ||||
if ctx.rev() is None: | ||||
dirty = '' | ||||
if ctx.dirty(missing=True): | ||||
dirty = '+' | ||||
fm.data(dirty=dirty) | ||||
fm.end() | ||||
return out.getvalue() | ||||
|
r11557 | |||
|
r8778 | class tarit(object): | ||
|
r2112 | '''write archive to tar file or stream. can write uncompressed, | ||
or compress with gzip or bzip2.''' | ||||
|
r4652 | class GzipFileWithTime(gzip.GzipFile): | ||
def __init__(self, *args, **kw): | ||||
timestamp = None | ||||
|
r36744 | if r'timestamp' in kw: | ||
|
r35350 | timestamp = kw.pop(r'timestamp') | ||
|
r8527 | if timestamp is None: | ||
|
r4652 | self.timestamp = time.time() | ||
else: | ||||
self.timestamp = timestamp | ||||
gzip.GzipFile.__init__(self, *args, **kw) | ||||
def _write_gzip_header(self): | ||||
self.fileobj.write('\037\213') # magic header | ||||
self.fileobj.write('\010') # compression method | ||||
|
r26198 | fname = self.name | ||
|
r13102 | if fname and fname.endswith('.gz'): | ||
fname = fname[:-3] | ||||
|
r4652 | flags = 0 | ||
if fname: | ||||
flags = gzip.FNAME | ||||
|
r36746 | self.fileobj.write(pycompat.bytechr(flags)) | ||
gzip.write32u(self.fileobj, int(self.timestamp)) | ||||
|
r4652 | self.fileobj.write('\002') | ||
self.fileobj.write('\377') | ||||
if fname: | ||||
self.fileobj.write(fname + '\000') | ||||
|
r11558 | def __init__(self, dest, mtime, kind=''): | ||
|
r2477 | self.mtime = mtime | ||
|
r13400 | self.fileobj = None | ||
|
r4652 | |||
|
r30479 | def taropen(mode, name='', fileobj=None): | ||
|
r4652 | if kind == 'gz': | ||
|
r36465 | mode = mode[0:1] | ||
|
r4652 | if not fileobj: | ||
|
r4731 | fileobj = open(name, mode + 'b') | ||
|
r36745 | gzfileobj = self.GzipFileWithTime(name, | ||
pycompat.sysstr(mode + 'b'), | ||||
|
r4652 | zlib.Z_BEST_COMPRESSION, | ||
fileobj, timestamp=mtime) | ||||
|
r13400 | self.fileobj = gzfileobj | ||
|
r36725 | return tarfile.TarFile.taropen( | ||
name, pycompat.sysstr(mode), gzfileobj) | ||||
|
r4652 | else: | ||
|
r36725 | return tarfile.open( | ||
name, pycompat.sysstr(mode + kind), fileobj) | ||||
|
r4652 | |||
|
r36726 | if isinstance(dest, bytes): | ||
|
r30479 | self.z = taropen('w:', name=dest) | ||
|
r2112 | else: | ||
|
r30479 | self.z = taropen('w|', fileobj=dest) | ||
|
r2112 | |||
|
r4831 | def addfile(self, name, mode, islink, data): | ||
|
r36724 | name = pycompat.fsdecode(name) | ||
|
r11558 | i = tarfile.TarInfo(name) | ||
|
r2112 | i.mtime = self.mtime | ||
i.size = len(data) | ||||
|
r4831 | if islink: | ||
i.type = tarfile.SYMTYPE | ||||
|
r25658 | i.mode = 0o777 | ||
|
r36724 | i.linkname = pycompat.fsdecode(data) | ||
|
r4831 | data = None | ||
|
r7770 | i.size = 0 | ||
|
r4831 | else: | ||
i.mode = mode | ||||
|
r28861 | data = stringio(data) | ||
|
r4831 | self.z.addfile(i, data) | ||
|
r2112 | |||
def done(self): | ||||
self.z.close() | ||||
|
r13400 | if self.fileobj: | ||
self.fileobj.close() | ||||
|
r2112 | |||
|
r8778 | class zipit(object): | ||
|
r2112 | '''write archive to zip file or stream. can write uncompressed, | ||
or compressed with deflate.''' | ||||
|
r11558 | def __init__(self, dest, mtime, compress=True): | ||
|
r36724 | self.z = zipfile.ZipFile(pycompat.fsdecode(dest), r'w', | ||
|
r2112 | compress and zipfile.ZIP_DEFLATED or | ||
zipfile.ZIP_STORED) | ||||
|
r12319 | |||
# Python's zipfile module emits deprecation warnings if we try | ||||
# to store files with a date before 1980. | ||||
epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0)) | ||||
if mtime < epoch: | ||||
mtime = epoch | ||||
|
r17628 | self.mtime = mtime | ||
|
r2477 | self.date_time = time.gmtime(mtime)[:6] | ||
|
r2112 | |||
|
r4831 | def addfile(self, name, mode, islink, data): | ||
|
r36724 | i = zipfile.ZipInfo(pycompat.fsdecode(name), self.date_time) | ||
|
r2112 | i.compress_type = self.z.compression | ||
# unzip will not honor unix file modes unless file creator is | ||||
# set to unix (id 3). | ||||
i.create_system = 3 | ||||
|
r17429 | ftype = _UNX_IFREG | ||
|
r4831 | if islink: | ||
|
r25658 | mode = 0o777 | ||
|
r17429 | ftype = _UNX_IFLNK | ||
|
r29890 | i.external_attr = (mode | ftype) << 16 | ||
|
r17628 | # add "extended-timestamp" extra block, because zip archives | ||
# without this will be extracted with unexpected timestamp, | ||||
# if TZ is not configured as GMT | ||||
i.extra += struct.pack('<hhBl', | ||||
0x5455, # block type: "extended-timestamp" | ||||
1 + 4, # size of this block | ||||
1, # "modification time is present" | ||||
|
r18301 | int(self.mtime)) # last modification (UTC) | ||
|
r2112 | self.z.writestr(i, data) | ||
def done(self): | ||||
self.z.close() | ||||
|
r8778 | class fileit(object): | ||
|
r2112 | '''write archive as files in directory.''' | ||
|
r11558 | def __init__(self, name, mtime): | ||
|
r2112 | self.basedir = name | ||
|
r31235 | self.opener = vfsmod.vfs(self.basedir) | ||
|
r35203 | self.mtime = mtime | ||
|
r2112 | |||
|
r4831 | def addfile(self, name, mode, islink, data): | ||
if islink: | ||||
self.opener.symlink(data, name) | ||||
return | ||||
|
r36785 | f = self.opener(name, "w", atomictemp=False) | ||
|
r4830 | f.write(data) | ||
|
r15057 | f.close() | ||
|
r2112 | destfile = os.path.join(self.basedir, name) | ||
|
r4830 | os.chmod(destfile, mode) | ||
|
r35203 | if self.mtime is not None: | ||
os.utime(destfile, (self.mtime, self.mtime)) | ||||
|
r2112 | |||
def done(self): | ||||
pass | ||||
archivers = { | ||||
'files': fileit, | ||||
'tar': tarit, | ||||
|
r11558 | 'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'), | ||
'tgz': lambda name, mtime: tarit(name, mtime, 'gz'), | ||||
'uzip': lambda name, mtime: zipit(name, mtime, False), | ||||
|
r2112 | 'zip': zipit, | ||
} | ||||
def archive(repo, dest, node, kind, decode=True, matchfn=None, | ||||
|
r24172 | prefix='', mtime=None, subrepos=False): | ||
|
r2112 | '''create archive of repo as it was at node. | ||
dest can be name of directory, name of archive file, or file | ||||
object to write archive to. | ||||
kind is type of archive to create. | ||||
decode tells whether to put files through decode filters from | ||||
hgrc. | ||||
matchfn is function to filter names of files to write to archive. | ||||
|
r35203 | prefix is name of path to put before every archive member. | ||
mtime is the modified time, in seconds, or None to use the changeset time. | ||||
subrepos tells whether to include subrepos. | ||||
''' | ||||
|
r2112 | |||
|
r11558 | if kind == 'files': | ||
if prefix: | ||||
|
r26587 | raise error.Abort(_('cannot give prefix when archiving to files')) | ||
|
r11558 | else: | ||
prefix = tidyprefix(dest, kind, prefix) | ||||
|
r4951 | def write(name, mode, islink, getdata): | ||
data = getdata() | ||||
|
r2112 | if decode: | ||
|
r4005 | data = repo.wwritedata(name, data) | ||
|
r11558 | archiver.addfile(prefix + name, mode, islink, data) | ||
|
r2112 | |||
|
r6019 | if kind not in archivers: | ||
|
r26587 | raise error.Abort(_("unknown archive type '%s'") % kind) | ||
|
r6749 | |||
ctx = repo[node] | ||||
|
r11558 | archiver = archivers[kind](dest, mtime or ctx.date()[0]) | ||
|
r6749 | |||
|
r33499 | if repo.ui.configbool("ui", "archivemeta"): | ||
|
r16919 | name = '.hg_archival.txt' | ||
if not matchfn or matchfn(name): | ||||
|
r25658 | write(name, 0o644, False, lambda: buildmetadata(ctx)) | ||
|
r9614 | |||
|
r16919 | if matchfn: | ||
files = [f for f in ctx.manifest().keys() if matchfn(f)] | ||||
else: | ||||
files = ctx.manifest().keys() | ||||
total = len(files) | ||||
|
r18967 | if total: | ||
files.sort() | ||||
|
r37780 | scmutil.prefetchfiles(repo, [ctx.rev()], | ||
scmutil.matchfiles(repo, files)) | ||||
|
r38400 | progress = scmutil.progress(repo.ui, _('archiving'), unit=_('files'), | ||
total=total) | ||||
progress.update(0) | ||||
for f in files: | ||||
|
r18967 | ff = ctx.flags(f) | ||
|
r25658 | write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data) | ||
|
r38400 | progress.increment(item=f) | ||
progress.complete() | ||||
|
r12323 | |||
if subrepos: | ||||
|
r18364 | for subpath in sorted(ctx.substate): | ||
|
r25601 | sub = ctx.workingsub(subpath) | ||
|
r28017 | submatch = matchmod.subdirmatcher(subpath, matchfn) | ||
|
r31099 | total += sub.archive(archiver, prefix, submatch, decode) | ||
|
r18967 | |||
if total == 0: | ||||
raise error.Abort(_('no files match the archive pattern')) | ||||
|
r12323 | |||
|
r2112 | archiver.done() | ||
|
r18967 | return total | ||