archival.py
390 lines
| 11.3 KiB
| text/x-python
|
PythonLexer
/ mercurial / archival.py
|
r2112 | # archival.py - revision archival for mercurial | ||
# | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r2112 | |||
|
r25916 | from __future__ import absolute_import | ||
import gzip | ||||
import os | ||||
|
r17628 | import struct | ||
|
r25916 | import tarfile | ||
import time | ||||
import zipfile | ||||
import zlib | ||||
from .i18n import _ | ||||
|
r43346 | from .node import nullrev | ||
|
r43355 | from .pycompat import open | ||
|
r25916 | |||
from . import ( | ||||
error, | ||||
|
r33544 | formatter, | ||
|
r25916 | match as matchmod, | ||
|
r36724 | pycompat, | ||
|
r36156 | scmutil, | ||
|
r25916 | util, | ||
|
r31235 | vfs as vfsmod, | ||
|
r25916 | ) | ||
|
r43346 | |||
|
r28861 | stringio = util.stringio | ||
|
r2112 | |||
|
r17429 | # from unzip source code: | ||
_UNX_IFREG = 0x8000 | ||||
|
r43346 | _UNX_IFLNK = 0xA000 | ||
|
r17429 | |||
|
r11558 | def tidyprefix(dest, kind, prefix): | ||
|
r2112 | '''choose prefix to use for names in archive. make sure prefix is | ||
safe for consumers.''' | ||||
if prefix: | ||||
|
r5842 | prefix = util.normpath(prefix) | ||
|
r2112 | else: | ||
|
r36455 | if not isinstance(dest, bytes): | ||
|
r43347 | raise ValueError(b'dest must be string if no prefix') | ||
|
r2112 | prefix = os.path.basename(dest) | ||
lower = prefix.lower() | ||||
|
r11558 | for sfx in exts.get(kind, []): | ||
|
r2112 | if lower.endswith(sfx): | ||
|
r43346 | prefix = prefix[: -len(sfx)] | ||
|
r2112 | break | ||
lpfx = os.path.normpath(util.localpath(prefix)) | ||||
prefix = util.pconvert(lpfx) | ||||
|
r43347 | if not prefix.endswith(b'/'): | ||
prefix += b'/' | ||||
|
r24953 | # Drop the leading '.' path component if present, so Windows can read the | ||
# zip files (issue4634) | ||||
|
r43347 | if prefix.startswith(b'./'): | ||
|
r24953 | prefix = prefix[2:] | ||
|
r43347 | if prefix.startswith(b'../') or os.path.isabs(lpfx) or b'/../' in prefix: | ||
raise error.Abort(_(b'archive prefix contains illegal components')) | ||||
|
r2112 | return prefix | ||
|
r43346 | |||
|
r11557 | exts = { | ||
|
r43347 | b'tar': [b'.tar'], | ||
b'tbz2': [b'.tbz2', b'.tar.bz2'], | ||||
b'tgz': [b'.tgz', b'.tar.gz'], | ||||
b'zip': [b'.zip'], | ||||
b'txz': [b'.txz', b'.tar.xz'], | ||||
|
r43346 | } | ||
|
r11557 | |||
def guesskind(dest): | ||||
|
r43376 | for kind, extensions in pycompat.iteritems(exts): | ||
|
r25149 | if any(dest.endswith(ext) for ext in extensions): | ||
|
r11557 | return kind | ||
return None | ||||
|
r43346 | |||
|
r24681 | def _rootctx(repo): | ||
# repo[0] may be hidden | ||||
for rev in repo: | ||||
return repo[rev] | ||||
|
r39930 | return repo[nullrev] | ||
|
r24681 | |||
|
r43346 | |||
|
r35923 | # {tags} on ctx includes local tags and 'tip', with no current way to limit | ||
# that to global tags. Therefore, use {latesttag} as a substitute when | ||||
# the distance is 0, since that will be the list of global tags on ctx. | ||||
_defaultmetatemplate = br''' | ||||
repo: {root} | ||||
node: {ifcontains(rev, revset("wdir()"), "{p1node}{dirty}", "{node}")} | ||||
branch: {branch|utf8} | ||||
{ifeq(latesttagdistance, 0, join(latesttag % "tag: {tag}", "\n"), | ||||
separate("\n", | ||||
join(latesttag % "latesttag: {tag}", "\n"), | ||||
"latesttagdistance: {latesttagdistance}", | ||||
"changessincelatesttag: {changessincelatesttag}"))} | ||||
|
r43346 | '''[ | ||
1: | ||||
] # drop leading '\n' | ||||
|
r35923 | |||
|
r24678 | def buildmetadata(ctx): | ||
'''build content of .hg_archival.txt''' | ||||
repo = ctx.repo() | ||||
|
r33544 | |||
opts = { | ||||
|
r43347 | b'template': repo.ui.config( | ||
b'experimental', b'archivemetatemplate', _defaultmetatemplate | ||||
|
r43346 | ) | ||
|
r33544 | } | ||
out = util.stringio() | ||||
|
r24678 | |||
|
r43347 | fm = formatter.formatter(repo.ui, out, b'archive', opts) | ||
|
r33544 | fm.startitem() | ||
fm.context(ctx=ctx) | ||||
fm.data(root=_rootctx(repo).hex()) | ||||
if ctx.rev() is None: | ||||
|
r43347 | dirty = b'' | ||
|
r33544 | if ctx.dirty(missing=True): | ||
|
r43347 | dirty = b'+' | ||
|
r33544 | fm.data(dirty=dirty) | ||
fm.end() | ||||
return out.getvalue() | ||||
|
r11557 | |||
|
r43346 | |||
|
r8778 | class tarit(object): | ||
|
r2112 | '''write archive to tar file or stream. can write uncompressed, | ||
or compress with gzip or bzip2.''' | ||||
r44982 | if pycompat.ispy3: | |||
GzipFileWithTime = gzip.GzipFile # camelcase-required | ||||
else: | ||||
r44981 | ||||
class GzipFileWithTime(gzip.GzipFile): | ||||
def __init__(self, *args, **kw): | ||||
timestamp = None | ||||
if 'mtime' in kw: | ||||
timestamp = kw.pop('mtime') | ||||
if timestamp is None: | ||||
self.timestamp = time.time() | ||||
else: | ||||
self.timestamp = timestamp | ||||
gzip.GzipFile.__init__(self, *args, **kw) | ||||
|
r4652 | |||
r44981 | def _write_gzip_header(self): | |||
self.fileobj.write(b'\037\213') # magic header | ||||
self.fileobj.write(b'\010') # compression method | ||||
fname = self.name | ||||
if fname and fname.endswith(b'.gz'): | ||||
fname = fname[:-3] | ||||
flags = 0 | ||||
if fname: | ||||
flags = gzip.FNAME # pytype: disable=module-attr | ||||
self.fileobj.write(pycompat.bytechr(flags)) | ||||
gzip.write32u( # pytype: disable=module-attr | ||||
self.fileobj, int(self.timestamp) | ||||
) | ||||
self.fileobj.write(b'\002') | ||||
self.fileobj.write(b'\377') | ||||
if fname: | ||||
self.fileobj.write(fname + b'\000') | ||||
|
r4652 | |||
|
r43347 | def __init__(self, dest, mtime, kind=b''): | ||
|
r2477 | self.mtime = mtime | ||
|
r13400 | self.fileobj = None | ||
|
r4652 | |||
|
r43347 | def taropen(mode, name=b'', fileobj=None): | ||
if kind == b'gz': | ||||
|
r36465 | mode = mode[0:1] | ||
|
r4652 | if not fileobj: | ||
|
r43347 | fileobj = open(name, mode + b'b') | ||
|
r43346 | gzfileobj = self.GzipFileWithTime( | ||
name, | ||||
|
r43347 | pycompat.sysstr(mode + b'b'), | ||
|
r43346 | zlib.Z_BEST_COMPRESSION, | ||
fileobj, | ||||
r44980 | mtime=mtime, | |||
|
r43346 | ) | ||
|
r13400 | self.fileobj = gzfileobj | ||
|
r43788 | return tarfile.TarFile.taropen( # pytype: disable=attribute-error | ||
|
r43346 | name, pycompat.sysstr(mode), gzfileobj | ||
) | ||||
|
r4652 | else: | ||
|
r43346 | return tarfile.open(name, pycompat.sysstr(mode + kind), fileobj) | ||
|
r4652 | |||
|
r36726 | if isinstance(dest, bytes): | ||
|
r43347 | self.z = taropen(b'w:', name=dest) | ||
|
r2112 | else: | ||
|
r43347 | self.z = taropen(b'w|', fileobj=dest) | ||
|
r2112 | |||
|
r4831 | def addfile(self, name, mode, islink, data): | ||
|
r36724 | name = pycompat.fsdecode(name) | ||
|
r11558 | i = tarfile.TarInfo(name) | ||
|
r2112 | i.mtime = self.mtime | ||
i.size = len(data) | ||||
|
r4831 | if islink: | ||
i.type = tarfile.SYMTYPE | ||||
|
r25658 | i.mode = 0o777 | ||
|
r36724 | i.linkname = pycompat.fsdecode(data) | ||
|
r4831 | data = None | ||
|
r7770 | i.size = 0 | ||
|
r4831 | else: | ||
i.mode = mode | ||||
|
r28861 | data = stringio(data) | ||
|
r4831 | self.z.addfile(i, data) | ||
|
r2112 | |||
def done(self): | ||||
self.z.close() | ||||
|
r13400 | if self.fileobj: | ||
self.fileobj.close() | ||||
|
r2112 | |||
|
r43346 | |||
|
r8778 | class zipit(object): | ||
|
r2112 | '''write archive to zip file or stream. can write uncompressed, | ||
or compressed with deflate.''' | ||||
|
r11558 | def __init__(self, dest, mtime, compress=True): | ||
|
r40283 | if isinstance(dest, bytes): | ||
dest = pycompat.fsdecode(dest) | ||||
|
r43346 | self.z = zipfile.ZipFile( | ||
|
r43906 | dest, 'w', compress and zipfile.ZIP_DEFLATED or zipfile.ZIP_STORED | ||
|
r43346 | ) | ||
|
r12319 | |||
# Python's zipfile module emits deprecation warnings if we try | ||||
# to store files with a date before 1980. | ||||
|
r43346 | epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0)) | ||
|
r12319 | if mtime < epoch: | ||
mtime = epoch | ||||
|
r17628 | self.mtime = mtime | ||
|
r2477 | self.date_time = time.gmtime(mtime)[:6] | ||
|
r2112 | |||
|
r4831 | def addfile(self, name, mode, islink, data): | ||
|
r36724 | i = zipfile.ZipInfo(pycompat.fsdecode(name), self.date_time) | ||
|
r43788 | i.compress_type = self.z.compression # pytype: disable=attribute-error | ||
|
r2112 | # unzip will not honor unix file modes unless file creator is | ||
# set to unix (id 3). | ||||
i.create_system = 3 | ||||
|
r17429 | ftype = _UNX_IFREG | ||
|
r4831 | if islink: | ||
|
r25658 | mode = 0o777 | ||
|
r17429 | ftype = _UNX_IFLNK | ||
|
r29890 | i.external_attr = (mode | ftype) << 16 | ||
|
r17628 | # add "extended-timestamp" extra block, because zip archives | ||
# without this will be extracted with unexpected timestamp, | ||||
# if TZ is not configured as GMT | ||||
|
r43346 | i.extra += struct.pack( | ||
|
r43347 | b'<hhBl', | ||
|
r43346 | 0x5455, # block type: "extended-timestamp" | ||
1 + 4, # size of this block | ||||
1, # "modification time is present" | ||||
int(self.mtime), | ||||
) # last modification (UTC) | ||||
|
r2112 | self.z.writestr(i, data) | ||
def done(self): | ||||
self.z.close() | ||||
|
r43346 | |||
|
r8778 | class fileit(object): | ||
|
r2112 | '''write archive as files in directory.''' | ||
|
r11558 | def __init__(self, name, mtime): | ||
|
r2112 | self.basedir = name | ||
|
r31235 | self.opener = vfsmod.vfs(self.basedir) | ||
|
r35203 | self.mtime = mtime | ||
|
r2112 | |||
|
r4831 | def addfile(self, name, mode, islink, data): | ||
if islink: | ||||
self.opener.symlink(data, name) | ||||
return | ||||
|
r43347 | f = self.opener(name, b"w", atomictemp=False) | ||
|
r4830 | f.write(data) | ||
|
r15057 | f.close() | ||
|
r2112 | destfile = os.path.join(self.basedir, name) | ||
|
r4830 | os.chmod(destfile, mode) | ||
|
r35203 | if self.mtime is not None: | ||
os.utime(destfile, (self.mtime, self.mtime)) | ||||
|
r2112 | |||
def done(self): | ||||
pass | ||||
|
r43346 | |||
|
r2112 | archivers = { | ||
|
r43347 | b'files': fileit, | ||
b'tar': tarit, | ||||
b'tbz2': lambda name, mtime: tarit(name, mtime, b'bz2'), | ||||
b'tgz': lambda name, mtime: tarit(name, mtime, b'gz'), | ||||
b'txz': lambda name, mtime: tarit(name, mtime, b'xz'), | ||||
b'uzip': lambda name, mtime: zipit(name, mtime, False), | ||||
b'zip': zipit, | ||||
|
r43346 | } | ||
|
r2112 | |||
|
r43346 | def archive( | ||
repo, | ||||
dest, | ||||
node, | ||||
kind, | ||||
decode=True, | ||||
match=None, | ||||
|
r43347 | prefix=b'', | ||
|
r43346 | mtime=None, | ||
subrepos=False, | ||||
): | ||||
|
r2112 | '''create archive of repo as it was at node. | ||
dest can be name of directory, name of archive file, or file | ||||
object to write archive to. | ||||
kind is type of archive to create. | ||||
decode tells whether to put files through decode filters from | ||||
hgrc. | ||||
|
r40443 | match is a matcher to filter names of files to write to archive. | ||
|
r2112 | |||
|
r35203 | prefix is name of path to put before every archive member. | ||
mtime is the modified time, in seconds, or None to use the changeset time. | ||||
subrepos tells whether to include subrepos. | ||||
''' | ||||
|
r2112 | |||
|
r43347 | if kind == b'txz' and not pycompat.ispy3: | ||
raise error.Abort(_(b'xz compression is only available in Python 3')) | ||||
|
r43210 | |||
|
r43347 | if kind == b'files': | ||
|
r11558 | if prefix: | ||
|
r43347 | raise error.Abort(_(b'cannot give prefix when archiving to files')) | ||
|
r11558 | else: | ||
prefix = tidyprefix(dest, kind, prefix) | ||||
|
r4951 | def write(name, mode, islink, getdata): | ||
data = getdata() | ||||
|
r2112 | if decode: | ||
|
r4005 | data = repo.wwritedata(name, data) | ||
|
r11558 | archiver.addfile(prefix + name, mode, islink, data) | ||
|
r2112 | |||
|
r6019 | if kind not in archivers: | ||
|
r43347 | raise error.Abort(_(b"unknown archive type '%s'") % kind) | ||
|
r6749 | |||
ctx = repo[node] | ||||
|
r11558 | archiver = archivers[kind](dest, mtime or ctx.date()[0]) | ||
|
r6749 | |||
|
r40444 | if not match: | ||
match = scmutil.matchall(repo) | ||||
|
r43347 | if repo.ui.configbool(b"ui", b"archivemeta"): | ||
name = b'.hg_archival.txt' | ||||
|
r40444 | if match(name): | ||
|
r25658 | write(name, 0o644, False, lambda: buildmetadata(ctx)) | ||
|
r9614 | |||
|
r44766 | files = list(ctx.manifest().walk(match)) | ||
|
r16919 | total = len(files) | ||
|
r18967 | if total: | ||
files.sort() | ||||
|
r43346 | scmutil.prefetchfiles( | ||
repo, [ctx.rev()], scmutil.matchfiles(repo, files) | ||||
) | ||||
progress = repo.ui.makeprogress( | ||||
|
r43347 | _(b'archiving'), unit=_(b'files'), total=total | ||
|
r43346 | ) | ||
|
r38400 | progress.update(0) | ||
for f in files: | ||||
|
r18967 | ff = ctx.flags(f) | ||
|
r43347 | write(f, b'x' in ff and 0o755 or 0o644, b'l' in ff, ctx[f].data) | ||
|
r38400 | progress.increment(item=f) | ||
progress.complete() | ||||
|
r12323 | |||
if subrepos: | ||||
|
r18364 | for subpath in sorted(ctx.substate): | ||
|
r25601 | sub = ctx.workingsub(subpath) | ||
|
r40443 | submatch = matchmod.subdirmatcher(subpath, match) | ||
|
r43347 | subprefix = prefix + subpath + b'/' | ||
|
r41780 | total += sub.archive(archiver, subprefix, submatch, decode) | ||
|
r18967 | |||
if total == 0: | ||||
|
r43347 | raise error.Abort(_(b'no files match the archive pattern')) | ||
|
r12323 | |||
|
r2112 | archiver.done() | ||
|
r18967 | return total | ||