##// END OF EJS Templates
archive: use a templater to build the metadata file...
Matt Harbison -
r33544:4c4e95ca default
parent child Browse files
Show More
@@ -1,340 +1,351 b''
1 # archival.py - revision archival for mercurial
1 # archival.py - revision archival for mercurial
2 #
2 #
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import gzip
10 import gzip
11 import os
11 import os
12 import struct
12 import struct
13 import tarfile
13 import tarfile
14 import time
14 import time
15 import zipfile
15 import zipfile
16 import zlib
16 import zlib
17
17
18 from .i18n import _
18 from .i18n import _
19
19
20 from . import (
20 from . import (
21 cmdutil,
22 encoding,
23 error,
21 error,
22 formatter,
24 match as matchmod,
23 match as matchmod,
25 util,
24 util,
26 vfs as vfsmod,
25 vfs as vfsmod,
27 )
26 )
28 stringio = util.stringio
27 stringio = util.stringio
29
28
30 # from unzip source code:
29 # from unzip source code:
31 _UNX_IFREG = 0x8000
30 _UNX_IFREG = 0x8000
32 _UNX_IFLNK = 0xa000
31 _UNX_IFLNK = 0xa000
33
32
34 def tidyprefix(dest, kind, prefix):
33 def tidyprefix(dest, kind, prefix):
35 '''choose prefix to use for names in archive. make sure prefix is
34 '''choose prefix to use for names in archive. make sure prefix is
36 safe for consumers.'''
35 safe for consumers.'''
37
36
38 if prefix:
37 if prefix:
39 prefix = util.normpath(prefix)
38 prefix = util.normpath(prefix)
40 else:
39 else:
41 if not isinstance(dest, str):
40 if not isinstance(dest, str):
42 raise ValueError('dest must be string if no prefix')
41 raise ValueError('dest must be string if no prefix')
43 prefix = os.path.basename(dest)
42 prefix = os.path.basename(dest)
44 lower = prefix.lower()
43 lower = prefix.lower()
45 for sfx in exts.get(kind, []):
44 for sfx in exts.get(kind, []):
46 if lower.endswith(sfx):
45 if lower.endswith(sfx):
47 prefix = prefix[:-len(sfx)]
46 prefix = prefix[:-len(sfx)]
48 break
47 break
49 lpfx = os.path.normpath(util.localpath(prefix))
48 lpfx = os.path.normpath(util.localpath(prefix))
50 prefix = util.pconvert(lpfx)
49 prefix = util.pconvert(lpfx)
51 if not prefix.endswith('/'):
50 if not prefix.endswith('/'):
52 prefix += '/'
51 prefix += '/'
53 # Drop the leading '.' path component if present, so Windows can read the
52 # Drop the leading '.' path component if present, so Windows can read the
54 # zip files (issue4634)
53 # zip files (issue4634)
55 if prefix.startswith('./'):
54 if prefix.startswith('./'):
56 prefix = prefix[2:]
55 prefix = prefix[2:]
57 if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
56 if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
58 raise error.Abort(_('archive prefix contains illegal components'))
57 raise error.Abort(_('archive prefix contains illegal components'))
59 return prefix
58 return prefix
60
59
61 exts = {
60 exts = {
62 'tar': ['.tar'],
61 'tar': ['.tar'],
63 'tbz2': ['.tbz2', '.tar.bz2'],
62 'tbz2': ['.tbz2', '.tar.bz2'],
64 'tgz': ['.tgz', '.tar.gz'],
63 'tgz': ['.tgz', '.tar.gz'],
65 'zip': ['.zip'],
64 'zip': ['.zip'],
66 }
65 }
67
66
68 def guesskind(dest):
67 def guesskind(dest):
69 for kind, extensions in exts.iteritems():
68 for kind, extensions in exts.iteritems():
70 if any(dest.endswith(ext) for ext in extensions):
69 if any(dest.endswith(ext) for ext in extensions):
71 return kind
70 return kind
72 return None
71 return None
73
72
74 def _rootctx(repo):
73 def _rootctx(repo):
75 # repo[0] may be hidden
74 # repo[0] may be hidden
76 for rev in repo:
75 for rev in repo:
77 return repo[rev]
76 return repo[rev]
78 return repo['null']
77 return repo['null']
79
78
80 def buildmetadata(ctx):
79 def buildmetadata(ctx):
81 '''build content of .hg_archival.txt'''
80 '''build content of .hg_archival.txt'''
82 repo = ctx.repo()
81 repo = ctx.repo()
83 hex = ctx.hex()
82
84 if ctx.rev() is None:
83 default = (
85 hex = ctx.p1().hex()
84 r'repo: {root}\n'
86 if ctx.dirty(missing=True):
85 r'node: {ifcontains(rev, revset("wdir()"),'
87 hex += '+'
86 r'"{p1node}{dirty}", "{node}")}\n'
87 r'branch: {branch|utf8}\n'
88
88
89 base = 'repo: %s\nnode: %s\nbranch: %s\n' % (
89 # {tags} on ctx includes local tags and 'tip', with no current way to
90 _rootctx(repo).hex(), hex, encoding.fromlocal(ctx.branch()))
90 # limit that to global tags. Therefore, use {latesttag} as a substitute
91 # when the distance is 0, since that will be the list of global tags on
92 # ctx.
93 r'{ifeq(latesttagdistance, 0, latesttag % "tag: {tag}\n",'
94 r'"{latesttag % "latesttag: {tag}\n"}'
95 r'latesttagdistance: {latesttagdistance}\n'
96 r'changessincelatesttag: {changessincelatesttag}\n")}'
97 )
91
98
92 tags = ''.join('tag: %s\n' % t for t in ctx.tags()
99 opts = {
93 if repo.tagtype(t) == 'global')
100 'template': default
94 if not tags:
101 }
95 repo.ui.pushbuffer()
102
96 opts = {'template': '{latesttag}\n{latesttagdistance}\n'
103 out = util.stringio()
97 '{changessincelatesttag}',
98 'style': '', 'patch': None, 'git': None}
99 cmdutil.show_changeset(repo.ui, repo, opts).show(ctx)
100 ltags, dist, changessince = repo.ui.popbuffer().split('\n')
101 ltags = ltags.split(':')
102 tags = ''.join('latesttag: %s\n' % t for t in ltags)
103 tags += 'latesttagdistance: %s\n' % dist
104 tags += 'changessincelatesttag: %s\n' % changessince
105
104
106 return base + tags
105 fm = formatter.formatter(repo.ui, out, 'archive', opts)
106 fm.startitem()
107 fm.context(ctx=ctx)
108 fm.data(root=_rootctx(repo).hex())
109
110 if ctx.rev() is None:
111 dirty = ''
112 if ctx.dirty(missing=True):
113 dirty = '+'
114 fm.data(dirty=dirty)
115 fm.end()
116
117 return out.getvalue()
107
118
108 class tarit(object):
119 class tarit(object):
109 '''write archive to tar file or stream. can write uncompressed,
120 '''write archive to tar file or stream. can write uncompressed,
110 or compress with gzip or bzip2.'''
121 or compress with gzip or bzip2.'''
111
122
112 class GzipFileWithTime(gzip.GzipFile):
123 class GzipFileWithTime(gzip.GzipFile):
113
124
114 def __init__(self, *args, **kw):
125 def __init__(self, *args, **kw):
115 timestamp = None
126 timestamp = None
116 if 'timestamp' in kw:
127 if 'timestamp' in kw:
117 timestamp = kw.pop('timestamp')
128 timestamp = kw.pop('timestamp')
118 if timestamp is None:
129 if timestamp is None:
119 self.timestamp = time.time()
130 self.timestamp = time.time()
120 else:
131 else:
121 self.timestamp = timestamp
132 self.timestamp = timestamp
122 gzip.GzipFile.__init__(self, *args, **kw)
133 gzip.GzipFile.__init__(self, *args, **kw)
123
134
124 def _write_gzip_header(self):
135 def _write_gzip_header(self):
125 self.fileobj.write('\037\213') # magic header
136 self.fileobj.write('\037\213') # magic header
126 self.fileobj.write('\010') # compression method
137 self.fileobj.write('\010') # compression method
127 fname = self.name
138 fname = self.name
128 if fname and fname.endswith('.gz'):
139 if fname and fname.endswith('.gz'):
129 fname = fname[:-3]
140 fname = fname[:-3]
130 flags = 0
141 flags = 0
131 if fname:
142 if fname:
132 flags = gzip.FNAME
143 flags = gzip.FNAME
133 self.fileobj.write(chr(flags))
144 self.fileobj.write(chr(flags))
134 gzip.write32u(self.fileobj, long(self.timestamp))
145 gzip.write32u(self.fileobj, long(self.timestamp))
135 self.fileobj.write('\002')
146 self.fileobj.write('\002')
136 self.fileobj.write('\377')
147 self.fileobj.write('\377')
137 if fname:
148 if fname:
138 self.fileobj.write(fname + '\000')
149 self.fileobj.write(fname + '\000')
139
150
140 def __init__(self, dest, mtime, kind=''):
151 def __init__(self, dest, mtime, kind=''):
141 self.mtime = mtime
152 self.mtime = mtime
142 self.fileobj = None
153 self.fileobj = None
143
154
144 def taropen(mode, name='', fileobj=None):
155 def taropen(mode, name='', fileobj=None):
145 if kind == 'gz':
156 if kind == 'gz':
146 mode = mode[0]
157 mode = mode[0]
147 if not fileobj:
158 if not fileobj:
148 fileobj = open(name, mode + 'b')
159 fileobj = open(name, mode + 'b')
149 gzfileobj = self.GzipFileWithTime(name, mode + 'b',
160 gzfileobj = self.GzipFileWithTime(name, mode + 'b',
150 zlib.Z_BEST_COMPRESSION,
161 zlib.Z_BEST_COMPRESSION,
151 fileobj, timestamp=mtime)
162 fileobj, timestamp=mtime)
152 self.fileobj = gzfileobj
163 self.fileobj = gzfileobj
153 return tarfile.TarFile.taropen(name, mode, gzfileobj)
164 return tarfile.TarFile.taropen(name, mode, gzfileobj)
154 else:
165 else:
155 return tarfile.open(name, mode + kind, fileobj)
166 return tarfile.open(name, mode + kind, fileobj)
156
167
157 if isinstance(dest, str):
168 if isinstance(dest, str):
158 self.z = taropen('w:', name=dest)
169 self.z = taropen('w:', name=dest)
159 else:
170 else:
160 self.z = taropen('w|', fileobj=dest)
171 self.z = taropen('w|', fileobj=dest)
161
172
162 def addfile(self, name, mode, islink, data):
173 def addfile(self, name, mode, islink, data):
163 i = tarfile.TarInfo(name)
174 i = tarfile.TarInfo(name)
164 i.mtime = self.mtime
175 i.mtime = self.mtime
165 i.size = len(data)
176 i.size = len(data)
166 if islink:
177 if islink:
167 i.type = tarfile.SYMTYPE
178 i.type = tarfile.SYMTYPE
168 i.mode = 0o777
179 i.mode = 0o777
169 i.linkname = data
180 i.linkname = data
170 data = None
181 data = None
171 i.size = 0
182 i.size = 0
172 else:
183 else:
173 i.mode = mode
184 i.mode = mode
174 data = stringio(data)
185 data = stringio(data)
175 self.z.addfile(i, data)
186 self.z.addfile(i, data)
176
187
177 def done(self):
188 def done(self):
178 self.z.close()
189 self.z.close()
179 if self.fileobj:
190 if self.fileobj:
180 self.fileobj.close()
191 self.fileobj.close()
181
192
182 class tellable(object):
193 class tellable(object):
183 '''provide tell method for zipfile.ZipFile when writing to http
194 '''provide tell method for zipfile.ZipFile when writing to http
184 response file object.'''
195 response file object.'''
185
196
186 def __init__(self, fp):
197 def __init__(self, fp):
187 self.fp = fp
198 self.fp = fp
188 self.offset = 0
199 self.offset = 0
189
200
190 def __getattr__(self, key):
201 def __getattr__(self, key):
191 return getattr(self.fp, key)
202 return getattr(self.fp, key)
192
203
193 def write(self, s):
204 def write(self, s):
194 self.fp.write(s)
205 self.fp.write(s)
195 self.offset += len(s)
206 self.offset += len(s)
196
207
197 def tell(self):
208 def tell(self):
198 return self.offset
209 return self.offset
199
210
200 class zipit(object):
211 class zipit(object):
201 '''write archive to zip file or stream. can write uncompressed,
212 '''write archive to zip file or stream. can write uncompressed,
202 or compressed with deflate.'''
213 or compressed with deflate.'''
203
214
204 def __init__(self, dest, mtime, compress=True):
215 def __init__(self, dest, mtime, compress=True):
205 if not isinstance(dest, str):
216 if not isinstance(dest, str):
206 try:
217 try:
207 dest.tell()
218 dest.tell()
208 except (AttributeError, IOError):
219 except (AttributeError, IOError):
209 dest = tellable(dest)
220 dest = tellable(dest)
210 self.z = zipfile.ZipFile(dest, 'w',
221 self.z = zipfile.ZipFile(dest, 'w',
211 compress and zipfile.ZIP_DEFLATED or
222 compress and zipfile.ZIP_DEFLATED or
212 zipfile.ZIP_STORED)
223 zipfile.ZIP_STORED)
213
224
214 # Python's zipfile module emits deprecation warnings if we try
225 # Python's zipfile module emits deprecation warnings if we try
215 # to store files with a date before 1980.
226 # to store files with a date before 1980.
216 epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
227 epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
217 if mtime < epoch:
228 if mtime < epoch:
218 mtime = epoch
229 mtime = epoch
219
230
220 self.mtime = mtime
231 self.mtime = mtime
221 self.date_time = time.gmtime(mtime)[:6]
232 self.date_time = time.gmtime(mtime)[:6]
222
233
223 def addfile(self, name, mode, islink, data):
234 def addfile(self, name, mode, islink, data):
224 i = zipfile.ZipInfo(name, self.date_time)
235 i = zipfile.ZipInfo(name, self.date_time)
225 i.compress_type = self.z.compression
236 i.compress_type = self.z.compression
226 # unzip will not honor unix file modes unless file creator is
237 # unzip will not honor unix file modes unless file creator is
227 # set to unix (id 3).
238 # set to unix (id 3).
228 i.create_system = 3
239 i.create_system = 3
229 ftype = _UNX_IFREG
240 ftype = _UNX_IFREG
230 if islink:
241 if islink:
231 mode = 0o777
242 mode = 0o777
232 ftype = _UNX_IFLNK
243 ftype = _UNX_IFLNK
233 i.external_attr = (mode | ftype) << 16
244 i.external_attr = (mode | ftype) << 16
234 # add "extended-timestamp" extra block, because zip archives
245 # add "extended-timestamp" extra block, because zip archives
235 # without this will be extracted with unexpected timestamp,
246 # without this will be extracted with unexpected timestamp,
236 # if TZ is not configured as GMT
247 # if TZ is not configured as GMT
237 i.extra += struct.pack('<hhBl',
248 i.extra += struct.pack('<hhBl',
238 0x5455, # block type: "extended-timestamp"
249 0x5455, # block type: "extended-timestamp"
239 1 + 4, # size of this block
250 1 + 4, # size of this block
240 1, # "modification time is present"
251 1, # "modification time is present"
241 int(self.mtime)) # last modification (UTC)
252 int(self.mtime)) # last modification (UTC)
242 self.z.writestr(i, data)
253 self.z.writestr(i, data)
243
254
244 def done(self):
255 def done(self):
245 self.z.close()
256 self.z.close()
246
257
247 class fileit(object):
258 class fileit(object):
248 '''write archive as files in directory.'''
259 '''write archive as files in directory.'''
249
260
250 def __init__(self, name, mtime):
261 def __init__(self, name, mtime):
251 self.basedir = name
262 self.basedir = name
252 self.opener = vfsmod.vfs(self.basedir)
263 self.opener = vfsmod.vfs(self.basedir)
253
264
254 def addfile(self, name, mode, islink, data):
265 def addfile(self, name, mode, islink, data):
255 if islink:
266 if islink:
256 self.opener.symlink(data, name)
267 self.opener.symlink(data, name)
257 return
268 return
258 f = self.opener(name, "w", atomictemp=True)
269 f = self.opener(name, "w", atomictemp=True)
259 f.write(data)
270 f.write(data)
260 f.close()
271 f.close()
261 destfile = os.path.join(self.basedir, name)
272 destfile = os.path.join(self.basedir, name)
262 os.chmod(destfile, mode)
273 os.chmod(destfile, mode)
263
274
264 def done(self):
275 def done(self):
265 pass
276 pass
266
277
267 archivers = {
278 archivers = {
268 'files': fileit,
279 'files': fileit,
269 'tar': tarit,
280 'tar': tarit,
270 'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
281 'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
271 'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
282 'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
272 'uzip': lambda name, mtime: zipit(name, mtime, False),
283 'uzip': lambda name, mtime: zipit(name, mtime, False),
273 'zip': zipit,
284 'zip': zipit,
274 }
285 }
275
286
276 def archive(repo, dest, node, kind, decode=True, matchfn=None,
287 def archive(repo, dest, node, kind, decode=True, matchfn=None,
277 prefix='', mtime=None, subrepos=False):
288 prefix='', mtime=None, subrepos=False):
278 '''create archive of repo as it was at node.
289 '''create archive of repo as it was at node.
279
290
280 dest can be name of directory, name of archive file, or file
291 dest can be name of directory, name of archive file, or file
281 object to write archive to.
292 object to write archive to.
282
293
283 kind is type of archive to create.
294 kind is type of archive to create.
284
295
285 decode tells whether to put files through decode filters from
296 decode tells whether to put files through decode filters from
286 hgrc.
297 hgrc.
287
298
288 matchfn is function to filter names of files to write to archive.
299 matchfn is function to filter names of files to write to archive.
289
300
290 prefix is name of path to put before every archive member.'''
301 prefix is name of path to put before every archive member.'''
291
302
292 if kind == 'files':
303 if kind == 'files':
293 if prefix:
304 if prefix:
294 raise error.Abort(_('cannot give prefix when archiving to files'))
305 raise error.Abort(_('cannot give prefix when archiving to files'))
295 else:
306 else:
296 prefix = tidyprefix(dest, kind, prefix)
307 prefix = tidyprefix(dest, kind, prefix)
297
308
298 def write(name, mode, islink, getdata):
309 def write(name, mode, islink, getdata):
299 data = getdata()
310 data = getdata()
300 if decode:
311 if decode:
301 data = repo.wwritedata(name, data)
312 data = repo.wwritedata(name, data)
302 archiver.addfile(prefix + name, mode, islink, data)
313 archiver.addfile(prefix + name, mode, islink, data)
303
314
304 if kind not in archivers:
315 if kind not in archivers:
305 raise error.Abort(_("unknown archive type '%s'") % kind)
316 raise error.Abort(_("unknown archive type '%s'") % kind)
306
317
307 ctx = repo[node]
318 ctx = repo[node]
308 archiver = archivers[kind](dest, mtime or ctx.date()[0])
319 archiver = archivers[kind](dest, mtime or ctx.date()[0])
309
320
310 if repo.ui.configbool("ui", "archivemeta"):
321 if repo.ui.configbool("ui", "archivemeta"):
311 name = '.hg_archival.txt'
322 name = '.hg_archival.txt'
312 if not matchfn or matchfn(name):
323 if not matchfn or matchfn(name):
313 write(name, 0o644, False, lambda: buildmetadata(ctx))
324 write(name, 0o644, False, lambda: buildmetadata(ctx))
314
325
315 if matchfn:
326 if matchfn:
316 files = [f for f in ctx.manifest().keys() if matchfn(f)]
327 files = [f for f in ctx.manifest().keys() if matchfn(f)]
317 else:
328 else:
318 files = ctx.manifest().keys()
329 files = ctx.manifest().keys()
319 total = len(files)
330 total = len(files)
320 if total:
331 if total:
321 files.sort()
332 files.sort()
322 repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
333 repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
323 for i, f in enumerate(files):
334 for i, f in enumerate(files):
324 ff = ctx.flags(f)
335 ff = ctx.flags(f)
325 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data)
336 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data)
326 repo.ui.progress(_('archiving'), i + 1, item=f,
337 repo.ui.progress(_('archiving'), i + 1, item=f,
327 unit=_('files'), total=total)
338 unit=_('files'), total=total)
328 repo.ui.progress(_('archiving'), None)
339 repo.ui.progress(_('archiving'), None)
329
340
330 if subrepos:
341 if subrepos:
331 for subpath in sorted(ctx.substate):
342 for subpath in sorted(ctx.substate):
332 sub = ctx.workingsub(subpath)
343 sub = ctx.workingsub(subpath)
333 submatch = matchmod.subdirmatcher(subpath, matchfn)
344 submatch = matchmod.subdirmatcher(subpath, matchfn)
334 total += sub.archive(archiver, prefix, submatch, decode)
345 total += sub.archive(archiver, prefix, submatch, decode)
335
346
336 if total == 0:
347 if total == 0:
337 raise error.Abort(_('no files match the archive pattern'))
348 raise error.Abort(_('no files match the archive pattern'))
338
349
339 archiver.done()
350 archiver.done()
340 return total
351 return total
General Comments 0
You need to be logged in to leave comments. Login now