##// END OF EJS Templates
archival: use progress helper...
Martin von Zweigbergk -
r38400:1a2ff11e default
parent child Browse files
Show More
@@ -1,343 +1,344 b''
1 1 # archival.py - revision archival for mercurial
2 2 #
3 3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import gzip
11 11 import os
12 12 import struct
13 13 import tarfile
14 14 import time
15 15 import zipfile
16 16 import zlib
17 17
18 18 from .i18n import _
19 19
20 20 from . import (
21 21 error,
22 22 formatter,
23 23 match as matchmod,
24 24 pycompat,
25 25 scmutil,
26 26 util,
27 27 vfs as vfsmod,
28 28 )
29 29 stringio = util.stringio
30 30
31 31 # from unzip source code:
32 32 _UNX_IFREG = 0x8000
33 33 _UNX_IFLNK = 0xa000
34 34
35 35 def tidyprefix(dest, kind, prefix):
36 36 '''choose prefix to use for names in archive. make sure prefix is
37 37 safe for consumers.'''
38 38
39 39 if prefix:
40 40 prefix = util.normpath(prefix)
41 41 else:
42 42 if not isinstance(dest, bytes):
43 43 raise ValueError('dest must be string if no prefix')
44 44 prefix = os.path.basename(dest)
45 45 lower = prefix.lower()
46 46 for sfx in exts.get(kind, []):
47 47 if lower.endswith(sfx):
48 48 prefix = prefix[:-len(sfx)]
49 49 break
50 50 lpfx = os.path.normpath(util.localpath(prefix))
51 51 prefix = util.pconvert(lpfx)
52 52 if not prefix.endswith('/'):
53 53 prefix += '/'
54 54 # Drop the leading '.' path component if present, so Windows can read the
55 55 # zip files (issue4634)
56 56 if prefix.startswith('./'):
57 57 prefix = prefix[2:]
58 58 if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
59 59 raise error.Abort(_('archive prefix contains illegal components'))
60 60 return prefix
61 61
62 62 exts = {
63 63 'tar': ['.tar'],
64 64 'tbz2': ['.tbz2', '.tar.bz2'],
65 65 'tgz': ['.tgz', '.tar.gz'],
66 66 'zip': ['.zip'],
67 67 }
68 68
69 69 def guesskind(dest):
70 70 for kind, extensions in exts.iteritems():
71 71 if any(dest.endswith(ext) for ext in extensions):
72 72 return kind
73 73 return None
74 74
75 75 def _rootctx(repo):
76 76 # repo[0] may be hidden
77 77 for rev in repo:
78 78 return repo[rev]
79 79 return repo['null']
80 80
81 81 # {tags} on ctx includes local tags and 'tip', with no current way to limit
82 82 # that to global tags. Therefore, use {latesttag} as a substitute when
83 83 # the distance is 0, since that will be the list of global tags on ctx.
84 84 _defaultmetatemplate = br'''
85 85 repo: {root}
86 86 node: {ifcontains(rev, revset("wdir()"), "{p1node}{dirty}", "{node}")}
87 87 branch: {branch|utf8}
88 88 {ifeq(latesttagdistance, 0, join(latesttag % "tag: {tag}", "\n"),
89 89 separate("\n",
90 90 join(latesttag % "latesttag: {tag}", "\n"),
91 91 "latesttagdistance: {latesttagdistance}",
92 92 "changessincelatesttag: {changessincelatesttag}"))}
93 93 '''[1:] # drop leading '\n'
94 94
95 95 def buildmetadata(ctx):
96 96 '''build content of .hg_archival.txt'''
97 97 repo = ctx.repo()
98 98
99 99 opts = {
100 100 'template': repo.ui.config('experimental', 'archivemetatemplate',
101 101 _defaultmetatemplate)
102 102 }
103 103
104 104 out = util.stringio()
105 105
106 106 fm = formatter.formatter(repo.ui, out, 'archive', opts)
107 107 fm.startitem()
108 108 fm.context(ctx=ctx)
109 109 fm.data(root=_rootctx(repo).hex())
110 110
111 111 if ctx.rev() is None:
112 112 dirty = ''
113 113 if ctx.dirty(missing=True):
114 114 dirty = '+'
115 115 fm.data(dirty=dirty)
116 116 fm.end()
117 117
118 118 return out.getvalue()
119 119
120 120 class tarit(object):
121 121 '''write archive to tar file or stream. can write uncompressed,
122 122 or compress with gzip or bzip2.'''
123 123
124 124 class GzipFileWithTime(gzip.GzipFile):
125 125
126 126 def __init__(self, *args, **kw):
127 127 timestamp = None
128 128 if r'timestamp' in kw:
129 129 timestamp = kw.pop(r'timestamp')
130 130 if timestamp is None:
131 131 self.timestamp = time.time()
132 132 else:
133 133 self.timestamp = timestamp
134 134 gzip.GzipFile.__init__(self, *args, **kw)
135 135
136 136 def _write_gzip_header(self):
137 137 self.fileobj.write('\037\213') # magic header
138 138 self.fileobj.write('\010') # compression method
139 139 fname = self.name
140 140 if fname and fname.endswith('.gz'):
141 141 fname = fname[:-3]
142 142 flags = 0
143 143 if fname:
144 144 flags = gzip.FNAME
145 145 self.fileobj.write(pycompat.bytechr(flags))
146 146 gzip.write32u(self.fileobj, int(self.timestamp))
147 147 self.fileobj.write('\002')
148 148 self.fileobj.write('\377')
149 149 if fname:
150 150 self.fileobj.write(fname + '\000')
151 151
152 152 def __init__(self, dest, mtime, kind=''):
153 153 self.mtime = mtime
154 154 self.fileobj = None
155 155
156 156 def taropen(mode, name='', fileobj=None):
157 157 if kind == 'gz':
158 158 mode = mode[0:1]
159 159 if not fileobj:
160 160 fileobj = open(name, mode + 'b')
161 161 gzfileobj = self.GzipFileWithTime(name,
162 162 pycompat.sysstr(mode + 'b'),
163 163 zlib.Z_BEST_COMPRESSION,
164 164 fileobj, timestamp=mtime)
165 165 self.fileobj = gzfileobj
166 166 return tarfile.TarFile.taropen(
167 167 name, pycompat.sysstr(mode), gzfileobj)
168 168 else:
169 169 return tarfile.open(
170 170 name, pycompat.sysstr(mode + kind), fileobj)
171 171
172 172 if isinstance(dest, bytes):
173 173 self.z = taropen('w:', name=dest)
174 174 else:
175 175 self.z = taropen('w|', fileobj=dest)
176 176
177 177 def addfile(self, name, mode, islink, data):
178 178 name = pycompat.fsdecode(name)
179 179 i = tarfile.TarInfo(name)
180 180 i.mtime = self.mtime
181 181 i.size = len(data)
182 182 if islink:
183 183 i.type = tarfile.SYMTYPE
184 184 i.mode = 0o777
185 185 i.linkname = pycompat.fsdecode(data)
186 186 data = None
187 187 i.size = 0
188 188 else:
189 189 i.mode = mode
190 190 data = stringio(data)
191 191 self.z.addfile(i, data)
192 192
193 193 def done(self):
194 194 self.z.close()
195 195 if self.fileobj:
196 196 self.fileobj.close()
197 197
198 198 class zipit(object):
199 199 '''write archive to zip file or stream. can write uncompressed,
200 200 or compressed with deflate.'''
201 201
202 202 def __init__(self, dest, mtime, compress=True):
203 203 self.z = zipfile.ZipFile(pycompat.fsdecode(dest), r'w',
204 204 compress and zipfile.ZIP_DEFLATED or
205 205 zipfile.ZIP_STORED)
206 206
207 207 # Python's zipfile module emits deprecation warnings if we try
208 208 # to store files with a date before 1980.
209 209 epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
210 210 if mtime < epoch:
211 211 mtime = epoch
212 212
213 213 self.mtime = mtime
214 214 self.date_time = time.gmtime(mtime)[:6]
215 215
216 216 def addfile(self, name, mode, islink, data):
217 217 i = zipfile.ZipInfo(pycompat.fsdecode(name), self.date_time)
218 218 i.compress_type = self.z.compression
219 219 # unzip will not honor unix file modes unless file creator is
220 220 # set to unix (id 3).
221 221 i.create_system = 3
222 222 ftype = _UNX_IFREG
223 223 if islink:
224 224 mode = 0o777
225 225 ftype = _UNX_IFLNK
226 226 i.external_attr = (mode | ftype) << 16
227 227 # add "extended-timestamp" extra block, because zip archives
228 228 # without this will be extracted with unexpected timestamp,
229 229 # if TZ is not configured as GMT
230 230 i.extra += struct.pack('<hhBl',
231 231 0x5455, # block type: "extended-timestamp"
232 232 1 + 4, # size of this block
233 233 1, # "modification time is present"
234 234 int(self.mtime)) # last modification (UTC)
235 235 self.z.writestr(i, data)
236 236
237 237 def done(self):
238 238 self.z.close()
239 239
240 240 class fileit(object):
241 241 '''write archive as files in directory.'''
242 242
243 243 def __init__(self, name, mtime):
244 244 self.basedir = name
245 245 self.opener = vfsmod.vfs(self.basedir)
246 246 self.mtime = mtime
247 247
248 248 def addfile(self, name, mode, islink, data):
249 249 if islink:
250 250 self.opener.symlink(data, name)
251 251 return
252 252 f = self.opener(name, "w", atomictemp=False)
253 253 f.write(data)
254 254 f.close()
255 255 destfile = os.path.join(self.basedir, name)
256 256 os.chmod(destfile, mode)
257 257 if self.mtime is not None:
258 258 os.utime(destfile, (self.mtime, self.mtime))
259 259
260 260 def done(self):
261 261 pass
262 262
263 263 archivers = {
264 264 'files': fileit,
265 265 'tar': tarit,
266 266 'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
267 267 'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
268 268 'uzip': lambda name, mtime: zipit(name, mtime, False),
269 269 'zip': zipit,
270 270 }
271 271
272 272 def archive(repo, dest, node, kind, decode=True, matchfn=None,
273 273 prefix='', mtime=None, subrepos=False):
274 274 '''create archive of repo as it was at node.
275 275
276 276 dest can be name of directory, name of archive file, or file
277 277 object to write archive to.
278 278
279 279 kind is type of archive to create.
280 280
281 281 decode tells whether to put files through decode filters from
282 282 hgrc.
283 283
284 284 matchfn is function to filter names of files to write to archive.
285 285
286 286 prefix is name of path to put before every archive member.
287 287
288 288 mtime is the modified time, in seconds, or None to use the changeset time.
289 289
290 290 subrepos tells whether to include subrepos.
291 291 '''
292 292
293 293 if kind == 'files':
294 294 if prefix:
295 295 raise error.Abort(_('cannot give prefix when archiving to files'))
296 296 else:
297 297 prefix = tidyprefix(dest, kind, prefix)
298 298
299 299 def write(name, mode, islink, getdata):
300 300 data = getdata()
301 301 if decode:
302 302 data = repo.wwritedata(name, data)
303 303 archiver.addfile(prefix + name, mode, islink, data)
304 304
305 305 if kind not in archivers:
306 306 raise error.Abort(_("unknown archive type '%s'") % kind)
307 307
308 308 ctx = repo[node]
309 309 archiver = archivers[kind](dest, mtime or ctx.date()[0])
310 310
311 311 if repo.ui.configbool("ui", "archivemeta"):
312 312 name = '.hg_archival.txt'
313 313 if not matchfn or matchfn(name):
314 314 write(name, 0o644, False, lambda: buildmetadata(ctx))
315 315
316 316 if matchfn:
317 317 files = [f for f in ctx.manifest().keys() if matchfn(f)]
318 318 else:
319 319 files = ctx.manifest().keys()
320 320 total = len(files)
321 321 if total:
322 322 files.sort()
323 323 scmutil.prefetchfiles(repo, [ctx.rev()],
324 324 scmutil.matchfiles(repo, files))
325 repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
326 for i, f in enumerate(files):
325 progress = scmutil.progress(repo.ui, _('archiving'), unit=_('files'),
326 total=total)
327 progress.update(0)
328 for f in files:
327 329 ff = ctx.flags(f)
328 330 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data)
329 repo.ui.progress(_('archiving'), i + 1, item=f,
330 unit=_('files'), total=total)
331 repo.ui.progress(_('archiving'), None)
331 progress.increment(item=f)
332 progress.complete()
332 333
333 334 if subrepos:
334 335 for subpath in sorted(ctx.substate):
335 336 sub = ctx.workingsub(subpath)
336 337 submatch = matchmod.subdirmatcher(subpath, matchfn)
337 338 total += sub.archive(archiver, prefix, submatch, decode)
338 339
339 340 if total == 0:
340 341 raise error.Abort(_('no files match the archive pattern'))
341 342
342 343 archiver.done()
343 344 return total
General Comments 0
You need to be logged in to leave comments. Login now