##// END OF EJS Templates
py3: use raw string for file open mode...
Gregory Szorc -
r36053:788b2e85 default
parent child Browse files
Show More
@@ -1,360 +1,360 b''
1 # archival.py - revision archival for mercurial
1 # archival.py - revision archival for mercurial
2 #
2 #
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import gzip
10 import gzip
11 import os
11 import os
12 import struct
12 import struct
13 import tarfile
13 import tarfile
14 import time
14 import time
15 import zipfile
15 import zipfile
16 import zlib
16 import zlib
17
17
18 from .i18n import _
18 from .i18n import _
19
19
20 from . import (
20 from . import (
21 cmdutil,
21 cmdutil,
22 error,
22 error,
23 formatter,
23 formatter,
24 match as matchmod,
24 match as matchmod,
25 util,
25 util,
26 vfs as vfsmod,
26 vfs as vfsmod,
27 )
27 )
28 stringio = util.stringio
28 stringio = util.stringio
29
29
30 # from unzip source code:
30 # from unzip source code:
31 _UNX_IFREG = 0x8000
31 _UNX_IFREG = 0x8000
32 _UNX_IFLNK = 0xa000
32 _UNX_IFLNK = 0xa000
33
33
34 def tidyprefix(dest, kind, prefix):
34 def tidyprefix(dest, kind, prefix):
35 '''choose prefix to use for names in archive. make sure prefix is
35 '''choose prefix to use for names in archive. make sure prefix is
36 safe for consumers.'''
36 safe for consumers.'''
37
37
38 if prefix:
38 if prefix:
39 prefix = util.normpath(prefix)
39 prefix = util.normpath(prefix)
40 else:
40 else:
41 if not isinstance(dest, str):
41 if not isinstance(dest, str):
42 raise ValueError('dest must be string if no prefix')
42 raise ValueError('dest must be string if no prefix')
43 prefix = os.path.basename(dest)
43 prefix = os.path.basename(dest)
44 lower = prefix.lower()
44 lower = prefix.lower()
45 for sfx in exts.get(kind, []):
45 for sfx in exts.get(kind, []):
46 if lower.endswith(sfx):
46 if lower.endswith(sfx):
47 prefix = prefix[:-len(sfx)]
47 prefix = prefix[:-len(sfx)]
48 break
48 break
49 lpfx = os.path.normpath(util.localpath(prefix))
49 lpfx = os.path.normpath(util.localpath(prefix))
50 prefix = util.pconvert(lpfx)
50 prefix = util.pconvert(lpfx)
51 if not prefix.endswith('/'):
51 if not prefix.endswith('/'):
52 prefix += '/'
52 prefix += '/'
53 # Drop the leading '.' path component if present, so Windows can read the
53 # Drop the leading '.' path component if present, so Windows can read the
54 # zip files (issue4634)
54 # zip files (issue4634)
55 if prefix.startswith('./'):
55 if prefix.startswith('./'):
56 prefix = prefix[2:]
56 prefix = prefix[2:]
57 if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
57 if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
58 raise error.Abort(_('archive prefix contains illegal components'))
58 raise error.Abort(_('archive prefix contains illegal components'))
59 return prefix
59 return prefix
60
60
61 exts = {
61 exts = {
62 'tar': ['.tar'],
62 'tar': ['.tar'],
63 'tbz2': ['.tbz2', '.tar.bz2'],
63 'tbz2': ['.tbz2', '.tar.bz2'],
64 'tgz': ['.tgz', '.tar.gz'],
64 'tgz': ['.tgz', '.tar.gz'],
65 'zip': ['.zip'],
65 'zip': ['.zip'],
66 }
66 }
67
67
68 def guesskind(dest):
68 def guesskind(dest):
69 for kind, extensions in exts.iteritems():
69 for kind, extensions in exts.iteritems():
70 if any(dest.endswith(ext) for ext in extensions):
70 if any(dest.endswith(ext) for ext in extensions):
71 return kind
71 return kind
72 return None
72 return None
73
73
74 def _rootctx(repo):
74 def _rootctx(repo):
75 # repo[0] may be hidden
75 # repo[0] may be hidden
76 for rev in repo:
76 for rev in repo:
77 return repo[rev]
77 return repo[rev]
78 return repo['null']
78 return repo['null']
79
79
80 # {tags} on ctx includes local tags and 'tip', with no current way to limit
80 # {tags} on ctx includes local tags and 'tip', with no current way to limit
81 # that to global tags. Therefore, use {latesttag} as a substitute when
81 # that to global tags. Therefore, use {latesttag} as a substitute when
82 # the distance is 0, since that will be the list of global tags on ctx.
82 # the distance is 0, since that will be the list of global tags on ctx.
83 _defaultmetatemplate = br'''
83 _defaultmetatemplate = br'''
84 repo: {root}
84 repo: {root}
85 node: {ifcontains(rev, revset("wdir()"), "{p1node}{dirty}", "{node}")}
85 node: {ifcontains(rev, revset("wdir()"), "{p1node}{dirty}", "{node}")}
86 branch: {branch|utf8}
86 branch: {branch|utf8}
87 {ifeq(latesttagdistance, 0, join(latesttag % "tag: {tag}", "\n"),
87 {ifeq(latesttagdistance, 0, join(latesttag % "tag: {tag}", "\n"),
88 separate("\n",
88 separate("\n",
89 join(latesttag % "latesttag: {tag}", "\n"),
89 join(latesttag % "latesttag: {tag}", "\n"),
90 "latesttagdistance: {latesttagdistance}",
90 "latesttagdistance: {latesttagdistance}",
91 "changessincelatesttag: {changessincelatesttag}"))}
91 "changessincelatesttag: {changessincelatesttag}"))}
92 '''[1:] # drop leading '\n'
92 '''[1:] # drop leading '\n'
93
93
94 def buildmetadata(ctx):
94 def buildmetadata(ctx):
95 '''build content of .hg_archival.txt'''
95 '''build content of .hg_archival.txt'''
96 repo = ctx.repo()
96 repo = ctx.repo()
97
97
98 opts = {
98 opts = {
99 'template': repo.ui.config('experimental', 'archivemetatemplate',
99 'template': repo.ui.config('experimental', 'archivemetatemplate',
100 _defaultmetatemplate)
100 _defaultmetatemplate)
101 }
101 }
102
102
103 out = util.stringio()
103 out = util.stringio()
104
104
105 fm = formatter.formatter(repo.ui, out, 'archive', opts)
105 fm = formatter.formatter(repo.ui, out, 'archive', opts)
106 fm.startitem()
106 fm.startitem()
107 fm.context(ctx=ctx)
107 fm.context(ctx=ctx)
108 fm.data(root=_rootctx(repo).hex())
108 fm.data(root=_rootctx(repo).hex())
109
109
110 if ctx.rev() is None:
110 if ctx.rev() is None:
111 dirty = ''
111 dirty = ''
112 if ctx.dirty(missing=True):
112 if ctx.dirty(missing=True):
113 dirty = '+'
113 dirty = '+'
114 fm.data(dirty=dirty)
114 fm.data(dirty=dirty)
115 fm.end()
115 fm.end()
116
116
117 return out.getvalue()
117 return out.getvalue()
118
118
119 class tarit(object):
119 class tarit(object):
120 '''write archive to tar file or stream. can write uncompressed,
120 '''write archive to tar file or stream. can write uncompressed,
121 or compress with gzip or bzip2.'''
121 or compress with gzip or bzip2.'''
122
122
123 class GzipFileWithTime(gzip.GzipFile):
123 class GzipFileWithTime(gzip.GzipFile):
124
124
125 def __init__(self, *args, **kw):
125 def __init__(self, *args, **kw):
126 timestamp = None
126 timestamp = None
127 if 'timestamp' in kw:
127 if 'timestamp' in kw:
128 timestamp = kw.pop(r'timestamp')
128 timestamp = kw.pop(r'timestamp')
129 if timestamp is None:
129 if timestamp is None:
130 self.timestamp = time.time()
130 self.timestamp = time.time()
131 else:
131 else:
132 self.timestamp = timestamp
132 self.timestamp = timestamp
133 gzip.GzipFile.__init__(self, *args, **kw)
133 gzip.GzipFile.__init__(self, *args, **kw)
134
134
135 def _write_gzip_header(self):
135 def _write_gzip_header(self):
136 self.fileobj.write('\037\213') # magic header
136 self.fileobj.write('\037\213') # magic header
137 self.fileobj.write('\010') # compression method
137 self.fileobj.write('\010') # compression method
138 fname = self.name
138 fname = self.name
139 if fname and fname.endswith('.gz'):
139 if fname and fname.endswith('.gz'):
140 fname = fname[:-3]
140 fname = fname[:-3]
141 flags = 0
141 flags = 0
142 if fname:
142 if fname:
143 flags = gzip.FNAME
143 flags = gzip.FNAME
144 self.fileobj.write(chr(flags))
144 self.fileobj.write(chr(flags))
145 gzip.write32u(self.fileobj, long(self.timestamp))
145 gzip.write32u(self.fileobj, long(self.timestamp))
146 self.fileobj.write('\002')
146 self.fileobj.write('\002')
147 self.fileobj.write('\377')
147 self.fileobj.write('\377')
148 if fname:
148 if fname:
149 self.fileobj.write(fname + '\000')
149 self.fileobj.write(fname + '\000')
150
150
151 def __init__(self, dest, mtime, kind=''):
151 def __init__(self, dest, mtime, kind=''):
152 self.mtime = mtime
152 self.mtime = mtime
153 self.fileobj = None
153 self.fileobj = None
154
154
155 def taropen(mode, name='', fileobj=None):
155 def taropen(mode, name='', fileobj=None):
156 if kind == 'gz':
156 if kind == 'gz':
157 mode = mode[0]
157 mode = mode[0]
158 if not fileobj:
158 if not fileobj:
159 fileobj = open(name, mode + 'b')
159 fileobj = open(name, mode + 'b')
160 gzfileobj = self.GzipFileWithTime(name, mode + 'b',
160 gzfileobj = self.GzipFileWithTime(name, mode + 'b',
161 zlib.Z_BEST_COMPRESSION,
161 zlib.Z_BEST_COMPRESSION,
162 fileobj, timestamp=mtime)
162 fileobj, timestamp=mtime)
163 self.fileobj = gzfileobj
163 self.fileobj = gzfileobj
164 return tarfile.TarFile.taropen(name, mode, gzfileobj)
164 return tarfile.TarFile.taropen(name, mode, gzfileobj)
165 else:
165 else:
166 return tarfile.open(name, mode + kind, fileobj)
166 return tarfile.open(name, mode + kind, fileobj)
167
167
168 if isinstance(dest, str):
168 if isinstance(dest, str):
169 self.z = taropen('w:', name=dest)
169 self.z = taropen('w:', name=dest)
170 else:
170 else:
171 self.z = taropen('w|', fileobj=dest)
171 self.z = taropen('w|', fileobj=dest)
172
172
173 def addfile(self, name, mode, islink, data):
173 def addfile(self, name, mode, islink, data):
174 i = tarfile.TarInfo(name)
174 i = tarfile.TarInfo(name)
175 i.mtime = self.mtime
175 i.mtime = self.mtime
176 i.size = len(data)
176 i.size = len(data)
177 if islink:
177 if islink:
178 i.type = tarfile.SYMTYPE
178 i.type = tarfile.SYMTYPE
179 i.mode = 0o777
179 i.mode = 0o777
180 i.linkname = data
180 i.linkname = data
181 data = None
181 data = None
182 i.size = 0
182 i.size = 0
183 else:
183 else:
184 i.mode = mode
184 i.mode = mode
185 data = stringio(data)
185 data = stringio(data)
186 self.z.addfile(i, data)
186 self.z.addfile(i, data)
187
187
188 def done(self):
188 def done(self):
189 self.z.close()
189 self.z.close()
190 if self.fileobj:
190 if self.fileobj:
191 self.fileobj.close()
191 self.fileobj.close()
192
192
193 class tellable(object):
193 class tellable(object):
194 '''provide tell method for zipfile.ZipFile when writing to http
194 '''provide tell method for zipfile.ZipFile when writing to http
195 response file object.'''
195 response file object.'''
196
196
197 def __init__(self, fp):
197 def __init__(self, fp):
198 self.fp = fp
198 self.fp = fp
199 self.offset = 0
199 self.offset = 0
200
200
201 def __getattr__(self, key):
201 def __getattr__(self, key):
202 return getattr(self.fp, key)
202 return getattr(self.fp, key)
203
203
204 def write(self, s):
204 def write(self, s):
205 self.fp.write(s)
205 self.fp.write(s)
206 self.offset += len(s)
206 self.offset += len(s)
207
207
208 def tell(self):
208 def tell(self):
209 return self.offset
209 return self.offset
210
210
211 class zipit(object):
211 class zipit(object):
212 '''write archive to zip file or stream. can write uncompressed,
212 '''write archive to zip file or stream. can write uncompressed,
213 or compressed with deflate.'''
213 or compressed with deflate.'''
214
214
215 def __init__(self, dest, mtime, compress=True):
215 def __init__(self, dest, mtime, compress=True):
216 if not isinstance(dest, str):
216 if not isinstance(dest, str):
217 try:
217 try:
218 dest.tell()
218 dest.tell()
219 except (AttributeError, IOError):
219 except (AttributeError, IOError):
220 dest = tellable(dest)
220 dest = tellable(dest)
221 self.z = zipfile.ZipFile(dest, 'w',
221 self.z = zipfile.ZipFile(dest, r'w',
222 compress and zipfile.ZIP_DEFLATED or
222 compress and zipfile.ZIP_DEFLATED or
223 zipfile.ZIP_STORED)
223 zipfile.ZIP_STORED)
224
224
225 # Python's zipfile module emits deprecation warnings if we try
225 # Python's zipfile module emits deprecation warnings if we try
226 # to store files with a date before 1980.
226 # to store files with a date before 1980.
227 epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
227 epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
228 if mtime < epoch:
228 if mtime < epoch:
229 mtime = epoch
229 mtime = epoch
230
230
231 self.mtime = mtime
231 self.mtime = mtime
232 self.date_time = time.gmtime(mtime)[:6]
232 self.date_time = time.gmtime(mtime)[:6]
233
233
234 def addfile(self, name, mode, islink, data):
234 def addfile(self, name, mode, islink, data):
235 i = zipfile.ZipInfo(name, self.date_time)
235 i = zipfile.ZipInfo(name, self.date_time)
236 i.compress_type = self.z.compression
236 i.compress_type = self.z.compression
237 # unzip will not honor unix file modes unless file creator is
237 # unzip will not honor unix file modes unless file creator is
238 # set to unix (id 3).
238 # set to unix (id 3).
239 i.create_system = 3
239 i.create_system = 3
240 ftype = _UNX_IFREG
240 ftype = _UNX_IFREG
241 if islink:
241 if islink:
242 mode = 0o777
242 mode = 0o777
243 ftype = _UNX_IFLNK
243 ftype = _UNX_IFLNK
244 i.external_attr = (mode | ftype) << 16
244 i.external_attr = (mode | ftype) << 16
245 # add "extended-timestamp" extra block, because zip archives
245 # add "extended-timestamp" extra block, because zip archives
246 # without this will be extracted with unexpected timestamp,
246 # without this will be extracted with unexpected timestamp,
247 # if TZ is not configured as GMT
247 # if TZ is not configured as GMT
248 i.extra += struct.pack('<hhBl',
248 i.extra += struct.pack('<hhBl',
249 0x5455, # block type: "extended-timestamp"
249 0x5455, # block type: "extended-timestamp"
250 1 + 4, # size of this block
250 1 + 4, # size of this block
251 1, # "modification time is present"
251 1, # "modification time is present"
252 int(self.mtime)) # last modification (UTC)
252 int(self.mtime)) # last modification (UTC)
253 self.z.writestr(i, data)
253 self.z.writestr(i, data)
254
254
255 def done(self):
255 def done(self):
256 self.z.close()
256 self.z.close()
257
257
258 class fileit(object):
258 class fileit(object):
259 '''write archive as files in directory.'''
259 '''write archive as files in directory.'''
260
260
261 def __init__(self, name, mtime):
261 def __init__(self, name, mtime):
262 self.basedir = name
262 self.basedir = name
263 self.opener = vfsmod.vfs(self.basedir)
263 self.opener = vfsmod.vfs(self.basedir)
264 self.mtime = mtime
264 self.mtime = mtime
265
265
266 def addfile(self, name, mode, islink, data):
266 def addfile(self, name, mode, islink, data):
267 if islink:
267 if islink:
268 self.opener.symlink(data, name)
268 self.opener.symlink(data, name)
269 return
269 return
270 f = self.opener(name, "w", atomictemp=True)
270 f = self.opener(name, "w", atomictemp=True)
271 f.write(data)
271 f.write(data)
272 f.close()
272 f.close()
273 destfile = os.path.join(self.basedir, name)
273 destfile = os.path.join(self.basedir, name)
274 os.chmod(destfile, mode)
274 os.chmod(destfile, mode)
275 if self.mtime is not None:
275 if self.mtime is not None:
276 os.utime(destfile, (self.mtime, self.mtime))
276 os.utime(destfile, (self.mtime, self.mtime))
277
277
278 def done(self):
278 def done(self):
279 pass
279 pass
280
280
281 archivers = {
281 archivers = {
282 'files': fileit,
282 'files': fileit,
283 'tar': tarit,
283 'tar': tarit,
284 'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
284 'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
285 'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
285 'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
286 'uzip': lambda name, mtime: zipit(name, mtime, False),
286 'uzip': lambda name, mtime: zipit(name, mtime, False),
287 'zip': zipit,
287 'zip': zipit,
288 }
288 }
289
289
290 def archive(repo, dest, node, kind, decode=True, matchfn=None,
290 def archive(repo, dest, node, kind, decode=True, matchfn=None,
291 prefix='', mtime=None, subrepos=False):
291 prefix='', mtime=None, subrepos=False):
292 '''create archive of repo as it was at node.
292 '''create archive of repo as it was at node.
293
293
294 dest can be name of directory, name of archive file, or file
294 dest can be name of directory, name of archive file, or file
295 object to write archive to.
295 object to write archive to.
296
296
297 kind is type of archive to create.
297 kind is type of archive to create.
298
298
299 decode tells whether to put files through decode filters from
299 decode tells whether to put files through decode filters from
300 hgrc.
300 hgrc.
301
301
302 matchfn is function to filter names of files to write to archive.
302 matchfn is function to filter names of files to write to archive.
303
303
304 prefix is name of path to put before every archive member.
304 prefix is name of path to put before every archive member.
305
305
306 mtime is the modified time, in seconds, or None to use the changeset time.
306 mtime is the modified time, in seconds, or None to use the changeset time.
307
307
308 subrepos tells whether to include subrepos.
308 subrepos tells whether to include subrepos.
309 '''
309 '''
310
310
311 if kind == 'files':
311 if kind == 'files':
312 if prefix:
312 if prefix:
313 raise error.Abort(_('cannot give prefix when archiving to files'))
313 raise error.Abort(_('cannot give prefix when archiving to files'))
314 else:
314 else:
315 prefix = tidyprefix(dest, kind, prefix)
315 prefix = tidyprefix(dest, kind, prefix)
316
316
317 def write(name, mode, islink, getdata):
317 def write(name, mode, islink, getdata):
318 data = getdata()
318 data = getdata()
319 if decode:
319 if decode:
320 data = repo.wwritedata(name, data)
320 data = repo.wwritedata(name, data)
321 archiver.addfile(prefix + name, mode, islink, data)
321 archiver.addfile(prefix + name, mode, islink, data)
322
322
323 if kind not in archivers:
323 if kind not in archivers:
324 raise error.Abort(_("unknown archive type '%s'") % kind)
324 raise error.Abort(_("unknown archive type '%s'") % kind)
325
325
326 ctx = repo[node]
326 ctx = repo[node]
327 archiver = archivers[kind](dest, mtime or ctx.date()[0])
327 archiver = archivers[kind](dest, mtime or ctx.date()[0])
328
328
329 if repo.ui.configbool("ui", "archivemeta"):
329 if repo.ui.configbool("ui", "archivemeta"):
330 name = '.hg_archival.txt'
330 name = '.hg_archival.txt'
331 if not matchfn or matchfn(name):
331 if not matchfn or matchfn(name):
332 write(name, 0o644, False, lambda: buildmetadata(ctx))
332 write(name, 0o644, False, lambda: buildmetadata(ctx))
333
333
334 if matchfn:
334 if matchfn:
335 files = [f for f in ctx.manifest().keys() if matchfn(f)]
335 files = [f for f in ctx.manifest().keys() if matchfn(f)]
336 else:
336 else:
337 files = ctx.manifest().keys()
337 files = ctx.manifest().keys()
338 total = len(files)
338 total = len(files)
339 if total:
339 if total:
340 files.sort()
340 files.sort()
341 cmdutil._prefetchfiles(repo, ctx, files)
341 cmdutil._prefetchfiles(repo, ctx, files)
342 repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
342 repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
343 for i, f in enumerate(files):
343 for i, f in enumerate(files):
344 ff = ctx.flags(f)
344 ff = ctx.flags(f)
345 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data)
345 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data)
346 repo.ui.progress(_('archiving'), i + 1, item=f,
346 repo.ui.progress(_('archiving'), i + 1, item=f,
347 unit=_('files'), total=total)
347 unit=_('files'), total=total)
348 repo.ui.progress(_('archiving'), None)
348 repo.ui.progress(_('archiving'), None)
349
349
350 if subrepos:
350 if subrepos:
351 for subpath in sorted(ctx.substate):
351 for subpath in sorted(ctx.substate):
352 sub = ctx.workingsub(subpath)
352 sub = ctx.workingsub(subpath)
353 submatch = matchmod.subdirmatcher(subpath, matchfn)
353 submatch = matchmod.subdirmatcher(subpath, matchfn)
354 total += sub.archive(archiver, prefix, submatch, decode)
354 total += sub.archive(archiver, prefix, submatch, decode)
355
355
356 if total == 0:
356 if total == 0:
357 raise error.Abort(_('no files match the archive pattern'))
357 raise error.Abort(_('no files match the archive pattern'))
358
358
359 archiver.done()
359 archiver.done()
360 return total
360 return total
General Comments 0
You need to be logged in to leave comments. Login now