##// END OF EJS Templates
largefiles: fix uppercase in abort message
Martin Geisler -
r15380:a5388868 stable
parent child Browse files
Show More
@@ -1,473 +1,473 b''
1 1 # Copyright 2009-2010 Gregory P. Ward
2 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 3 # Copyright 2010-2011 Fog Creek Software
4 4 # Copyright 2010-2011 Unity Technologies
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 '''High-level command function for lfconvert, plus the cmdtable.'''
10 10
11 11 import os
12 12 import shutil
13 13
14 14 from mercurial import util, match as match_, hg, node, context, error
15 15 from mercurial.i18n import _
16 16
17 17 import lfutil
18 18 import basestore
19 19
20 20 # -- Commands ----------------------------------------------------------
21 21
22 22 def lfconvert(ui, src, dest, *pats, **opts):
23 23 '''convert a normal repository to a largefiles repository
24 24
25 25 Convert repository SOURCE to a new repository DEST, identical to
26 26 SOURCE except that certain files will be converted as largefiles:
27 27 specifically, any file that matches any PATTERN *or* whose size is
28 28 above the minimum size threshold is converted as a largefile. The
29 29 size used to determine whether or not to track a file as a
30 30 largefile is the size of the first version of the file. The
31 31 minimum size can be specified either with --size or in
32 32 configuration as ``largefiles.size``.
33 33
34 34 After running this command you will need to make sure that
35 35 largefiles is enabled anywhere you intend to push the new
36 36 repository.
37 37
38 38 Use --to-normal to convert largefiles back to normal files; after
39 39 this, the DEST repository can be used without largefiles at all.'''
40 40
41 41 if opts['to_normal']:
42 42 tolfile = False
43 43 else:
44 44 tolfile = True
45 45 size = lfutil.getminsize(ui, True, opts.get('size'), default=None)
46 46
47 47 if not hg.islocal(src):
48 48 raise util.Abort(_('%s is not a local Mercurial repo') % src)
49 49 if not hg.islocal(dest):
50 50 raise util.Abort(_('%s is not a local Mercurial repo') % dest)
51 51
52 52 rsrc = hg.repository(ui, src)
53 53 ui.status(_('initializing destination %s\n') % dest)
54 54 rdst = hg.repository(ui, dest, create=True)
55 55
56 56 success = False
57 57 try:
58 58 # Lock destination to prevent modification while it is converted to.
59 59 # Don't need to lock src because we are just reading from its history
60 60 # which can't change.
61 61 dst_lock = rdst.lock()
62 62
63 63 # Get a list of all changesets in the source. The easy way to do this
64 64 # is to simply walk the changelog, using changelog.nodesbewteen().
65 65 # Take a look at mercurial/revlog.py:639 for more details.
66 66 # Use a generator instead of a list to decrease memory usage
67 67 ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None,
68 68 rsrc.heads())[0])
69 69 revmap = {node.nullid: node.nullid}
70 70 if tolfile:
71 71 lfiles = set()
72 72 normalfiles = set()
73 73 if not pats:
74 74 pats = ui.config(lfutil.longname, 'patterns', default=())
75 75 if pats:
76 76 pats = pats.split(' ')
77 77 if pats:
78 78 matcher = match_.match(rsrc.root, '', list(pats))
79 79 else:
80 80 matcher = None
81 81
82 82 lfiletohash = {}
83 83 for ctx in ctxs:
84 84 ui.progress(_('converting revisions'), ctx.rev(),
85 85 unit=_('revision'), total=rsrc['tip'].rev())
86 86 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap,
87 87 lfiles, normalfiles, matcher, size, lfiletohash)
88 88 ui.progress(_('converting revisions'), None)
89 89
90 90 if os.path.exists(rdst.wjoin(lfutil.shortname)):
91 91 shutil.rmtree(rdst.wjoin(lfutil.shortname))
92 92
93 93 for f in lfiletohash.keys():
94 94 if os.path.isfile(rdst.wjoin(f)):
95 95 os.unlink(rdst.wjoin(f))
96 96 try:
97 97 os.removedirs(os.path.dirname(rdst.wjoin(f)))
98 98 except OSError:
99 99 pass
100 100
101 101 # If there were any files converted to largefiles, add largefiles
102 102 # to the destination repository's requirements.
103 103 if lfiles:
104 104 rdst.requirements.add('largefiles')
105 105 rdst._writerequirements()
106 106 else:
107 107 for ctx in ctxs:
108 108 ui.progress(_('converting revisions'), ctx.rev(),
109 109 unit=_('revision'), total=rsrc['tip'].rev())
110 110 _addchangeset(ui, rsrc, rdst, ctx, revmap)
111 111
112 112 ui.progress(_('converting revisions'), None)
113 113 success = True
114 114 finally:
115 115 if not success:
116 116 # we failed, remove the new directory
117 117 shutil.rmtree(rdst.root)
118 118 dst_lock.release()
119 119
120 120 def _addchangeset(ui, rsrc, rdst, ctx, revmap):
121 121 # Convert src parents to dst parents
122 122 parents = []
123 123 for p in ctx.parents():
124 124 parents.append(revmap[p.node()])
125 125 while len(parents) < 2:
126 126 parents.append(node.nullid)
127 127
128 128 # Generate list of changed files
129 129 files = set(ctx.files())
130 130 if node.nullid not in parents:
131 131 mc = ctx.manifest()
132 132 mp1 = ctx.parents()[0].manifest()
133 133 mp2 = ctx.parents()[1].manifest()
134 134 files |= (set(mp1) | set(mp2)) - set(mc)
135 135 for f in mc:
136 136 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
137 137 files.add(f)
138 138
139 139 def getfilectx(repo, memctx, f):
140 140 if lfutil.standin(f) in files:
141 141 # if the file isn't in the manifest then it was removed
142 142 # or renamed, raise IOError to indicate this
143 143 try:
144 144 fctx = ctx.filectx(lfutil.standin(f))
145 145 except error.LookupError:
146 146 raise IOError()
147 147 renamed = fctx.renamed()
148 148 if renamed:
149 149 renamed = lfutil.splitstandin(renamed[0])
150 150
151 151 hash = fctx.data().strip()
152 152 path = lfutil.findfile(rsrc, hash)
153 153 ### TODO: What if the file is not cached?
154 154 data = ''
155 155 fd = None
156 156 try:
157 157 fd = open(path, 'rb')
158 158 data = fd.read()
159 159 finally:
160 160 if fd:
161 161 fd.close()
162 162 return context.memfilectx(f, data, 'l' in fctx.flags(),
163 163 'x' in fctx.flags(), renamed)
164 164 else:
165 165 try:
166 166 fctx = ctx.filectx(f)
167 167 except error.LookupError:
168 168 raise IOError()
169 169 renamed = fctx.renamed()
170 170 if renamed:
171 171 renamed = renamed[0]
172 172 data = fctx.data()
173 173 if f == '.hgtags':
174 174 newdata = []
175 175 for line in data.splitlines():
176 176 id, name = line.split(' ', 1)
177 177 newdata.append('%s %s\n' % (node.hex(revmap[node.bin(id)]),
178 178 name))
179 179 data = ''.join(newdata)
180 180 return context.memfilectx(f, data, 'l' in fctx.flags(),
181 181 'x' in fctx.flags(), renamed)
182 182
183 183 dstfiles = []
184 184 for file in files:
185 185 if lfutil.isstandin(file):
186 186 dstfiles.append(lfutil.splitstandin(file))
187 187 else:
188 188 dstfiles.append(file)
189 189 # Commit
190 190 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
191 191 getfilectx, ctx.user(), ctx.date(), ctx.extra())
192 192 ret = rdst.commitctx(mctx)
193 193 rdst.dirstate.setparents(ret)
194 194 revmap[ctx.node()] = rdst.changelog.tip()
195 195
196 196 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles,
197 197 matcher, size, lfiletohash):
198 198 # Convert src parents to dst parents
199 199 parents = []
200 200 for p in ctx.parents():
201 201 parents.append(revmap[p.node()])
202 202 while len(parents) < 2:
203 203 parents.append(node.nullid)
204 204
205 205 # Generate list of changed files
206 206 files = set(ctx.files())
207 207 if node.nullid not in parents:
208 208 mc = ctx.manifest()
209 209 mp1 = ctx.parents()[0].manifest()
210 210 mp2 = ctx.parents()[1].manifest()
211 211 files |= (set(mp1) | set(mp2)) - set(mc)
212 212 for f in mc:
213 213 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
214 214 files.add(f)
215 215
216 216 dstfiles = []
217 217 for f in files:
218 218 if f not in lfiles and f not in normalfiles:
219 219 islfile = _islfile(f, ctx, matcher, size)
220 220 # If this file was renamed or copied then copy
221 221 # the lfileness of its predecessor
222 222 if f in ctx.manifest():
223 223 fctx = ctx.filectx(f)
224 224 renamed = fctx.renamed()
225 225 renamedlfile = renamed and renamed[0] in lfiles
226 226 islfile |= renamedlfile
227 227 if 'l' in fctx.flags():
228 228 if renamedlfile:
229 229 raise util.Abort(
230 _('Renamed/copied largefile %s becomes symlink')
230 _('renamed/copied largefile %s becomes symlink')
231 231 % f)
232 232 islfile = False
233 233 if islfile:
234 234 lfiles.add(f)
235 235 else:
236 236 normalfiles.add(f)
237 237
238 238 if f in lfiles:
239 239 dstfiles.append(lfutil.standin(f))
240 240 # largefile in manifest if it has not been removed/renamed
241 241 if f in ctx.manifest():
242 242 if 'l' in ctx.filectx(f).flags():
243 243 if renamed and renamed[0] in lfiles:
244 244 raise util.Abort(_('largefile %s becomes symlink') % f)
245 245
246 246 # largefile was modified, update standins
247 247 fullpath = rdst.wjoin(f)
248 248 util.makedirs(os.path.dirname(fullpath))
249 249 m = util.sha1('')
250 250 m.update(ctx[f].data())
251 251 hash = m.hexdigest()
252 252 if f not in lfiletohash or lfiletohash[f] != hash:
253 253 try:
254 254 fd = open(fullpath, 'wb')
255 255 fd.write(ctx[f].data())
256 256 finally:
257 257 if fd:
258 258 fd.close()
259 259 executable = 'x' in ctx[f].flags()
260 260 os.chmod(fullpath, lfutil.getmode(executable))
261 261 lfutil.writestandin(rdst, lfutil.standin(f), hash,
262 262 executable)
263 263 lfiletohash[f] = hash
264 264 else:
265 265 # normal file
266 266 dstfiles.append(f)
267 267
268 268 def getfilectx(repo, memctx, f):
269 269 if lfutil.isstandin(f):
270 270 # if the file isn't in the manifest then it was removed
271 271 # or renamed, raise IOError to indicate this
272 272 srcfname = lfutil.splitstandin(f)
273 273 try:
274 274 fctx = ctx.filectx(srcfname)
275 275 except error.LookupError:
276 276 raise IOError()
277 277 renamed = fctx.renamed()
278 278 if renamed:
279 279 # standin is always a largefile because largefile-ness
280 280 # doesn't change after rename or copy
281 281 renamed = lfutil.standin(renamed[0])
282 282
283 283 return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in
284 284 fctx.flags(), 'x' in fctx.flags(), renamed)
285 285 else:
286 286 try:
287 287 fctx = ctx.filectx(f)
288 288 except error.LookupError:
289 289 raise IOError()
290 290 renamed = fctx.renamed()
291 291 if renamed:
292 292 renamed = renamed[0]
293 293
294 294 data = fctx.data()
295 295 if f == '.hgtags':
296 296 newdata = []
297 297 for line in data.splitlines():
298 298 id, name = line.split(' ', 1)
299 299 newdata.append('%s %s\n' % (node.hex(revmap[node.bin(id)]),
300 300 name))
301 301 data = ''.join(newdata)
302 302 return context.memfilectx(f, data, 'l' in fctx.flags(),
303 303 'x' in fctx.flags(), renamed)
304 304
305 305 # Commit
306 306 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
307 307 getfilectx, ctx.user(), ctx.date(), ctx.extra())
308 308 ret = rdst.commitctx(mctx)
309 309 rdst.dirstate.setparents(ret)
310 310 revmap[ctx.node()] = rdst.changelog.tip()
311 311
312 312 def _islfile(file, ctx, matcher, size):
313 313 '''Return true if file should be considered a largefile, i.e.
314 314 matcher matches it or it is larger than size.'''
315 315 # never store special .hg* files as largefiles
316 316 if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs':
317 317 return False
318 318 if matcher and matcher(file):
319 319 return True
320 320 try:
321 321 return ctx.filectx(file).size() >= size * 1024 * 1024
322 322 except error.LookupError:
323 323 return False
324 324
325 325 def uploadlfiles(ui, rsrc, rdst, files):
326 326 '''upload largefiles to the central store'''
327 327
328 328 if not files:
329 329 return
330 330
331 331 store = basestore._openstore(rsrc, rdst, put=True)
332 332
333 333 at = 0
334 334 files = filter(lambda h: not store.exists(h), files)
335 335 for hash in files:
336 336 ui.progress(_('uploading largefiles'), at, unit='largefile',
337 337 total=len(files))
338 338 source = lfutil.findfile(rsrc, hash)
339 339 if not source:
340 340 raise util.Abort(_('largefile %s missing from store'
341 341 ' (needs to be uploaded)') % hash)
342 342 # XXX check for errors here
343 343 store.put(source, hash)
344 344 at += 1
345 345 ui.progress(_('uploading largefiles'), None)
346 346
347 347 def verifylfiles(ui, repo, all=False, contents=False):
348 348 '''Verify that every big file revision in the current changeset
349 349 exists in the central store. With --contents, also verify that
350 350 the contents of each big file revision are correct (SHA-1 hash
351 351 matches the revision ID). With --all, check every changeset in
352 352 this repository.'''
353 353 if all:
354 354 # Pass a list to the function rather than an iterator because we know a
355 355 # list will work.
356 356 revs = range(len(repo))
357 357 else:
358 358 revs = ['.']
359 359
360 360 store = basestore._openstore(repo)
361 361 return store.verify(revs, contents=contents)
362 362
363 363 def cachelfiles(ui, repo, node):
364 364 '''cachelfiles ensures that all largefiles needed by the specified revision
365 365 are present in the repository's largefile cache.
366 366
367 367 returns a tuple (cached, missing). cached is the list of files downloaded
368 368 by this operation; missing is the list of files that were needed but could
369 369 not be found.'''
370 370 lfiles = lfutil.listlfiles(repo, node)
371 371 toget = []
372 372
373 373 for lfile in lfiles:
374 374 expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
375 375 # if it exists and its hash matches, it might have been locally
376 376 # modified before updating and the user chose 'local'. in this case,
377 377 # it will not be in any store, so don't look for it.
378 378 if ((not os.path.exists(repo.wjoin(lfile)) or
379 379 expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and
380 380 not lfutil.findfile(repo, expectedhash)):
381 381 toget.append((lfile, expectedhash))
382 382
383 383 if toget:
384 384 store = basestore._openstore(repo)
385 385 ret = store.get(toget)
386 386 return ret
387 387
388 388 return ([], [])
389 389
390 390 def updatelfiles(ui, repo, filelist=None, printmessage=True):
391 391 wlock = repo.wlock()
392 392 try:
393 393 lfdirstate = lfutil.openlfdirstate(ui, repo)
394 394 lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate)
395 395
396 396 if filelist is not None:
397 397 lfiles = [f for f in lfiles if f in filelist]
398 398
399 399 printed = False
400 400 if printmessage and lfiles:
401 401 ui.status(_('getting changed largefiles\n'))
402 402 printed = True
403 403 cachelfiles(ui, repo, '.')
404 404
405 405 updated, removed = 0, 0
406 406 for i in map(lambda f: _updatelfile(repo, lfdirstate, f), lfiles):
407 407 # increment the appropriate counter according to _updatelfile's
408 408 # return value
409 409 updated += i > 0 and i or 0
410 410 removed -= i < 0 and i or 0
411 411 if printmessage and (removed or updated) and not printed:
412 412 ui.status(_('getting changed largefiles\n'))
413 413 printed = True
414 414
415 415 lfdirstate.write()
416 416 if printed and printmessage:
417 417 ui.status(_('%d largefiles updated, %d removed\n') % (updated,
418 418 removed))
419 419 finally:
420 420 wlock.release()
421 421
422 422 def _updatelfile(repo, lfdirstate, lfile):
423 423 '''updates a single largefile and copies the state of its standin from
424 424 the repository's dirstate to its state in the lfdirstate.
425 425
426 426 returns 1 if the file was modified, -1 if the file was removed, 0 if the
427 427 file was unchanged, and None if the needed largefile was missing from the
428 428 cache.'''
429 429 ret = 0
430 430 abslfile = repo.wjoin(lfile)
431 431 absstandin = repo.wjoin(lfutil.standin(lfile))
432 432 if os.path.exists(absstandin):
433 433 if os.path.exists(absstandin+'.orig'):
434 434 shutil.copyfile(abslfile, abslfile+'.orig')
435 435 expecthash = lfutil.readstandin(repo, lfile)
436 436 if (expecthash != '' and
437 437 (not os.path.exists(abslfile) or
438 438 expecthash != lfutil.hashfile(abslfile))):
439 439 if not lfutil.copyfromcache(repo, expecthash, lfile):
440 440 return None # don't try to set the mode or update the dirstate
441 441 ret = 1
442 442 mode = os.stat(absstandin).st_mode
443 443 if mode != os.stat(abslfile).st_mode:
444 444 os.chmod(abslfile, mode)
445 445 ret = 1
446 446 else:
447 447 if os.path.exists(abslfile):
448 448 os.unlink(abslfile)
449 449 ret = -1
450 450 state = repo.dirstate[lfutil.standin(lfile)]
451 451 if state == 'n':
452 452 lfdirstate.normal(lfile)
453 453 elif state == 'r':
454 454 lfdirstate.remove(lfile)
455 455 elif state == 'a':
456 456 lfdirstate.add(lfile)
457 457 elif state == '?':
458 458 lfdirstate.drop(lfile)
459 459 return ret
460 460
461 461 # -- hg commands declarations ------------------------------------------------
462 462
463 463 cmdtable = {
464 464 'lfconvert': (lfconvert,
465 465 [('s', 'size', '',
466 466 _('minimum size (MB) for files to be converted '
467 467 'as largefiles'),
468 468 'SIZE'),
469 469 ('', 'to-normal', False,
470 470 _('convert from a largefiles repo to a normal repo')),
471 471 ],
472 472 _('hg lfconvert SOURCE DEST [FILE ...]')),
473 473 }
General Comments 0
You need to be logged in to leave comments. Login now