##// END OF EJS Templates
largefiles: convert EOL of hgrc before appending to bytes IO...
Yuya Nishihara -
r35641:f56f8abb default
parent child Browse files
Show More
@@ -1,1492 +1,1492
1 1 # Copyright 2009-2010 Gregory P. Ward
2 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 3 # Copyright 2010-2011 Fog Creek Software
4 4 # Copyright 2010-2011 Unity Technologies
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 '''Overridden Mercurial commands and functions for the largefiles extension'''
10 10 from __future__ import absolute_import
11 11
12 12 import copy
13 13 import os
14 14
15 15 from mercurial.i18n import _
16 16
17 17 from mercurial import (
18 18 archival,
19 19 cmdutil,
20 20 error,
21 21 hg,
22 22 match as matchmod,
23 23 pathutil,
24 24 pycompat,
25 25 registrar,
26 26 scmutil,
27 27 smartset,
28 28 util,
29 29 )
30 30
31 31 from . import (
32 32 lfcommands,
33 33 lfutil,
34 34 storefactory,
35 35 )
36 36
37 37 # -- Utility functions: commonly/repeatedly needed functionality ---------------
38 38
39 39 def composelargefilematcher(match, manifest):
40 40 '''create a matcher that matches only the largefiles in the original
41 41 matcher'''
42 42 m = copy.copy(match)
43 43 lfile = lambda f: lfutil.standin(f) in manifest
44 44 m._files = filter(lfile, m._files)
45 45 m._fileset = set(m._files)
46 46 m.always = lambda: False
47 47 origmatchfn = m.matchfn
48 48 m.matchfn = lambda f: lfile(f) and origmatchfn(f)
49 49 return m
50 50
51 51 def composenormalfilematcher(match, manifest, exclude=None):
52 52 excluded = set()
53 53 if exclude is not None:
54 54 excluded.update(exclude)
55 55
56 56 m = copy.copy(match)
57 57 notlfile = lambda f: not (lfutil.isstandin(f) or lfutil.standin(f) in
58 58 manifest or f in excluded)
59 59 m._files = filter(notlfile, m._files)
60 60 m._fileset = set(m._files)
61 61 m.always = lambda: False
62 62 origmatchfn = m.matchfn
63 63 m.matchfn = lambda f: notlfile(f) and origmatchfn(f)
64 64 return m
65 65
66 66 def installnormalfilesmatchfn(manifest):
67 67 '''installmatchfn with a matchfn that ignores all largefiles'''
68 68 def overridematch(ctx, pats=(), opts=None, globbed=False,
69 69 default='relpath', badfn=None):
70 70 if opts is None:
71 71 opts = {}
72 72 match = oldmatch(ctx, pats, opts, globbed, default, badfn=badfn)
73 73 return composenormalfilematcher(match, manifest)
74 74 oldmatch = installmatchfn(overridematch)
75 75
76 76 def installmatchfn(f):
77 77 '''monkey patch the scmutil module with a custom match function.
78 78 Warning: it is monkey patching the _module_ on runtime! Not thread safe!'''
79 79 oldmatch = scmutil.match
80 80 setattr(f, 'oldmatch', oldmatch)
81 81 scmutil.match = f
82 82 return oldmatch
83 83
84 84 def restorematchfn():
85 85 '''restores scmutil.match to what it was before installmatchfn
86 86 was called. no-op if scmutil.match is its original function.
87 87
88 88 Note that n calls to installmatchfn will require n calls to
89 89 restore the original matchfn.'''
90 90 scmutil.match = getattr(scmutil.match, 'oldmatch')
91 91
92 92 def installmatchandpatsfn(f):
93 93 oldmatchandpats = scmutil.matchandpats
94 94 setattr(f, 'oldmatchandpats', oldmatchandpats)
95 95 scmutil.matchandpats = f
96 96 return oldmatchandpats
97 97
98 98 def restorematchandpatsfn():
99 99 '''restores scmutil.matchandpats to what it was before
100 100 installmatchandpatsfn was called. No-op if scmutil.matchandpats
101 101 is its original function.
102 102
103 103 Note that n calls to installmatchandpatsfn will require n calls
104 104 to restore the original matchfn.'''
105 105 scmutil.matchandpats = getattr(scmutil.matchandpats, 'oldmatchandpats',
106 106 scmutil.matchandpats)
107 107
108 108 def addlargefiles(ui, repo, isaddremove, matcher, **opts):
109 109 large = opts.get(r'large')
110 110 lfsize = lfutil.getminsize(
111 111 ui, lfutil.islfilesrepo(repo), opts.get(r'lfsize'))
112 112
113 113 lfmatcher = None
114 114 if lfutil.islfilesrepo(repo):
115 115 lfpats = ui.configlist(lfutil.longname, 'patterns')
116 116 if lfpats:
117 117 lfmatcher = matchmod.match(repo.root, '', list(lfpats))
118 118
119 119 lfnames = []
120 120 m = matcher
121 121
122 122 wctx = repo[None]
123 123 for f in wctx.walk(matchmod.badmatch(m, lambda x, y: None)):
124 124 exact = m.exact(f)
125 125 lfile = lfutil.standin(f) in wctx
126 126 nfile = f in wctx
127 127 exists = lfile or nfile
128 128
129 129 # addremove in core gets fancy with the name, add doesn't
130 130 if isaddremove:
131 131 name = m.uipath(f)
132 132 else:
133 133 name = m.rel(f)
134 134
135 135 # Don't warn the user when they attempt to add a normal tracked file.
136 136 # The normal add code will do that for us.
137 137 if exact and exists:
138 138 if lfile:
139 139 ui.warn(_('%s already a largefile\n') % name)
140 140 continue
141 141
142 142 if (exact or not exists) and not lfutil.isstandin(f):
143 143 # In case the file was removed previously, but not committed
144 144 # (issue3507)
145 145 if not repo.wvfs.exists(f):
146 146 continue
147 147
148 148 abovemin = (lfsize and
149 149 repo.wvfs.lstat(f).st_size >= lfsize * 1024 * 1024)
150 150 if large or abovemin or (lfmatcher and lfmatcher(f)):
151 151 lfnames.append(f)
152 152 if ui.verbose or not exact:
153 153 ui.status(_('adding %s as a largefile\n') % name)
154 154
155 155 bad = []
156 156
157 157 # Need to lock, otherwise there could be a race condition between
158 158 # when standins are created and added to the repo.
159 159 with repo.wlock():
160 160 if not opts.get(r'dry_run'):
161 161 standins = []
162 162 lfdirstate = lfutil.openlfdirstate(ui, repo)
163 163 for f in lfnames:
164 164 standinname = lfutil.standin(f)
165 165 lfutil.writestandin(repo, standinname, hash='',
166 166 executable=lfutil.getexecutable(repo.wjoin(f)))
167 167 standins.append(standinname)
168 168 if lfdirstate[f] == 'r':
169 169 lfdirstate.normallookup(f)
170 170 else:
171 171 lfdirstate.add(f)
172 172 lfdirstate.write()
173 173 bad += [lfutil.splitstandin(f)
174 174 for f in repo[None].add(standins)
175 175 if f in m.files()]
176 176
177 177 added = [f for f in lfnames if f not in bad]
178 178 return added, bad
179 179
180 180 def removelargefiles(ui, repo, isaddremove, matcher, **opts):
181 181 after = opts.get(r'after')
182 182 m = composelargefilematcher(matcher, repo[None].manifest())
183 183 try:
184 184 repo.lfstatus = True
185 185 s = repo.status(match=m, clean=not isaddremove)
186 186 finally:
187 187 repo.lfstatus = False
188 188 manifest = repo[None].manifest()
189 189 modified, added, deleted, clean = [[f for f in list
190 190 if lfutil.standin(f) in manifest]
191 191 for list in (s.modified, s.added,
192 192 s.deleted, s.clean)]
193 193
194 194 def warn(files, msg):
195 195 for f in files:
196 196 ui.warn(msg % m.rel(f))
197 197 return int(len(files) > 0)
198 198
199 199 result = 0
200 200
201 201 if after:
202 202 remove = deleted
203 203 result = warn(modified + added + clean,
204 204 _('not removing %s: file still exists\n'))
205 205 else:
206 206 remove = deleted + clean
207 207 result = warn(modified, _('not removing %s: file is modified (use -f'
208 208 ' to force removal)\n'))
209 209 result = warn(added, _('not removing %s: file has been marked for add'
210 210 ' (use forget to undo)\n')) or result
211 211
212 212 # Need to lock because standin files are deleted then removed from the
213 213 # repository and we could race in-between.
214 214 with repo.wlock():
215 215 lfdirstate = lfutil.openlfdirstate(ui, repo)
216 216 for f in sorted(remove):
217 217 if ui.verbose or not m.exact(f):
218 218 # addremove in core gets fancy with the name, remove doesn't
219 219 if isaddremove:
220 220 name = m.uipath(f)
221 221 else:
222 222 name = m.rel(f)
223 223 ui.status(_('removing %s\n') % name)
224 224
225 225 if not opts.get(r'dry_run'):
226 226 if not after:
227 227 repo.wvfs.unlinkpath(f, ignoremissing=True)
228 228
229 229 if opts.get(r'dry_run'):
230 230 return result
231 231
232 232 remove = [lfutil.standin(f) for f in remove]
233 233 # If this is being called by addremove, let the original addremove
234 234 # function handle this.
235 235 if not isaddremove:
236 236 for f in remove:
237 237 repo.wvfs.unlinkpath(f, ignoremissing=True)
238 238 repo[None].forget(remove)
239 239
240 240 for f in remove:
241 241 lfutil.synclfdirstate(repo, lfdirstate, lfutil.splitstandin(f),
242 242 False)
243 243
244 244 lfdirstate.write()
245 245
246 246 return result
247 247
248 248 # For overriding mercurial.hgweb.webcommands so that largefiles will
249 249 # appear at their right place in the manifests.
250 250 def decodepath(orig, path):
251 251 return lfutil.splitstandin(path) or path
252 252
253 253 # -- Wrappers: modify existing commands --------------------------------
254 254
255 255 def overrideadd(orig, ui, repo, *pats, **opts):
256 256 if opts.get(r'normal') and opts.get(r'large'):
257 257 raise error.Abort(_('--normal cannot be used with --large'))
258 258 return orig(ui, repo, *pats, **opts)
259 259
260 260 def cmdutiladd(orig, ui, repo, matcher, prefix, explicitonly, **opts):
261 261 # The --normal flag short circuits this override
262 262 if opts.get(r'normal'):
263 263 return orig(ui, repo, matcher, prefix, explicitonly, **opts)
264 264
265 265 ladded, lbad = addlargefiles(ui, repo, False, matcher, **opts)
266 266 normalmatcher = composenormalfilematcher(matcher, repo[None].manifest(),
267 267 ladded)
268 268 bad = orig(ui, repo, normalmatcher, prefix, explicitonly, **opts)
269 269
270 270 bad.extend(f for f in lbad)
271 271 return bad
272 272
273 273 def cmdutilremove(orig, ui, repo, matcher, prefix, after, force, subrepos):
274 274 normalmatcher = composenormalfilematcher(matcher, repo[None].manifest())
275 275 result = orig(ui, repo, normalmatcher, prefix, after, force, subrepos)
276 276 return removelargefiles(ui, repo, False, matcher, after=after,
277 277 force=force) or result
278 278
279 279 def overridestatusfn(orig, repo, rev2, **opts):
280 280 try:
281 281 repo._repo.lfstatus = True
282 282 return orig(repo, rev2, **opts)
283 283 finally:
284 284 repo._repo.lfstatus = False
285 285
286 286 def overridestatus(orig, ui, repo, *pats, **opts):
287 287 try:
288 288 repo.lfstatus = True
289 289 return orig(ui, repo, *pats, **opts)
290 290 finally:
291 291 repo.lfstatus = False
292 292
293 293 def overridedirty(orig, repo, ignoreupdate=False, missing=False):
294 294 try:
295 295 repo._repo.lfstatus = True
296 296 return orig(repo, ignoreupdate=ignoreupdate, missing=missing)
297 297 finally:
298 298 repo._repo.lfstatus = False
299 299
300 300 def overridelog(orig, ui, repo, *pats, **opts):
301 301 def overridematchandpats(ctx, pats=(), opts=None, globbed=False,
302 302 default='relpath', badfn=None):
303 303 """Matcher that merges root directory with .hglf, suitable for log.
304 304 It is still possible to match .hglf directly.
305 305 For any listed files run log on the standin too.
306 306 matchfn tries both the given filename and with .hglf stripped.
307 307 """
308 308 if opts is None:
309 309 opts = {}
310 310 matchandpats = oldmatchandpats(ctx, pats, opts, globbed, default,
311 311 badfn=badfn)
312 312 m, p = copy.copy(matchandpats)
313 313
314 314 if m.always():
315 315 # We want to match everything anyway, so there's no benefit trying
316 316 # to add standins.
317 317 return matchandpats
318 318
319 319 pats = set(p)
320 320
321 321 def fixpats(pat, tostandin=lfutil.standin):
322 322 if pat.startswith('set:'):
323 323 return pat
324 324
325 325 kindpat = matchmod._patsplit(pat, None)
326 326
327 327 if kindpat[0] is not None:
328 328 return kindpat[0] + ':' + tostandin(kindpat[1])
329 329 return tostandin(kindpat[1])
330 330
331 331 if m._cwd:
332 332 hglf = lfutil.shortname
333 333 back = util.pconvert(m.rel(hglf)[:-len(hglf)])
334 334
335 335 def tostandin(f):
336 336 # The file may already be a standin, so truncate the back
337 337 # prefix and test before mangling it. This avoids turning
338 338 # 'glob:../.hglf/foo*' into 'glob:../.hglf/../.hglf/foo*'.
339 339 if f.startswith(back) and lfutil.splitstandin(f[len(back):]):
340 340 return f
341 341
342 342 # An absolute path is from outside the repo, so truncate the
343 343 # path to the root before building the standin. Otherwise cwd
344 344 # is somewhere in the repo, relative to root, and needs to be
345 345 # prepended before building the standin.
346 346 if os.path.isabs(m._cwd):
347 347 f = f[len(back):]
348 348 else:
349 349 f = m._cwd + '/' + f
350 350 return back + lfutil.standin(f)
351 351 else:
352 352 def tostandin(f):
353 353 if lfutil.isstandin(f):
354 354 return f
355 355 return lfutil.standin(f)
356 356 pats.update(fixpats(f, tostandin) for f in p)
357 357
358 358 for i in range(0, len(m._files)):
359 359 # Don't add '.hglf' to m.files, since that is already covered by '.'
360 360 if m._files[i] == '.':
361 361 continue
362 362 standin = lfutil.standin(m._files[i])
363 363 # If the "standin" is a directory, append instead of replace to
364 364 # support naming a directory on the command line with only
365 365 # largefiles. The original directory is kept to support normal
366 366 # files.
367 367 if standin in ctx:
368 368 m._files[i] = standin
369 369 elif m._files[i] not in ctx and repo.wvfs.isdir(standin):
370 370 m._files.append(standin)
371 371
372 372 m._fileset = set(m._files)
373 373 m.always = lambda: False
374 374 origmatchfn = m.matchfn
375 375 def lfmatchfn(f):
376 376 lf = lfutil.splitstandin(f)
377 377 if lf is not None and origmatchfn(lf):
378 378 return True
379 379 r = origmatchfn(f)
380 380 return r
381 381 m.matchfn = lfmatchfn
382 382
383 383 ui.debug('updated patterns: %s\n' % ', '.join(sorted(pats)))
384 384 return m, pats
385 385
386 386 # For hg log --patch, the match object is used in two different senses:
387 387 # (1) to determine what revisions should be printed out, and
388 388 # (2) to determine what files to print out diffs for.
389 389 # The magic matchandpats override should be used for case (1) but not for
390 390 # case (2).
391 391 def overridemakelogfilematcher(repo, pats, opts, badfn=None):
392 392 wctx = repo[None]
393 393 match, pats = oldmatchandpats(wctx, pats, opts, badfn=badfn)
394 394 return lambda rev: match
395 395
396 396 oldmatchandpats = installmatchandpatsfn(overridematchandpats)
397 397 oldmakelogfilematcher = cmdutil._makenofollowlogfilematcher
398 398 setattr(cmdutil, '_makenofollowlogfilematcher', overridemakelogfilematcher)
399 399
400 400 try:
401 401 return orig(ui, repo, *pats, **opts)
402 402 finally:
403 403 restorematchandpatsfn()
404 404 setattr(cmdutil, '_makenofollowlogfilematcher', oldmakelogfilematcher)
405 405
406 406 def overrideverify(orig, ui, repo, *pats, **opts):
407 407 large = opts.pop(r'large', False)
408 408 all = opts.pop(r'lfa', False)
409 409 contents = opts.pop(r'lfc', False)
410 410
411 411 result = orig(ui, repo, *pats, **opts)
412 412 if large or all or contents:
413 413 result = result or lfcommands.verifylfiles(ui, repo, all, contents)
414 414 return result
415 415
416 416 def overridedebugstate(orig, ui, repo, *pats, **opts):
417 417 large = opts.pop(r'large', False)
418 418 if large:
419 419 class fakerepo(object):
420 420 dirstate = lfutil.openlfdirstate(ui, repo)
421 421 orig(ui, fakerepo, *pats, **opts)
422 422 else:
423 423 orig(ui, repo, *pats, **opts)
424 424
425 425 # Before starting the manifest merge, merge.updates will call
426 426 # _checkunknownfile to check if there are any files in the merged-in
427 427 # changeset that collide with unknown files in the working copy.
428 428 #
429 429 # The largefiles are seen as unknown, so this prevents us from merging
430 430 # in a file 'foo' if we already have a largefile with the same name.
431 431 #
432 432 # The overridden function filters the unknown files by removing any
433 433 # largefiles. This makes the merge proceed and we can then handle this
434 434 # case further in the overridden calculateupdates function below.
435 435 def overridecheckunknownfile(origfn, repo, wctx, mctx, f, f2=None):
436 436 if lfutil.standin(repo.dirstate.normalize(f)) in wctx:
437 437 return False
438 438 return origfn(repo, wctx, mctx, f, f2)
439 439
440 440 # The manifest merge handles conflicts on the manifest level. We want
441 441 # to handle changes in largefile-ness of files at this level too.
442 442 #
443 443 # The strategy is to run the original calculateupdates and then process
444 444 # the action list it outputs. There are two cases we need to deal with:
445 445 #
446 446 # 1. Normal file in p1, largefile in p2. Here the largefile is
447 447 # detected via its standin file, which will enter the working copy
448 448 # with a "get" action. It is not "merge" since the standin is all
449 449 # Mercurial is concerned with at this level -- the link to the
450 450 # existing normal file is not relevant here.
451 451 #
452 452 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
453 453 # since the largefile will be present in the working copy and
454 454 # different from the normal file in p2. Mercurial therefore
455 455 # triggers a merge action.
456 456 #
457 457 # In both cases, we prompt the user and emit new actions to either
458 458 # remove the standin (if the normal file was kept) or to remove the
459 459 # normal file and get the standin (if the largefile was kept). The
460 460 # default prompt answer is to use the largefile version since it was
461 461 # presumably changed on purpose.
462 462 #
463 463 # Finally, the merge.applyupdates function will then take care of
464 464 # writing the files into the working copy and lfcommands.updatelfiles
465 465 # will update the largefiles.
466 466 def overridecalculateupdates(origfn, repo, p1, p2, pas, branchmerge, force,
467 467 acceptremote, *args, **kwargs):
468 468 overwrite = force and not branchmerge
469 469 actions, diverge, renamedelete = origfn(
470 470 repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs)
471 471
472 472 if overwrite:
473 473 return actions, diverge, renamedelete
474 474
475 475 # Convert to dictionary with filename as key and action as value.
476 476 lfiles = set()
477 477 for f in actions:
478 478 splitstandin = lfutil.splitstandin(f)
479 479 if splitstandin in p1:
480 480 lfiles.add(splitstandin)
481 481 elif lfutil.standin(f) in p1:
482 482 lfiles.add(f)
483 483
484 484 for lfile in sorted(lfiles):
485 485 standin = lfutil.standin(lfile)
486 486 (lm, largs, lmsg) = actions.get(lfile, (None, None, None))
487 487 (sm, sargs, smsg) = actions.get(standin, (None, None, None))
488 488 if sm in ('g', 'dc') and lm != 'r':
489 489 if sm == 'dc':
490 490 f1, f2, fa, move, anc = sargs
491 491 sargs = (p2[f2].flags(), False)
492 492 # Case 1: normal file in the working copy, largefile in
493 493 # the second parent
494 494 usermsg = _('remote turned local normal file %s into a largefile\n'
495 495 'use (l)argefile or keep (n)ormal file?'
496 496 '$$ &Largefile $$ &Normal file') % lfile
497 497 if repo.ui.promptchoice(usermsg, 0) == 0: # pick remote largefile
498 498 actions[lfile] = ('r', None, 'replaced by standin')
499 499 actions[standin] = ('g', sargs, 'replaces standin')
500 500 else: # keep local normal file
501 501 actions[lfile] = ('k', None, 'replaces standin')
502 502 if branchmerge:
503 503 actions[standin] = ('k', None, 'replaced by non-standin')
504 504 else:
505 505 actions[standin] = ('r', None, 'replaced by non-standin')
506 506 elif lm in ('g', 'dc') and sm != 'r':
507 507 if lm == 'dc':
508 508 f1, f2, fa, move, anc = largs
509 509 largs = (p2[f2].flags(), False)
510 510 # Case 2: largefile in the working copy, normal file in
511 511 # the second parent
512 512 usermsg = _('remote turned local largefile %s into a normal file\n'
513 513 'keep (l)argefile or use (n)ormal file?'
514 514 '$$ &Largefile $$ &Normal file') % lfile
515 515 if repo.ui.promptchoice(usermsg, 0) == 0: # keep local largefile
516 516 if branchmerge:
517 517 # largefile can be restored from standin safely
518 518 actions[lfile] = ('k', None, 'replaced by standin')
519 519 actions[standin] = ('k', None, 'replaces standin')
520 520 else:
521 521 # "lfile" should be marked as "removed" without
522 522 # removal of itself
523 523 actions[lfile] = ('lfmr', None,
524 524 'forget non-standin largefile')
525 525
526 526 # linear-merge should treat this largefile as 're-added'
527 527 actions[standin] = ('a', None, 'keep standin')
528 528 else: # pick remote normal file
529 529 actions[lfile] = ('g', largs, 'replaces standin')
530 530 actions[standin] = ('r', None, 'replaced by non-standin')
531 531
532 532 return actions, diverge, renamedelete
533 533
534 534 def mergerecordupdates(orig, repo, actions, branchmerge):
535 535 if 'lfmr' in actions:
536 536 lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
537 537 for lfile, args, msg in actions['lfmr']:
538 538 # this should be executed before 'orig', to execute 'remove'
539 539 # before all other actions
540 540 repo.dirstate.remove(lfile)
541 541 # make sure lfile doesn't get synclfdirstate'd as normal
542 542 lfdirstate.add(lfile)
543 543 lfdirstate.write()
544 544
545 545 return orig(repo, actions, branchmerge)
546 546
547 547 # Override filemerge to prompt the user about how they wish to merge
548 548 # largefiles. This will handle identical edits without prompting the user.
549 549 def overridefilemerge(origfn, premerge, repo, wctx, mynode, orig, fcd, fco, fca,
550 550 labels=None):
551 551 if not lfutil.isstandin(orig) or fcd.isabsent() or fco.isabsent():
552 552 return origfn(premerge, repo, wctx, mynode, orig, fcd, fco, fca,
553 553 labels=labels)
554 554
555 555 ahash = lfutil.readasstandin(fca).lower()
556 556 dhash = lfutil.readasstandin(fcd).lower()
557 557 ohash = lfutil.readasstandin(fco).lower()
558 558 if (ohash != ahash and
559 559 ohash != dhash and
560 560 (dhash == ahash or
561 561 repo.ui.promptchoice(
562 562 _('largefile %s has a merge conflict\nancestor was %s\n'
563 563 'keep (l)ocal %s or\ntake (o)ther %s?'
564 564 '$$ &Local $$ &Other') %
565 565 (lfutil.splitstandin(orig), ahash, dhash, ohash),
566 566 0) == 1)):
567 567 repo.wwrite(fcd.path(), fco.data(), fco.flags())
568 568 return True, 0, False
569 569
570 570 def copiespathcopies(orig, ctx1, ctx2, match=None):
571 571 copies = orig(ctx1, ctx2, match=match)
572 572 updated = {}
573 573
574 574 for k, v in copies.iteritems():
575 575 updated[lfutil.splitstandin(k) or k] = lfutil.splitstandin(v) or v
576 576
577 577 return updated
578 578
579 579 # Copy first changes the matchers to match standins instead of
580 580 # largefiles. Then it overrides util.copyfile in that function it
581 581 # checks if the destination largefile already exists. It also keeps a
582 582 # list of copied files so that the largefiles can be copied and the
583 583 # dirstate updated.
584 584 def overridecopy(orig, ui, repo, pats, opts, rename=False):
585 585 # doesn't remove largefile on rename
586 586 if len(pats) < 2:
587 587 # this isn't legal, let the original function deal with it
588 588 return orig(ui, repo, pats, opts, rename)
589 589
590 590 # This could copy both lfiles and normal files in one command,
591 591 # but we don't want to do that. First replace their matcher to
592 592 # only match normal files and run it, then replace it to just
593 593 # match largefiles and run it again.
594 594 nonormalfiles = False
595 595 nolfiles = False
596 596 installnormalfilesmatchfn(repo[None].manifest())
597 597 try:
598 598 result = orig(ui, repo, pats, opts, rename)
599 599 except error.Abort as e:
600 600 if str(e) != _('no files to copy'):
601 601 raise e
602 602 else:
603 603 nonormalfiles = True
604 604 result = 0
605 605 finally:
606 606 restorematchfn()
607 607
608 608 # The first rename can cause our current working directory to be removed.
609 609 # In that case there is nothing left to copy/rename so just quit.
610 610 try:
611 611 repo.getcwd()
612 612 except OSError:
613 613 return result
614 614
615 615 def makestandin(relpath):
616 616 path = pathutil.canonpath(repo.root, repo.getcwd(), relpath)
617 617 return repo.wvfs.join(lfutil.standin(path))
618 618
619 619 fullpats = scmutil.expandpats(pats)
620 620 dest = fullpats[-1]
621 621
622 622 if os.path.isdir(dest):
623 623 if not os.path.isdir(makestandin(dest)):
624 624 os.makedirs(makestandin(dest))
625 625
626 626 try:
627 627 # When we call orig below it creates the standins but we don't add
628 628 # them to the dir state until later so lock during that time.
629 629 wlock = repo.wlock()
630 630
631 631 manifest = repo[None].manifest()
632 632 def overridematch(ctx, pats=(), opts=None, globbed=False,
633 633 default='relpath', badfn=None):
634 634 if opts is None:
635 635 opts = {}
636 636 newpats = []
637 637 # The patterns were previously mangled to add the standin
638 638 # directory; we need to remove that now
639 639 for pat in pats:
640 640 if matchmod.patkind(pat) is None and lfutil.shortname in pat:
641 641 newpats.append(pat.replace(lfutil.shortname, ''))
642 642 else:
643 643 newpats.append(pat)
644 644 match = oldmatch(ctx, newpats, opts, globbed, default, badfn=badfn)
645 645 m = copy.copy(match)
646 646 lfile = lambda f: lfutil.standin(f) in manifest
647 647 m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
648 648 m._fileset = set(m._files)
649 649 origmatchfn = m.matchfn
650 650 def matchfn(f):
651 651 lfile = lfutil.splitstandin(f)
652 652 return (lfile is not None and
653 653 (f in manifest) and
654 654 origmatchfn(lfile) or
655 655 None)
656 656 m.matchfn = matchfn
657 657 return m
658 658 oldmatch = installmatchfn(overridematch)
659 659 listpats = []
660 660 for pat in pats:
661 661 if matchmod.patkind(pat) is not None:
662 662 listpats.append(pat)
663 663 else:
664 664 listpats.append(makestandin(pat))
665 665
666 666 try:
667 667 origcopyfile = util.copyfile
668 668 copiedfiles = []
669 669 def overridecopyfile(src, dest):
670 670 if (lfutil.shortname in src and
671 671 dest.startswith(repo.wjoin(lfutil.shortname))):
672 672 destlfile = dest.replace(lfutil.shortname, '')
673 673 if not opts['force'] and os.path.exists(destlfile):
674 674 raise IOError('',
675 675 _('destination largefile already exists'))
676 676 copiedfiles.append((src, dest))
677 677 origcopyfile(src, dest)
678 678
679 679 util.copyfile = overridecopyfile
680 680 result += orig(ui, repo, listpats, opts, rename)
681 681 finally:
682 682 util.copyfile = origcopyfile
683 683
684 684 lfdirstate = lfutil.openlfdirstate(ui, repo)
685 685 for (src, dest) in copiedfiles:
686 686 if (lfutil.shortname in src and
687 687 dest.startswith(repo.wjoin(lfutil.shortname))):
688 688 srclfile = src.replace(repo.wjoin(lfutil.standin('')), '')
689 689 destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '')
690 690 destlfiledir = repo.wvfs.dirname(repo.wjoin(destlfile)) or '.'
691 691 if not os.path.isdir(destlfiledir):
692 692 os.makedirs(destlfiledir)
693 693 if rename:
694 694 os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
695 695
696 696 # The file is gone, but this deletes any empty parent
697 697 # directories as a side-effect.
698 698 repo.wvfs.unlinkpath(srclfile, ignoremissing=True)
699 699 lfdirstate.remove(srclfile)
700 700 else:
701 701 util.copyfile(repo.wjoin(srclfile),
702 702 repo.wjoin(destlfile))
703 703
704 704 lfdirstate.add(destlfile)
705 705 lfdirstate.write()
706 706 except error.Abort as e:
707 707 if str(e) != _('no files to copy'):
708 708 raise e
709 709 else:
710 710 nolfiles = True
711 711 finally:
712 712 restorematchfn()
713 713 wlock.release()
714 714
715 715 if nolfiles and nonormalfiles:
716 716 raise error.Abort(_('no files to copy'))
717 717
718 718 return result
719 719
720 720 # When the user calls revert, we have to be careful to not revert any
721 721 # changes to other largefiles accidentally. This means we have to keep
722 722 # track of the largefiles that are being reverted so we only pull down
723 723 # the necessary largefiles.
724 724 #
725 725 # Standins are only updated (to match the hash of largefiles) before
726 726 # commits. Update the standins then run the original revert, changing
727 727 # the matcher to hit standins instead of largefiles. Based on the
728 728 # resulting standins update the largefiles.
729 729 def overriderevert(orig, ui, repo, ctx, parents, *pats, **opts):
730 730 # Because we put the standins in a bad state (by updating them)
731 731 # and then return them to a correct state we need to lock to
732 732 # prevent others from changing them in their incorrect state.
733 733 with repo.wlock():
734 734 lfdirstate = lfutil.openlfdirstate(ui, repo)
735 735 s = lfutil.lfdirstatestatus(lfdirstate, repo)
736 736 lfdirstate.write()
737 737 for lfile in s.modified:
738 738 lfutil.updatestandin(repo, lfile, lfutil.standin(lfile))
739 739 for lfile in s.deleted:
740 740 fstandin = lfutil.standin(lfile)
741 741 if (repo.wvfs.exists(fstandin)):
742 742 repo.wvfs.unlink(fstandin)
743 743
744 744 oldstandins = lfutil.getstandinsstate(repo)
745 745
746 746 def overridematch(mctx, pats=(), opts=None, globbed=False,
747 747 default='relpath', badfn=None):
748 748 if opts is None:
749 749 opts = {}
750 750 match = oldmatch(mctx, pats, opts, globbed, default, badfn=badfn)
751 751 m = copy.copy(match)
752 752
753 753 # revert supports recursing into subrepos, and though largefiles
754 754 # currently doesn't work correctly in that case, this match is
755 755 # called, so the lfdirstate above may not be the correct one for
756 756 # this invocation of match.
757 757 lfdirstate = lfutil.openlfdirstate(mctx.repo().ui, mctx.repo(),
758 758 False)
759 759
760 760 wctx = repo[None]
761 761 matchfiles = []
762 762 for f in m._files:
763 763 standin = lfutil.standin(f)
764 764 if standin in ctx or standin in mctx:
765 765 matchfiles.append(standin)
766 766 elif standin in wctx or lfdirstate[f] == 'r':
767 767 continue
768 768 else:
769 769 matchfiles.append(f)
770 770 m._files = matchfiles
771 771 m._fileset = set(m._files)
772 772 origmatchfn = m.matchfn
773 773 def matchfn(f):
774 774 lfile = lfutil.splitstandin(f)
775 775 if lfile is not None:
776 776 return (origmatchfn(lfile) and
777 777 (f in ctx or f in mctx))
778 778 return origmatchfn(f)
779 779 m.matchfn = matchfn
780 780 return m
781 781 oldmatch = installmatchfn(overridematch)
782 782 try:
783 783 orig(ui, repo, ctx, parents, *pats, **opts)
784 784 finally:
785 785 restorematchfn()
786 786
787 787 newstandins = lfutil.getstandinsstate(repo)
788 788 filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
789 789 # lfdirstate should be 'normallookup'-ed for updated files,
790 790 # because reverting doesn't touch dirstate for 'normal' files
791 791 # when target revision is explicitly specified: in such case,
792 792 # 'n' and valid timestamp in dirstate doesn't ensure 'clean'
793 793 # of target (standin) file.
794 794 lfcommands.updatelfiles(ui, repo, filelist, printmessage=False,
795 795 normallookup=True)
796 796
797 797 # after pulling changesets, we need to take some extra care to get
798 798 # largefiles updated remotely
799 799 def overridepull(orig, ui, repo, source=None, **opts):
800 800 revsprepull = len(repo)
801 801 if not source:
802 802 source = 'default'
803 803 repo.lfpullsource = source
804 804 result = orig(ui, repo, source, **opts)
805 805 revspostpull = len(repo)
806 806 lfrevs = opts.get(r'lfrev', [])
807 807 if opts.get(r'all_largefiles'):
808 808 lfrevs.append('pulled()')
809 809 if lfrevs and revspostpull > revsprepull:
810 810 numcached = 0
811 811 repo.firstpulled = revsprepull # for pulled() revset expression
812 812 try:
813 813 for rev in scmutil.revrange(repo, lfrevs):
814 814 ui.note(_('pulling largefiles for revision %s\n') % rev)
815 815 (cached, missing) = lfcommands.cachelfiles(ui, repo, rev)
816 816 numcached += len(cached)
817 817 finally:
818 818 del repo.firstpulled
819 819 ui.status(_("%d largefiles cached\n") % numcached)
820 820 return result
821 821
822 822 def overridepush(orig, ui, repo, *args, **kwargs):
823 823 """Override push command and store --lfrev parameters in opargs"""
824 824 lfrevs = kwargs.pop(r'lfrev', None)
825 825 if lfrevs:
826 826 opargs = kwargs.setdefault('opargs', {})
827 827 opargs['lfrevs'] = scmutil.revrange(repo, lfrevs)
828 828 return orig(ui, repo, *args, **kwargs)
829 829
830 830 def exchangepushoperation(orig, *args, **kwargs):
831 831 """Override pushoperation constructor and store lfrevs parameter"""
832 832 lfrevs = kwargs.pop(r'lfrevs', None)
833 833 pushop = orig(*args, **kwargs)
834 834 pushop.lfrevs = lfrevs
835 835 return pushop
836 836
837 837 revsetpredicate = registrar.revsetpredicate()
838 838
839 839 @revsetpredicate('pulled()')
840 840 def pulledrevsetsymbol(repo, subset, x):
841 841 """Changesets that just has been pulled.
842 842
843 843 Only available with largefiles from pull --lfrev expressions.
844 844
845 845 .. container:: verbose
846 846
847 847 Some examples:
848 848
849 849 - pull largefiles for all new changesets::
850 850
851 851 hg pull -lfrev "pulled()"
852 852
853 853 - pull largefiles for all new branch heads::
854 854
855 855 hg pull -lfrev "head(pulled()) and not closed()"
856 856
857 857 """
858 858
859 859 try:
860 860 firstpulled = repo.firstpulled
861 861 except AttributeError:
862 862 raise error.Abort(_("pulled() only available in --lfrev"))
863 863 return smartset.baseset([r for r in subset if r >= firstpulled])
864 864
865 865 def overrideclone(orig, ui, source, dest=None, **opts):
866 866 d = dest
867 867 if d is None:
868 868 d = hg.defaultdest(source)
869 869 if opts.get(r'all_largefiles') and not hg.islocal(d):
870 870 raise error.Abort(_(
871 871 '--all-largefiles is incompatible with non-local destination %s') %
872 872 d)
873 873
874 874 return orig(ui, source, dest, **opts)
875 875
876 876 def hgclone(orig, ui, opts, *args, **kwargs):
877 877 result = orig(ui, opts, *args, **kwargs)
878 878
879 879 if result is not None:
880 880 sourcerepo, destrepo = result
881 881 repo = destrepo.local()
882 882
883 883 # When cloning to a remote repo (like through SSH), no repo is available
884 884 # from the peer. Therefore the largefiles can't be downloaded and the
885 885 # hgrc can't be updated.
886 886 if not repo:
887 887 return result
888 888
889 889 # If largefiles is required for this repo, permanently enable it locally
890 890 if 'largefiles' in repo.requirements:
891 with repo.vfs('hgrc', 'a', text=True) as fp:
892 fp.write('\n[extensions]\nlargefiles=\n')
891 repo.vfs.append('hgrc',
892 util.tonativeeol('\n[extensions]\nlargefiles=\n'))
893 893
894 894 # Caching is implicitly limited to 'rev' option, since the dest repo was
895 895 # truncated at that point. The user may expect a download count with
896 896 # this option, so attempt whether or not this is a largefile repo.
897 897 if opts.get(r'all_largefiles'):
898 898 success, missing = lfcommands.downloadlfiles(ui, repo, None)
899 899
900 900 if missing != 0:
901 901 return None
902 902
903 903 return result
904 904
905 905 def hgpostshare(orig, sourcerepo, destrepo, bookmarks=True, defaultpath=None):
906 906 orig(sourcerepo, destrepo, bookmarks, defaultpath)
907 907
908 908 # If largefiles is required for this repo, permanently enable it locally
909 909 if 'largefiles' in destrepo.requirements:
910 with destrepo.vfs('hgrc', 'a+', text=True) as fp:
911 fp.write('\n[extensions]\nlargefiles=\n')
910 destrepo.vfs.append('hgrc',
911 util.tonativeeol('\n[extensions]\nlargefiles=\n'))
912 912
913 913 def overriderebase(orig, ui, repo, **opts):
914 914 if not util.safehasattr(repo, '_largefilesenabled'):
915 915 return orig(ui, repo, **opts)
916 916
917 917 resuming = opts.get(r'continue')
918 918 repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
919 919 repo._lfstatuswriters.append(lambda *msg, **opts: None)
920 920 try:
921 921 return orig(ui, repo, **opts)
922 922 finally:
923 923 repo._lfstatuswriters.pop()
924 924 repo._lfcommithooks.pop()
925 925
926 926 def overridearchivecmd(orig, ui, repo, dest, **opts):
927 927 repo.unfiltered().lfstatus = True
928 928
929 929 try:
930 930 return orig(ui, repo.unfiltered(), dest, **opts)
931 931 finally:
932 932 repo.unfiltered().lfstatus = False
933 933
934 934 def hgwebarchive(orig, web, req, tmpl):
935 935 web.repo.lfstatus = True
936 936
937 937 try:
938 938 return orig(web, req, tmpl)
939 939 finally:
940 940 web.repo.lfstatus = False
941 941
942 942 def overridearchive(orig, repo, dest, node, kind, decode=True, matchfn=None,
943 943 prefix='', mtime=None, subrepos=None):
944 944 # For some reason setting repo.lfstatus in hgwebarchive only changes the
945 945 # unfiltered repo's attr, so check that as well.
946 946 if not repo.lfstatus and not repo.unfiltered().lfstatus:
947 947 return orig(repo, dest, node, kind, decode, matchfn, prefix, mtime,
948 948 subrepos)
949 949
950 950 # No need to lock because we are only reading history and
951 951 # largefile caches, neither of which are modified.
952 952 if node is not None:
953 953 lfcommands.cachelfiles(repo.ui, repo, node)
954 954
955 955 if kind not in archival.archivers:
956 956 raise error.Abort(_("unknown archive type '%s'") % kind)
957 957
958 958 ctx = repo[node]
959 959
960 960 if kind == 'files':
961 961 if prefix:
962 962 raise error.Abort(
963 963 _('cannot give prefix when archiving to files'))
964 964 else:
965 965 prefix = archival.tidyprefix(dest, kind, prefix)
966 966
967 967 def write(name, mode, islink, getdata):
968 968 if matchfn and not matchfn(name):
969 969 return
970 970 data = getdata()
971 971 if decode:
972 972 data = repo.wwritedata(name, data)
973 973 archiver.addfile(prefix + name, mode, islink, data)
974 974
975 975 archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
976 976
977 977 if repo.ui.configbool("ui", "archivemeta"):
978 978 write('.hg_archival.txt', 0o644, False,
979 979 lambda: archival.buildmetadata(ctx))
980 980
981 981 for f in ctx:
982 982 ff = ctx.flags(f)
983 983 getdata = ctx[f].data
984 984 lfile = lfutil.splitstandin(f)
985 985 if lfile is not None:
986 986 if node is not None:
987 987 path = lfutil.findfile(repo, getdata().strip())
988 988
989 989 if path is None:
990 990 raise error.Abort(
991 991 _('largefile %s not found in repo store or system cache')
992 992 % lfile)
993 993 else:
994 994 path = lfile
995 995
996 996 f = lfile
997 997
998 998 getdata = lambda: util.readfile(path)
999 999 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, getdata)
1000 1000
1001 1001 if subrepos:
1002 1002 for subpath in sorted(ctx.substate):
1003 1003 sub = ctx.workingsub(subpath)
1004 1004 submatch = matchmod.subdirmatcher(subpath, matchfn)
1005 1005 sub._repo.lfstatus = True
1006 1006 sub.archive(archiver, prefix, submatch)
1007 1007
1008 1008 archiver.done()
1009 1009
1010 1010 def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
1011 1011 lfenabled = util.safehasattr(repo._repo, '_largefilesenabled')
1012 1012 if not lfenabled or not repo._repo.lfstatus:
1013 1013 return orig(repo, archiver, prefix, match, decode)
1014 1014
1015 1015 repo._get(repo._state + ('hg',))
1016 1016 rev = repo._state[1]
1017 1017 ctx = repo._repo[rev]
1018 1018
1019 1019 if ctx.node() is not None:
1020 1020 lfcommands.cachelfiles(repo.ui, repo._repo, ctx.node())
1021 1021
1022 1022 def write(name, mode, islink, getdata):
1023 1023 # At this point, the standin has been replaced with the largefile name,
1024 1024 # so the normal matcher works here without the lfutil variants.
1025 1025 if match and not match(f):
1026 1026 return
1027 1027 data = getdata()
1028 1028 if decode:
1029 1029 data = repo._repo.wwritedata(name, data)
1030 1030
1031 1031 archiver.addfile(prefix + repo._path + '/' + name, mode, islink, data)
1032 1032
1033 1033 for f in ctx:
1034 1034 ff = ctx.flags(f)
1035 1035 getdata = ctx[f].data
1036 1036 lfile = lfutil.splitstandin(f)
1037 1037 if lfile is not None:
1038 1038 if ctx.node() is not None:
1039 1039 path = lfutil.findfile(repo._repo, getdata().strip())
1040 1040
1041 1041 if path is None:
1042 1042 raise error.Abort(
1043 1043 _('largefile %s not found in repo store or system cache')
1044 1044 % lfile)
1045 1045 else:
1046 1046 path = lfile
1047 1047
1048 1048 f = lfile
1049 1049
1050 1050 getdata = lambda: util.readfile(os.path.join(prefix, path))
1051 1051
1052 1052 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, getdata)
1053 1053
1054 1054 for subpath in sorted(ctx.substate):
1055 1055 sub = ctx.workingsub(subpath)
1056 1056 submatch = matchmod.subdirmatcher(subpath, match)
1057 1057 sub._repo.lfstatus = True
1058 1058 sub.archive(archiver, prefix + repo._path + '/', submatch, decode)
1059 1059
1060 1060 # If a largefile is modified, the change is not reflected in its
1061 1061 # standin until a commit. cmdutil.bailifchanged() raises an exception
1062 1062 # if the repo has uncommitted changes. Wrap it to also check if
1063 1063 # largefiles were changed. This is used by bisect, backout and fetch.
1064 1064 def overridebailifchanged(orig, repo, *args, **kwargs):
1065 1065 orig(repo, *args, **kwargs)
1066 1066 repo.lfstatus = True
1067 1067 s = repo.status()
1068 1068 repo.lfstatus = False
1069 1069 if s.modified or s.added or s.removed or s.deleted:
1070 1070 raise error.Abort(_('uncommitted changes'))
1071 1071
1072 1072 def postcommitstatus(orig, repo, *args, **kwargs):
1073 1073 repo.lfstatus = True
1074 1074 try:
1075 1075 return orig(repo, *args, **kwargs)
1076 1076 finally:
1077 1077 repo.lfstatus = False
1078 1078
1079 1079 def cmdutilforget(orig, ui, repo, match, prefix, explicitonly):
1080 1080 normalmatcher = composenormalfilematcher(match, repo[None].manifest())
1081 1081 bad, forgot = orig(ui, repo, normalmatcher, prefix, explicitonly)
1082 1082 m = composelargefilematcher(match, repo[None].manifest())
1083 1083
1084 1084 try:
1085 1085 repo.lfstatus = True
1086 1086 s = repo.status(match=m, clean=True)
1087 1087 finally:
1088 1088 repo.lfstatus = False
1089 1089 manifest = repo[None].manifest()
1090 1090 forget = sorted(s.modified + s.added + s.deleted + s.clean)
1091 1091 forget = [f for f in forget if lfutil.standin(f) in manifest]
1092 1092
1093 1093 for f in forget:
1094 1094 fstandin = lfutil.standin(f)
1095 1095 if fstandin not in repo.dirstate and not repo.wvfs.isdir(fstandin):
1096 1096 ui.warn(_('not removing %s: file is already untracked\n')
1097 1097 % m.rel(f))
1098 1098 bad.append(f)
1099 1099
1100 1100 for f in forget:
1101 1101 if ui.verbose or not m.exact(f):
1102 1102 ui.status(_('removing %s\n') % m.rel(f))
1103 1103
1104 1104 # Need to lock because standin files are deleted then removed from the
1105 1105 # repository and we could race in-between.
1106 1106 with repo.wlock():
1107 1107 lfdirstate = lfutil.openlfdirstate(ui, repo)
1108 1108 for f in forget:
1109 1109 if lfdirstate[f] == 'a':
1110 1110 lfdirstate.drop(f)
1111 1111 else:
1112 1112 lfdirstate.remove(f)
1113 1113 lfdirstate.write()
1114 1114 standins = [lfutil.standin(f) for f in forget]
1115 1115 for f in standins:
1116 1116 repo.wvfs.unlinkpath(f, ignoremissing=True)
1117 1117 rejected = repo[None].forget(standins)
1118 1118
1119 1119 bad.extend(f for f in rejected if f in m.files())
1120 1120 forgot.extend(f for f in forget if f not in rejected)
1121 1121 return bad, forgot
1122 1122
1123 1123 def _getoutgoings(repo, other, missing, addfunc):
1124 1124 """get pairs of filename and largefile hash in outgoing revisions
1125 1125 in 'missing'.
1126 1126
1127 1127 largefiles already existing on 'other' repository are ignored.
1128 1128
1129 1129 'addfunc' is invoked with each unique pairs of filename and
1130 1130 largefile hash value.
1131 1131 """
1132 1132 knowns = set()
1133 1133 lfhashes = set()
1134 1134 def dedup(fn, lfhash):
1135 1135 k = (fn, lfhash)
1136 1136 if k not in knowns:
1137 1137 knowns.add(k)
1138 1138 lfhashes.add(lfhash)
1139 1139 lfutil.getlfilestoupload(repo, missing, dedup)
1140 1140 if lfhashes:
1141 1141 lfexists = storefactory.openstore(repo, other).exists(lfhashes)
1142 1142 for fn, lfhash in knowns:
1143 1143 if not lfexists[lfhash]: # lfhash doesn't exist on "other"
1144 1144 addfunc(fn, lfhash)
1145 1145
1146 1146 def outgoinghook(ui, repo, other, opts, missing):
1147 1147 if opts.pop('large', None):
1148 1148 lfhashes = set()
1149 1149 if ui.debugflag:
1150 1150 toupload = {}
1151 1151 def addfunc(fn, lfhash):
1152 1152 if fn not in toupload:
1153 1153 toupload[fn] = []
1154 1154 toupload[fn].append(lfhash)
1155 1155 lfhashes.add(lfhash)
1156 1156 def showhashes(fn):
1157 1157 for lfhash in sorted(toupload[fn]):
1158 1158 ui.debug(' %s\n' % (lfhash))
1159 1159 else:
1160 1160 toupload = set()
1161 1161 def addfunc(fn, lfhash):
1162 1162 toupload.add(fn)
1163 1163 lfhashes.add(lfhash)
1164 1164 def showhashes(fn):
1165 1165 pass
1166 1166 _getoutgoings(repo, other, missing, addfunc)
1167 1167
1168 1168 if not toupload:
1169 1169 ui.status(_('largefiles: no files to upload\n'))
1170 1170 else:
1171 1171 ui.status(_('largefiles to upload (%d entities):\n')
1172 1172 % (len(lfhashes)))
1173 1173 for file in sorted(toupload):
1174 1174 ui.status(lfutil.splitstandin(file) + '\n')
1175 1175 showhashes(file)
1176 1176 ui.status('\n')
1177 1177
1178 1178 def summaryremotehook(ui, repo, opts, changes):
1179 1179 largeopt = opts.get('large', False)
1180 1180 if changes is None:
1181 1181 if largeopt:
1182 1182 return (False, True) # only outgoing check is needed
1183 1183 else:
1184 1184 return (False, False)
1185 1185 elif largeopt:
1186 1186 url, branch, peer, outgoing = changes[1]
1187 1187 if peer is None:
1188 1188 # i18n: column positioning for "hg summary"
1189 1189 ui.status(_('largefiles: (no remote repo)\n'))
1190 1190 return
1191 1191
1192 1192 toupload = set()
1193 1193 lfhashes = set()
1194 1194 def addfunc(fn, lfhash):
1195 1195 toupload.add(fn)
1196 1196 lfhashes.add(lfhash)
1197 1197 _getoutgoings(repo, peer, outgoing.missing, addfunc)
1198 1198
1199 1199 if not toupload:
1200 1200 # i18n: column positioning for "hg summary"
1201 1201 ui.status(_('largefiles: (no files to upload)\n'))
1202 1202 else:
1203 1203 # i18n: column positioning for "hg summary"
1204 1204 ui.status(_('largefiles: %d entities for %d files to upload\n')
1205 1205 % (len(lfhashes), len(toupload)))
1206 1206
1207 1207 def overridesummary(orig, ui, repo, *pats, **opts):
1208 1208 try:
1209 1209 repo.lfstatus = True
1210 1210 orig(ui, repo, *pats, **opts)
1211 1211 finally:
1212 1212 repo.lfstatus = False
1213 1213
1214 1214 def scmutiladdremove(orig, repo, matcher, prefix, opts=None, dry_run=None,
1215 1215 similarity=None):
1216 1216 if opts is None:
1217 1217 opts = {}
1218 1218 if not lfutil.islfilesrepo(repo):
1219 1219 return orig(repo, matcher, prefix, opts, dry_run, similarity)
1220 1220 # Get the list of missing largefiles so we can remove them
1221 1221 lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
1222 1222 unsure, s = lfdirstate.status(matchmod.always(repo.root, repo.getcwd()),
1223 1223 subrepos=[], ignored=False, clean=False,
1224 1224 unknown=False)
1225 1225
1226 1226 # Call into the normal remove code, but the removing of the standin, we want
1227 1227 # to have handled by original addremove. Monkey patching here makes sure
1228 1228 # we don't remove the standin in the largefiles code, preventing a very
1229 1229 # confused state later.
1230 1230 if s.deleted:
1231 1231 m = copy.copy(matcher)
1232 1232
1233 1233 # The m._files and m._map attributes are not changed to the deleted list
1234 1234 # because that affects the m.exact() test, which in turn governs whether
1235 1235 # or not the file name is printed, and how. Simply limit the original
1236 1236 # matches to those in the deleted status list.
1237 1237 matchfn = m.matchfn
1238 1238 m.matchfn = lambda f: f in s.deleted and matchfn(f)
1239 1239
1240 1240 removelargefiles(repo.ui, repo, True, m, **opts)
1241 1241 # Call into the normal add code, and any files that *should* be added as
1242 1242 # largefiles will be
1243 1243 added, bad = addlargefiles(repo.ui, repo, True, matcher, **opts)
1244 1244 # Now that we've handled largefiles, hand off to the original addremove
1245 1245 # function to take care of the rest. Make sure it doesn't do anything with
1246 1246 # largefiles by passing a matcher that will ignore them.
1247 1247 matcher = composenormalfilematcher(matcher, repo[None].manifest(), added)
1248 1248 return orig(repo, matcher, prefix, opts, dry_run, similarity)
1249 1249
1250 1250 # Calling purge with --all will cause the largefiles to be deleted.
1251 1251 # Override repo.status to prevent this from happening.
1252 1252 def overridepurge(orig, ui, repo, *dirs, **opts):
1253 1253 # XXX Monkey patching a repoview will not work. The assigned attribute will
1254 1254 # be set on the unfiltered repo, but we will only lookup attributes in the
1255 1255 # unfiltered repo if the lookup in the repoview object itself fails. As the
1256 1256 # monkey patched method exists on the repoview class the lookup will not
1257 1257 # fail. As a result, the original version will shadow the monkey patched
1258 1258 # one, defeating the monkey patch.
1259 1259 #
1260 1260 # As a work around we use an unfiltered repo here. We should do something
1261 1261 # cleaner instead.
1262 1262 repo = repo.unfiltered()
1263 1263 oldstatus = repo.status
1264 1264 def overridestatus(node1='.', node2=None, match=None, ignored=False,
1265 1265 clean=False, unknown=False, listsubrepos=False):
1266 1266 r = oldstatus(node1, node2, match, ignored, clean, unknown,
1267 1267 listsubrepos)
1268 1268 lfdirstate = lfutil.openlfdirstate(ui, repo)
1269 1269 unknown = [f for f in r.unknown if lfdirstate[f] == '?']
1270 1270 ignored = [f for f in r.ignored if lfdirstate[f] == '?']
1271 1271 return scmutil.status(r.modified, r.added, r.removed, r.deleted,
1272 1272 unknown, ignored, r.clean)
1273 1273 repo.status = overridestatus
1274 1274 orig(ui, repo, *dirs, **opts)
1275 1275 repo.status = oldstatus
1276 1276
1277 1277 def overriderollback(orig, ui, repo, **opts):
1278 1278 with repo.wlock():
1279 1279 before = repo.dirstate.parents()
1280 1280 orphans = set(f for f in repo.dirstate
1281 1281 if lfutil.isstandin(f) and repo.dirstate[f] != 'r')
1282 1282 result = orig(ui, repo, **opts)
1283 1283 after = repo.dirstate.parents()
1284 1284 if before == after:
1285 1285 return result # no need to restore standins
1286 1286
1287 1287 pctx = repo['.']
1288 1288 for f in repo.dirstate:
1289 1289 if lfutil.isstandin(f):
1290 1290 orphans.discard(f)
1291 1291 if repo.dirstate[f] == 'r':
1292 1292 repo.wvfs.unlinkpath(f, ignoremissing=True)
1293 1293 elif f in pctx:
1294 1294 fctx = pctx[f]
1295 1295 repo.wwrite(f, fctx.data(), fctx.flags())
1296 1296 else:
1297 1297 # content of standin is not so important in 'a',
1298 1298 # 'm' or 'n' (coming from the 2nd parent) cases
1299 1299 lfutil.writestandin(repo, f, '', False)
1300 1300 for standin in orphans:
1301 1301 repo.wvfs.unlinkpath(standin, ignoremissing=True)
1302 1302
1303 1303 lfdirstate = lfutil.openlfdirstate(ui, repo)
1304 1304 orphans = set(lfdirstate)
1305 1305 lfiles = lfutil.listlfiles(repo)
1306 1306 for file in lfiles:
1307 1307 lfutil.synclfdirstate(repo, lfdirstate, file, True)
1308 1308 orphans.discard(file)
1309 1309 for lfile in orphans:
1310 1310 lfdirstate.drop(lfile)
1311 1311 lfdirstate.write()
1312 1312 return result
1313 1313
1314 1314 def overridetransplant(orig, ui, repo, *revs, **opts):
1315 1315 resuming = opts.get(r'continue')
1316 1316 repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
1317 1317 repo._lfstatuswriters.append(lambda *msg, **opts: None)
1318 1318 try:
1319 1319 result = orig(ui, repo, *revs, **opts)
1320 1320 finally:
1321 1321 repo._lfstatuswriters.pop()
1322 1322 repo._lfcommithooks.pop()
1323 1323 return result
1324 1324
1325 1325 def overridecat(orig, ui, repo, file1, *pats, **opts):
1326 1326 opts = pycompat.byteskwargs(opts)
1327 1327 ctx = scmutil.revsingle(repo, opts.get('rev'))
1328 1328 err = 1
1329 1329 notbad = set()
1330 1330 m = scmutil.match(ctx, (file1,) + pats, opts)
1331 1331 origmatchfn = m.matchfn
1332 1332 def lfmatchfn(f):
1333 1333 if origmatchfn(f):
1334 1334 return True
1335 1335 lf = lfutil.splitstandin(f)
1336 1336 if lf is None:
1337 1337 return False
1338 1338 notbad.add(lf)
1339 1339 return origmatchfn(lf)
1340 1340 m.matchfn = lfmatchfn
1341 1341 origbadfn = m.bad
1342 1342 def lfbadfn(f, msg):
1343 1343 if not f in notbad:
1344 1344 origbadfn(f, msg)
1345 1345 m.bad = lfbadfn
1346 1346
1347 1347 origvisitdirfn = m.visitdir
1348 1348 def lfvisitdirfn(dir):
1349 1349 if dir == lfutil.shortname:
1350 1350 return True
1351 1351 ret = origvisitdirfn(dir)
1352 1352 if ret:
1353 1353 return ret
1354 1354 lf = lfutil.splitstandin(dir)
1355 1355 if lf is None:
1356 1356 return False
1357 1357 return origvisitdirfn(lf)
1358 1358 m.visitdir = lfvisitdirfn
1359 1359
1360 1360 for f in ctx.walk(m):
1361 1361 with cmdutil.makefileobj(repo, opts.get('output'), ctx.node(),
1362 1362 pathname=f) as fp:
1363 1363 lf = lfutil.splitstandin(f)
1364 1364 if lf is None or origmatchfn(f):
1365 1365 # duplicating unreachable code from commands.cat
1366 1366 data = ctx[f].data()
1367 1367 if opts.get('decode'):
1368 1368 data = repo.wwritedata(f, data)
1369 1369 fp.write(data)
1370 1370 else:
1371 1371 hash = lfutil.readasstandin(ctx[f])
1372 1372 if not lfutil.inusercache(repo.ui, hash):
1373 1373 store = storefactory.openstore(repo)
1374 1374 success, missing = store.get([(lf, hash)])
1375 1375 if len(success) != 1:
1376 1376 raise error.Abort(
1377 1377 _('largefile %s is not in cache and could not be '
1378 1378 'downloaded') % lf)
1379 1379 path = lfutil.usercachepath(repo.ui, hash)
1380 1380 with open(path, "rb") as fpin:
1381 1381 for chunk in util.filechunkiter(fpin):
1382 1382 fp.write(chunk)
1383 1383 err = 0
1384 1384 return err
1385 1385
1386 1386 def mergeupdate(orig, repo, node, branchmerge, force,
1387 1387 *args, **kwargs):
1388 1388 matcher = kwargs.get(r'matcher', None)
1389 1389 # note if this is a partial update
1390 1390 partial = matcher and not matcher.always()
1391 1391 with repo.wlock():
1392 1392 # branch | | |
1393 1393 # merge | force | partial | action
1394 1394 # -------+-------+---------+--------------
1395 1395 # x | x | x | linear-merge
1396 1396 # o | x | x | branch-merge
1397 1397 # x | o | x | overwrite (as clean update)
1398 1398 # o | o | x | force-branch-merge (*1)
1399 1399 # x | x | o | (*)
1400 1400 # o | x | o | (*)
1401 1401 # x | o | o | overwrite (as revert)
1402 1402 # o | o | o | (*)
1403 1403 #
1404 1404 # (*) don't care
1405 1405 # (*1) deprecated, but used internally (e.g: "rebase --collapse")
1406 1406
1407 1407 lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
1408 1408 unsure, s = lfdirstate.status(matchmod.always(repo.root,
1409 1409 repo.getcwd()),
1410 1410 subrepos=[], ignored=False,
1411 1411 clean=True, unknown=False)
1412 1412 oldclean = set(s.clean)
1413 1413 pctx = repo['.']
1414 1414 dctx = repo[node]
1415 1415 for lfile in unsure + s.modified:
1416 1416 lfileabs = repo.wvfs.join(lfile)
1417 1417 if not repo.wvfs.exists(lfileabs):
1418 1418 continue
1419 1419 lfhash = lfutil.hashfile(lfileabs)
1420 1420 standin = lfutil.standin(lfile)
1421 1421 lfutil.writestandin(repo, standin, lfhash,
1422 1422 lfutil.getexecutable(lfileabs))
1423 1423 if (standin in pctx and
1424 1424 lfhash == lfutil.readasstandin(pctx[standin])):
1425 1425 oldclean.add(lfile)
1426 1426 for lfile in s.added:
1427 1427 fstandin = lfutil.standin(lfile)
1428 1428 if fstandin not in dctx:
1429 1429 # in this case, content of standin file is meaningless
1430 1430 # (in dctx, lfile is unknown, or normal file)
1431 1431 continue
1432 1432 lfutil.updatestandin(repo, lfile, fstandin)
1433 1433 # mark all clean largefiles as dirty, just in case the update gets
1434 1434 # interrupted before largefiles and lfdirstate are synchronized
1435 1435 for lfile in oldclean:
1436 1436 lfdirstate.normallookup(lfile)
1437 1437 lfdirstate.write()
1438 1438
1439 1439 oldstandins = lfutil.getstandinsstate(repo)
1440 1440 # Make sure the merge runs on disk, not in-memory. largefiles is not a
1441 1441 # good candidate for in-memory merge (large files, custom dirstate,
1442 1442 # matcher usage).
1443 1443 kwargs[r'wc'] = repo[None]
1444 1444 result = orig(repo, node, branchmerge, force, *args, **kwargs)
1445 1445
1446 1446 newstandins = lfutil.getstandinsstate(repo)
1447 1447 filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
1448 1448
1449 1449 # to avoid leaving all largefiles as dirty and thus rehash them, mark
1450 1450 # all the ones that didn't change as clean
1451 1451 for lfile in oldclean.difference(filelist):
1452 1452 lfdirstate.normal(lfile)
1453 1453 lfdirstate.write()
1454 1454
1455 1455 if branchmerge or force or partial:
1456 1456 filelist.extend(s.deleted + s.removed)
1457 1457
1458 1458 lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
1459 1459 normallookup=partial)
1460 1460
1461 1461 return result
1462 1462
1463 1463 def scmutilmarktouched(orig, repo, files, *args, **kwargs):
1464 1464 result = orig(repo, files, *args, **kwargs)
1465 1465
1466 1466 filelist = []
1467 1467 for f in files:
1468 1468 lf = lfutil.splitstandin(f)
1469 1469 if lf is not None:
1470 1470 filelist.append(lf)
1471 1471 if filelist:
1472 1472 lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
1473 1473 printmessage=False, normallookup=True)
1474 1474
1475 1475 return result
1476 1476
1477 1477 def upgraderequirements(orig, repo):
1478 1478 reqs = orig(repo)
1479 1479 if 'largefiles' in repo.requirements:
1480 1480 reqs.add('largefiles')
1481 1481 return reqs
1482 1482
1483 1483 _lfscheme = 'largefile://'
1484 1484 def openlargefile(orig, ui, url_, data=None):
1485 1485 if url_.startswith(_lfscheme):
1486 1486 if data:
1487 1487 msg = "cannot use data on a 'largefile://' url"
1488 1488 raise error.ProgrammingError(msg)
1489 1489 lfid = url_[len(_lfscheme):]
1490 1490 return storefactory.getlfile(ui, lfid)
1491 1491 else:
1492 1492 return orig(ui, url_, data=data)
General Comments 0
You need to be logged in to leave comments. Login now