##// END OF EJS Templates
largefiles: call readstandin() with changectx itself instead of rev or node...
FUJIWARA Katsunori -
r31616:10561eb9 default
parent child Browse files
Show More
@@ -1,668 +1,673 b''
1 1 # Copyright 2009-2010 Gregory P. Ward
2 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 3 # Copyright 2010-2011 Fog Creek Software
4 4 # Copyright 2010-2011 Unity Technologies
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 '''largefiles utility code: must not import other modules in this package.'''
10 10 from __future__ import absolute_import
11 11
12 12 import copy
13 13 import hashlib
14 14 import os
15 15 import platform
16 16 import stat
17 17
18 18 from mercurial.i18n import _
19 19
20 20 from mercurial import (
21 21 dirstate,
22 22 encoding,
23 23 error,
24 24 httpconnection,
25 25 match as matchmod,
26 26 node,
27 27 pycompat,
28 28 scmutil,
29 29 util,
30 30 vfs as vfsmod,
31 31 )
32 32
33 33 shortname = '.hglf'
34 34 shortnameslash = shortname + '/'
35 35 longname = 'largefiles'
36 36
37 37 # -- Private worker functions ------------------------------------------
38 38
39 39 def getminsize(ui, assumelfiles, opt, default=10):
40 40 lfsize = opt
41 41 if not lfsize and assumelfiles:
42 42 lfsize = ui.config(longname, 'minsize', default=default)
43 43 if lfsize:
44 44 try:
45 45 lfsize = float(lfsize)
46 46 except ValueError:
47 47 raise error.Abort(_('largefiles: size must be number (not %s)\n')
48 48 % lfsize)
49 49 if lfsize is None:
50 50 raise error.Abort(_('minimum size for largefiles must be specified'))
51 51 return lfsize
52 52
53 53 def link(src, dest):
54 54 """Try to create hardlink - if that fails, efficiently make a copy."""
55 55 util.makedirs(os.path.dirname(dest))
56 56 try:
57 57 util.oslink(src, dest)
58 58 except OSError:
59 59 # if hardlinks fail, fallback on atomic copy
60 60 with open(src, 'rb') as srcf:
61 61 with util.atomictempfile(dest) as dstf:
62 62 for chunk in util.filechunkiter(srcf):
63 63 dstf.write(chunk)
64 64 os.chmod(dest, os.stat(src).st_mode)
65 65
66 66 def usercachepath(ui, hash):
67 67 '''Return the correct location in the "global" largefiles cache for a file
68 68 with the given hash.
69 69 This cache is used for sharing of largefiles across repositories - both
70 70 to preserve download bandwidth and storage space.'''
71 71 return os.path.join(_usercachedir(ui), hash)
72 72
73 73 def _usercachedir(ui):
74 74 '''Return the location of the "global" largefiles cache.'''
75 75 path = ui.configpath(longname, 'usercache', None)
76 76 if path:
77 77 return path
78 78 if pycompat.osname == 'nt':
79 79 appdata = encoding.environ.get('LOCALAPPDATA',\
80 80 encoding.environ.get('APPDATA'))
81 81 if appdata:
82 82 return os.path.join(appdata, longname)
83 83 elif platform.system() == 'Darwin':
84 84 home = encoding.environ.get('HOME')
85 85 if home:
86 86 return os.path.join(home, 'Library', 'Caches', longname)
87 87 elif pycompat.osname == 'posix':
88 88 path = encoding.environ.get('XDG_CACHE_HOME')
89 89 if path:
90 90 return os.path.join(path, longname)
91 91 home = encoding.environ.get('HOME')
92 92 if home:
93 93 return os.path.join(home, '.cache', longname)
94 94 else:
95 95 raise error.Abort(_('unknown operating system: %s\n')
96 96 % pycompat.osname)
97 97 raise error.Abort(_('unknown %s usercache location') % longname)
98 98
99 99 def inusercache(ui, hash):
100 100 path = usercachepath(ui, hash)
101 101 return os.path.exists(path)
102 102
103 103 def findfile(repo, hash):
104 104 '''Return store path of the largefile with the specified hash.
105 105 As a side effect, the file might be linked from user cache.
106 106 Return None if the file can't be found locally.'''
107 107 path, exists = findstorepath(repo, hash)
108 108 if exists:
109 109 repo.ui.note(_('found %s in store\n') % hash)
110 110 return path
111 111 elif inusercache(repo.ui, hash):
112 112 repo.ui.note(_('found %s in system cache\n') % hash)
113 113 path = storepath(repo, hash)
114 114 link(usercachepath(repo.ui, hash), path)
115 115 return path
116 116 return None
117 117
118 118 class largefilesdirstate(dirstate.dirstate):
119 119 def __getitem__(self, key):
120 120 return super(largefilesdirstate, self).__getitem__(unixpath(key))
121 121 def normal(self, f):
122 122 return super(largefilesdirstate, self).normal(unixpath(f))
123 123 def remove(self, f):
124 124 return super(largefilesdirstate, self).remove(unixpath(f))
125 125 def add(self, f):
126 126 return super(largefilesdirstate, self).add(unixpath(f))
127 127 def drop(self, f):
128 128 return super(largefilesdirstate, self).drop(unixpath(f))
129 129 def forget(self, f):
130 130 return super(largefilesdirstate, self).forget(unixpath(f))
131 131 def normallookup(self, f):
132 132 return super(largefilesdirstate, self).normallookup(unixpath(f))
133 133 def _ignore(self, f):
134 134 return False
135 135 def write(self, tr=False):
136 136 # (1) disable PENDING mode always
137 137 # (lfdirstate isn't yet managed as a part of the transaction)
138 138 # (2) avoid develwarn 'use dirstate.write with ....'
139 139 super(largefilesdirstate, self).write(None)
140 140
141 141 def openlfdirstate(ui, repo, create=True):
142 142 '''
143 143 Return a dirstate object that tracks largefiles: i.e. its root is
144 144 the repo root, but it is saved in .hg/largefiles/dirstate.
145 145 '''
146 146 vfs = repo.vfs
147 147 lfstoredir = longname
148 148 opener = vfsmod.vfs(vfs.join(lfstoredir))
149 149 lfdirstate = largefilesdirstate(opener, ui, repo.root,
150 150 repo.dirstate._validate)
151 151
152 152 # If the largefiles dirstate does not exist, populate and create
153 153 # it. This ensures that we create it on the first meaningful
154 154 # largefiles operation in a new clone.
155 155 if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')):
156 156 matcher = getstandinmatcher(repo)
157 157 standins = repo.dirstate.walk(matcher, [], False, False)
158 158
159 159 if len(standins) > 0:
160 160 vfs.makedirs(lfstoredir)
161 161
162 162 for standin in standins:
163 163 lfile = splitstandin(standin)
164 164 lfdirstate.normallookup(lfile)
165 165 return lfdirstate
166 166
167 167 def lfdirstatestatus(lfdirstate, repo):
168 168 wctx = repo['.']
169 169 match = matchmod.always(repo.root, repo.getcwd())
170 170 unsure, s = lfdirstate.status(match, [], False, False, False)
171 171 modified, clean = s.modified, s.clean
172 172 for lfile in unsure:
173 173 try:
174 174 fctx = wctx[standin(lfile)]
175 175 except LookupError:
176 176 fctx = None
177 177 if not fctx or fctx.data().strip() != hashfile(repo.wjoin(lfile)):
178 178 modified.append(lfile)
179 179 else:
180 180 clean.append(lfile)
181 181 lfdirstate.normal(lfile)
182 182 return s
183 183
184 184 def listlfiles(repo, rev=None, matcher=None):
185 185 '''return a list of largefiles in the working copy or the
186 186 specified changeset'''
187 187
188 188 if matcher is None:
189 189 matcher = getstandinmatcher(repo)
190 190
191 191 # ignore unknown files in working directory
192 192 return [splitstandin(f)
193 193 for f in repo[rev].walk(matcher)
194 194 if rev is not None or repo.dirstate[f] != '?']
195 195
196 196 def instore(repo, hash, forcelocal=False):
197 197 '''Return true if a largefile with the given hash exists in the store'''
198 198 return os.path.exists(storepath(repo, hash, forcelocal))
199 199
200 200 def storepath(repo, hash, forcelocal=False):
201 201 '''Return the correct location in the repository largefiles store for a
202 202 file with the given hash.'''
203 203 if not forcelocal and repo.shared():
204 204 return repo.vfs.reljoin(repo.sharedpath, longname, hash)
205 205 return repo.vfs.join(longname, hash)
206 206
207 207 def findstorepath(repo, hash):
208 208 '''Search through the local store path(s) to find the file for the given
209 209 hash. If the file is not found, its path in the primary store is returned.
210 210 The return value is a tuple of (path, exists(path)).
211 211 '''
212 212 # For shared repos, the primary store is in the share source. But for
213 213 # backward compatibility, force a lookup in the local store if it wasn't
214 214 # found in the share source.
215 215 path = storepath(repo, hash, False)
216 216
217 217 if instore(repo, hash):
218 218 return (path, True)
219 219 elif repo.shared() and instore(repo, hash, True):
220 220 return storepath(repo, hash, True), True
221 221
222 222 return (path, False)
223 223
224 224 def copyfromcache(repo, hash, filename):
225 225 '''Copy the specified largefile from the repo or system cache to
226 226 filename in the repository. Return true on success or false if the
227 227 file was not found in either cache (which should not happened:
228 228 this is meant to be called only after ensuring that the needed
229 229 largefile exists in the cache).'''
230 230 wvfs = repo.wvfs
231 231 path = findfile(repo, hash)
232 232 if path is None:
233 233 return False
234 234 wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
235 235 # The write may fail before the file is fully written, but we
236 236 # don't use atomic writes in the working copy.
237 237 with open(path, 'rb') as srcfd:
238 238 with wvfs(filename, 'wb') as destfd:
239 239 gothash = copyandhash(
240 240 util.filechunkiter(srcfd), destfd)
241 241 if gothash != hash:
242 242 repo.ui.warn(_('%s: data corruption in %s with hash %s\n')
243 243 % (filename, path, gothash))
244 244 wvfs.unlink(filename)
245 245 return False
246 246 return True
247 247
248 def copytostore(repo, rev, file, uploaded=False):
248 def copytostore(repo, revorctx, file, uploaded=False):
249 249 wvfs = repo.wvfs
250 hash = readstandin(repo, file, rev)
250 hash = readstandin(repo, file, revorctx)
251 251 if instore(repo, hash):
252 252 return
253 253 if wvfs.exists(file):
254 254 copytostoreabsolute(repo, wvfs.join(file), hash)
255 255 else:
256 256 repo.ui.warn(_("%s: largefile %s not available from local store\n") %
257 257 (file, hash))
258 258
259 259 def copyalltostore(repo, node):
260 260 '''Copy all largefiles in a given revision to the store'''
261 261
262 262 ctx = repo[node]
263 263 for filename in ctx.files():
264 264 realfile = splitstandin(filename)
265 265 if realfile is not None and filename in ctx.manifest():
266 copytostore(repo, ctx.node(), realfile)
266 copytostore(repo, ctx, realfile)
267 267
268 268 def copytostoreabsolute(repo, file, hash):
269 269 if inusercache(repo.ui, hash):
270 270 link(usercachepath(repo.ui, hash), storepath(repo, hash))
271 271 else:
272 272 util.makedirs(os.path.dirname(storepath(repo, hash)))
273 273 with open(file, 'rb') as srcf:
274 274 with util.atomictempfile(storepath(repo, hash),
275 275 createmode=repo.store.createmode) as dstf:
276 276 for chunk in util.filechunkiter(srcf):
277 277 dstf.write(chunk)
278 278 linktousercache(repo, hash)
279 279
280 280 def linktousercache(repo, hash):
281 281 '''Link / copy the largefile with the specified hash from the store
282 282 to the cache.'''
283 283 path = usercachepath(repo.ui, hash)
284 284 link(storepath(repo, hash), path)
285 285
286 286 def getstandinmatcher(repo, rmatcher=None):
287 287 '''Return a match object that applies rmatcher to the standin directory'''
288 288 wvfs = repo.wvfs
289 289 standindir = shortname
290 290
291 291 # no warnings about missing files or directories
292 292 badfn = lambda f, msg: None
293 293
294 294 if rmatcher and not rmatcher.always():
295 295 pats = [wvfs.join(standindir, pat) for pat in rmatcher.files()]
296 296 if not pats:
297 297 pats = [wvfs.join(standindir)]
298 298 match = scmutil.match(repo[None], pats, badfn=badfn)
299 299 # if pats is empty, it would incorrectly always match, so clear _always
300 300 match._always = False
301 301 else:
302 302 # no patterns: relative to repo root
303 303 match = scmutil.match(repo[None], [wvfs.join(standindir)], badfn=badfn)
304 304 return match
305 305
306 306 def composestandinmatcher(repo, rmatcher):
307 307 '''Return a matcher that accepts standins corresponding to the
308 308 files accepted by rmatcher. Pass the list of files in the matcher
309 309 as the paths specified by the user.'''
310 310 smatcher = getstandinmatcher(repo, rmatcher)
311 311 isstandin = smatcher.matchfn
312 312 def composedmatchfn(f):
313 313 return isstandin(f) and rmatcher.matchfn(splitstandin(f))
314 314 smatcher.matchfn = composedmatchfn
315 315
316 316 return smatcher
317 317
318 318 def standin(filename):
319 319 '''Return the repo-relative path to the standin for the specified big
320 320 file.'''
321 321 # Notes:
322 322 # 1) Some callers want an absolute path, but for instance addlargefiles
323 323 # needs it repo-relative so it can be passed to repo[None].add(). So
324 324 # leave it up to the caller to use repo.wjoin() to get an absolute path.
325 325 # 2) Join with '/' because that's what dirstate always uses, even on
326 326 # Windows. Change existing separator to '/' first in case we are
327 327 # passed filenames from an external source (like the command line).
328 328 return shortnameslash + util.pconvert(filename)
329 329
330 330 def isstandin(filename):
331 331 '''Return true if filename is a big file standin. filename must be
332 332 in Mercurial's internal form (slash-separated).'''
333 333 return filename.startswith(shortnameslash)
334 334
335 335 def splitstandin(filename):
336 336 # Split on / because that's what dirstate always uses, even on Windows.
337 337 # Change local separator to / first just in case we are passed filenames
338 338 # from an external source (like the command line).
339 339 bits = util.pconvert(filename).split('/', 1)
340 340 if len(bits) == 2 and bits[0] == shortname:
341 341 return bits[1]
342 342 else:
343 343 return None
344 344
345 345 def updatestandin(repo, standin):
346 346 lfile = splitstandin(standin)
347 347 file = repo.wjoin(lfile)
348 348 if repo.wvfs.exists(lfile):
349 349 hash = hashfile(file)
350 350 executable = getexecutable(file)
351 351 writestandin(repo, standin, hash, executable)
352 352 else:
353 353 raise error.Abort(_('%s: file not found!') % lfile)
354 354
355 355 def readstandin(repo, filename, node=None):
356 356 '''read hex hash from standin for filename at given node, or working
357 357 directory if no node is given'''
358 358 return repo[node][standin(filename)].data().strip()
359 359
360 360 def writestandin(repo, standin, hash, executable):
361 361 '''write hash to <repo.root>/<standin>'''
362 362 repo.wwrite(standin, hash + '\n', executable and 'x' or '')
363 363
364 364 def copyandhash(instream, outfile):
365 365 '''Read bytes from instream (iterable) and write them to outfile,
366 366 computing the SHA-1 hash of the data along the way. Return the hash.'''
367 367 hasher = hashlib.sha1('')
368 368 for data in instream:
369 369 hasher.update(data)
370 370 outfile.write(data)
371 371 return hasher.hexdigest()
372 372
373 373 def hashrepofile(repo, file):
374 374 return hashfile(repo.wjoin(file))
375 375
376 376 def hashfile(file):
377 377 if not os.path.exists(file):
378 378 return ''
379 379 hasher = hashlib.sha1('')
380 380 with open(file, 'rb') as fd:
381 381 for data in util.filechunkiter(fd):
382 382 hasher.update(data)
383 383 return hasher.hexdigest()
384 384
385 385 def getexecutable(filename):
386 386 mode = os.stat(filename).st_mode
387 387 return ((mode & stat.S_IXUSR) and
388 388 (mode & stat.S_IXGRP) and
389 389 (mode & stat.S_IXOTH))
390 390
391 391 def urljoin(first, second, *arg):
392 392 def join(left, right):
393 393 if not left.endswith('/'):
394 394 left += '/'
395 395 if right.startswith('/'):
396 396 right = right[1:]
397 397 return left + right
398 398
399 399 url = join(first, second)
400 400 for a in arg:
401 401 url = join(url, a)
402 402 return url
403 403
404 404 def hexsha1(data):
405 405 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
406 406 object data"""
407 407 h = hashlib.sha1()
408 408 for chunk in util.filechunkiter(data):
409 409 h.update(chunk)
410 410 return h.hexdigest()
411 411
412 412 def httpsendfile(ui, filename):
413 413 return httpconnection.httpsendfile(ui, filename, 'rb')
414 414
415 415 def unixpath(path):
416 416 '''Return a version of path normalized for use with the lfdirstate.'''
417 417 return util.pconvert(os.path.normpath(path))
418 418
419 419 def islfilesrepo(repo):
420 420 '''Return true if the repo is a largefile repo.'''
421 421 if ('largefiles' in repo.requirements and
422 422 any(shortnameslash in f[0] for f in repo.store.datafiles())):
423 423 return True
424 424
425 425 return any(openlfdirstate(repo.ui, repo, False))
426 426
427 427 class storeprotonotcapable(Exception):
428 428 def __init__(self, storetypes):
429 429 self.storetypes = storetypes
430 430
431 431 def getstandinsstate(repo):
432 432 standins = []
433 433 matcher = getstandinmatcher(repo)
434 434 for standin in repo.dirstate.walk(matcher, [], False, False):
435 435 lfile = splitstandin(standin)
436 436 try:
437 437 hash = readstandin(repo, lfile)
438 438 except IOError:
439 439 hash = None
440 440 standins.append((lfile, hash))
441 441 return standins
442 442
443 443 def synclfdirstate(repo, lfdirstate, lfile, normallookup):
444 444 lfstandin = standin(lfile)
445 445 if lfstandin in repo.dirstate:
446 446 stat = repo.dirstate._map[lfstandin]
447 447 state, mtime = stat[0], stat[3]
448 448 else:
449 449 state, mtime = '?', -1
450 450 if state == 'n':
451 451 if (normallookup or mtime < 0 or
452 452 not repo.wvfs.exists(lfile)):
453 453 # state 'n' doesn't ensure 'clean' in this case
454 454 lfdirstate.normallookup(lfile)
455 455 else:
456 456 lfdirstate.normal(lfile)
457 457 elif state == 'm':
458 458 lfdirstate.normallookup(lfile)
459 459 elif state == 'r':
460 460 lfdirstate.remove(lfile)
461 461 elif state == 'a':
462 462 lfdirstate.add(lfile)
463 463 elif state == '?':
464 464 lfdirstate.drop(lfile)
465 465
466 466 def markcommitted(orig, ctx, node):
467 467 repo = ctx.repo()
468 468
469 469 orig(node)
470 470
471 471 # ATTENTION: "ctx.files()" may differ from "repo[node].files()"
472 472 # because files coming from the 2nd parent are omitted in the latter.
473 473 #
474 474 # The former should be used to get targets of "synclfdirstate",
475 475 # because such files:
476 476 # - are marked as "a" by "patch.patch()" (e.g. via transplant), and
477 477 # - have to be marked as "n" after commit, but
478 478 # - aren't listed in "repo[node].files()"
479 479
480 480 lfdirstate = openlfdirstate(repo.ui, repo)
481 481 for f in ctx.files():
482 482 lfile = splitstandin(f)
483 483 if lfile is not None:
484 484 synclfdirstate(repo, lfdirstate, lfile, False)
485 485 lfdirstate.write()
486 486
487 487 # As part of committing, copy all of the largefiles into the cache.
488 #
489 # Using "node" instead of "ctx" implies additional "repo[node]"
490 # lookup while copyalltostore(), but can omit redundant check for
491 # files comming from the 2nd parent, which should exist in store
492 # at merging.
488 493 copyalltostore(repo, node)
489 494
490 495 def getlfilestoupdate(oldstandins, newstandins):
491 496 changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
492 497 filelist = []
493 498 for f in changedstandins:
494 499 if f[0] not in filelist:
495 500 filelist.append(f[0])
496 501 return filelist
497 502
498 503 def getlfilestoupload(repo, missing, addfunc):
499 504 for i, n in enumerate(missing):
500 505 repo.ui.progress(_('finding outgoing largefiles'), i,
501 506 unit=_('revisions'), total=len(missing))
502 507 parents = [p for p in repo[n].parents() if p != node.nullid]
503 508
504 509 oldlfstatus = repo.lfstatus
505 510 repo.lfstatus = False
506 511 try:
507 512 ctx = repo[n]
508 513 finally:
509 514 repo.lfstatus = oldlfstatus
510 515
511 516 files = set(ctx.files())
512 517 if len(parents) == 2:
513 518 mc = ctx.manifest()
514 519 mp1 = ctx.parents()[0].manifest()
515 520 mp2 = ctx.parents()[1].manifest()
516 521 for f in mp1:
517 522 if f not in mc:
518 523 files.add(f)
519 524 for f in mp2:
520 525 if f not in mc:
521 526 files.add(f)
522 527 for f in mc:
523 528 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
524 529 files.add(f)
525 530 for fn in files:
526 531 if isstandin(fn) and fn in ctx:
527 532 addfunc(fn, ctx[fn].data().strip())
528 533 repo.ui.progress(_('finding outgoing largefiles'), None)
529 534
530 535 def updatestandinsbymatch(repo, match):
531 536 '''Update standins in the working directory according to specified match
532 537
533 538 This returns (possibly modified) ``match`` object to be used for
534 539 subsequent commit process.
535 540 '''
536 541
537 542 ui = repo.ui
538 543
539 544 # Case 1: user calls commit with no specific files or
540 545 # include/exclude patterns: refresh and commit all files that
541 546 # are "dirty".
542 547 if match is None or match.always():
543 548 # Spend a bit of time here to get a list of files we know
544 549 # are modified so we can compare only against those.
545 550 # It can cost a lot of time (several seconds)
546 551 # otherwise to update all standins if the largefiles are
547 552 # large.
548 553 lfdirstate = openlfdirstate(ui, repo)
549 554 dirtymatch = matchmod.always(repo.root, repo.getcwd())
550 555 unsure, s = lfdirstate.status(dirtymatch, [], False, False,
551 556 False)
552 557 modifiedfiles = unsure + s.modified + s.added + s.removed
553 558 lfiles = listlfiles(repo)
554 559 # this only loops through largefiles that exist (not
555 560 # removed/renamed)
556 561 for lfile in lfiles:
557 562 if lfile in modifiedfiles:
558 563 if repo.wvfs.exists(standin(lfile)):
559 564 # this handles the case where a rebase is being
560 565 # performed and the working copy is not updated
561 566 # yet.
562 567 if repo.wvfs.exists(lfile):
563 568 updatestandin(repo,
564 569 standin(lfile))
565 570
566 571 return match
567 572
568 573 lfiles = listlfiles(repo)
569 574 match._files = repo._subdirlfs(match.files(), lfiles)
570 575
571 576 # Case 2: user calls commit with specified patterns: refresh
572 577 # any matching big files.
573 578 smatcher = composestandinmatcher(repo, match)
574 579 standins = repo.dirstate.walk(smatcher, [], False, False)
575 580
576 581 # No matching big files: get out of the way and pass control to
577 582 # the usual commit() method.
578 583 if not standins:
579 584 return match
580 585
581 586 # Refresh all matching big files. It's possible that the
582 587 # commit will end up failing, in which case the big files will
583 588 # stay refreshed. No harm done: the user modified them and
584 589 # asked to commit them, so sooner or later we're going to
585 590 # refresh the standins. Might as well leave them refreshed.
586 591 lfdirstate = openlfdirstate(ui, repo)
587 592 for fstandin in standins:
588 593 lfile = splitstandin(fstandin)
589 594 if lfdirstate[lfile] != 'r':
590 595 updatestandin(repo, fstandin)
591 596
592 597 # Cook up a new matcher that only matches regular files or
593 598 # standins corresponding to the big files requested by the
594 599 # user. Have to modify _files to prevent commit() from
595 600 # complaining "not tracked" for big files.
596 601 match = copy.copy(match)
597 602 origmatchfn = match.matchfn
598 603
599 604 # Check both the list of largefiles and the list of
600 605 # standins because if a largefile was removed, it
601 606 # won't be in the list of largefiles at this point
602 607 match._files += sorted(standins)
603 608
604 609 actualfiles = []
605 610 for f in match._files:
606 611 fstandin = standin(f)
607 612
608 613 # For largefiles, only one of the normal and standin should be
609 614 # committed (except if one of them is a remove). In the case of a
610 615 # standin removal, drop the normal file if it is unknown to dirstate.
611 616 # Thus, skip plain largefile names but keep the standin.
612 617 if f in lfiles or fstandin in standins:
613 618 if repo.dirstate[fstandin] != 'r':
614 619 if repo.dirstate[f] != 'r':
615 620 continue
616 621 elif repo.dirstate[f] == '?':
617 622 continue
618 623
619 624 actualfiles.append(f)
620 625 match._files = actualfiles
621 626
622 627 def matchfn(f):
623 628 if origmatchfn(f):
624 629 return f not in lfiles
625 630 else:
626 631 return f in standins
627 632
628 633 match.matchfn = matchfn
629 634
630 635 return match
631 636
632 637 class automatedcommithook(object):
633 638 '''Stateful hook to update standins at the 1st commit of resuming
634 639
635 640 For efficiency, updating standins in the working directory should
636 641 be avoided while automated committing (like rebase, transplant and
637 642 so on), because they should be updated before committing.
638 643
639 644 But the 1st commit of resuming automated committing (e.g. ``rebase
640 645 --continue``) should update them, because largefiles may be
641 646 modified manually.
642 647 '''
643 648 def __init__(self, resuming):
644 649 self.resuming = resuming
645 650
646 651 def __call__(self, repo, match):
647 652 if self.resuming:
648 653 self.resuming = False # avoids updating at subsequent commits
649 654 return updatestandinsbymatch(repo, match)
650 655 else:
651 656 return match
652 657
653 658 def getstatuswriter(ui, repo, forcibly=None):
654 659 '''Return the function to write largefiles specific status out
655 660
656 661 If ``forcibly`` is ``None``, this returns the last element of
657 662 ``repo._lfstatuswriters`` as "default" writer function.
658 663
659 664 Otherwise, this returns the function to always write out (or
660 665 ignore if ``not forcibly``) status.
661 666 '''
662 667 if forcibly is None and util.safehasattr(repo, '_largefilesenabled'):
663 668 return repo._lfstatuswriters[-1]
664 669 else:
665 670 if forcibly:
666 671 return ui.status # forcibly WRITE OUT
667 672 else:
668 673 return lambda *msg, **opts: None # forcibly IGNORE
@@ -1,1451 +1,1451 b''
1 1 # Copyright 2009-2010 Gregory P. Ward
2 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 3 # Copyright 2010-2011 Fog Creek Software
4 4 # Copyright 2010-2011 Unity Technologies
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 '''Overridden Mercurial commands and functions for the largefiles extension'''
10 10 from __future__ import absolute_import
11 11
12 12 import copy
13 13 import os
14 14
15 15 from mercurial.i18n import _
16 16
17 17 from mercurial import (
18 18 archival,
19 19 cmdutil,
20 20 error,
21 21 hg,
22 22 match as matchmod,
23 23 pathutil,
24 24 registrar,
25 25 scmutil,
26 26 smartset,
27 27 util,
28 28 )
29 29
30 30 from . import (
31 31 lfcommands,
32 32 lfutil,
33 33 storefactory,
34 34 )
35 35
36 36 # -- Utility functions: commonly/repeatedly needed functionality ---------------
37 37
38 38 def composelargefilematcher(match, manifest):
39 39 '''create a matcher that matches only the largefiles in the original
40 40 matcher'''
41 41 m = copy.copy(match)
42 42 lfile = lambda f: lfutil.standin(f) in manifest
43 43 m._files = filter(lfile, m._files)
44 44 m._fileroots = set(m._files)
45 45 m._always = False
46 46 origmatchfn = m.matchfn
47 47 m.matchfn = lambda f: lfile(f) and origmatchfn(f)
48 48 return m
49 49
50 50 def composenormalfilematcher(match, manifest, exclude=None):
51 51 excluded = set()
52 52 if exclude is not None:
53 53 excluded.update(exclude)
54 54
55 55 m = copy.copy(match)
56 56 notlfile = lambda f: not (lfutil.isstandin(f) or lfutil.standin(f) in
57 57 manifest or f in excluded)
58 58 m._files = filter(notlfile, m._files)
59 59 m._fileroots = set(m._files)
60 60 m._always = False
61 61 origmatchfn = m.matchfn
62 62 m.matchfn = lambda f: notlfile(f) and origmatchfn(f)
63 63 return m
64 64
65 65 def installnormalfilesmatchfn(manifest):
66 66 '''installmatchfn with a matchfn that ignores all largefiles'''
67 67 def overridematch(ctx, pats=(), opts=None, globbed=False,
68 68 default='relpath', badfn=None):
69 69 if opts is None:
70 70 opts = {}
71 71 match = oldmatch(ctx, pats, opts, globbed, default, badfn=badfn)
72 72 return composenormalfilematcher(match, manifest)
73 73 oldmatch = installmatchfn(overridematch)
74 74
75 75 def installmatchfn(f):
76 76 '''monkey patch the scmutil module with a custom match function.
77 77 Warning: it is monkey patching the _module_ on runtime! Not thread safe!'''
78 78 oldmatch = scmutil.match
79 79 setattr(f, 'oldmatch', oldmatch)
80 80 scmutil.match = f
81 81 return oldmatch
82 82
83 83 def restorematchfn():
84 84 '''restores scmutil.match to what it was before installmatchfn
85 85 was called. no-op if scmutil.match is its original function.
86 86
87 87 Note that n calls to installmatchfn will require n calls to
88 88 restore the original matchfn.'''
89 89 scmutil.match = getattr(scmutil.match, 'oldmatch')
90 90
91 91 def installmatchandpatsfn(f):
92 92 oldmatchandpats = scmutil.matchandpats
93 93 setattr(f, 'oldmatchandpats', oldmatchandpats)
94 94 scmutil.matchandpats = f
95 95 return oldmatchandpats
96 96
97 97 def restorematchandpatsfn():
98 98 '''restores scmutil.matchandpats to what it was before
99 99 installmatchandpatsfn was called. No-op if scmutil.matchandpats
100 100 is its original function.
101 101
102 102 Note that n calls to installmatchandpatsfn will require n calls
103 103 to restore the original matchfn.'''
104 104 scmutil.matchandpats = getattr(scmutil.matchandpats, 'oldmatchandpats',
105 105 scmutil.matchandpats)
106 106
107 107 def addlargefiles(ui, repo, isaddremove, matcher, **opts):
108 108 large = opts.get('large')
109 109 lfsize = lfutil.getminsize(
110 110 ui, lfutil.islfilesrepo(repo), opts.get('lfsize'))
111 111
112 112 lfmatcher = None
113 113 if lfutil.islfilesrepo(repo):
114 114 lfpats = ui.configlist(lfutil.longname, 'patterns', default=[])
115 115 if lfpats:
116 116 lfmatcher = matchmod.match(repo.root, '', list(lfpats))
117 117
118 118 lfnames = []
119 119 m = matcher
120 120
121 121 wctx = repo[None]
122 122 for f in repo.walk(matchmod.badmatch(m, lambda x, y: None)):
123 123 exact = m.exact(f)
124 124 lfile = lfutil.standin(f) in wctx
125 125 nfile = f in wctx
126 126 exists = lfile or nfile
127 127
128 128 # addremove in core gets fancy with the name, add doesn't
129 129 if isaddremove:
130 130 name = m.uipath(f)
131 131 else:
132 132 name = m.rel(f)
133 133
134 134 # Don't warn the user when they attempt to add a normal tracked file.
135 135 # The normal add code will do that for us.
136 136 if exact and exists:
137 137 if lfile:
138 138 ui.warn(_('%s already a largefile\n') % name)
139 139 continue
140 140
141 141 if (exact or not exists) and not lfutil.isstandin(f):
142 142 # In case the file was removed previously, but not committed
143 143 # (issue3507)
144 144 if not repo.wvfs.exists(f):
145 145 continue
146 146
147 147 abovemin = (lfsize and
148 148 repo.wvfs.lstat(f).st_size >= lfsize * 1024 * 1024)
149 149 if large or abovemin or (lfmatcher and lfmatcher(f)):
150 150 lfnames.append(f)
151 151 if ui.verbose or not exact:
152 152 ui.status(_('adding %s as a largefile\n') % name)
153 153
154 154 bad = []
155 155
156 156 # Need to lock, otherwise there could be a race condition between
157 157 # when standins are created and added to the repo.
158 158 with repo.wlock():
159 159 if not opts.get('dry_run'):
160 160 standins = []
161 161 lfdirstate = lfutil.openlfdirstate(ui, repo)
162 162 for f in lfnames:
163 163 standinname = lfutil.standin(f)
164 164 lfutil.writestandin(repo, standinname, hash='',
165 165 executable=lfutil.getexecutable(repo.wjoin(f)))
166 166 standins.append(standinname)
167 167 if lfdirstate[f] == 'r':
168 168 lfdirstate.normallookup(f)
169 169 else:
170 170 lfdirstate.add(f)
171 171 lfdirstate.write()
172 172 bad += [lfutil.splitstandin(f)
173 173 for f in repo[None].add(standins)
174 174 if f in m.files()]
175 175
176 176 added = [f for f in lfnames if f not in bad]
177 177 return added, bad
178 178
179 179 def removelargefiles(ui, repo, isaddremove, matcher, **opts):
180 180 after = opts.get('after')
181 181 m = composelargefilematcher(matcher, repo[None].manifest())
182 182 try:
183 183 repo.lfstatus = True
184 184 s = repo.status(match=m, clean=not isaddremove)
185 185 finally:
186 186 repo.lfstatus = False
187 187 manifest = repo[None].manifest()
188 188 modified, added, deleted, clean = [[f for f in list
189 189 if lfutil.standin(f) in manifest]
190 190 for list in (s.modified, s.added,
191 191 s.deleted, s.clean)]
192 192
193 193 def warn(files, msg):
194 194 for f in files:
195 195 ui.warn(msg % m.rel(f))
196 196 return int(len(files) > 0)
197 197
198 198 result = 0
199 199
200 200 if after:
201 201 remove = deleted
202 202 result = warn(modified + added + clean,
203 203 _('not removing %s: file still exists\n'))
204 204 else:
205 205 remove = deleted + clean
206 206 result = warn(modified, _('not removing %s: file is modified (use -f'
207 207 ' to force removal)\n'))
208 208 result = warn(added, _('not removing %s: file has been marked for add'
209 209 ' (use forget to undo)\n')) or result
210 210
211 211 # Need to lock because standin files are deleted then removed from the
212 212 # repository and we could race in-between.
213 213 with repo.wlock():
214 214 lfdirstate = lfutil.openlfdirstate(ui, repo)
215 215 for f in sorted(remove):
216 216 if ui.verbose or not m.exact(f):
217 217 # addremove in core gets fancy with the name, remove doesn't
218 218 if isaddremove:
219 219 name = m.uipath(f)
220 220 else:
221 221 name = m.rel(f)
222 222 ui.status(_('removing %s\n') % name)
223 223
224 224 if not opts.get('dry_run'):
225 225 if not after:
226 226 repo.wvfs.unlinkpath(f, ignoremissing=True)
227 227
228 228 if opts.get('dry_run'):
229 229 return result
230 230
231 231 remove = [lfutil.standin(f) for f in remove]
232 232 # If this is being called by addremove, let the original addremove
233 233 # function handle this.
234 234 if not isaddremove:
235 235 for f in remove:
236 236 repo.wvfs.unlinkpath(f, ignoremissing=True)
237 237 repo[None].forget(remove)
238 238
239 239 for f in remove:
240 240 lfutil.synclfdirstate(repo, lfdirstate, lfutil.splitstandin(f),
241 241 False)
242 242
243 243 lfdirstate.write()
244 244
245 245 return result
246 246
247 247 # For overriding mercurial.hgweb.webcommands so that largefiles will
248 248 # appear at their right place in the manifests.
249 249 def decodepath(orig, path):
250 250 return lfutil.splitstandin(path) or path
251 251
252 252 # -- Wrappers: modify existing commands --------------------------------
253 253
254 254 def overrideadd(orig, ui, repo, *pats, **opts):
255 255 if opts.get('normal') and opts.get('large'):
256 256 raise error.Abort(_('--normal cannot be used with --large'))
257 257 return orig(ui, repo, *pats, **opts)
258 258
259 259 def cmdutiladd(orig, ui, repo, matcher, prefix, explicitonly, **opts):
260 260 # The --normal flag short circuits this override
261 261 if opts.get('normal'):
262 262 return orig(ui, repo, matcher, prefix, explicitonly, **opts)
263 263
264 264 ladded, lbad = addlargefiles(ui, repo, False, matcher, **opts)
265 265 normalmatcher = composenormalfilematcher(matcher, repo[None].manifest(),
266 266 ladded)
267 267 bad = orig(ui, repo, normalmatcher, prefix, explicitonly, **opts)
268 268
269 269 bad.extend(f for f in lbad)
270 270 return bad
271 271
272 272 def cmdutilremove(orig, ui, repo, matcher, prefix, after, force, subrepos):
273 273 normalmatcher = composenormalfilematcher(matcher, repo[None].manifest())
274 274 result = orig(ui, repo, normalmatcher, prefix, after, force, subrepos)
275 275 return removelargefiles(ui, repo, False, matcher, after=after,
276 276 force=force) or result
277 277
278 278 def overridestatusfn(orig, repo, rev2, **opts):
279 279 try:
280 280 repo._repo.lfstatus = True
281 281 return orig(repo, rev2, **opts)
282 282 finally:
283 283 repo._repo.lfstatus = False
284 284
285 285 def overridestatus(orig, ui, repo, *pats, **opts):
286 286 try:
287 287 repo.lfstatus = True
288 288 return orig(ui, repo, *pats, **opts)
289 289 finally:
290 290 repo.lfstatus = False
291 291
292 292 def overridedirty(orig, repo, ignoreupdate=False):
293 293 try:
294 294 repo._repo.lfstatus = True
295 295 return orig(repo, ignoreupdate)
296 296 finally:
297 297 repo._repo.lfstatus = False
298 298
299 299 def overridelog(orig, ui, repo, *pats, **opts):
300 300 def overridematchandpats(ctx, pats=(), opts=None, globbed=False,
301 301 default='relpath', badfn=None):
302 302 """Matcher that merges root directory with .hglf, suitable for log.
303 303 It is still possible to match .hglf directly.
304 304 For any listed files run log on the standin too.
305 305 matchfn tries both the given filename and with .hglf stripped.
306 306 """
307 307 if opts is None:
308 308 opts = {}
309 309 matchandpats = oldmatchandpats(ctx, pats, opts, globbed, default,
310 310 badfn=badfn)
311 311 m, p = copy.copy(matchandpats)
312 312
313 313 if m.always():
314 314 # We want to match everything anyway, so there's no benefit trying
315 315 # to add standins.
316 316 return matchandpats
317 317
318 318 pats = set(p)
319 319
320 320 def fixpats(pat, tostandin=lfutil.standin):
321 321 if pat.startswith('set:'):
322 322 return pat
323 323
324 324 kindpat = matchmod._patsplit(pat, None)
325 325
326 326 if kindpat[0] is not None:
327 327 return kindpat[0] + ':' + tostandin(kindpat[1])
328 328 return tostandin(kindpat[1])
329 329
330 330 if m._cwd:
331 331 hglf = lfutil.shortname
332 332 back = util.pconvert(m.rel(hglf)[:-len(hglf)])
333 333
334 334 def tostandin(f):
335 335 # The file may already be a standin, so truncate the back
336 336 # prefix and test before mangling it. This avoids turning
337 337 # 'glob:../.hglf/foo*' into 'glob:../.hglf/../.hglf/foo*'.
338 338 if f.startswith(back) and lfutil.splitstandin(f[len(back):]):
339 339 return f
340 340
341 341 # An absolute path is from outside the repo, so truncate the
342 342 # path to the root before building the standin. Otherwise cwd
343 343 # is somewhere in the repo, relative to root, and needs to be
344 344 # prepended before building the standin.
345 345 if os.path.isabs(m._cwd):
346 346 f = f[len(back):]
347 347 else:
348 348 f = m._cwd + '/' + f
349 349 return back + lfutil.standin(f)
350 350
351 351 pats.update(fixpats(f, tostandin) for f in p)
352 352 else:
353 353 def tostandin(f):
354 354 if lfutil.isstandin(f):
355 355 return f
356 356 return lfutil.standin(f)
357 357 pats.update(fixpats(f, tostandin) for f in p)
358 358
359 359 for i in range(0, len(m._files)):
360 360 # Don't add '.hglf' to m.files, since that is already covered by '.'
361 361 if m._files[i] == '.':
362 362 continue
363 363 standin = lfutil.standin(m._files[i])
364 364 # If the "standin" is a directory, append instead of replace to
365 365 # support naming a directory on the command line with only
366 366 # largefiles. The original directory is kept to support normal
367 367 # files.
368 368 if standin in repo[ctx.node()]:
369 369 m._files[i] = standin
370 370 elif m._files[i] not in repo[ctx.node()] \
371 371 and repo.wvfs.isdir(standin):
372 372 m._files.append(standin)
373 373
374 374 m._fileroots = set(m._files)
375 375 m._always = False
376 376 origmatchfn = m.matchfn
377 377 def lfmatchfn(f):
378 378 lf = lfutil.splitstandin(f)
379 379 if lf is not None and origmatchfn(lf):
380 380 return True
381 381 r = origmatchfn(f)
382 382 return r
383 383 m.matchfn = lfmatchfn
384 384
385 385 ui.debug('updated patterns: %s\n' % sorted(pats))
386 386 return m, pats
387 387
388 388 # For hg log --patch, the match object is used in two different senses:
389 389 # (1) to determine what revisions should be printed out, and
390 390 # (2) to determine what files to print out diffs for.
391 391 # The magic matchandpats override should be used for case (1) but not for
392 392 # case (2).
393 393 def overridemakelogfilematcher(repo, pats, opts, badfn=None):
394 394 wctx = repo[None]
395 395 match, pats = oldmatchandpats(wctx, pats, opts, badfn=badfn)
396 396 return lambda rev: match
397 397
398 398 oldmatchandpats = installmatchandpatsfn(overridematchandpats)
399 399 oldmakelogfilematcher = cmdutil._makenofollowlogfilematcher
400 400 setattr(cmdutil, '_makenofollowlogfilematcher', overridemakelogfilematcher)
401 401
402 402 try:
403 403 return orig(ui, repo, *pats, **opts)
404 404 finally:
405 405 restorematchandpatsfn()
406 406 setattr(cmdutil, '_makenofollowlogfilematcher', oldmakelogfilematcher)
407 407
408 408 def overrideverify(orig, ui, repo, *pats, **opts):
409 409 large = opts.pop('large', False)
410 410 all = opts.pop('lfa', False)
411 411 contents = opts.pop('lfc', False)
412 412
413 413 result = orig(ui, repo, *pats, **opts)
414 414 if large or all or contents:
415 415 result = result or lfcommands.verifylfiles(ui, repo, all, contents)
416 416 return result
417 417
418 418 def overridedebugstate(orig, ui, repo, *pats, **opts):
419 419 large = opts.pop('large', False)
420 420 if large:
421 421 class fakerepo(object):
422 422 dirstate = lfutil.openlfdirstate(ui, repo)
423 423 orig(ui, fakerepo, *pats, **opts)
424 424 else:
425 425 orig(ui, repo, *pats, **opts)
426 426
427 427 # Before starting the manifest merge, merge.updates will call
428 428 # _checkunknownfile to check if there are any files in the merged-in
429 429 # changeset that collide with unknown files in the working copy.
430 430 #
431 431 # The largefiles are seen as unknown, so this prevents us from merging
432 432 # in a file 'foo' if we already have a largefile with the same name.
433 433 #
434 434 # The overridden function filters the unknown files by removing any
435 435 # largefiles. This makes the merge proceed and we can then handle this
436 436 # case further in the overridden calculateupdates function below.
437 437 def overridecheckunknownfile(origfn, repo, wctx, mctx, f, f2=None):
438 438 if lfutil.standin(repo.dirstate.normalize(f)) in wctx:
439 439 return False
440 440 return origfn(repo, wctx, mctx, f, f2)
441 441
442 442 # The manifest merge handles conflicts on the manifest level. We want
443 443 # to handle changes in largefile-ness of files at this level too.
444 444 #
445 445 # The strategy is to run the original calculateupdates and then process
446 446 # the action list it outputs. There are two cases we need to deal with:
447 447 #
448 448 # 1. Normal file in p1, largefile in p2. Here the largefile is
449 449 # detected via its standin file, which will enter the working copy
450 450 # with a "get" action. It is not "merge" since the standin is all
451 451 # Mercurial is concerned with at this level -- the link to the
452 452 # existing normal file is not relevant here.
453 453 #
454 454 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
455 455 # since the largefile will be present in the working copy and
456 456 # different from the normal file in p2. Mercurial therefore
457 457 # triggers a merge action.
458 458 #
459 459 # In both cases, we prompt the user and emit new actions to either
460 460 # remove the standin (if the normal file was kept) or to remove the
461 461 # normal file and get the standin (if the largefile was kept). The
462 462 # default prompt answer is to use the largefile version since it was
463 463 # presumably changed on purpose.
464 464 #
465 465 # Finally, the merge.applyupdates function will then take care of
466 466 # writing the files into the working copy and lfcommands.updatelfiles
467 467 # will update the largefiles.
468 468 def overridecalculateupdates(origfn, repo, p1, p2, pas, branchmerge, force,
469 469 acceptremote, *args, **kwargs):
470 470 overwrite = force and not branchmerge
471 471 actions, diverge, renamedelete = origfn(
472 472 repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs)
473 473
474 474 if overwrite:
475 475 return actions, diverge, renamedelete
476 476
477 477 # Convert to dictionary with filename as key and action as value.
478 478 lfiles = set()
479 479 for f in actions:
480 480 splitstandin = lfutil.splitstandin(f)
481 481 if splitstandin in p1:
482 482 lfiles.add(splitstandin)
483 483 elif lfutil.standin(f) in p1:
484 484 lfiles.add(f)
485 485
486 486 for lfile in sorted(lfiles):
487 487 standin = lfutil.standin(lfile)
488 488 (lm, largs, lmsg) = actions.get(lfile, (None, None, None))
489 489 (sm, sargs, smsg) = actions.get(standin, (None, None, None))
490 490 if sm in ('g', 'dc') and lm != 'r':
491 491 if sm == 'dc':
492 492 f1, f2, fa, move, anc = sargs
493 493 sargs = (p2[f2].flags(), False)
494 494 # Case 1: normal file in the working copy, largefile in
495 495 # the second parent
496 496 usermsg = _('remote turned local normal file %s into a largefile\n'
497 497 'use (l)argefile or keep (n)ormal file?'
498 498 '$$ &Largefile $$ &Normal file') % lfile
499 499 if repo.ui.promptchoice(usermsg, 0) == 0: # pick remote largefile
500 500 actions[lfile] = ('r', None, 'replaced by standin')
501 501 actions[standin] = ('g', sargs, 'replaces standin')
502 502 else: # keep local normal file
503 503 actions[lfile] = ('k', None, 'replaces standin')
504 504 if branchmerge:
505 505 actions[standin] = ('k', None, 'replaced by non-standin')
506 506 else:
507 507 actions[standin] = ('r', None, 'replaced by non-standin')
508 508 elif lm in ('g', 'dc') and sm != 'r':
509 509 if lm == 'dc':
510 510 f1, f2, fa, move, anc = largs
511 511 largs = (p2[f2].flags(), False)
512 512 # Case 2: largefile in the working copy, normal file in
513 513 # the second parent
514 514 usermsg = _('remote turned local largefile %s into a normal file\n'
515 515 'keep (l)argefile or use (n)ormal file?'
516 516 '$$ &Largefile $$ &Normal file') % lfile
517 517 if repo.ui.promptchoice(usermsg, 0) == 0: # keep local largefile
518 518 if branchmerge:
519 519 # largefile can be restored from standin safely
520 520 actions[lfile] = ('k', None, 'replaced by standin')
521 521 actions[standin] = ('k', None, 'replaces standin')
522 522 else:
523 523 # "lfile" should be marked as "removed" without
524 524 # removal of itself
525 525 actions[lfile] = ('lfmr', None,
526 526 'forget non-standin largefile')
527 527
528 528 # linear-merge should treat this largefile as 're-added'
529 529 actions[standin] = ('a', None, 'keep standin')
530 530 else: # pick remote normal file
531 531 actions[lfile] = ('g', largs, 'replaces standin')
532 532 actions[standin] = ('r', None, 'replaced by non-standin')
533 533
534 534 return actions, diverge, renamedelete
535 535
536 536 def mergerecordupdates(orig, repo, actions, branchmerge):
537 537 if 'lfmr' in actions:
538 538 lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
539 539 for lfile, args, msg in actions['lfmr']:
540 540 # this should be executed before 'orig', to execute 'remove'
541 541 # before all other actions
542 542 repo.dirstate.remove(lfile)
543 543 # make sure lfile doesn't get synclfdirstate'd as normal
544 544 lfdirstate.add(lfile)
545 545 lfdirstate.write()
546 546
547 547 return orig(repo, actions, branchmerge)
548 548
549 549 # Override filemerge to prompt the user about how they wish to merge
550 550 # largefiles. This will handle identical edits without prompting the user.
551 551 def overridefilemerge(origfn, premerge, repo, mynode, orig, fcd, fco, fca,
552 552 labels=None):
553 553 if not lfutil.isstandin(orig) or fcd.isabsent() or fco.isabsent():
554 554 return origfn(premerge, repo, mynode, orig, fcd, fco, fca,
555 555 labels=labels)
556 556
557 557 ahash = fca.data().strip().lower()
558 558 dhash = fcd.data().strip().lower()
559 559 ohash = fco.data().strip().lower()
560 560 if (ohash != ahash and
561 561 ohash != dhash and
562 562 (dhash == ahash or
563 563 repo.ui.promptchoice(
564 564 _('largefile %s has a merge conflict\nancestor was %s\n'
565 565 'keep (l)ocal %s or\ntake (o)ther %s?'
566 566 '$$ &Local $$ &Other') %
567 567 (lfutil.splitstandin(orig), ahash, dhash, ohash),
568 568 0) == 1)):
569 569 repo.wwrite(fcd.path(), fco.data(), fco.flags())
570 570 return True, 0, False
571 571
572 572 def copiespathcopies(orig, ctx1, ctx2, match=None):
573 573 copies = orig(ctx1, ctx2, match=match)
574 574 updated = {}
575 575
576 576 for k, v in copies.iteritems():
577 577 updated[lfutil.splitstandin(k) or k] = lfutil.splitstandin(v) or v
578 578
579 579 return updated
580 580
581 581 # Copy first changes the matchers to match standins instead of
582 582 # largefiles. Then it overrides util.copyfile in that function it
583 583 # checks if the destination largefile already exists. It also keeps a
584 584 # list of copied files so that the largefiles can be copied and the
585 585 # dirstate updated.
586 586 def overridecopy(orig, ui, repo, pats, opts, rename=False):
587 587 # doesn't remove largefile on rename
588 588 if len(pats) < 2:
589 589 # this isn't legal, let the original function deal with it
590 590 return orig(ui, repo, pats, opts, rename)
591 591
592 592 # This could copy both lfiles and normal files in one command,
593 593 # but we don't want to do that. First replace their matcher to
594 594 # only match normal files and run it, then replace it to just
595 595 # match largefiles and run it again.
596 596 nonormalfiles = False
597 597 nolfiles = False
598 598 installnormalfilesmatchfn(repo[None].manifest())
599 599 try:
600 600 result = orig(ui, repo, pats, opts, rename)
601 601 except error.Abort as e:
602 602 if str(e) != _('no files to copy'):
603 603 raise e
604 604 else:
605 605 nonormalfiles = True
606 606 result = 0
607 607 finally:
608 608 restorematchfn()
609 609
610 610 # The first rename can cause our current working directory to be removed.
611 611 # In that case there is nothing left to copy/rename so just quit.
612 612 try:
613 613 repo.getcwd()
614 614 except OSError:
615 615 return result
616 616
617 617 def makestandin(relpath):
618 618 path = pathutil.canonpath(repo.root, repo.getcwd(), relpath)
619 619 return repo.wvfs.join(lfutil.standin(path))
620 620
621 621 fullpats = scmutil.expandpats(pats)
622 622 dest = fullpats[-1]
623 623
624 624 if os.path.isdir(dest):
625 625 if not os.path.isdir(makestandin(dest)):
626 626 os.makedirs(makestandin(dest))
627 627
628 628 try:
629 629 # When we call orig below it creates the standins but we don't add
630 630 # them to the dir state until later so lock during that time.
631 631 wlock = repo.wlock()
632 632
633 633 manifest = repo[None].manifest()
634 634 def overridematch(ctx, pats=(), opts=None, globbed=False,
635 635 default='relpath', badfn=None):
636 636 if opts is None:
637 637 opts = {}
638 638 newpats = []
639 639 # The patterns were previously mangled to add the standin
640 640 # directory; we need to remove that now
641 641 for pat in pats:
642 642 if matchmod.patkind(pat) is None and lfutil.shortname in pat:
643 643 newpats.append(pat.replace(lfutil.shortname, ''))
644 644 else:
645 645 newpats.append(pat)
646 646 match = oldmatch(ctx, newpats, opts, globbed, default, badfn=badfn)
647 647 m = copy.copy(match)
648 648 lfile = lambda f: lfutil.standin(f) in manifest
649 649 m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
650 650 m._fileroots = set(m._files)
651 651 origmatchfn = m.matchfn
652 652 def matchfn(f):
653 653 lfile = lfutil.splitstandin(f)
654 654 return (lfile is not None and
655 655 (f in manifest) and
656 656 origmatchfn(lfile) or
657 657 None)
658 658 m.matchfn = matchfn
659 659 return m
660 660 oldmatch = installmatchfn(overridematch)
661 661 listpats = []
662 662 for pat in pats:
663 663 if matchmod.patkind(pat) is not None:
664 664 listpats.append(pat)
665 665 else:
666 666 listpats.append(makestandin(pat))
667 667
668 668 try:
669 669 origcopyfile = util.copyfile
670 670 copiedfiles = []
671 671 def overridecopyfile(src, dest):
672 672 if (lfutil.shortname in src and
673 673 dest.startswith(repo.wjoin(lfutil.shortname))):
674 674 destlfile = dest.replace(lfutil.shortname, '')
675 675 if not opts['force'] and os.path.exists(destlfile):
676 676 raise IOError('',
677 677 _('destination largefile already exists'))
678 678 copiedfiles.append((src, dest))
679 679 origcopyfile(src, dest)
680 680
681 681 util.copyfile = overridecopyfile
682 682 result += orig(ui, repo, listpats, opts, rename)
683 683 finally:
684 684 util.copyfile = origcopyfile
685 685
686 686 lfdirstate = lfutil.openlfdirstate(ui, repo)
687 687 for (src, dest) in copiedfiles:
688 688 if (lfutil.shortname in src and
689 689 dest.startswith(repo.wjoin(lfutil.shortname))):
690 690 srclfile = src.replace(repo.wjoin(lfutil.standin('')), '')
691 691 destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '')
692 692 destlfiledir = repo.wvfs.dirname(repo.wjoin(destlfile)) or '.'
693 693 if not os.path.isdir(destlfiledir):
694 694 os.makedirs(destlfiledir)
695 695 if rename:
696 696 os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
697 697
698 698 # The file is gone, but this deletes any empty parent
699 699 # directories as a side-effect.
700 700 repo.wvfs.unlinkpath(srclfile, ignoremissing=True)
701 701 lfdirstate.remove(srclfile)
702 702 else:
703 703 util.copyfile(repo.wjoin(srclfile),
704 704 repo.wjoin(destlfile))
705 705
706 706 lfdirstate.add(destlfile)
707 707 lfdirstate.write()
708 708 except error.Abort as e:
709 709 if str(e) != _('no files to copy'):
710 710 raise e
711 711 else:
712 712 nolfiles = True
713 713 finally:
714 714 restorematchfn()
715 715 wlock.release()
716 716
717 717 if nolfiles and nonormalfiles:
718 718 raise error.Abort(_('no files to copy'))
719 719
720 720 return result
721 721
722 722 # When the user calls revert, we have to be careful to not revert any
723 723 # changes to other largefiles accidentally. This means we have to keep
724 724 # track of the largefiles that are being reverted so we only pull down
725 725 # the necessary largefiles.
726 726 #
727 727 # Standins are only updated (to match the hash of largefiles) before
728 728 # commits. Update the standins then run the original revert, changing
729 729 # the matcher to hit standins instead of largefiles. Based on the
730 730 # resulting standins update the largefiles.
731 731 def overriderevert(orig, ui, repo, ctx, parents, *pats, **opts):
732 732 # Because we put the standins in a bad state (by updating them)
733 733 # and then return them to a correct state we need to lock to
734 734 # prevent others from changing them in their incorrect state.
735 735 with repo.wlock():
736 736 lfdirstate = lfutil.openlfdirstate(ui, repo)
737 737 s = lfutil.lfdirstatestatus(lfdirstate, repo)
738 738 lfdirstate.write()
739 739 for lfile in s.modified:
740 740 lfutil.updatestandin(repo, lfutil.standin(lfile))
741 741 for lfile in s.deleted:
742 742 if (repo.wvfs.exists(lfutil.standin(lfile))):
743 743 repo.wvfs.unlink(lfutil.standin(lfile))
744 744
745 745 oldstandins = lfutil.getstandinsstate(repo)
746 746
747 747 def overridematch(mctx, pats=(), opts=None, globbed=False,
748 748 default='relpath', badfn=None):
749 749 if opts is None:
750 750 opts = {}
751 751 match = oldmatch(mctx, pats, opts, globbed, default, badfn=badfn)
752 752 m = copy.copy(match)
753 753
754 754 # revert supports recursing into subrepos, and though largefiles
755 755 # currently doesn't work correctly in that case, this match is
756 756 # called, so the lfdirstate above may not be the correct one for
757 757 # this invocation of match.
758 758 lfdirstate = lfutil.openlfdirstate(mctx.repo().ui, mctx.repo(),
759 759 False)
760 760
761 761 def tostandin(f):
762 762 standin = lfutil.standin(f)
763 763 if standin in ctx or standin in mctx:
764 764 return standin
765 765 elif standin in repo[None] or lfdirstate[f] == 'r':
766 766 return None
767 767 return f
768 768 m._files = [tostandin(f) for f in m._files]
769 769 m._files = [f for f in m._files if f is not None]
770 770 m._fileroots = set(m._files)
771 771 origmatchfn = m.matchfn
772 772 def matchfn(f):
773 773 lfile = lfutil.splitstandin(f)
774 774 if lfile is not None:
775 775 return (origmatchfn(lfile) and
776 776 (f in ctx or f in mctx))
777 777 return origmatchfn(f)
778 778 m.matchfn = matchfn
779 779 return m
780 780 oldmatch = installmatchfn(overridematch)
781 781 try:
782 782 orig(ui, repo, ctx, parents, *pats, **opts)
783 783 finally:
784 784 restorematchfn()
785 785
786 786 newstandins = lfutil.getstandinsstate(repo)
787 787 filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
788 788 # lfdirstate should be 'normallookup'-ed for updated files,
789 789 # because reverting doesn't touch dirstate for 'normal' files
790 790 # when target revision is explicitly specified: in such case,
791 791 # 'n' and valid timestamp in dirstate doesn't ensure 'clean'
792 792 # of target (standin) file.
793 793 lfcommands.updatelfiles(ui, repo, filelist, printmessage=False,
794 794 normallookup=True)
795 795
796 796 # after pulling changesets, we need to take some extra care to get
797 797 # largefiles updated remotely
798 798 def overridepull(orig, ui, repo, source=None, **opts):
799 799 revsprepull = len(repo)
800 800 if not source:
801 801 source = 'default'
802 802 repo.lfpullsource = source
803 803 result = orig(ui, repo, source, **opts)
804 804 revspostpull = len(repo)
805 805 lfrevs = opts.get('lfrev', [])
806 806 if opts.get('all_largefiles'):
807 807 lfrevs.append('pulled()')
808 808 if lfrevs and revspostpull > revsprepull:
809 809 numcached = 0
810 810 repo.firstpulled = revsprepull # for pulled() revset expression
811 811 try:
812 812 for rev in scmutil.revrange(repo, lfrevs):
813 813 ui.note(_('pulling largefiles for revision %s\n') % rev)
814 814 (cached, missing) = lfcommands.cachelfiles(ui, repo, rev)
815 815 numcached += len(cached)
816 816 finally:
817 817 del repo.firstpulled
818 818 ui.status(_("%d largefiles cached\n") % numcached)
819 819 return result
820 820
821 821 def overridepush(orig, ui, repo, *args, **kwargs):
822 822 """Override push command and store --lfrev parameters in opargs"""
823 823 lfrevs = kwargs.pop('lfrev', None)
824 824 if lfrevs:
825 825 opargs = kwargs.setdefault('opargs', {})
826 826 opargs['lfrevs'] = scmutil.revrange(repo, lfrevs)
827 827 return orig(ui, repo, *args, **kwargs)
828 828
829 829 def exchangepushoperation(orig, *args, **kwargs):
830 830 """Override pushoperation constructor and store lfrevs parameter"""
831 831 lfrevs = kwargs.pop('lfrevs', None)
832 832 pushop = orig(*args, **kwargs)
833 833 pushop.lfrevs = lfrevs
834 834 return pushop
835 835
836 836 revsetpredicate = registrar.revsetpredicate()
837 837
838 838 @revsetpredicate('pulled()')
839 839 def pulledrevsetsymbol(repo, subset, x):
840 840 """Changesets that just has been pulled.
841 841
842 842 Only available with largefiles from pull --lfrev expressions.
843 843
844 844 .. container:: verbose
845 845
846 846 Some examples:
847 847
848 848 - pull largefiles for all new changesets::
849 849
850 850 hg pull -lfrev "pulled()"
851 851
852 852 - pull largefiles for all new branch heads::
853 853
854 854 hg pull -lfrev "head(pulled()) and not closed()"
855 855
856 856 """
857 857
858 858 try:
859 859 firstpulled = repo.firstpulled
860 860 except AttributeError:
861 861 raise error.Abort(_("pulled() only available in --lfrev"))
862 862 return smartset.baseset([r for r in subset if r >= firstpulled])
863 863
864 864 def overrideclone(orig, ui, source, dest=None, **opts):
865 865 d = dest
866 866 if d is None:
867 867 d = hg.defaultdest(source)
868 868 if opts.get('all_largefiles') and not hg.islocal(d):
869 869 raise error.Abort(_(
870 870 '--all-largefiles is incompatible with non-local destination %s') %
871 871 d)
872 872
873 873 return orig(ui, source, dest, **opts)
874 874
875 875 def hgclone(orig, ui, opts, *args, **kwargs):
876 876 result = orig(ui, opts, *args, **kwargs)
877 877
878 878 if result is not None:
879 879 sourcerepo, destrepo = result
880 880 repo = destrepo.local()
881 881
882 882 # When cloning to a remote repo (like through SSH), no repo is available
883 883 # from the peer. Therefore the largefiles can't be downloaded and the
884 884 # hgrc can't be updated.
885 885 if not repo:
886 886 return result
887 887
888 888 # If largefiles is required for this repo, permanently enable it locally
889 889 if 'largefiles' in repo.requirements:
890 890 with repo.vfs('hgrc', 'a', text=True) as fp:
891 891 fp.write('\n[extensions]\nlargefiles=\n')
892 892
893 893 # Caching is implicitly limited to 'rev' option, since the dest repo was
894 894 # truncated at that point. The user may expect a download count with
895 895 # this option, so attempt whether or not this is a largefile repo.
896 896 if opts.get('all_largefiles'):
897 897 success, missing = lfcommands.downloadlfiles(ui, repo, None)
898 898
899 899 if missing != 0:
900 900 return None
901 901
902 902 return result
903 903
904 904 def overriderebase(orig, ui, repo, **opts):
905 905 if not util.safehasattr(repo, '_largefilesenabled'):
906 906 return orig(ui, repo, **opts)
907 907
908 908 resuming = opts.get('continue')
909 909 repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
910 910 repo._lfstatuswriters.append(lambda *msg, **opts: None)
911 911 try:
912 912 return orig(ui, repo, **opts)
913 913 finally:
914 914 repo._lfstatuswriters.pop()
915 915 repo._lfcommithooks.pop()
916 916
917 917 def overridearchivecmd(orig, ui, repo, dest, **opts):
918 918 repo.unfiltered().lfstatus = True
919 919
920 920 try:
921 921 return orig(ui, repo.unfiltered(), dest, **opts)
922 922 finally:
923 923 repo.unfiltered().lfstatus = False
924 924
925 925 def hgwebarchive(orig, web, req, tmpl):
926 926 web.repo.lfstatus = True
927 927
928 928 try:
929 929 return orig(web, req, tmpl)
930 930 finally:
931 931 web.repo.lfstatus = False
932 932
933 933 def overridearchive(orig, repo, dest, node, kind, decode=True, matchfn=None,
934 934 prefix='', mtime=None, subrepos=None):
935 935 # For some reason setting repo.lfstatus in hgwebarchive only changes the
936 936 # unfiltered repo's attr, so check that as well.
937 937 if not repo.lfstatus and not repo.unfiltered().lfstatus:
938 938 return orig(repo, dest, node, kind, decode, matchfn, prefix, mtime,
939 939 subrepos)
940 940
941 941 # No need to lock because we are only reading history and
942 942 # largefile caches, neither of which are modified.
943 943 if node is not None:
944 944 lfcommands.cachelfiles(repo.ui, repo, node)
945 945
946 946 if kind not in archival.archivers:
947 947 raise error.Abort(_("unknown archive type '%s'") % kind)
948 948
949 949 ctx = repo[node]
950 950
951 951 if kind == 'files':
952 952 if prefix:
953 953 raise error.Abort(
954 954 _('cannot give prefix when archiving to files'))
955 955 else:
956 956 prefix = archival.tidyprefix(dest, kind, prefix)
957 957
958 958 def write(name, mode, islink, getdata):
959 959 if matchfn and not matchfn(name):
960 960 return
961 961 data = getdata()
962 962 if decode:
963 963 data = repo.wwritedata(name, data)
964 964 archiver.addfile(prefix + name, mode, islink, data)
965 965
966 966 archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
967 967
968 968 if repo.ui.configbool("ui", "archivemeta", True):
969 969 write('.hg_archival.txt', 0o644, False,
970 970 lambda: archival.buildmetadata(ctx))
971 971
972 972 for f in ctx:
973 973 ff = ctx.flags(f)
974 974 getdata = ctx[f].data
975 975 lfile = lfutil.splitstandin(f)
976 976 if lfile is not None:
977 977 if node is not None:
978 978 path = lfutil.findfile(repo, getdata().strip())
979 979
980 980 if path is None:
981 981 raise error.Abort(
982 982 _('largefile %s not found in repo store or system cache')
983 983 % lfile)
984 984 else:
985 985 path = lfile
986 986
987 987 f = lfile
988 988
989 989 getdata = lambda: util.readfile(path)
990 990 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, getdata)
991 991
992 992 if subrepos:
993 993 for subpath in sorted(ctx.substate):
994 994 sub = ctx.workingsub(subpath)
995 995 submatch = matchmod.subdirmatcher(subpath, matchfn)
996 996 sub._repo.lfstatus = True
997 997 sub.archive(archiver, prefix, submatch)
998 998
999 999 archiver.done()
1000 1000
1001 1001 def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
1002 1002 if not repo._repo.lfstatus:
1003 1003 return orig(repo, archiver, prefix, match, decode)
1004 1004
1005 1005 repo._get(repo._state + ('hg',))
1006 1006 rev = repo._state[1]
1007 1007 ctx = repo._repo[rev]
1008 1008
1009 1009 if ctx.node() is not None:
1010 1010 lfcommands.cachelfiles(repo.ui, repo._repo, ctx.node())
1011 1011
1012 1012 def write(name, mode, islink, getdata):
1013 1013 # At this point, the standin has been replaced with the largefile name,
1014 1014 # so the normal matcher works here without the lfutil variants.
1015 1015 if match and not match(f):
1016 1016 return
1017 1017 data = getdata()
1018 1018 if decode:
1019 1019 data = repo._repo.wwritedata(name, data)
1020 1020
1021 1021 archiver.addfile(prefix + repo._path + '/' + name, mode, islink, data)
1022 1022
1023 1023 for f in ctx:
1024 1024 ff = ctx.flags(f)
1025 1025 getdata = ctx[f].data
1026 1026 lfile = lfutil.splitstandin(f)
1027 1027 if lfile is not None:
1028 1028 if ctx.node() is not None:
1029 1029 path = lfutil.findfile(repo._repo, getdata().strip())
1030 1030
1031 1031 if path is None:
1032 1032 raise error.Abort(
1033 1033 _('largefile %s not found in repo store or system cache')
1034 1034 % lfile)
1035 1035 else:
1036 1036 path = lfile
1037 1037
1038 1038 f = lfile
1039 1039
1040 1040 getdata = lambda: util.readfile(os.path.join(prefix, path))
1041 1041
1042 1042 write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, getdata)
1043 1043
1044 1044 for subpath in sorted(ctx.substate):
1045 1045 sub = ctx.workingsub(subpath)
1046 1046 submatch = matchmod.subdirmatcher(subpath, match)
1047 1047 sub._repo.lfstatus = True
1048 1048 sub.archive(archiver, prefix + repo._path + '/', submatch, decode)
1049 1049
1050 1050 # If a largefile is modified, the change is not reflected in its
1051 1051 # standin until a commit. cmdutil.bailifchanged() raises an exception
1052 1052 # if the repo has uncommitted changes. Wrap it to also check if
1053 1053 # largefiles were changed. This is used by bisect, backout and fetch.
1054 1054 def overridebailifchanged(orig, repo, *args, **kwargs):
1055 1055 orig(repo, *args, **kwargs)
1056 1056 repo.lfstatus = True
1057 1057 s = repo.status()
1058 1058 repo.lfstatus = False
1059 1059 if s.modified or s.added or s.removed or s.deleted:
1060 1060 raise error.Abort(_('uncommitted changes'))
1061 1061
1062 1062 def postcommitstatus(orig, repo, *args, **kwargs):
1063 1063 repo.lfstatus = True
1064 1064 try:
1065 1065 return orig(repo, *args, **kwargs)
1066 1066 finally:
1067 1067 repo.lfstatus = False
1068 1068
1069 1069 def cmdutilforget(orig, ui, repo, match, prefix, explicitonly):
1070 1070 normalmatcher = composenormalfilematcher(match, repo[None].manifest())
1071 1071 bad, forgot = orig(ui, repo, normalmatcher, prefix, explicitonly)
1072 1072 m = composelargefilematcher(match, repo[None].manifest())
1073 1073
1074 1074 try:
1075 1075 repo.lfstatus = True
1076 1076 s = repo.status(match=m, clean=True)
1077 1077 finally:
1078 1078 repo.lfstatus = False
1079 1079 forget = sorted(s.modified + s.added + s.deleted + s.clean)
1080 1080 forget = [f for f in forget if lfutil.standin(f) in repo[None].manifest()]
1081 1081
1082 1082 for f in forget:
1083 1083 if lfutil.standin(f) not in repo.dirstate and not \
1084 1084 repo.wvfs.isdir(lfutil.standin(f)):
1085 1085 ui.warn(_('not removing %s: file is already untracked\n')
1086 1086 % m.rel(f))
1087 1087 bad.append(f)
1088 1088
1089 1089 for f in forget:
1090 1090 if ui.verbose or not m.exact(f):
1091 1091 ui.status(_('removing %s\n') % m.rel(f))
1092 1092
1093 1093 # Need to lock because standin files are deleted then removed from the
1094 1094 # repository and we could race in-between.
1095 1095 with repo.wlock():
1096 1096 lfdirstate = lfutil.openlfdirstate(ui, repo)
1097 1097 for f in forget:
1098 1098 if lfdirstate[f] == 'a':
1099 1099 lfdirstate.drop(f)
1100 1100 else:
1101 1101 lfdirstate.remove(f)
1102 1102 lfdirstate.write()
1103 1103 standins = [lfutil.standin(f) for f in forget]
1104 1104 for f in standins:
1105 1105 repo.wvfs.unlinkpath(f, ignoremissing=True)
1106 1106 rejected = repo[None].forget(standins)
1107 1107
1108 1108 bad.extend(f for f in rejected if f in m.files())
1109 1109 forgot.extend(f for f in forget if f not in rejected)
1110 1110 return bad, forgot
1111 1111
1112 1112 def _getoutgoings(repo, other, missing, addfunc):
1113 1113 """get pairs of filename and largefile hash in outgoing revisions
1114 1114 in 'missing'.
1115 1115
1116 1116 largefiles already existing on 'other' repository are ignored.
1117 1117
1118 1118 'addfunc' is invoked with each unique pairs of filename and
1119 1119 largefile hash value.
1120 1120 """
1121 1121 knowns = set()
1122 1122 lfhashes = set()
1123 1123 def dedup(fn, lfhash):
1124 1124 k = (fn, lfhash)
1125 1125 if k not in knowns:
1126 1126 knowns.add(k)
1127 1127 lfhashes.add(lfhash)
1128 1128 lfutil.getlfilestoupload(repo, missing, dedup)
1129 1129 if lfhashes:
1130 1130 lfexists = storefactory.openstore(repo, other).exists(lfhashes)
1131 1131 for fn, lfhash in knowns:
1132 1132 if not lfexists[lfhash]: # lfhash doesn't exist on "other"
1133 1133 addfunc(fn, lfhash)
1134 1134
1135 1135 def outgoinghook(ui, repo, other, opts, missing):
1136 1136 if opts.pop('large', None):
1137 1137 lfhashes = set()
1138 1138 if ui.debugflag:
1139 1139 toupload = {}
1140 1140 def addfunc(fn, lfhash):
1141 1141 if fn not in toupload:
1142 1142 toupload[fn] = []
1143 1143 toupload[fn].append(lfhash)
1144 1144 lfhashes.add(lfhash)
1145 1145 def showhashes(fn):
1146 1146 for lfhash in sorted(toupload[fn]):
1147 1147 ui.debug(' %s\n' % (lfhash))
1148 1148 else:
1149 1149 toupload = set()
1150 1150 def addfunc(fn, lfhash):
1151 1151 toupload.add(fn)
1152 1152 lfhashes.add(lfhash)
1153 1153 def showhashes(fn):
1154 1154 pass
1155 1155 _getoutgoings(repo, other, missing, addfunc)
1156 1156
1157 1157 if not toupload:
1158 1158 ui.status(_('largefiles: no files to upload\n'))
1159 1159 else:
1160 1160 ui.status(_('largefiles to upload (%d entities):\n')
1161 1161 % (len(lfhashes)))
1162 1162 for file in sorted(toupload):
1163 1163 ui.status(lfutil.splitstandin(file) + '\n')
1164 1164 showhashes(file)
1165 1165 ui.status('\n')
1166 1166
1167 1167 def summaryremotehook(ui, repo, opts, changes):
1168 1168 largeopt = opts.get('large', False)
1169 1169 if changes is None:
1170 1170 if largeopt:
1171 1171 return (False, True) # only outgoing check is needed
1172 1172 else:
1173 1173 return (False, False)
1174 1174 elif largeopt:
1175 1175 url, branch, peer, outgoing = changes[1]
1176 1176 if peer is None:
1177 1177 # i18n: column positioning for "hg summary"
1178 1178 ui.status(_('largefiles: (no remote repo)\n'))
1179 1179 return
1180 1180
1181 1181 toupload = set()
1182 1182 lfhashes = set()
1183 1183 def addfunc(fn, lfhash):
1184 1184 toupload.add(fn)
1185 1185 lfhashes.add(lfhash)
1186 1186 _getoutgoings(repo, peer, outgoing.missing, addfunc)
1187 1187
1188 1188 if not toupload:
1189 1189 # i18n: column positioning for "hg summary"
1190 1190 ui.status(_('largefiles: (no files to upload)\n'))
1191 1191 else:
1192 1192 # i18n: column positioning for "hg summary"
1193 1193 ui.status(_('largefiles: %d entities for %d files to upload\n')
1194 1194 % (len(lfhashes), len(toupload)))
1195 1195
1196 1196 def overridesummary(orig, ui, repo, *pats, **opts):
1197 1197 try:
1198 1198 repo.lfstatus = True
1199 1199 orig(ui, repo, *pats, **opts)
1200 1200 finally:
1201 1201 repo.lfstatus = False
1202 1202
1203 1203 def scmutiladdremove(orig, repo, matcher, prefix, opts=None, dry_run=None,
1204 1204 similarity=None):
1205 1205 if opts is None:
1206 1206 opts = {}
1207 1207 if not lfutil.islfilesrepo(repo):
1208 1208 return orig(repo, matcher, prefix, opts, dry_run, similarity)
1209 1209 # Get the list of missing largefiles so we can remove them
1210 1210 lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
1211 1211 unsure, s = lfdirstate.status(matchmod.always(repo.root, repo.getcwd()), [],
1212 1212 False, False, False)
1213 1213
1214 1214 # Call into the normal remove code, but the removing of the standin, we want
1215 1215 # to have handled by original addremove. Monkey patching here makes sure
1216 1216 # we don't remove the standin in the largefiles code, preventing a very
1217 1217 # confused state later.
1218 1218 if s.deleted:
1219 1219 m = copy.copy(matcher)
1220 1220
1221 1221 # The m._files and m._map attributes are not changed to the deleted list
1222 1222 # because that affects the m.exact() test, which in turn governs whether
1223 1223 # or not the file name is printed, and how. Simply limit the original
1224 1224 # matches to those in the deleted status list.
1225 1225 matchfn = m.matchfn
1226 1226 m.matchfn = lambda f: f in s.deleted and matchfn(f)
1227 1227
1228 1228 removelargefiles(repo.ui, repo, True, m, **opts)
1229 1229 # Call into the normal add code, and any files that *should* be added as
1230 1230 # largefiles will be
1231 1231 added, bad = addlargefiles(repo.ui, repo, True, matcher, **opts)
1232 1232 # Now that we've handled largefiles, hand off to the original addremove
1233 1233 # function to take care of the rest. Make sure it doesn't do anything with
1234 1234 # largefiles by passing a matcher that will ignore them.
1235 1235 matcher = composenormalfilematcher(matcher, repo[None].manifest(), added)
1236 1236 return orig(repo, matcher, prefix, opts, dry_run, similarity)
1237 1237
1238 1238 # Calling purge with --all will cause the largefiles to be deleted.
1239 1239 # Override repo.status to prevent this from happening.
1240 1240 def overridepurge(orig, ui, repo, *dirs, **opts):
1241 1241 # XXX Monkey patching a repoview will not work. The assigned attribute will
1242 1242 # be set on the unfiltered repo, but we will only lookup attributes in the
1243 1243 # unfiltered repo if the lookup in the repoview object itself fails. As the
1244 1244 # monkey patched method exists on the repoview class the lookup will not
1245 1245 # fail. As a result, the original version will shadow the monkey patched
1246 1246 # one, defeating the monkey patch.
1247 1247 #
1248 1248 # As a work around we use an unfiltered repo here. We should do something
1249 1249 # cleaner instead.
1250 1250 repo = repo.unfiltered()
1251 1251 oldstatus = repo.status
1252 1252 def overridestatus(node1='.', node2=None, match=None, ignored=False,
1253 1253 clean=False, unknown=False, listsubrepos=False):
1254 1254 r = oldstatus(node1, node2, match, ignored, clean, unknown,
1255 1255 listsubrepos)
1256 1256 lfdirstate = lfutil.openlfdirstate(ui, repo)
1257 1257 unknown = [f for f in r.unknown if lfdirstate[f] == '?']
1258 1258 ignored = [f for f in r.ignored if lfdirstate[f] == '?']
1259 1259 return scmutil.status(r.modified, r.added, r.removed, r.deleted,
1260 1260 unknown, ignored, r.clean)
1261 1261 repo.status = overridestatus
1262 1262 orig(ui, repo, *dirs, **opts)
1263 1263 repo.status = oldstatus
1264 1264 def overriderollback(orig, ui, repo, **opts):
1265 1265 with repo.wlock():
1266 1266 before = repo.dirstate.parents()
1267 1267 orphans = set(f for f in repo.dirstate
1268 1268 if lfutil.isstandin(f) and repo.dirstate[f] != 'r')
1269 1269 result = orig(ui, repo, **opts)
1270 1270 after = repo.dirstate.parents()
1271 1271 if before == after:
1272 1272 return result # no need to restore standins
1273 1273
1274 1274 pctx = repo['.']
1275 1275 for f in repo.dirstate:
1276 1276 if lfutil.isstandin(f):
1277 1277 orphans.discard(f)
1278 1278 if repo.dirstate[f] == 'r':
1279 1279 repo.wvfs.unlinkpath(f, ignoremissing=True)
1280 1280 elif f in pctx:
1281 1281 fctx = pctx[f]
1282 1282 repo.wwrite(f, fctx.data(), fctx.flags())
1283 1283 else:
1284 1284 # content of standin is not so important in 'a',
1285 1285 # 'm' or 'n' (coming from the 2nd parent) cases
1286 1286 lfutil.writestandin(repo, f, '', False)
1287 1287 for standin in orphans:
1288 1288 repo.wvfs.unlinkpath(standin, ignoremissing=True)
1289 1289
1290 1290 lfdirstate = lfutil.openlfdirstate(ui, repo)
1291 1291 orphans = set(lfdirstate)
1292 1292 lfiles = lfutil.listlfiles(repo)
1293 1293 for file in lfiles:
1294 1294 lfutil.synclfdirstate(repo, lfdirstate, file, True)
1295 1295 orphans.discard(file)
1296 1296 for lfile in orphans:
1297 1297 lfdirstate.drop(lfile)
1298 1298 lfdirstate.write()
1299 1299 return result
1300 1300
1301 1301 def overridetransplant(orig, ui, repo, *revs, **opts):
1302 1302 resuming = opts.get('continue')
1303 1303 repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
1304 1304 repo._lfstatuswriters.append(lambda *msg, **opts: None)
1305 1305 try:
1306 1306 result = orig(ui, repo, *revs, **opts)
1307 1307 finally:
1308 1308 repo._lfstatuswriters.pop()
1309 1309 repo._lfcommithooks.pop()
1310 1310 return result
1311 1311
1312 1312 def overridecat(orig, ui, repo, file1, *pats, **opts):
1313 1313 ctx = scmutil.revsingle(repo, opts.get('rev'))
1314 1314 err = 1
1315 1315 notbad = set()
1316 1316 m = scmutil.match(ctx, (file1,) + pats, opts)
1317 1317 origmatchfn = m.matchfn
1318 1318 def lfmatchfn(f):
1319 1319 if origmatchfn(f):
1320 1320 return True
1321 1321 lf = lfutil.splitstandin(f)
1322 1322 if lf is None:
1323 1323 return False
1324 1324 notbad.add(lf)
1325 1325 return origmatchfn(lf)
1326 1326 m.matchfn = lfmatchfn
1327 1327 origbadfn = m.bad
1328 1328 def lfbadfn(f, msg):
1329 1329 if not f in notbad:
1330 1330 origbadfn(f, msg)
1331 1331 m.bad = lfbadfn
1332 1332
1333 1333 origvisitdirfn = m.visitdir
1334 1334 def lfvisitdirfn(dir):
1335 1335 if dir == lfutil.shortname:
1336 1336 return True
1337 1337 ret = origvisitdirfn(dir)
1338 1338 if ret:
1339 1339 return ret
1340 1340 lf = lfutil.splitstandin(dir)
1341 1341 if lf is None:
1342 1342 return False
1343 1343 return origvisitdirfn(lf)
1344 1344 m.visitdir = lfvisitdirfn
1345 1345
1346 1346 for f in ctx.walk(m):
1347 1347 with cmdutil.makefileobj(repo, opts.get('output'), ctx.node(),
1348 1348 pathname=f) as fp:
1349 1349 lf = lfutil.splitstandin(f)
1350 1350 if lf is None or origmatchfn(f):
1351 1351 # duplicating unreachable code from commands.cat
1352 1352 data = ctx[f].data()
1353 1353 if opts.get('decode'):
1354 1354 data = repo.wwritedata(f, data)
1355 1355 fp.write(data)
1356 1356 else:
1357 hash = lfutil.readstandin(repo, lf, ctx.rev())
1357 hash = lfutil.readstandin(repo, lf, ctx)
1358 1358 if not lfutil.inusercache(repo.ui, hash):
1359 1359 store = storefactory.openstore(repo)
1360 1360 success, missing = store.get([(lf, hash)])
1361 1361 if len(success) != 1:
1362 1362 raise error.Abort(
1363 1363 _('largefile %s is not in cache and could not be '
1364 1364 'downloaded') % lf)
1365 1365 path = lfutil.usercachepath(repo.ui, hash)
1366 1366 with open(path, "rb") as fpin:
1367 1367 for chunk in util.filechunkiter(fpin):
1368 1368 fp.write(chunk)
1369 1369 err = 0
1370 1370 return err
1371 1371
1372 1372 def mergeupdate(orig, repo, node, branchmerge, force,
1373 1373 *args, **kwargs):
1374 1374 matcher = kwargs.get('matcher', None)
1375 1375 # note if this is a partial update
1376 1376 partial = matcher and not matcher.always()
1377 1377 with repo.wlock():
1378 1378 # branch | | |
1379 1379 # merge | force | partial | action
1380 1380 # -------+-------+---------+--------------
1381 1381 # x | x | x | linear-merge
1382 1382 # o | x | x | branch-merge
1383 1383 # x | o | x | overwrite (as clean update)
1384 1384 # o | o | x | force-branch-merge (*1)
1385 1385 # x | x | o | (*)
1386 1386 # o | x | o | (*)
1387 1387 # x | o | o | overwrite (as revert)
1388 1388 # o | o | o | (*)
1389 1389 #
1390 1390 # (*) don't care
1391 1391 # (*1) deprecated, but used internally (e.g: "rebase --collapse")
1392 1392
1393 1393 lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
1394 1394 unsure, s = lfdirstate.status(matchmod.always(repo.root,
1395 1395 repo.getcwd()),
1396 1396 [], False, True, False)
1397 1397 oldclean = set(s.clean)
1398 1398 pctx = repo['.']
1399 1399 for lfile in unsure + s.modified:
1400 1400 lfileabs = repo.wvfs.join(lfile)
1401 1401 if not repo.wvfs.exists(lfileabs):
1402 1402 continue
1403 1403 lfhash = lfutil.hashrepofile(repo, lfile)
1404 1404 standin = lfutil.standin(lfile)
1405 1405 lfutil.writestandin(repo, standin, lfhash,
1406 1406 lfutil.getexecutable(lfileabs))
1407 1407 if (standin in pctx and
1408 lfhash == lfutil.readstandin(repo, lfile, '.')):
1408 lfhash == lfutil.readstandin(repo, lfile, pctx)):
1409 1409 oldclean.add(lfile)
1410 1410 for lfile in s.added:
1411 1411 lfutil.updatestandin(repo, lfutil.standin(lfile))
1412 1412 # mark all clean largefiles as dirty, just in case the update gets
1413 1413 # interrupted before largefiles and lfdirstate are synchronized
1414 1414 for lfile in oldclean:
1415 1415 lfdirstate.normallookup(lfile)
1416 1416 lfdirstate.write()
1417 1417
1418 1418 oldstandins = lfutil.getstandinsstate(repo)
1419 1419
1420 1420 result = orig(repo, node, branchmerge, force, *args, **kwargs)
1421 1421
1422 1422 newstandins = lfutil.getstandinsstate(repo)
1423 1423 filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
1424 1424
1425 1425 # to avoid leaving all largefiles as dirty and thus rehash them, mark
1426 1426 # all the ones that didn't change as clean
1427 1427 for lfile in oldclean.difference(filelist):
1428 1428 lfdirstate.normal(lfile)
1429 1429 lfdirstate.write()
1430 1430
1431 1431 if branchmerge or force or partial:
1432 1432 filelist.extend(s.deleted + s.removed)
1433 1433
1434 1434 lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
1435 1435 normallookup=partial)
1436 1436
1437 1437 return result
1438 1438
1439 1439 def scmutilmarktouched(orig, repo, files, *args, **kwargs):
1440 1440 result = orig(repo, files, *args, **kwargs)
1441 1441
1442 1442 filelist = []
1443 1443 for f in files:
1444 1444 lf = lfutil.splitstandin(f)
1445 1445 if lf is not None:
1446 1446 filelist.append(lf)
1447 1447 if filelist:
1448 1448 lfcommands.updatelfiles(repo.ui, repo, filelist=filelist,
1449 1449 printmessage=False, normallookup=True)
1450 1450
1451 1451 return result
General Comments 0
You need to be logged in to leave comments. Login now