lfutil.py
394 lines
| 13.5 KiB
| text/x-python
|
PythonLexer
|
r15168 | # Copyright 2009-2010 Gregory P. Ward | ||
# Copyright 2009-2010 Intelerad Medical Systems Incorporated | ||||
# Copyright 2010-2011 Fog Creek Software | ||||
# Copyright 2010-2011 Unity Technologies | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
'''largefiles utility code: must not import other modules in this package.''' | ||||
import os | ||||
|
r15320 | import platform | ||
|
r15168 | import shutil | ||
import stat | ||||
|
r15226 | from mercurial import dirstate, httpconnection, match as match_, util, scmutil | ||
|
r15168 | from mercurial.i18n import _ | ||
|
r21042 | from mercurial import node | ||
|
r15168 | |||
shortname = '.hglf' | ||||
|
r18151 | shortnameslash = shortname + '/' | ||
|
r15168 | longname = 'largefiles' | ||
# -- Private worker functions ------------------------------------------ | ||||
|
r15227 | def getminsize(ui, assumelfiles, opt, default=10): | ||
lfsize = opt | ||||
if not lfsize and assumelfiles: | ||||
|
r15304 | lfsize = ui.config(longname, 'minsize', default=default) | ||
|
r15227 | if lfsize: | ||
try: | ||||
|
r15228 | lfsize = float(lfsize) | ||
|
r15227 | except ValueError: | ||
|
r15228 | raise util.Abort(_('largefiles: size must be number (not %s)\n') | ||
|
r15227 | % lfsize) | ||
if lfsize is None: | ||||
raise util.Abort(_('minimum size for largefiles must be specified')) | ||||
return lfsize | ||||
|
r15168 | def link(src, dest): | ||
|
r18998 | util.makedirs(os.path.dirname(dest)) | ||
|
r15168 | try: | ||
|
r15206 | util.oslink(src, dest) | ||
|
r15168 | except OSError: | ||
|
r15572 | # if hardlinks fail, fallback on atomic copy | ||
dst = util.atomictempfile(dest) | ||||
|
r15699 | for chunk in util.filechunkiter(open(src, 'rb')): | ||
|
r15572 | dst.write(chunk) | ||
dst.close() | ||||
|
r15168 | os.chmod(dest, os.stat(src).st_mode) | ||
|
r15316 | def usercachepath(ui, hash): | ||
|
r15350 | path = ui.configpath(longname, 'usercache', None) | ||
|
r15168 | if path: | ||
path = os.path.join(path, hash) | ||||
else: | ||||
if os.name == 'nt': | ||||
|
r15255 | appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) | ||
|
r15658 | if appdata: | ||
path = os.path.join(appdata, longname, hash) | ||||
|
r15320 | elif platform.system() == 'Darwin': | ||
|
r15658 | home = os.getenv('HOME') | ||
if home: | ||||
path = os.path.join(home, 'Library', 'Caches', | ||||
longname, hash) | ||||
|
r15168 | elif os.name == 'posix': | ||
|
r15320 | path = os.getenv('XDG_CACHE_HOME') | ||
if path: | ||||
path = os.path.join(path, longname, hash) | ||||
else: | ||||
|
r15658 | home = os.getenv('HOME') | ||
if home: | ||||
path = os.path.join(home, '.cache', longname, hash) | ||||
|
r15168 | else: | ||
|
r15253 | raise util.Abort(_('unknown operating system: %s\n') % os.name) | ||
|
r15168 | return path | ||
|
r15316 | def inusercache(ui, hash): | ||
|
r15658 | path = usercachepath(ui, hash) | ||
return path and os.path.exists(path) | ||||
|
r15168 | |||
def findfile(repo, hash): | ||||
|
r15316 | if instore(repo, hash): | ||
|
r16928 | repo.ui.note(_('found %s in store\n') % hash) | ||
|
r15913 | return storepath(repo, hash) | ||
|
r15317 | elif inusercache(repo.ui, hash): | ||
|
r16928 | repo.ui.note(_('found %s in system cache\n') % hash) | ||
|
r15408 | path = storepath(repo, hash) | ||
link(usercachepath(repo.ui, hash), path) | ||||
|
r15913 | return path | ||
return None | ||||
|
r15168 | |||
|
r16247 | class largefilesdirstate(dirstate.dirstate): | ||
|
r15168 | def __getitem__(self, key): | ||
|
r16247 | return super(largefilesdirstate, self).__getitem__(unixpath(key)) | ||
|
r15168 | def normal(self, f): | ||
|
r16247 | return super(largefilesdirstate, self).normal(unixpath(f)) | ||
|
r15168 | def remove(self, f): | ||
|
r16247 | return super(largefilesdirstate, self).remove(unixpath(f)) | ||
|
r15168 | def add(self, f): | ||
|
r16247 | return super(largefilesdirstate, self).add(unixpath(f)) | ||
|
r15168 | def drop(self, f): | ||
|
r16247 | return super(largefilesdirstate, self).drop(unixpath(f)) | ||
|
r15168 | def forget(self, f): | ||
|
r16247 | return super(largefilesdirstate, self).forget(unixpath(f)) | ||
|
r15793 | def normallookup(self, f): | ||
|
r16247 | return super(largefilesdirstate, self).normallookup(unixpath(f)) | ||
|
r21085 | def _ignore(self, f): | ||
|
r18148 | return False | ||
|
r15168 | |||
|
r17659 | def openlfdirstate(ui, repo, create=True): | ||
|
r15168 | ''' | ||
|
r15252 | Return a dirstate object that tracks largefiles: i.e. its root is | ||
the repo root, but it is saved in .hg/largefiles/dirstate. | ||||
|
r15168 | ''' | ||
|
r18147 | lfstoredir = repo.join(longname) | ||
opener = scmutil.opener(lfstoredir) | ||||
|
r16247 | lfdirstate = largefilesdirstate(opener, ui, repo.root, | ||
|
r15349 | repo.dirstate._validate) | ||
|
r15168 | |||
|
r15252 | # If the largefiles dirstate does not exist, populate and create | ||
# it. This ensures that we create it on the first meaningful | ||||
|
r15794 | # largefiles operation in a new clone. | ||
|
r18147 | if create and not os.path.exists(os.path.join(lfstoredir, 'dirstate')): | ||
|
r15168 | matcher = getstandinmatcher(repo) | ||
|
r21917 | standins = repo.dirstate.walk(matcher, [], False, False) | ||
if len(standins) > 0: | ||||
util.makedirs(lfstoredir) | ||||
for standin in standins: | ||||
|
r15168 | lfile = splitstandin(standin) | ||
lfdirstate.normallookup(lfile) | ||||
return lfdirstate | ||||
|
r16247 | def lfdirstatestatus(lfdirstate, repo, rev): | ||
|
r15794 | match = match_.always(repo.root, repo.getcwd()) | ||
s = lfdirstate.status(match, [], False, False, False) | ||||
unsure, modified, added, removed, missing, unknown, ignored, clean = s | ||||
for lfile in unsure: | ||||
|
r18299 | try: | ||
fctx = repo[rev][standin(lfile)] | ||||
except LookupError: | ||||
fctx = None | ||||
if not fctx or fctx.data().strip() != hashfile(repo.wjoin(lfile)): | ||||
|
r15794 | modified.append(lfile) | ||
else: | ||||
clean.append(lfile) | ||||
lfdirstate.normal(lfile) | ||||
|
r15168 | return (modified, added, removed, missing, unknown, ignored, clean) | ||
def listlfiles(repo, rev=None, matcher=None): | ||||
|
r15252 | '''return a list of largefiles in the working copy or the | ||
specified changeset''' | ||||
|
r15168 | |||
if matcher is None: | ||||
matcher = getstandinmatcher(repo) | ||||
# ignore unknown files in working directory | ||||
|
r15255 | return [splitstandin(f) | ||
for f in repo[rev].walk(matcher) | ||||
|
r15168 | if rev is not None or repo.dirstate[f] != '?'] | ||
|
r15316 | def instore(repo, hash): | ||
return os.path.exists(storepath(repo, hash)) | ||||
|
r15168 | |||
|
r15316 | def storepath(repo, hash): | ||
|
r15168 | return repo.join(os.path.join(longname, hash)) | ||
def copyfromcache(repo, hash, filename): | ||||
|
r15252 | '''Copy the specified largefile from the repo or system cache to | ||
filename in the repository. Return true on success or false if the | ||||
file was not found in either cache (which should not happened: | ||||
this is meant to be called only after ensuring that the needed | ||||
largefile exists in the cache).''' | ||||
|
r15168 | path = findfile(repo, hash) | ||
if path is None: | ||||
return False | ||||
util.makedirs(os.path.dirname(repo.wjoin(filename))) | ||||
|
r15570 | # The write may fail before the file is fully written, but we | ||
# don't use atomic writes in the working copy. | ||||
|
r15168 | shutil.copy(path, repo.wjoin(filename)) | ||
return True | ||||
|
r15316 | def copytostore(repo, rev, file, uploaded=False): | ||
|
r17877 | hash = readstandin(repo, file, rev) | ||
|
r15316 | if instore(repo, hash): | ||
|
r15168 | return | ||
|
r15316 | copytostoreabsolute(repo, repo.wjoin(file), hash) | ||
|
r15168 | |||
|
r15796 | def copyalltostore(repo, node): | ||
'''Copy all largefiles in a given revision to the store''' | ||||
ctx = repo[node] | ||||
for filename in ctx.files(): | ||||
if isstandin(filename) and filename in ctx.manifest(): | ||||
realfile = splitstandin(filename) | ||||
copytostore(repo, ctx.node(), realfile) | ||||
|
r15316 | def copytostoreabsolute(repo, file, hash): | ||
if inusercache(repo.ui, hash): | ||||
link(usercachepath(repo.ui, hash), storepath(repo, hash)) | ||||
|
r17878 | elif not getattr(repo, "_isconverting", False): | ||
|
r18998 | util.makedirs(os.path.dirname(storepath(repo, hash))) | ||
|
r16153 | dst = util.atomictempfile(storepath(repo, hash), | ||
createmode=repo.store.createmode) | ||||
|
r15699 | for chunk in util.filechunkiter(open(file, 'rb')): | ||
|
r15571 | dst.write(chunk) | ||
dst.close() | ||||
|
r15316 | linktousercache(repo, hash) | ||
|
r15168 | |||
|
r15316 | def linktousercache(repo, hash): | ||
|
r15658 | path = usercachepath(repo.ui, hash) | ||
if path: | ||||
link(storepath(repo, hash), path) | ||||
|
r15168 | |||
def getstandinmatcher(repo, pats=[], opts={}): | ||||
'''Return a match object that applies pats to the standin directory''' | ||||
|
r18150 | standindir = repo.wjoin(shortname) | ||
|
r15168 | if pats: | ||
|
r18490 | pats = [os.path.join(standindir, pat) for pat in pats] | ||
|
r18724 | else: | ||
|
r15168 | # no patterns: relative to repo root | ||
pats = [standindir] | ||||
|
r18146 | # no warnings about missing files or directories | ||
|
r15224 | match = scmutil.match(repo[None], pats, opts) | ||
|
r18146 | match.bad = lambda f, msg: None | ||
|
r15168 | return match | ||
def composestandinmatcher(repo, rmatcher): | ||||
|
r15252 | '''Return a matcher that accepts standins corresponding to the | ||
files accepted by rmatcher. Pass the list of files in the matcher | ||||
as the paths specified by the user.''' | ||||
|
r15168 | smatcher = getstandinmatcher(repo, rmatcher.files()) | ||
isstandin = smatcher.matchfn | ||||
|
r16247 | def composedmatchfn(f): | ||
|
r15168 | return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | ||
|
r16247 | smatcher.matchfn = composedmatchfn | ||
|
r15168 | |||
return smatcher | ||||
def standin(filename): | ||||
'''Return the repo-relative path to the standin for the specified big | ||||
file.''' | ||||
# Notes: | ||||
|
r17425 | # 1) Some callers want an absolute path, but for instance addlargefiles | ||
|
r18154 | # needs it repo-relative so it can be passed to repo[None].add(). So | ||
# leave it up to the caller to use repo.wjoin() to get an absolute path. | ||||
|
r15168 | # 2) Join with '/' because that's what dirstate always uses, even on | ||
# Windows. Change existing separator to '/' first in case we are | ||||
# passed filenames from an external source (like the command line). | ||||
|
r18151 | return shortnameslash + util.pconvert(filename) | ||
|
r15168 | |||
def isstandin(filename): | ||||
|
r15252 | '''Return true if filename is a big file standin. filename must be | ||
in Mercurial's internal form (slash-separated).''' | ||||
|
r18151 | return filename.startswith(shortnameslash) | ||
|
r15168 | |||
def splitstandin(filename): | ||||
# Split on / because that's what dirstate always uses, even on Windows. | ||||
# Change local separator to / first just in case we are passed filenames | ||||
# from an external source (like the command line). | ||||
|
r16066 | bits = util.pconvert(filename).split('/', 1) | ||
|
r15168 | if len(bits) == 2 and bits[0] == shortname: | ||
return bits[1] | ||||
else: | ||||
return None | ||||
def updatestandin(repo, standin): | ||||
file = repo.wjoin(splitstandin(standin)) | ||||
if os.path.exists(file): | ||||
hash = hashfile(file) | ||||
executable = getexecutable(file) | ||||
writestandin(repo, standin, hash, executable) | ||||
def readstandin(repo, filename, node=None): | ||||
'''read hex hash from standin for filename at given node, or working | ||||
directory if no node is given''' | ||||
return repo[node][standin(filename)].data().strip() | ||||
def writestandin(repo, standin, hash, executable): | ||||
|
r15252 | '''write hash to <repo.root>/<standin>''' | ||
|
r19089 | repo.wwrite(standin, hash + '\n', executable and 'x' or '') | ||
|
r15168 | |||
def copyandhash(instream, outfile): | ||||
'''Read bytes from instream (iterable) and write them to outfile, | ||||
|
r19002 | computing the SHA-1 hash of the data along the way. Return the hash.''' | ||
|
r15168 | hasher = util.sha1('') | ||
for data in instream: | ||||
hasher.update(data) | ||||
outfile.write(data) | ||||
|
r18999 | return hasher.hexdigest() | ||
|
r15168 | |||
def hashrepofile(repo, file): | ||||
return hashfile(repo.wjoin(file)) | ||||
def hashfile(file): | ||||
if not os.path.exists(file): | ||||
return '' | ||||
hasher = util.sha1('') | ||||
fd = open(file, 'rb') | ||||
|
r19001 | for data in util.filechunkiter(fd, 128 * 1024): | ||
|
r15168 | hasher.update(data) | ||
fd.close() | ||||
return hasher.hexdigest() | ||||
def getexecutable(filename): | ||||
mode = os.stat(filename).st_mode | ||||
|
r15255 | return ((mode & stat.S_IXUSR) and | ||
(mode & stat.S_IXGRP) and | ||||
(mode & stat.S_IXOTH)) | ||||
|
r15168 | |||
def urljoin(first, second, *arg): | ||||
def join(left, right): | ||||
if not left.endswith('/'): | ||||
left += '/' | ||||
if right.startswith('/'): | ||||
right = right[1:] | ||||
return left + right | ||||
url = join(first, second) | ||||
for a in arg: | ||||
url = join(url, a) | ||||
return url | ||||
def hexsha1(data): | ||||
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | ||||
object data""" | ||||
|
r15347 | h = util.sha1() | ||
|
r15168 | for chunk in util.filechunkiter(data): | ||
h.update(chunk) | ||||
return h.hexdigest() | ||||
def httpsendfile(ui, filename): | ||||
|
r15224 | return httpconnection.httpsendfile(ui, filename, 'rb') | ||
|
r15168 | |||
def unixpath(path): | ||||
|
r15252 | '''Return a version of path normalized for use with the lfdirstate.''' | ||
|
r16066 | return util.pconvert(os.path.normpath(path)) | ||
|
r15168 | |||
def islfilesrepo(repo): | ||||
|
r17659 | if ('largefiles' in repo.requirements and | ||
|
r18151 | util.any(shortnameslash in f[0] for f in repo.store.datafiles())): | ||
|
r17659 | return True | ||
return util.any(openlfdirstate(repo.ui, repo, False)) | ||||
|
r15168 | |||
|
r15333 | class storeprotonotcapable(Exception): | ||
|
r15168 | def __init__(self, storetypes): | ||
self.storetypes = storetypes | ||||
|
r16103 | |||
|
r16120 | def getstandinsstate(repo): | ||
standins = [] | ||||
matcher = getstandinmatcher(repo) | ||||
|
r18154 | for standin in repo.dirstate.walk(matcher, [], False, False): | ||
|
r16120 | lfile = splitstandin(standin) | ||
|
r18300 | try: | ||
hash = readstandin(repo, lfile) | ||||
except IOError: | ||||
hash = None | ||||
standins.append((lfile, hash)) | ||||
|
r16120 | return standins | ||
|
r16245 | |||
def getlfilestoupdate(oldstandins, newstandins): | ||||
changedstandins = set(oldstandins).symmetric_difference(set(newstandins)) | ||||
filelist = [] | ||||
for f in changedstandins: | ||||
if f[0] not in filelist: | ||||
filelist.append(f[0]) | ||||
return filelist | ||||
|
r21042 | |||
def getlfilestoupload(repo, missing, addfunc): | ||||
for n in missing: | ||||
parents = [p for p in repo.changelog.parents(n) if p != node.nullid] | ||||
ctx = repo[n] | ||||
files = set(ctx.files()) | ||||
if len(parents) == 2: | ||||
mc = ctx.manifest() | ||||
mp1 = ctx.parents()[0].manifest() | ||||
mp2 = ctx.parents()[1].manifest() | ||||
for f in mp1: | ||||
if f not in mc: | ||||
files.add(f) | ||||
for f in mp2: | ||||
if f not in mc: | ||||
files.add(f) | ||||
for f in mc: | ||||
if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): | ||||
files.add(f) | ||||
for fn in files: | ||||
if isstandin(fn) and fn in ctx: | ||||
addfunc(fn, ctx[fn].data().strip()) | ||||