lfutil.py
451 lines
| 15.2 KiB
| text/x-python
|
PythonLexer
|
r15168 | # Copyright 2009-2010 Gregory P. Ward | ||
# Copyright 2009-2010 Intelerad Medical Systems Incorporated | ||||
# Copyright 2010-2011 Fog Creek Software | ||||
# Copyright 2010-2011 Unity Technologies | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
'''largefiles utility code: must not import other modules in this package.''' | ||||
import os | ||||
import errno | ||||
|
r15320 | import platform | ||
|
r15168 | import shutil | ||
import stat | ||||
|
r15391 | import tempfile | ||
|
r15168 | |||
|
r15226 | from mercurial import dirstate, httpconnection, match as match_, util, scmutil | ||
|
r15168 | from mercurial.i18n import _ | ||
shortname = '.hglf' | ||||
longname = 'largefiles' | ||||
# -- Portability wrappers ---------------------------------------------- | ||||
|
r15224 | def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): | ||
return dirstate.walk(matcher, [], unknown, ignored) | ||||
|
r15168 | |||
def repo_add(repo, list): | ||||
|
r15224 | add = repo[None].add | ||
|
r15168 | return add(list) | ||
def repo_remove(repo, list, unlink=False): | ||||
|
r15224 | def remove(list, unlink): | ||
wlock = repo.wlock() | ||||
|
r15168 | try: | ||
|
r15224 | if unlink: | ||
for f in list: | ||||
try: | ||||
util.unlinkpath(repo.wjoin(f)) | ||||
except OSError, inst: | ||||
if inst.errno != errno.ENOENT: | ||||
raise | ||||
repo[None].forget(list) | ||||
finally: | ||||
wlock.release() | ||||
|
r15168 | return remove(list, unlink=unlink) | ||
def repo_forget(repo, list): | ||||
|
r15224 | forget = repo[None].forget | ||
|
r15168 | return forget(list) | ||
def findoutgoing(repo, remote, force): | ||||
|
r15224 | from mercurial import discovery | ||
common, _anyinc, _heads = discovery.findcommonincoming(repo, | ||||
remote, force=force) | ||||
return repo.changelog.findmissing(common) | ||||
|
r15168 | |||
# -- Private worker functions ------------------------------------------ | ||||
|
r15227 | def getminsize(ui, assumelfiles, opt, default=10): | ||
lfsize = opt | ||||
if not lfsize and assumelfiles: | ||||
|
r15304 | lfsize = ui.config(longname, 'minsize', default=default) | ||
|
r15227 | if lfsize: | ||
try: | ||||
|
r15228 | lfsize = float(lfsize) | ||
|
r15227 | except ValueError: | ||
|
r15228 | raise util.Abort(_('largefiles: size must be number (not %s)\n') | ||
|
r15227 | % lfsize) | ||
if lfsize is None: | ||||
raise util.Abort(_('minimum size for largefiles must be specified')) | ||||
return lfsize | ||||
|
r15168 | def link(src, dest): | ||
try: | ||||
|
r15206 | util.oslink(src, dest) | ||
|
r15168 | except OSError: | ||
|
r15572 | # if hardlinks fail, fallback on atomic copy | ||
dst = util.atomictempfile(dest) | ||||
|
r15699 | for chunk in util.filechunkiter(open(src, 'rb')): | ||
|
r15572 | dst.write(chunk) | ||
dst.close() | ||||
|
r15168 | os.chmod(dest, os.stat(src).st_mode) | ||
|
r15316 | def usercachepath(ui, hash): | ||
|
r15350 | path = ui.configpath(longname, 'usercache', None) | ||
|
r15168 | if path: | ||
path = os.path.join(path, hash) | ||||
else: | ||||
if os.name == 'nt': | ||||
|
r15255 | appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) | ||
|
r15658 | if appdata: | ||
path = os.path.join(appdata, longname, hash) | ||||
|
r15320 | elif platform.system() == 'Darwin': | ||
|
r15658 | home = os.getenv('HOME') | ||
if home: | ||||
path = os.path.join(home, 'Library', 'Caches', | ||||
longname, hash) | ||||
|
r15168 | elif os.name == 'posix': | ||
|
r15320 | path = os.getenv('XDG_CACHE_HOME') | ||
if path: | ||||
path = os.path.join(path, longname, hash) | ||||
else: | ||||
|
r15658 | home = os.getenv('HOME') | ||
if home: | ||||
path = os.path.join(home, '.cache', longname, hash) | ||||
|
r15168 | else: | ||
|
r15253 | raise util.Abort(_('unknown operating system: %s\n') % os.name) | ||
|
r15168 | return path | ||
|
r15316 | def inusercache(ui, hash): | ||
|
r15658 | path = usercachepath(ui, hash) | ||
return path and os.path.exists(path) | ||||
|
r15168 | |||
def findfile(repo, hash): | ||||
|
r15316 | if instore(repo, hash): | ||
repo.ui.note(_('Found %s in store\n') % hash) | ||||
|
r15913 | return storepath(repo, hash) | ||
|
r15317 | elif inusercache(repo.ui, hash): | ||
|
r15168 | repo.ui.note(_('Found %s in system cache\n') % hash) | ||
|
r15408 | path = storepath(repo, hash) | ||
util.makedirs(os.path.dirname(path)) | ||||
link(usercachepath(repo.ui, hash), path) | ||||
|
r15913 | return path | ||
return None | ||||
|
r15168 | |||
class largefiles_dirstate(dirstate.dirstate): | ||||
def __getitem__(self, key): | ||||
return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | ||||
def normal(self, f): | ||||
return super(largefiles_dirstate, self).normal(unixpath(f)) | ||||
def remove(self, f): | ||||
return super(largefiles_dirstate, self).remove(unixpath(f)) | ||||
def add(self, f): | ||||
return super(largefiles_dirstate, self).add(unixpath(f)) | ||||
def drop(self, f): | ||||
return super(largefiles_dirstate, self).drop(unixpath(f)) | ||||
def forget(self, f): | ||||
return super(largefiles_dirstate, self).forget(unixpath(f)) | ||||
|
r15793 | def normallookup(self, f): | ||
return super(largefiles_dirstate, self).normallookup(unixpath(f)) | ||||
|
r15168 | |||
def openlfdirstate(ui, repo): | ||||
''' | ||||
|
r15252 | Return a dirstate object that tracks largefiles: i.e. its root is | ||
the repo root, but it is saved in .hg/largefiles/dirstate. | ||||
|
r15168 | ''' | ||
admin = repo.join(longname) | ||||
|
r15224 | opener = scmutil.opener(admin) | ||
|
r15349 | lfdirstate = largefiles_dirstate(opener, ui, repo.root, | ||
repo.dirstate._validate) | ||||
|
r15168 | |||
|
r15252 | # If the largefiles dirstate does not exist, populate and create | ||
# it. This ensures that we create it on the first meaningful | ||||
|
r15794 | # largefiles operation in a new clone. | ||
|
r15168 | if not os.path.exists(os.path.join(admin, 'dirstate')): | ||
util.makedirs(admin) | ||||
matcher = getstandinmatcher(repo) | ||||
for standin in dirstate_walk(repo.dirstate, matcher): | ||||
lfile = splitstandin(standin) | ||||
hash = readstandin(repo, lfile) | ||||
lfdirstate.normallookup(lfile) | ||||
try: | ||||
|
r15553 | if hash == hashfile(repo.wjoin(lfile)): | ||
|
r15168 | lfdirstate.normal(lfile) | ||
|
r15548 | except OSError, err: | ||
|
r15168 | if err.errno != errno.ENOENT: | ||
raise | ||||
return lfdirstate | ||||
def lfdirstate_status(lfdirstate, repo, rev): | ||||
|
r15794 | match = match_.always(repo.root, repo.getcwd()) | ||
s = lfdirstate.status(match, [], False, False, False) | ||||
unsure, modified, added, removed, missing, unknown, ignored, clean = s | ||||
for lfile in unsure: | ||||
if repo[rev][standin(lfile)].data().strip() != \ | ||||
hashfile(repo.wjoin(lfile)): | ||||
modified.append(lfile) | ||||
else: | ||||
clean.append(lfile) | ||||
lfdirstate.normal(lfile) | ||||
|
r15168 | return (modified, added, removed, missing, unknown, ignored, clean) | ||
def listlfiles(repo, rev=None, matcher=None): | ||||
|
r15252 | '''return a list of largefiles in the working copy or the | ||
specified changeset''' | ||||
|
r15168 | |||
if matcher is None: | ||||
matcher = getstandinmatcher(repo) | ||||
# ignore unknown files in working directory | ||||
|
r15255 | return [splitstandin(f) | ||
for f in repo[rev].walk(matcher) | ||||
|
r15168 | if rev is not None or repo.dirstate[f] != '?'] | ||
|
r15316 | def instore(repo, hash): | ||
return os.path.exists(storepath(repo, hash)) | ||||
|
r15168 | |||
|
r15316 | def storepath(repo, hash): | ||
|
r15168 | return repo.join(os.path.join(longname, hash)) | ||
def copyfromcache(repo, hash, filename): | ||||
|
r15252 | '''Copy the specified largefile from the repo or system cache to | ||
filename in the repository. Return true on success or false if the | ||||
file was not found in either cache (which should not happened: | ||||
this is meant to be called only after ensuring that the needed | ||||
largefile exists in the cache).''' | ||||
|
r15168 | path = findfile(repo, hash) | ||
if path is None: | ||||
return False | ||||
util.makedirs(os.path.dirname(repo.wjoin(filename))) | ||||
|
r15570 | # The write may fail before the file is fully written, but we | ||
# don't use atomic writes in the working copy. | ||||
|
r15168 | shutil.copy(path, repo.wjoin(filename)) | ||
return True | ||||
|
r15316 | def copytostore(repo, rev, file, uploaded=False): | ||
|
r15168 | hash = readstandin(repo, file) | ||
|
r15316 | if instore(repo, hash): | ||
|
r15168 | return | ||
|
r15316 | copytostoreabsolute(repo, repo.wjoin(file), hash) | ||
|
r15168 | |||
|
r15796 | def copyalltostore(repo, node): | ||
'''Copy all largefiles in a given revision to the store''' | ||||
ctx = repo[node] | ||||
for filename in ctx.files(): | ||||
if isstandin(filename) and filename in ctx.manifest(): | ||||
realfile = splitstandin(filename) | ||||
copytostore(repo, ctx.node(), realfile) | ||||
|
r15316 | def copytostoreabsolute(repo, file, hash): | ||
|
r15371 | util.makedirs(os.path.dirname(storepath(repo, hash))) | ||
|
r15316 | if inusercache(repo.ui, hash): | ||
link(usercachepath(repo.ui, hash), storepath(repo, hash)) | ||||
|
r15168 | else: | ||
|
r15571 | dst = util.atomictempfile(storepath(repo, hash)) | ||
|
r15699 | for chunk in util.filechunkiter(open(file, 'rb')): | ||
|
r15571 | dst.write(chunk) | ||
dst.close() | ||||
util.copymode(file, storepath(repo, hash)) | ||||
|
r15316 | linktousercache(repo, hash) | ||
|
r15168 | |||
|
r15316 | def linktousercache(repo, hash): | ||
|
r15658 | path = usercachepath(repo.ui, hash) | ||
if path: | ||||
util.makedirs(os.path.dirname(path)) | ||||
link(storepath(repo, hash), path) | ||||
|
r15168 | |||
def getstandinmatcher(repo, pats=[], opts={}): | ||||
'''Return a match object that applies pats to the standin directory''' | ||||
standindir = repo.pathto(shortname) | ||||
if pats: | ||||
# patterns supplied: search standin directory relative to current dir | ||||
cwd = repo.getcwd() | ||||
if os.path.isabs(cwd): | ||||
# cwd is an absolute path for hg -R <reponame> | ||||
# work relative to the repository root in this case | ||||
cwd = '' | ||||
pats = [os.path.join(standindir, cwd, pat) for pat in pats] | ||||
elif os.path.isdir(standindir): | ||||
# no patterns: relative to repo root | ||||
pats = [standindir] | ||||
else: | ||||
# no patterns and no standin dir: return matcher that matches nothing | ||||
match = match_.match(repo.root, None, [], exact=True) | ||||
match.matchfn = lambda f: False | ||||
return match | ||||
return getmatcher(repo, pats, opts, showbad=False) | ||||
def getmatcher(repo, pats=[], opts={}, showbad=True): | ||||
|
r15252 | '''Wrapper around scmutil.match() that adds showbad: if false, | ||
neuter the match object's bad() method so it does not print any | ||||
warnings about missing files or directories.''' | ||||
|
r15224 | match = scmutil.match(repo[None], pats, opts) | ||
|
r15168 | |||
if not showbad: | ||||
match.bad = lambda f, msg: None | ||||
return match | ||||
def composestandinmatcher(repo, rmatcher): | ||||
|
r15252 | '''Return a matcher that accepts standins corresponding to the | ||
files accepted by rmatcher. Pass the list of files in the matcher | ||||
as the paths specified by the user.''' | ||||
|
r15168 | smatcher = getstandinmatcher(repo, rmatcher.files()) | ||
isstandin = smatcher.matchfn | ||||
def composed_matchfn(f): | ||||
return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | ||||
smatcher.matchfn = composed_matchfn | ||||
return smatcher | ||||
def standin(filename): | ||||
'''Return the repo-relative path to the standin for the specified big | ||||
file.''' | ||||
# Notes: | ||||
# 1) Most callers want an absolute path, but _create_standin() needs | ||||
# it repo-relative so lfadd() can pass it to repo_add(). So leave | ||||
# it up to the caller to use repo.wjoin() to get an absolute path. | ||||
# 2) Join with '/' because that's what dirstate always uses, even on | ||||
# Windows. Change existing separator to '/' first in case we are | ||||
# passed filenames from an external source (like the command line). | ||||
|
r16066 | return shortname + '/' + util.pconvert(filename) | ||
|
r15168 | |||
def isstandin(filename): | ||||
|
r15252 | '''Return true if filename is a big file standin. filename must be | ||
in Mercurial's internal form (slash-separated).''' | ||||
|
r15168 | return filename.startswith(shortname + '/') | ||
def splitstandin(filename): | ||||
# Split on / because that's what dirstate always uses, even on Windows. | ||||
# Change local separator to / first just in case we are passed filenames | ||||
# from an external source (like the command line). | ||||
|
r16066 | bits = util.pconvert(filename).split('/', 1) | ||
|
r15168 | if len(bits) == 2 and bits[0] == shortname: | ||
return bits[1] | ||||
else: | ||||
return None | ||||
def updatestandin(repo, standin): | ||||
file = repo.wjoin(splitstandin(standin)) | ||||
if os.path.exists(file): | ||||
hash = hashfile(file) | ||||
executable = getexecutable(file) | ||||
writestandin(repo, standin, hash, executable) | ||||
def readstandin(repo, filename, node=None): | ||||
'''read hex hash from standin for filename at given node, or working | ||||
directory if no node is given''' | ||||
return repo[node][standin(filename)].data().strip() | ||||
def writestandin(repo, standin, hash, executable): | ||||
|
r15252 | '''write hash to <repo.root>/<standin>''' | ||
|
r15168 | writehash(hash, repo.wjoin(standin), executable) | ||
def copyandhash(instream, outfile): | ||||
'''Read bytes from instream (iterable) and write them to outfile, | ||||
computing the SHA-1 hash of the data along the way. Close outfile | ||||
when done and return the binary hash.''' | ||||
hasher = util.sha1('') | ||||
for data in instream: | ||||
hasher.update(data) | ||||
outfile.write(data) | ||||
# Blecch: closing a file that somebody else opened is rude and | ||||
|
r15252 | # wrong. But it's so darn convenient and practical! After all, | ||
|
r15168 | # outfile was opened just to copy and hash. | ||
outfile.close() | ||||
return hasher.digest() | ||||
def hashrepofile(repo, file): | ||||
return hashfile(repo.wjoin(file)) | ||||
def hashfile(file): | ||||
if not os.path.exists(file): | ||||
return '' | ||||
hasher = util.sha1('') | ||||
fd = open(file, 'rb') | ||||
for data in blockstream(fd): | ||||
hasher.update(data) | ||||
fd.close() | ||||
return hasher.hexdigest() | ||||
class limitreader(object): | ||||
def __init__(self, f, limit): | ||||
self.f = f | ||||
self.limit = limit | ||||
def read(self, length): | ||||
if self.limit == 0: | ||||
return '' | ||||
length = length > self.limit and self.limit or length | ||||
self.limit -= length | ||||
return self.f.read(length) | ||||
def close(self): | ||||
pass | ||||
def blockstream(infile, blocksize=128 * 1024): | ||||
"""Generator that yields blocks of data from infile and closes infile.""" | ||||
while True: | ||||
data = infile.read(blocksize) | ||||
if not data: | ||||
break | ||||
yield data | ||||
|
r15252 | # same blecch as copyandhash() above | ||
|
r15168 | infile.close() | ||
def writehash(hash, filename, executable): | ||||
util.makedirs(os.path.dirname(filename)) | ||||
|
r15574 | util.writefile(filename, hash + '\n') | ||
os.chmod(filename, getmode(executable)) | ||||
|
r15168 | |||
def getexecutable(filename): | ||||
mode = os.stat(filename).st_mode | ||||
|
r15255 | return ((mode & stat.S_IXUSR) and | ||
(mode & stat.S_IXGRP) and | ||||
(mode & stat.S_IXOTH)) | ||||
|
r15168 | |||
def getmode(executable): | ||||
if executable: | ||||
return 0755 | ||||
else: | ||||
return 0644 | ||||
def urljoin(first, second, *arg): | ||||
def join(left, right): | ||||
if not left.endswith('/'): | ||||
left += '/' | ||||
if right.startswith('/'): | ||||
right = right[1:] | ||||
return left + right | ||||
url = join(first, second) | ||||
for a in arg: | ||||
url = join(url, a) | ||||
return url | ||||
def hexsha1(data): | ||||
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | ||||
object data""" | ||||
|
r15347 | h = util.sha1() | ||
|
r15168 | for chunk in util.filechunkiter(data): | ||
h.update(chunk) | ||||
return h.hexdigest() | ||||
def httpsendfile(ui, filename): | ||||
|
r15224 | return httpconnection.httpsendfile(ui, filename, 'rb') | ||
|
r15168 | |||
def unixpath(path): | ||||
|
r15252 | '''Return a version of path normalized for use with the lfdirstate.''' | ||
|
r16066 | return util.pconvert(os.path.normpath(path)) | ||
|
r15168 | |||
def islfilesrepo(repo): | ||||
|
r15170 | return ('largefiles' in repo.requirements and | ||
|
r15319 | util.any(shortname + '/' in f[0] for f in repo.store.datafiles())) | ||
|
r15168 | |||
|
r15391 | def mkstemp(repo, prefix): | ||
'''Returns a file descriptor and a filename corresponding to a temporary | ||||
file in the repo's largefiles store.''' | ||||
path = repo.join(longname) | ||||
|
r15392 | util.makedirs(path) | ||
|
r15391 | return tempfile.mkstemp(prefix=prefix, dir=path) | ||
|
r15333 | class storeprotonotcapable(Exception): | ||
|
r15168 | def __init__(self, storetypes): | ||
self.storetypes = storetypes | ||||