merge with stable

contrib/check-code.py

@@ -1,563 +1,563 b''
 #!/usr/bin/env python
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """style and portability checker for Mercurial
 
 when a rule triggers wrong, do one of the following (prefer one from top):
  * do the work-around the rule suggests
  * doublecheck that it is a false match
  * improve the rule pattern
  * add an ignore pattern to the rule (3rd arg) which matches your good line
    (you can append a short comment and match this, like: #re-raises, # no-py24)
  * change the pattern to a warning and list the exception in test-check-code-hg
  * ONLY use no--check-code for skipping entire files from external sources
 """
 
 import re, glob, os, sys
 import keyword
 import optparse
 try:
     import re2
 except ImportError:
     re2 = None
 
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
     if re2:
         try:
             return re2.compile(pat)
         except re2.error:
             pass
     return re.compile(pat)
 
 def repquote(m):
     fromc = '.:'
     tochr = 'pq'
     def encodechr(i):
         if i > 255:
             return 'u'
         c = chr(i)
         if c in ' \n':
             return c
         if c.isalpha():
             return 'x'
         if c.isdigit():
             return 'n'
         try:
             return tochr[fromc.find(c)]
         except (ValueError, IndexError):
             return 'o'
     t = m.group('text')
     tt = ''.join(encodechr(i) for i in xrange(256))
     t = t.translate(tt)
     return m.group('quote') + t + m.group('quote')
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         l = len(comment.rstrip())
         return "#" * l + comment[l:]
     return repquote(m)
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
   [
     (r'pushd|popd', "don't use 'pushd' or 'popd', use 'cd'"),
     (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
     (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
     (r'(?<!hg )grep.*-a', "don't use 'grep -a', use in-line python"),
     (r'sed.*-i', "don't use 'sed -i', use a temporary file"),
     (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"),
     (r'echo -n', "don't use 'echo -n', use printf"),
     (r'(^| )wc[^|]*$\n(?!.*\(re\))', "filter wc output"),
     (r'head -c', "don't use 'head -c', use 'dd'"),
     (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
     (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
     (r'ls.*-\w*R', "don't use 'ls -R', use 'find'"),
     (r'printf.*[^\\]\\([1-9]|0\d)', "don't use 'printf \NNN', use Python"),
     (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
     (r'\$\(.*\)', "don't use $(expr), use `expr`"),
     (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
     (r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
      "use egrep for extended grep syntax"),
     (r'/bin/', "don't use explicit paths for tools"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'export.*=', "don't export and assign at once"),
     (r'^source\b', "don't use 'source', use '.'"),
     (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
     (r'ls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
     (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
     (r'^stop\(\)', "don't use 'stop' as a shell function name"),
     (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
     (r'^alias\b.*=', "don't use alias, use a function"),
     (r'if\s*!', "don't use '!' to negate exit status"),
     (r'/dev/u?random', "don't use entropy, use /dev/zero"),
     (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"),
     (r'^( *)\t', "don't use tabs to indent"),
     (r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)',
      "put a backslash-escaped newline after sed 'i' command"),
     (r'^diff *-\w*u.*$\n(^  \$ |^$)', "prefix diff -u with cmp"),
   ],
   # warnings
   [
     (r'^function', "don't use 'function', use old style"),
     (r'^diff.*-\w*N', "don't use 'diff -N'"),
     (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`"),
     (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"),
     (r'kill (`|\$\()', "don't use kill, use killdaemons.py")
   ]
 ]
 
 testfilters = [
     (r"( *)(#([^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 winglobmsg = "use (glob) to match Windows paths too"
 uprefix = r"^  \$ "
 utestpats = [
   [
     (r'^(\S.*||  [$>] .*)[ \t]\n', "trailing whitespace on non-output"),
     (uprefix + r'.*\|\s*sed[^|>\n]*\n',
      "use regex test output patterns instead of sed"),
     (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
     (uprefix + r'.*(?<!\[)\$\?', "explicit exit code checks unnecessary"),
     (uprefix + r'.*\|\| echo.*(fail|error)',
      "explicit exit code checks unnecessary"),
     (uprefix + r'set -e', "don't use set -e"),
     (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"),
     (uprefix + r'.*:\.\S*/', "x:.y in a path does not work on msys, rewrite "
      "as x://.y, or see `hg log -k msys` for alternatives", r'-\S+:\.|' #-Rxxx
      'hg pull -q file:../test'), # in test-pull.t which is skipped on windows
     (r'^  saved backup bundle to \$TESTTMP.*\.hg$', winglobmsg),
     (r'^  changeset .* references (corrupted|missing) \$TESTTMP/.*[^)]$',
      winglobmsg),
     (r'^  pulling from \$TESTTMP/.*[^)]$', winglobmsg,
      '\$TESTTMP/unix-repo$'), # in test-issue1802.t which skipped on windows
     (r'^  reverting .*/.*[^)]$', winglobmsg),
     (r'^  cloning subrepo \S+/.*[^)]$', winglobmsg),
     (r'^  pushing to \$TESTTMP/.*[^)]$', winglobmsg),
     (r'^  pushing subrepo \S+/\S+ to.*[^)]$', winglobmsg),
     (r'^  moving \S+/.*[^)]$', winglobmsg),
     (r'^  no changes made to subrepo since.*/.*[^)]$', winglobmsg),
     (r'^  .*: largefile \S+ not available from file:.*/.*[^)]$', winglobmsg),
     (r'^  .*file://\$TESTTMP',
      'write "file:/*/$TESTTMP" + (glob) to match on windows too'),
   ],
   # warnings
   [
     (r'^  [^*?/\n]* \(glob\)$',
      "glob match with no glob character (?*/)"),
   ]
 ]
 
 for i in [0, 1]:
     for p, m in testpats[i]:
         if p.startswith(r'^'):
             p = r"^  [$>] (%s)" % p[1:]
         else:
             p = r"^  [$>] .*(%s)" % p
         utestpats[i].append((p, m))
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
     (r"( *)(#([^\n]*\S)?)", repcomment),
 ]
 
 pypats = [
   [
     (r'\([^)]*\*\w[^()]+\w+=', "can't pass varargs with keyword in Py2.5"),
     (r'^\s*def\s*\w+\s*\(.*,\s*\(',
      "tuple parameter unpacking not available in Python 3+"),
     (r'lambda\s*\(.*,.*\)',
      "tuple parameter unpacking not available in Python 3+"),
     (r'import (.+,[^.]+\.[^.]+|[^.]+\.[^.]+,)',
      '2to3 can\'t always rewrite "import qux, foo.bar", '
      'use "import foo.bar" on its own line instead.'),
     (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
     (r'\breduce\s*\(.*', "reduce is not available in Python 3+"),
     (r'dict\(.*=', 'dict() is different in Py2 and 3 and is slower than {}',
      'dict-from-generator'),
     (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
     (r'\s<>\s', '<> operator is not available in Python 3+, use !='),
     (r'^\s*\t', "don't use tabs"),
     (r'\S;\s*\n', "semicolon"),
-    (r'[^_]_\("[^"]+"\s*%', "don't use % inside _()"),
-    (r"[^_]_\('[^']+'\s*%", "don't use % inside _()"),
+    (r'[^_]_\((?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"),
+    (r"[^_]_\((?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"),
     (r'(\w|\)),\w', "missing whitespace after ,"),
     (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"),
     (r'^\s+(\w|\.)+=\w[^,()\n]*$', "missing whitespace in assignment"),
     (r'(\s+)try:\n((?:\n|\1\s.*\n)+?)\1except.*?:\n'
      r'((?:\n|\1\s.*\n)+?)\1finally:', 'no try/except/finally in Python 2.4'),
     (r'(?<!def)(\s+|^|\()next\(.+\)',
      'no next(foo) in Python 2.4 and 2.5, use foo.next() instead'),
     (r'(\s+)try:\n((?:\n|\1\s.*\n)*?)\1\s*yield\b.*?'
      r'((?:\n|\1\s.*\n)+?)\1finally:',
      'no yield inside try/finally in Python 2.4'),
     (r'.{81}', "line too long"),
     (r' x+[xo][\'"]\n\s+[\'"]x', 'string join across lines with no space'),
     (r'[^\n]\Z', "no trailing newline"),
     (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
 #    (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=',
 #     "don't use underbars in identifiers"),
     (r'^\s+(self\.)?[A-za-z][a-z0-9]+[A-Z]\w* = ',
      "don't use camelcase in identifiers"),
     (r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
      "linebreak after :"),
     (r'class\s[^( \n]+:', "old-style class, use class foo(object)"),
     (r'class\s[^( \n]+\(\):',
      "class foo() not available in Python 2.4, use class foo(object)"),
     (r'\b(%s)\(' % '|'.join(keyword.kwlist),
      "Python keyword is not a function"),
     (r',]', "unneeded trailing ',' in list"),
 #    (r'class\s[A-Z][^\(]*\((?!Exception)',
 #     "don't capitalize non-exception classes"),
 #    (r'in range\(', "use xrange"),
 #    (r'^\s*print\s+', "avoid using print in core and extensions"),
     (r'[\x80-\xff]', "non-ASCII character literal"),
     (r'("\')\.format\(', "str.format() not available in Python 2.4"),
     (r'^\s*with\s+', "with not available in Python 2.4"),
     (r'\.isdisjoint\(', "set.isdisjoint not available in Python 2.4"),
     (r'^\s*except.* as .*:', "except as not available in Python 2.4"),
     (r'^\s*os\.path\.relpath', "relpath not available in Python 2.4"),
     (r'(?<!def)\s+(any|all|format)\(',
      "any/all/format not available in Python 2.4", 'no-py24'),
     (r'(?<!def)\s+(callable)\(',
      "callable not available in Python 3, use getattr(f, '__call__', None)"),
     (r'if\s.*\selse', "if ... else form not available in Python 2.4"),
     (r'^\s*(%s)\s\s' % '|'.join(keyword.kwlist),
      "gratuitous whitespace after Python keyword"),
     (r'([\(\[][ \t]\S)|(\S[ \t][\)\]])', "gratuitous whitespace in () or []"),
 #    (r'\s\s=', "gratuitous whitespace before ="),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
      "missing whitespace around operator"),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s',
      "missing whitespace around operator"),
     (r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
      "missing whitespace around operator"),
     (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]',
      "wrong whitespace around ="),
     (r'\([^()]*( =[^=]|[^<>!=]= )',
      "no whitespace around = for named parameters"),
     (r'raise Exception', "don't raise generic exceptions"),
     (r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
      "don't use old-style two-argument raise, use Exception(message)"),
     (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
     (r' [=!]=\s+(True|False|None)',
      "comparison with singleton, use 'is' or 'is not' instead"),
     (r'^\s*(while|if) [01]:',
      "use True/False for constant Boolean expression"),
     (r'(?:(?<!def)\s+|\()hasattr',
      'hasattr(foo, bar) is broken, use util.safehasattr(foo, bar) instead'),
     (r'opener\([^)]*\).read\(',
      "use opener.read() instead"),
     (r'BaseException', 'not in Python 2.4, use Exception'),
     (r'os\.path\.relpath', 'os.path.relpath is not in Python 2.5'),
     (r'opener\([^)]*\).write\(',
      "use opener.write() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.read\(',
      "use util.readfile() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.write\(',
      "use util.writefile() instead"),
     (r'^[\s\(]*(open(er)?|file)\([^)]*\)',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'[\s\(](open|file)\([^)]*\)\.',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'(?i)descendent', "the proper spelling is descendAnt"),
     (r'\.debug\(\_', "don't mark debug messages for translation"),
     (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
     (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
     (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
     (r'ui\.(status|progress|write|note|warn)\([\'\"]x',
      "missing _() in ui message (use () to hide false-positives)"),
     (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
   ],
   # warnings
   [
     (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
   ]
 ]
 
 pyfilters = [
     (r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""", reppython),
 ]
 
 txtfilters = []
 
 txtpats = [
   [
     ('\s$', 'trailing whitespace'),
     ('.. note::[ \n][^\n]', 'add two newlines after note::')
   ],
   []
 ]
 
 cpats = [
   [
     (r'//', "don't use //-style comments"),
     (r'^  ', "don't use spaces to indent"),
     (r'\S\t', "don't use tabs except for indent"),
     (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
     (r'.{81}', "line too long"),
     (r'(while|if|do|for)\(', "use space after while/if/do/for"),
     (r'return\(', "return is not a function"),
     (r' ;', "no space before ;"),
     (r'[)][{]', "space between ) and {"),
     (r'\w+\* \w+', "use int *foo, not int* foo"),
     (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
     (r'\w+ (\+\+|--)', "use foo++, not foo ++"),
     (r'\w,\w', "missing whitespace after ,"),
     (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
     (r'^#\s+\w', "use #foo, not # foo"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'^\s*#import\b', "use only #include in standard C code"),
   ],
   # warnings
   []
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
   [
     (r'\bui\.', "don't use ui in util"),
   ],
   # warnings
   []
 ]
 
 inrevlogpats = [
   [
     (r'\brepo\.', "don't use repo in revlog"),
   ],
   # warnings
   []
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', pyfilters, pypats),
     ('test script', r'(.*/)?test-[^.~]*$', testfilters, testpats),
     ('c', r'.*\.[ch]$', cfilters, cpats),
     ('unified test', r'.*\.t$', utestfilters, utestpats),
     ('layering violation repo in revlog', r'mercurial/revlog\.py', pyfilters,
      inrevlogpats),
     ('layering violation ui in util', r'mercurial/util\.py', pyfilters,
      inutilpats),
     ('txt', r'.*\.txt$', txtfilters, txtpats),
 ]
 
 def _preparepats():
     for c in checks:
         failandwarn = c[-1]
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
                 p = pseq[0]
                 # \s doesn't match \n
                 p = re.sub(r'(?<!\\)\\s', r'[ \\t]', p)
                 # [^...] doesn't match newline
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
         filters = c[2]
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
 _preparepats()
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print "%s:%d (%s):" % (fname, lineno, blame)
             else:
                 print "%s:%d:" % (fname, lineno)
             print " > %s" % line
             self._lastseen = msgid
         print " " + msg
 
 _defaultlogger = norepeatlogger()
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 def checkfile(f, logfunc=_defaultlogger.log, maxerr=None, warnings=False,
               blame=False, debug=False, lineno=True):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting.
              Set to false (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     blamecache = None
     result = True
     for name, match, filters, pats in checks:
         if debug:
             print name, f
         fc = 0
         if not re.match(match, f):
             if debug:
                 print "Skipping %s for %s it doesn't match %s" % (
                        name, match, f)
             continue
         try:
             fp = open(f)
         except IOError, e:
             print "Skipping %s, %s" % (f, str(e).split(':', 1)[0])
             continue
         pre = post = fp.read()
         fp.close()
         if "no-" "check-code" in pre:
             print "Skipping %s it has no-" "check-code" % f
             return "Skip" # skip checking this file
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0]) # nerr elements are errors
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
 
         if debug:
             print "Checking %s for %s" % (name, f)
 
         prelines = None
         errors = []
         for i, pat in enumerate(pats):
             if len(pat) == 3:
                 p, msg, ignore = pat
             else:
                 p, msg = pat
                 ignore = None
             if i >= nerrs:
                 msg = "warning: " + msg
 
             pos = 0
             n = 0
             for m in p.finditer(post):
                 if prelines is None:
                     prelines = pre.splitlines()
                     postlines = post.splitlines(True)
 
                 start = m.start()
                 while n < len(postlines):
                     step = len(postlines[n])
                     if pos + step > start:
                         break
                     pos += step
                     n += 1
                 l = prelines[n]
 
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print "Skipping %s for %s:%s (ignore pattern)" % (
                             name, f, n)
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
                     if not blamecache:
                         blamecache = getblame(f)
                     if n < len(blamecache):
                         bl, bu, br = blamecache[n]
                         if bl == l:
                             bd = '%s@%s' % (bu, br)
 
                 errors.append((f, lineno and n + 1, l, msg, bd))
                 result = False
 
         errors.sort()
         for e in errors:
             logfunc(*e)
             fc += 1
             if maxerr and fc >= maxerr:
                 print " (too many errors, giving up)"
                 break
 
     return result
 
 if __name__ == "__main__":
     parser = optparse.OptionParser("%prog [options] [files]")
     parser.add_option("-w", "--warnings", action="store_true",
                       help="include warning-level checks")
     parser.add_option("-p", "--per-file", type="int",
                       help="max warnings per file")
     parser.add_option("-b", "--blame", action="store_true",
                       help="use annotate to generate blame info")
     parser.add_option("", "--debug", action="store_true",
                       help="show debug information")
     parser.add_option("", "--nolineno", action="store_false",
                       dest='lineno', help="don't show line numbers")
 
     parser.set_defaults(per_file=15, warnings=False, blame=False, debug=False,
                         lineno=True)
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     else:
         check = args
 
     ret = 0
     for f in check:
         if not checkfile(f, maxerr=options.per_file, warnings=options.warnings,
                          blame=options.blame, debug=options.debug,
                          lineno=options.lineno):
             ret = 1
     sys.exit(ret)

hgext/eol.py

@@ -1,350 +1,350 b''
 """automatically manage newlines in repository files
 
 This extension allows you to manage the type of line endings (CRLF or
 LF) that are used in the repository and in the local working
 directory. That way you can get CRLF line endings on Windows and LF on
 Unix/Mac, thereby letting everybody use their OS native line endings.
 
 The extension reads its configuration from a versioned ``.hgeol``
 configuration file found in the root of the working copy. The
 ``.hgeol`` file use the same syntax as all other Mercurial
 configuration files. It uses two sections, ``[patterns]`` and
 ``[repository]``.
 
 The ``[patterns]`` section specifies how line endings should be
 converted between the working copy and the repository. The format is
 specified by a file pattern. The first match is used, so put more
 specific patterns first. The available line endings are ``LF``,
 ``CRLF``, and ``BIN``.
 
 Files with the declared format of ``CRLF`` or ``LF`` are always
 checked out and stored in the repository in that format and files
 declared to be binary (``BIN``) are left unchanged. Additionally,
 ``native`` is an alias for checking out in the platform's default line
 ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on
 Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's
 default behaviour; it is only needed if you need to override a later,
 more general pattern.
 
 The optional ``[repository]`` section specifies the line endings to
 use for files stored in the repository. It has a single setting,
 ``native``, which determines the storage line endings for files
 declared as ``native`` in the ``[patterns]`` section. It can be set to
 ``LF`` or ``CRLF``. The default is ``LF``. For example, this means
 that on Windows, files configured as ``native`` (``CRLF`` by default)
 will be converted to ``LF`` when stored in the repository. Files
 declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section
 are always stored as-is in the repository.
 
 Example versioned ``.hgeol`` file::
 
   [patterns]
   **.py = native
   **.vcproj = CRLF
   **.txt = native
   Makefile = LF
   **.jpg = BIN
 
   [repository]
   native = LF
 
 .. note::
 
    The rules will first apply when files are touched in the working
    copy, e.g. by updating to null and back to tip to touch all files.
 
 The extension uses an optional ``[eol]`` section read from both the
 normal Mercurial configuration files and the ``.hgeol`` file, with the
 latter overriding the former. You can use that section to control the
 overall behavior. There are three settings:
 
 - ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or
   ``CRLF`` to override the default interpretation of ``native`` for
   checkout. This can be used with :hg:`archive` on Unix, say, to
   generate an archive where files have line endings for Windows.
 
 - ``eol.only-consistent`` (default True) can be set to False to make
   the extension convert files with inconsistent EOLs. Inconsistent
   means that there is both ``CRLF`` and ``LF`` present in the file.
   Such files are normally not touched under the assumption that they
   have mixed EOLs on purpose.
 
 - ``eol.fix-trailing-newline`` (default False) can be set to True to
   ensure that converted files end with a EOL character (either ``\\n``
   or ``\\r\\n`` as per the configured patterns).
 
 The extension provides ``cleverencode:`` and ``cleverdecode:`` filters
 like the deprecated win32text extension does. This means that you can
 disable win32text and enable eol and your filters will still work. You
 only need to these filters until you have prepared a ``.hgeol`` file.
 
 The ``win32text.forbid*`` hooks provided by the win32text extension
 have been unified into a single hook named ``eol.checkheadshook``. The
 hook will lookup the expected line endings from the ``.hgeol`` file,
 which means you must migrate to a ``.hgeol`` file first before using
 the hook. ``eol.checkheadshook`` only checks heads, intermediate
 invalid revisions will be pushed. To forbid them completely, use the
 ``eol.checkallhook`` hook. These hooks are best used as
 ``pretxnchangegroup`` hooks.
 
 See :hg:`help patterns` for more information about the glob patterns
 used.
 """
 
 from mercurial.i18n import _
 from mercurial import util, config, extensions, match, error
 import re, os
 
 testedwith = 'internal'
 
 # Matches a lone LF, i.e., one that is not part of CRLF.
 singlelf = re.compile('(^|[^\r])\n')
 # Matches a single EOL which can either be a CRLF where repeated CR
 # are removed or a LF. We do not care about old Macintosh files, so a
 # stray CR is an error.
 eolre = re.compile('\r*\n')
 
 
 def inconsistenteol(data):
     return '\r\n' in data and singlelf.search(data)
 
 def tolf(s, params, ui, **kwargs):
     """Filter to convert to LF EOLs."""
     if util.binary(s):
         return s
     if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s):
         return s
     if (ui.configbool('eol', 'fix-trailing-newline', False)
         and s and s[-1] != '\n'):
         s = s + '\n'
     return eolre.sub('\n', s)
 
 def tocrlf(s, params, ui, **kwargs):
     """Filter to convert to CRLF EOLs."""
     if util.binary(s):
         return s
     if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s):
         return s
     if (ui.configbool('eol', 'fix-trailing-newline', False)
         and s and s[-1] != '\n'):
         s = s + '\n'
     return eolre.sub('\r\n', s)
 
 def isbinary(s, params):
     """Filter to do nothing with the file."""
     return s
 
 filters = {
     'to-lf': tolf,
     'to-crlf': tocrlf,
     'is-binary': isbinary,
     # The following provide backwards compatibility with win32text
     'cleverencode:': tolf,
     'cleverdecode:': tocrlf
 }
 
 class eolfile(object):
     def __init__(self, ui, root, data):
         self._decode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'}
         self._encode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'}
 
         self.cfg = config.config()
         # Our files should not be touched. The pattern must be
         # inserted first override a '** = native' pattern.
         self.cfg.set('patterns', '.hg*', 'BIN', 'eol')
         # We can then parse the user's patterns.
         self.cfg.parse('.hgeol', data)
 
         isrepolf = self.cfg.get('repository', 'native') != 'CRLF'
         self._encode['NATIVE'] = isrepolf and 'to-lf' or 'to-crlf'
         iswdlf = ui.config('eol', 'native', os.linesep) in ('LF', '\n')
         self._decode['NATIVE'] = iswdlf and 'to-lf' or 'to-crlf'
 
         include = []
         exclude = []
         for pattern, style in self.cfg.items('patterns'):
             key = style.upper()
             if key == 'BIN':
                 exclude.append(pattern)
             else:
                 include.append(pattern)
         # This will match the files for which we need to care
         # about inconsistent newlines.
         self.match = match.match(root, '', [], include, exclude)
 
     def copytoui(self, ui):
         for pattern, style in self.cfg.items('patterns'):
             key = style.upper()
             try:
                 ui.setconfig('decode', pattern, self._decode[key], 'eol')
                 ui.setconfig('encode', pattern, self._encode[key], 'eol')
             except KeyError:
                 ui.warn(_("ignoring unknown EOL style '%s' from %s\n")
                         % (style, self.cfg.source('patterns', pattern)))
         # eol.only-consistent can be specified in ~/.hgrc or .hgeol
         for k, v in self.cfg.items('eol'):
             ui.setconfig('eol', k, v, 'eol')
 
     def checkrev(self, repo, ctx, files):
         failed = []
         for f in (files or ctx.files()):
             if f not in ctx:
                 continue
             for pattern, style in self.cfg.items('patterns'):
                 if not match.match(repo.root, '', [pattern])(f):
                     continue
                 target = self._encode[style.upper()]
                 data = ctx[f].data()
                 if (target == "to-lf" and "\r\n" in data
                     or target == "to-crlf" and singlelf.search(data)):
                     failed.append((str(ctx), target, f))
                 break
         return failed
 
 def parseeol(ui, repo, nodes):
     try:
         for node in nodes:
             try:
                 if node is None:
                     # Cannot use workingctx.data() since it would load
                     # and cache the filters before we configure them.
                     data = repo.wfile('.hgeol').read()
                 else:
                     data = repo[node]['.hgeol'].data()
                 return eolfile(ui, repo.root, data)
             except (IOError, LookupError):
                 pass
     except error.ParseError, inst:
         ui.warn(_("warning: ignoring .hgeol file due to parse error "
                   "at %s: %s\n") % (inst.args[1], inst.args[0]))
     return None
 
 def _checkhook(ui, repo, node, headsonly):
     # Get revisions to check and touched files at the same time
     files = set()
     revs = set()
     for rev in xrange(repo[node].rev(), len(repo)):
         revs.add(rev)
         if headsonly:
             ctx = repo[rev]
             files.update(ctx.files())
             for pctx in ctx.parents():
                 revs.discard(pctx.rev())
     failed = []
     for rev in revs:
         ctx = repo[rev]
         eol = parseeol(ui, repo, [ctx.node()])
         if eol:
             failed.extend(eol.checkrev(repo, ctx, files))
 
     if failed:
         eols = {'to-lf': 'CRLF', 'to-crlf': 'LF'}
         msgs = []
         for node, target, f in failed:
             msgs.append(_("  %s in %s should not have %s line endings") %
                         (f, node, eols[target]))
         raise util.Abort(_("end-of-line check failed:\n") + "\n".join(msgs))
 
 def checkallhook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, False)
 
 def checkheadshook(ui, repo, node, hooktype, **kwargs):
     """verify that files have expected EOLs"""
     _checkhook(ui, repo, node, True)
 
 # "checkheadshook" used to be called "hook"
 hook = checkheadshook
 
 def preupdate(ui, repo, hooktype, parent1, parent2):
     repo.loadeol([parent1])
     return False
 
 def uisetup(ui):
     ui.setconfig('hooks', 'preupdate.eol', preupdate, 'eol')
 
 def extsetup(ui):
     try:
         extensions.find('win32text')
         ui.warn(_("the eol extension is incompatible with the "
                   "win32text extension\n"))
     except KeyError:
         pass
 
 
 def reposetup(ui, repo):
     uisetup(repo.ui)
 
     if not repo.local():
         return
     for name, fn in filters.iteritems():
         repo.adddatafilter(name, fn)
 
     ui.setconfig('patch', 'eol', 'auto', 'eol')
 
     class eolrepo(repo.__class__):
 
         def loadeol(self, nodes):
             eol = parseeol(self.ui, self, nodes)
             if eol is None:
                 return None
             eol.copytoui(self.ui)
             return eol.match
 
         def _hgcleardirstate(self):
             self._eolfile = self.loadeol([None, 'tip'])
             if not self._eolfile:
                 self._eolfile = util.never
                 return
 
             try:
                 cachemtime = os.path.getmtime(self.join("eol.cache"))
             except OSError:
                 cachemtime = 0
 
             try:
                 eolmtime = os.path.getmtime(self.wjoin(".hgeol"))
             except OSError:
                 eolmtime = 0
 
             if eolmtime > cachemtime:
                 self.ui.debug("eol: detected change in .hgeol\n")
                 wlock = None
                 try:
                     wlock = self.wlock()
                     for f in self.dirstate:
                         if self.dirstate[f] == 'n':
                             # all normal files need to be looked at
                             # again since the new .hgeol file might no
                             # longer match a file it matched before
                             self.dirstate.normallookup(f)
                     # Create or touch the cache to update mtime
                     self.opener("eol.cache", "w").close()
                     wlock.release()
                 except error.LockUnavailable:
                     # If we cannot lock the repository and clear the
                     # dirstate, then a commit might not see all files
                     # as modified. But if we cannot lock the
                     # repository, then we can also not make a commit,
                     # so ignore the error.
                     pass
 
         def commitctx(self, ctx, error=False):
             for f in sorted(ctx.added() + ctx.modified()):
                 if not self._eolfile(f):
                     continue
                 try:
                     data = ctx[f].data()
                 except IOError:
                     continue
                 if util.binary(data):
                     # We should not abort here, since the user should
                     # be able to say "** = native" to automatically
                     # have all non-binary files taken care of.
                     continue
                 if inconsistenteol(data):
                     raise util.Abort(_("inconsistent newline style "
-                                       "in %s\n" % f))
+                                       "in %s\n") % f)
             return super(eolrepo, self).commitctx(ctx, error)
     repo.__class__ = eolrepo
     repo._hgcleardirstate()

hgext/largefiles/lfcommands.py

@@ -1,582 +1,582 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''High-level command function for lfconvert, plus the cmdtable.'''
 
 import os, errno
 import shutil
 
 from mercurial import util, match as match_, hg, node, context, error, \
     cmdutil, scmutil, commands
 from mercurial.i18n import _
 from mercurial.lock import release
 
 import lfutil
 import basestore
 
 # -- Commands ----------------------------------------------------------
 
 def lfconvert(ui, src, dest, *pats, **opts):
     '''convert a normal repository to a largefiles repository
 
     Convert repository SOURCE to a new repository DEST, identical to
     SOURCE except that certain files will be converted as largefiles:
     specifically, any file that matches any PATTERN *or* whose size is
     above the minimum size threshold is converted as a largefile. The
     size used to determine whether or not to track a file as a
     largefile is the size of the first version of the file. The
     minimum size can be specified either with --size or in
     configuration as ``largefiles.size``.
 
     After running this command you will need to make sure that
     largefiles is enabled anywhere you intend to push the new
     repository.
 
     Use --to-normal to convert largefiles back to normal files; after
     this, the DEST repository can be used without largefiles at all.'''
 
     if opts['to_normal']:
         tolfile = False
     else:
         tolfile = True
         size = lfutil.getminsize(ui, True, opts.get('size'), default=None)
 
     if not hg.islocal(src):
         raise util.Abort(_('%s is not a local Mercurial repo') % src)
     if not hg.islocal(dest):
         raise util.Abort(_('%s is not a local Mercurial repo') % dest)
 
     rsrc = hg.repository(ui, src)
     ui.status(_('initializing destination %s\n') % dest)
     rdst = hg.repository(ui, dest, create=True)
 
     success = False
     dstwlock = dstlock = None
     try:
         # Lock destination to prevent modification while it is converted to.
         # Don't need to lock src because we are just reading from its history
         # which can't change.
         dstwlock = rdst.wlock()
         dstlock = rdst.lock()
 
         # Get a list of all changesets in the source.  The easy way to do this
         # is to simply walk the changelog, using changelog.nodesbetween().
         # Take a look at mercurial/revlog.py:639 for more details.
         # Use a generator instead of a list to decrease memory usage
         ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None,
             rsrc.heads())[0])
         revmap = {node.nullid: node.nullid}
         if tolfile:
             lfiles = set()
             normalfiles = set()
             if not pats:
                 pats = ui.configlist(lfutil.longname, 'patterns', default=[])
             if pats:
                 matcher = match_.match(rsrc.root, '', list(pats))
             else:
                 matcher = None
 
             lfiletohash = {}
             for ctx in ctxs:
                 ui.progress(_('converting revisions'), ctx.rev(),
                     unit=_('revision'), total=rsrc['tip'].rev())
                 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap,
                     lfiles, normalfiles, matcher, size, lfiletohash)
             ui.progress(_('converting revisions'), None)
 
             if os.path.exists(rdst.wjoin(lfutil.shortname)):
                 shutil.rmtree(rdst.wjoin(lfutil.shortname))
 
             for f in lfiletohash.keys():
                 if os.path.isfile(rdst.wjoin(f)):
                     os.unlink(rdst.wjoin(f))
                 try:
                     os.removedirs(os.path.dirname(rdst.wjoin(f)))
                 except OSError:
                     pass
 
             # If there were any files converted to largefiles, add largefiles
             # to the destination repository's requirements.
             if lfiles:
                 rdst.requirements.add('largefiles')
                 rdst._writerequirements()
         else:
             for ctx in ctxs:
                 ui.progress(_('converting revisions'), ctx.rev(),
                     unit=_('revision'), total=rsrc['tip'].rev())
                 _addchangeset(ui, rsrc, rdst, ctx, revmap)
 
             ui.progress(_('converting revisions'), None)
         success = True
     finally:
         rdst.dirstate.clear()
         release(dstlock, dstwlock)
         if not success:
             # we failed, remove the new directory
             shutil.rmtree(rdst.root)
 
 def _addchangeset(ui, rsrc, rdst, ctx, revmap):
     # Convert src parents to dst parents
     parents = _convertparents(ctx, revmap)
 
     # Generate list of changed files
     files = _getchangedfiles(ctx, parents)
 
     def getfilectx(repo, memctx, f):
         if lfutil.standin(f) in files:
             # if the file isn't in the manifest then it was removed
             # or renamed, raise IOError to indicate this
             try:
                 fctx = ctx.filectx(lfutil.standin(f))
             except error.LookupError:
                 raise IOError
             renamed = fctx.renamed()
             if renamed:
                 renamed = lfutil.splitstandin(renamed[0])
 
             hash = fctx.data().strip()
             path = lfutil.findfile(rsrc, hash)
 
             # If one file is missing, likely all files from this rev are
             if path is None:
                 cachelfiles(ui, rsrc, ctx.node())
                 path = lfutil.findfile(rsrc, hash)
 
                 if path is None:
                     raise util.Abort(
                         _("missing largefile \'%s\' from revision %s")
                          % (f, node.hex(ctx.node())))
 
             data = ''
             fd = None
             try:
                 fd = open(path, 'rb')
                 data = fd.read()
             finally:
                 if fd:
                     fd.close()
             return context.memfilectx(f, data, 'l' in fctx.flags(),
                                       'x' in fctx.flags(), renamed)
         else:
             return _getnormalcontext(repo.ui, ctx, f, revmap)
 
     dstfiles = []
     for file in files:
         if lfutil.isstandin(file):
             dstfiles.append(lfutil.splitstandin(file))
         else:
             dstfiles.append(file)
     # Commit
     _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
 
 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles,
         matcher, size, lfiletohash):
     # Convert src parents to dst parents
     parents = _convertparents(ctx, revmap)
 
     # Generate list of changed files
     files = _getchangedfiles(ctx, parents)
 
     dstfiles = []
     for f in files:
         if f not in lfiles and f not in normalfiles:
             islfile = _islfile(f, ctx, matcher, size)
             # If this file was renamed or copied then copy
             # the largefile-ness of its predecessor
             if f in ctx.manifest():
                 fctx = ctx.filectx(f)
                 renamed = fctx.renamed()
                 renamedlfile = renamed and renamed[0] in lfiles
                 islfile |= renamedlfile
                 if 'l' in fctx.flags():
                     if renamedlfile:
                         raise util.Abort(
                             _('renamed/copied largefile %s becomes symlink')
                             % f)
                     islfile = False
             if islfile:
                 lfiles.add(f)
             else:
                 normalfiles.add(f)
 
         if f in lfiles:
             dstfiles.append(lfutil.standin(f))
             # largefile in manifest if it has not been removed/renamed
             if f in ctx.manifest():
                 fctx = ctx.filectx(f)
                 if 'l' in fctx.flags():
                     renamed = fctx.renamed()
                     if renamed and renamed[0] in lfiles:
                         raise util.Abort(_('largefile %s becomes symlink') % f)
 
                 # largefile was modified, update standins
                 m = util.sha1('')
                 m.update(ctx[f].data())
                 hash = m.hexdigest()
                 if f not in lfiletohash or lfiletohash[f] != hash:
                     rdst.wwrite(f, ctx[f].data(), ctx[f].flags())
                     executable = 'x' in ctx[f].flags()
                     lfutil.writestandin(rdst, lfutil.standin(f), hash,
                         executable)
                     lfiletohash[f] = hash
         else:
             # normal file
             dstfiles.append(f)
 
     def getfilectx(repo, memctx, f):
         if lfutil.isstandin(f):
             # if the file isn't in the manifest then it was removed
             # or renamed, raise IOError to indicate this
             srcfname = lfutil.splitstandin(f)
             try:
                 fctx = ctx.filectx(srcfname)
             except error.LookupError:
                 raise IOError
             renamed = fctx.renamed()
             if renamed:
                 # standin is always a largefile because largefile-ness
                 # doesn't change after rename or copy
                 renamed = lfutil.standin(renamed[0])
 
             return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in
                 fctx.flags(), 'x' in fctx.flags(), renamed)
         else:
             return _getnormalcontext(repo.ui, ctx, f, revmap)
 
     # Commit
     _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
 
 def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap):
     mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
                           getfilectx, ctx.user(), ctx.date(), ctx.extra())
     ret = rdst.commitctx(mctx)
     rdst.setparents(ret)
     revmap[ctx.node()] = rdst.changelog.tip()
 
 # Generate list of changed files
 def _getchangedfiles(ctx, parents):
     files = set(ctx.files())
     if node.nullid not in parents:
         mc = ctx.manifest()
         mp1 = ctx.parents()[0].manifest()
         mp2 = ctx.parents()[1].manifest()
         files |= (set(mp1) | set(mp2)) - set(mc)
         for f in mc:
             if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
                 files.add(f)
     return files
 
 # Convert src parents to dst parents
 def _convertparents(ctx, revmap):
     parents = []
     for p in ctx.parents():
         parents.append(revmap[p.node()])
     while len(parents) < 2:
         parents.append(node.nullid)
     return parents
 
 # Get memfilectx for a normal file
 def _getnormalcontext(ui, ctx, f, revmap):
     try:
         fctx = ctx.filectx(f)
     except error.LookupError:
         raise IOError
     renamed = fctx.renamed()
     if renamed:
         renamed = renamed[0]
 
     data = fctx.data()
     if f == '.hgtags':
         data = _converttags (ui, revmap, data)
     return context.memfilectx(f, data, 'l' in fctx.flags(),
                               'x' in fctx.flags(), renamed)
 
 # Remap tag data using a revision map
 def _converttags(ui, revmap, data):
     newdata = []
     for line in data.splitlines():
         try:
             id, name = line.split(' ', 1)
         except ValueError:
-            ui.warn(_('skipping incorrectly formatted tag %s\n'
-                % line))
+            ui.warn(_('skipping incorrectly formatted tag %s\n')
+                % line)
             continue
         try:
             newid = node.bin(id)
         except TypeError:
-            ui.warn(_('skipping incorrectly formatted id %s\n'
-                % id))
+            ui.warn(_('skipping incorrectly formatted id %s\n')
+                % id)
             continue
         try:
             newdata.append('%s %s\n' % (node.hex(revmap[newid]),
                 name))
         except KeyError:
             ui.warn(_('no mapping for id %s\n') % id)
             continue
     return ''.join(newdata)
 
 def _islfile(file, ctx, matcher, size):
     '''Return true if file should be considered a largefile, i.e.
     matcher matches it or it is larger than size.'''
     # never store special .hg* files as largefiles
     if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs':
         return False
     if matcher and matcher(file):
         return True
     try:
         return ctx.filectx(file).size() >= size * 1024 * 1024
     except error.LookupError:
         return False
 
 def uploadlfiles(ui, rsrc, rdst, files):
     '''upload largefiles to the central store'''
 
     if not files:
         return
 
     store = basestore._openstore(rsrc, rdst, put=True)
 
     at = 0
     ui.debug("sending statlfile command for %d largefiles\n" % len(files))
     retval = store.exists(files)
     files = filter(lambda h: not retval[h], files)
     ui.debug("%d largefiles need to be uploaded\n" % len(files))
 
     for hash in files:
         ui.progress(_('uploading largefiles'), at, unit='largefile',
                     total=len(files))
         source = lfutil.findfile(rsrc, hash)
         if not source:
             raise util.Abort(_('largefile %s missing from store'
                                ' (needs to be uploaded)') % hash)
         # XXX check for errors here
         store.put(source, hash)
         at += 1
     ui.progress(_('uploading largefiles'), None)
 
 def verifylfiles(ui, repo, all=False, contents=False):
     '''Verify that every largefile revision in the current changeset
     exists in the central store.  With --contents, also verify that
     the contents of each local largefile file revision are correct (SHA-1 hash
     matches the revision ID).  With --all, check every changeset in
     this repository.'''
     if all:
         # Pass a list to the function rather than an iterator because we know a
         # list will work.
         revs = range(len(repo))
     else:
         revs = ['.']
 
     store = basestore._openstore(repo)
     return store.verify(revs, contents=contents)
 
 def debugdirstate(ui, repo):
     '''Show basic information for the largefiles dirstate'''
     lfdirstate = lfutil.openlfdirstate(ui, repo)
     for file_, ent in sorted(lfdirstate._map.iteritems()):
         mode = '%3o' % (ent[1] & 0777 & ~util.umask)
         ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_))
 
 def cachelfiles(ui, repo, node, filelist=None):
     '''cachelfiles ensures that all largefiles needed by the specified revision
     are present in the repository's largefile cache.
 
     returns a tuple (cached, missing).  cached is the list of files downloaded
     by this operation; missing is the list of files that were needed but could
     not be found.'''
     lfiles = lfutil.listlfiles(repo, node)
     if filelist:
         lfiles = set(lfiles) & set(filelist)
     toget = []
 
     for lfile in lfiles:
         try:
             expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
         except IOError, err:
             if err.errno == errno.ENOENT:
                 continue # node must be None and standin wasn't found in wctx
             raise
         if not lfutil.findfile(repo, expectedhash):
             toget.append((lfile, expectedhash))
 
     if toget:
         store = basestore._openstore(repo)
         ret = store.get(toget)
         return ret
 
     return ([], [])
 
 def downloadlfiles(ui, repo, rev=None):
     matchfn = scmutil.match(repo[None],
                             [repo.wjoin(lfutil.shortname)], {})
     def prepare(ctx, fns):
         pass
     totalsuccess = 0
     totalmissing = 0
     if rev != []: # walkchangerevs on empty list would return all revs
         for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev},
                                           prepare):
             success, missing = cachelfiles(ui, repo, ctx.node())
             totalsuccess += len(success)
             totalmissing += len(missing)
     ui.status(_("%d additional largefiles cached\n") % totalsuccess)
     if totalmissing > 0:
         ui.status(_("%d largefiles failed to download\n") % totalmissing)
     return totalsuccess, totalmissing
 
 def updatelfiles(ui, repo, filelist=None, printmessage=True):
     wlock = repo.wlock()
     try:
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate)
 
         if filelist is not None:
             lfiles = [f for f in lfiles if f in filelist]
 
         update = {}
         updated, removed = 0, 0
         for lfile in lfiles:
             abslfile = repo.wjoin(lfile)
             absstandin = repo.wjoin(lfutil.standin(lfile))
             if os.path.exists(absstandin):
                 if (os.path.exists(absstandin + '.orig') and
                     os.path.exists(abslfile)):
                     shutil.copyfile(abslfile, abslfile + '.orig')
                 expecthash = lfutil.readstandin(repo, lfile)
                 if (expecthash != '' and
                     (not os.path.exists(abslfile) or
                      expecthash != lfutil.hashfile(abslfile))):
                     if lfile not in repo[None]: # not switched to normal file
                         util.unlinkpath(abslfile, ignoremissing=True)
                     # use normallookup() to allocate entry in largefiles
                     # dirstate, because lack of it misleads
                     # lfilesrepo.status() into recognition that such cache
                     # missing files are REMOVED.
                     lfdirstate.normallookup(lfile)
                     update[lfile] = expecthash
             else:
                 # Remove lfiles for which the standin is deleted, unless the
                 # lfile is added to the repository again. This happens when a
                 # largefile is converted back to a normal file: the standin
                 # disappears, but a new (normal) file appears as the lfile.
                 if (os.path.exists(abslfile) and
                     repo.dirstate.normalize(lfile) not in repo[None]):
                     util.unlinkpath(abslfile)
                     removed += 1
 
         # largefile processing might be slow and be interrupted - be prepared
         lfdirstate.write()
 
         if lfiles:
             if printmessage:
                 ui.status(_('getting changed largefiles\n'))
             cachelfiles(ui, repo, None, lfiles)
 
         for lfile in lfiles:
             update1 = 0
 
             expecthash = update.get(lfile)
             if expecthash:
                 if not lfutil.copyfromcache(repo, expecthash, lfile):
                     # failed ... but already removed and set to normallookup
                     continue
                 # Synchronize largefile dirstate to the last modified
                 # time of the file
                 lfdirstate.normal(lfile)
                 update1 = 1
 
             # copy the state of largefile standin from the repository's
             # dirstate to its state in the lfdirstate.
             abslfile = repo.wjoin(lfile)
             absstandin = repo.wjoin(lfutil.standin(lfile))
             if os.path.exists(absstandin):
                 mode = os.stat(absstandin).st_mode
                 if mode != os.stat(abslfile).st_mode:
                     os.chmod(abslfile, mode)
                     update1 = 1
 
             updated += update1
 
             state = repo.dirstate[lfutil.standin(lfile)]
             if state == 'n':
                 # When rebasing, we need to synchronize the standin and the
                 # largefile, because otherwise the largefile will get reverted.
                 # But for commit's sake, we have to mark the file as unclean.
                 if getattr(repo, "_isrebasing", False):
                     lfdirstate.normallookup(lfile)
                 else:
                     lfdirstate.normal(lfile)
             elif state == 'r':
                 lfdirstate.remove(lfile)
             elif state == 'a':
                 lfdirstate.add(lfile)
             elif state == '?':
                 lfdirstate.drop(lfile)
 
         lfdirstate.write()
         if printmessage and lfiles:
             ui.status(_('%d largefiles updated, %d removed\n') % (updated,
                 removed))
     finally:
         wlock.release()
 
 def lfpull(ui, repo, source="default", **opts):
     """pull largefiles for the specified revisions from the specified source
 
     Pull largefiles that are referenced from local changesets but missing
     locally, pulling from a remote repository to the local cache.
 
     If SOURCE is omitted, the 'default' path will be used.
     See :hg:`help urls` for more information.
 
     .. container:: verbose
 
       Some examples:
 
       - pull largefiles for all branch heads::
 
           hg lfpull -r "head() and not closed()"
 
       - pull largefiles on the default branch::
 
           hg lfpull -r "branch(default)"
     """
     repo.lfpullsource = source
 
     revs = opts.get('rev', [])
     if not revs:
         raise util.Abort(_('no revisions specified'))
     revs = scmutil.revrange(repo, revs)
 
     numcached = 0
     for rev in revs:
         ui.note(_('pulling largefiles for revision %s\n') % rev)
         (cached, missing) = cachelfiles(ui, repo, rev)
         numcached += len(cached)
     ui.status(_("%d largefiles cached\n") % numcached)
 
 # -- hg commands declarations ------------------------------------------------
 
 cmdtable = {
     'lfconvert': (lfconvert,
                   [('s', 'size', '',
                     _('minimum size (MB) for files to be converted '
                       'as largefiles'),
                     'SIZE'),
                   ('', 'to-normal', False,
                    _('convert from a largefiles repo to a normal repo')),
                   ],
                   _('hg lfconvert SOURCE DEST [FILE ...]')),
     'lfpull': (lfpull,
                [('r', 'rev', [], _('pull largefiles for these revisions'))
                 ] +  commands.remoteopts,
                _('-r REV... [-e CMD] [--remotecmd CMD] [SOURCE]')
                ),
     }
 
 commands.inferrepo += " lfconvert"