match: make 'listfile:' split on LF and CRLF

We want util.readfile() to operate in binary mode, so EOLs have to be handled
correctly depending on the platform. It seems both easier and more convenient
to treat LF and CRLF the same way on all platforms.

mercurial/match.py

@@ -1,307 +1,310 b''
 # match.py - filename matching
 #
 #  Copyright 2008, 2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import re
 import scmutil, util
 from i18n import _
 
 class match(object):
     def __init__(self, root, cwd, patterns, include=[], exclude=[],
                  default='glob', exact=False, auditor=None):
         """build an object to match a set of file patterns
 
         arguments:
         root - the canonical root of the tree you're matching against
         cwd - the current working directory, if relevant
         patterns - patterns to find
         include - patterns to include
         exclude - patterns to exclude
         default - if a pattern in names has no explicit type, assume this one
         exact - patterns are actually literals
 
         a pattern is one of:
         'glob:<glob>' - a glob relative to cwd
         're:<regexp>' - a regular expression
         'path:<path>' - a path relative to canonroot
         'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs)
         'relpath:<path>' - a path relative to cwd
         'relre:<regexp>' - a regexp that needn't match the start of a name
         '<something>' - a pattern of the specified default type
         """
 
         self._root = root
         self._cwd = cwd
         self._files = []
         self._anypats = bool(include or exclude)
 
         if include:
             pats = _normalize(include, 'glob', root, cwd, auditor)
             self.includepat, im = _buildmatch(pats, '(?:/|$)')
         if exclude:
             pats = _normalize(exclude, 'glob', root, cwd, auditor)
             self.excludepat, em = _buildmatch(pats, '(?:/|$)')
         if exact:
             self._files = patterns
             pm = self.exact
         elif patterns:
             pats = _normalize(patterns, default, root, cwd, auditor)
             self._files = _roots(pats)
             self._anypats = self._anypats or _anypats(pats)
             self.patternspat, pm = _buildmatch(pats, '$')
 
         if patterns or exact:
             if include:
                 if exclude:
                     m = lambda f: im(f) and not em(f) and pm(f)
                 else:
                     m = lambda f: im(f) and pm(f)
             else:
                 if exclude:
                     m = lambda f: not em(f) and pm(f)
                 else:
                     m = pm
         else:
             if include:
                 if exclude:
                     m = lambda f: im(f) and not em(f)
                 else:
                     m = im
             else:
                 if exclude:
                     m = lambda f: not em(f)
                 else:
                     m = lambda f: True
 
         self.matchfn = m
         self._fmap = set(self._files)
 
     def __call__(self, fn):
         return self.matchfn(fn)
     def __iter__(self):
         for f in self._files:
             yield f
     def bad(self, f, msg):
         '''callback for each explicit file that can't be
         found/accessed, with an error message
         '''
         pass
     def dir(self, f):
         pass
     def missing(self, f):
         pass
     def exact(self, f):
         return f in self._fmap
     def rel(self, f):
         return util.pathto(self._root, self._cwd, f)
     def files(self):
         return self._files
     def anypats(self):
         return self._anypats
 
 class exact(match):
     def __init__(self, root, cwd, files):
         match.__init__(self, root, cwd, files, exact = True)
 
 class always(match):
     def __init__(self, root, cwd):
         match.__init__(self, root, cwd, [])
 
 class narrowmatcher(match):
     """Adapt a matcher to work on a subdirectory only.
 
     The paths are remapped to remove/insert the path as needed:
 
     >>> m1 = match('root', '', ['a.txt', 'sub/b.txt'])
     >>> m2 = narrowmatcher('sub', m1)
     >>> bool(m2('a.txt'))
     False
     >>> bool(m2('b.txt'))
     True
     >>> bool(m2.matchfn('a.txt'))
     False
     >>> bool(m2.matchfn('b.txt'))
     True
     >>> m2.files()
     ['b.txt']
     >>> m2.exact('b.txt')
     True
     >>> m2.rel('b.txt')
     'b.txt'
     >>> def bad(f, msg):
     ...     print "%s: %s" % (f, msg)
     >>> m1.bad = bad
     >>> m2.bad('x.txt', 'No such file')
     sub/x.txt: No such file
     """
 
     def __init__(self, path, matcher):
         self._root = matcher._root
         self._cwd = matcher._cwd
         self._path = path
         self._matcher = matcher
 
         self._files = [f[len(path) + 1:] for f in matcher._files
                        if f.startswith(path + "/")]
         self._anypats = matcher._anypats
         self.matchfn = lambda fn: matcher.matchfn(self._path + "/" + fn)
         self._fmap = set(self._files)
 
     def bad(self, f, msg):
         self._matcher.bad(self._path + "/" + f, msg)
 
 def patkind(pat):
     return _patsplit(pat, None)[0]
 
 def _patsplit(pat, default):
     """Split a string into an optional pattern kind prefix and the
     actual pattern."""
     if ':' in pat:
         kind, val = pat.split(':', 1)
         if kind in ('re', 'glob', 'path', 'relglob', 'relpath', 'relre',
                     'listfile', 'listfile0'):
             return kind, val
     return default, pat
 
 def _globre(pat):
     "convert a glob pattern into a regexp"
     i, n = 0, len(pat)
     res = ''
     group = 0
     escape = re.escape
     def peek():
         return i < n and pat[i]
     while i < n:
         c = pat[i]
         i += 1
         if c not in '*?[{},\\':
             res += escape(c)
         elif c == '*':
             if peek() == '*':
                 i += 1
                 res += '.*'
             else:
                 res += '[^/]*'
         elif c == '?':
             res += '.'
         elif c == '[':
             j = i
             if j < n and pat[j] in '!]':
                 j += 1
             while j < n and pat[j] != ']':
                 j += 1
             if j >= n:
                 res += '\\['
             else:
                 stuff = pat[i:j].replace('\\','\\\\')
                 i = j + 1
                 if stuff[0] == '!':
                     stuff = '^' + stuff[1:]
                 elif stuff[0] == '^':
                     stuff = '\\' + stuff
                 res = '%s[%s]' % (res, stuff)
         elif c == '{':
             group += 1
             res += '(?:'
         elif c == '}' and group:
             res += ')'
             group -= 1
         elif c == ',' and group:
             res += '|'
         elif c == '\\':
             p = peek()
             if p:
                 i += 1
                 res += escape(p)
             else:
                 res += escape(c)
         else:
             res += escape(c)
     return res
 
 def _regex(kind, name, tail):
     '''convert a pattern into a regular expression'''
     if not name:
         return ''
     if kind == 're':
         return name
     elif kind == 'path':
         return '^' + re.escape(name) + '(?:/|$)'
     elif kind == 'relglob':
         return '(?:|.*/)' + _globre(name) + tail
     elif kind == 'relpath':
         return re.escape(name) + '(?:/|$)'
     elif kind == 'relre':
         if name.startswith('^'):
             return name
         return '.*' + name
     return _globre(name) + tail
 
 def _buildmatch(pats, tail):
     """build a matching function from a set of patterns"""
     try:
         pat = '(?:%s)' % '|'.join([_regex(k, p, tail) for (k, p) in pats])
         if len(pat) > 20000:
             raise OverflowError()
         return pat, re.compile(pat).match
     except OverflowError:
         # We're using a Python with a tiny regex engine and we
         # made it explode, so we'll divide the pattern list in two
         # until it works
         l = len(pats)
         if l < 2:
             raise
         pata, a = _buildmatch(pats[:l//2], tail)
         patb, b = _buildmatch(pats[l//2:], tail)
         return pat, lambda s: a(s) or b(s)
     except re.error:
         for k, p in pats:
             try:
                 re.compile('(?:%s)' % _regex(k, p, tail))
             except re.error:
                 raise util.Abort(_("invalid pattern (%s): %s") % (k, p))
         raise util.Abort(_("invalid pattern"))
 
 def _normalize(names, default, root, cwd, auditor):
     pats = []
     for kind, name in [_patsplit(p, default) for p in names]:
         if kind in ('glob', 'relpath'):
             name = scmutil.canonpath(root, cwd, name, auditor)
         elif kind in ('relglob', 'path'):
             name = util.normpath(name)
         elif kind in ('listfile', 'listfile0'):
-            delimiter = kind == 'listfile0' and '\0' or '\n'
             try:
-                files = util.readfile(name).split(delimiter)
+                files = util.readfile(name)
+                if kind == 'listfile0':
+                    files = files.split('\0')
+                else:
+                    files = files.splitlines()
                 files = [f for f in files if f]
             except EnvironmentError:
                 raise util.Abort(_("unable to read file list (%s)") % name)
             pats += _normalize(files, default, root, cwd, auditor)
             continue
 
         pats.append((kind, name))
     return pats
 
 def _roots(patterns):
     r = []
     for kind, name in patterns:
         if kind == 'glob': # find the non-glob prefix
             root = []
             for p in name.split('/'):
                 if '[' in p or '{' in p or '*' in p or '?' in p:
                     break
                 root.append(p)
             r.append('/'.join(root) or '.')
         elif kind in ('relpath', 'path'):
             r.append(name or '.')
         elif kind == 'relglob':
             r.append('.')
     return r
 
 def _anypats(patterns):
     for kind, name in patterns:
         if kind in ('glob', 're', 'relglob', 'relre'):
             return True

tests/test-walk.t

@@ -1,306 +1,318 b''
   $ hg init t
   $ cd t
   $ mkdir -p beans
   $ for b in kidney navy turtle borlotti black pinto; do
   >     echo $b > beans/$b
   $ done
   $ mkdir -p mammals/Procyonidae
   $ for m in cacomistle coatimundi raccoon; do
   >     echo $m > mammals/Procyonidae/$m
   $ done
   $ echo skunk > mammals/skunk
   $ echo fennel > fennel
   $ echo fenugreek > fenugreek
   $ echo fiddlehead > fiddlehead
   $ echo glob:glob > glob:glob
   $ hg addremove
   adding beans/black
   adding beans/borlotti
   adding beans/kidney
   adding beans/navy
   adding beans/pinto
   adding beans/turtle
   adding fennel
   adding fenugreek
   adding fiddlehead
   adding glob:glob
   adding mammals/Procyonidae/cacomistle
   adding mammals/Procyonidae/coatimundi
   adding mammals/Procyonidae/raccoon
   adding mammals/skunk
   warning: filename contains ':', which is reserved on Windows: 'glob:glob'
   $ hg commit -m "commit #0"
 
   $ hg debugwalk
   f  beans/black                     beans/black
   f  beans/borlotti                  beans/borlotti
   f  beans/kidney                    beans/kidney
   f  beans/navy                      beans/navy
   f  beans/pinto                     beans/pinto
   f  beans/turtle                    beans/turtle
   f  fennel                          fennel
   f  fenugreek                       fenugreek
   f  fiddlehead                      fiddlehead
   f  glob:glob                       glob:glob
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   f  mammals/skunk                   mammals/skunk
   $ hg debugwalk -I.
   f  beans/black                     beans/black
   f  beans/borlotti                  beans/borlotti
   f  beans/kidney                    beans/kidney
   f  beans/navy                      beans/navy
   f  beans/pinto                     beans/pinto
   f  beans/turtle                    beans/turtle
   f  fennel                          fennel
   f  fenugreek                       fenugreek
   f  fiddlehead                      fiddlehead
   f  glob:glob                       glob:glob
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   f  mammals/skunk                   mammals/skunk
 
   $ cd mammals
   $ hg debugwalk
   f  beans/black                     ../beans/black
   f  beans/borlotti                  ../beans/borlotti
   f  beans/kidney                    ../beans/kidney
   f  beans/navy                      ../beans/navy
   f  beans/pinto                     ../beans/pinto
   f  beans/turtle                    ../beans/turtle
   f  fennel                          ../fennel
   f  fenugreek                       ../fenugreek
   f  fiddlehead                      ../fiddlehead
   f  glob:glob                       ../glob:glob
   f  mammals/Procyonidae/cacomistle  Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     Procyonidae/raccoon
   f  mammals/skunk                   skunk
   $ hg debugwalk -X ../beans
   f  fennel                          ../fennel
   f  fenugreek                       ../fenugreek
   f  fiddlehead                      ../fiddlehead
   f  glob:glob                       ../glob:glob
   f  mammals/Procyonidae/cacomistle  Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     Procyonidae/raccoon
   f  mammals/skunk                   skunk
   $ hg debugwalk -I '*k'
   f  mammals/skunk  skunk
   $ hg debugwalk -I 'glob:*k'
   f  mammals/skunk  skunk
   $ hg debugwalk -I 'relglob:*k'
   f  beans/black    ../beans/black
   f  fenugreek      ../fenugreek
   f  mammals/skunk  skunk
   $ hg debugwalk -I 'relglob:*k' .
   f  mammals/skunk  skunk
   $ hg debugwalk -I 're:.*k$'
   f  beans/black    ../beans/black
   f  fenugreek      ../fenugreek
   f  mammals/skunk  skunk
   $ hg debugwalk -I 'relre:.*k$'
   f  beans/black    ../beans/black
   f  fenugreek      ../fenugreek
   f  mammals/skunk  skunk
   $ hg debugwalk -I 'path:beans'
   f  beans/black     ../beans/black
   f  beans/borlotti  ../beans/borlotti
   f  beans/kidney    ../beans/kidney
   f  beans/navy      ../beans/navy
   f  beans/pinto     ../beans/pinto
   f  beans/turtle    ../beans/turtle
   $ hg debugwalk -I 'relpath:../beans'
   f  beans/black     ../beans/black
   f  beans/borlotti  ../beans/borlotti
   f  beans/kidney    ../beans/kidney
   f  beans/navy      ../beans/navy
   f  beans/pinto     ../beans/pinto
   f  beans/turtle    ../beans/turtle
   $ hg debugwalk .
   f  mammals/Procyonidae/cacomistle  Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     Procyonidae/raccoon
   f  mammals/skunk                   skunk
   $ hg debugwalk -I.
   f  mammals/Procyonidae/cacomistle  Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     Procyonidae/raccoon
   f  mammals/skunk                   skunk
   $ hg debugwalk Procyonidae
   f  mammals/Procyonidae/cacomistle  Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     Procyonidae/raccoon
 
   $ cd Procyonidae
   $ hg debugwalk .
   f  mammals/Procyonidae/cacomistle  cacomistle
   f  mammals/Procyonidae/coatimundi  coatimundi
   f  mammals/Procyonidae/raccoon     raccoon
   $ hg debugwalk ..
   f  mammals/Procyonidae/cacomistle  cacomistle
   f  mammals/Procyonidae/coatimundi  coatimundi
   f  mammals/Procyonidae/raccoon     raccoon
   f  mammals/skunk                   ../skunk
   $ cd ..
 
   $ hg debugwalk ../beans
   f  beans/black     ../beans/black
   f  beans/borlotti  ../beans/borlotti
   f  beans/kidney    ../beans/kidney
   f  beans/navy      ../beans/navy
   f  beans/pinto     ../beans/pinto
   f  beans/turtle    ../beans/turtle
   $ hg debugwalk .
   f  mammals/Procyonidae/cacomistle  Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     Procyonidae/raccoon
   f  mammals/skunk                   skunk
   $ hg debugwalk .hg
   abort: path 'mammals/.hg' is inside nested repo 'mammals'
   [255]
   $ hg debugwalk ../.hg
   abort: path contains illegal component: .hg
   [255]
   $ cd ..
 
   $ hg debugwalk -Ibeans
   f  beans/black     beans/black
   f  beans/borlotti  beans/borlotti
   f  beans/kidney    beans/kidney
   f  beans/navy      beans/navy
   f  beans/pinto     beans/pinto
   f  beans/turtle    beans/turtle
   $ hg debugwalk -I '{*,{b,m}*/*}k'
   f  beans/black    beans/black
   f  fenugreek      fenugreek
   f  mammals/skunk  mammals/skunk
   $ hg debugwalk 'glob:mammals/../beans/b*'
   f  beans/black     beans/black
   f  beans/borlotti  beans/borlotti
   $ hg debugwalk '-X*/Procyonidae' mammals
   f  mammals/skunk  mammals/skunk
   $ hg debugwalk path:mammals
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   f  mammals/skunk                   mammals/skunk
   $ hg debugwalk ..
   abort: .. not under root
   [255]
   $ hg debugwalk beans/../..
   abort: beans/../.. not under root
   [255]
   $ hg debugwalk .hg
   abort: path contains illegal component: .hg
   [255]
   $ hg debugwalk beans/../.hg
   abort: path contains illegal component: .hg
   [255]
   $ hg debugwalk beans/../.hg/data
   abort: path contains illegal component: .hg/data
   [255]
   $ hg debugwalk beans/.hg
   abort: path 'beans/.hg' is inside nested repo 'beans'
   [255]
 
 Test absolute paths:
 
   $ hg debugwalk `pwd`/beans
   f  beans/black     beans/black
   f  beans/borlotti  beans/borlotti
   f  beans/kidney    beans/kidney
   f  beans/navy      beans/navy
   f  beans/pinto     beans/pinto
   f  beans/turtle    beans/turtle
   $ hg debugwalk `pwd`/..
   abort: $TESTTMP/t/.. not under root
   [255]
 
 Test patterns:
 
   $ hg debugwalk glob:\*
   f  fennel      fennel
   f  fenugreek   fenugreek
   f  fiddlehead  fiddlehead
   f  glob:glob   glob:glob
 
   $ hg debugwalk 'glob:**e'
   f  beans/turtle                    beans/turtle
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
 
   $ hg debugwalk 're:.*[kb]$'
   f  beans/black    beans/black
   f  fenugreek      fenugreek
   f  glob:glob      glob:glob
   f  mammals/skunk  mammals/skunk
 
   $ hg debugwalk path:beans/black
   f  beans/black  beans/black  exact
   $ hg debugwalk path:beans//black
   f  beans/black  beans/black  exact
 
   $ hg debugwalk relglob:Procyonidae
   $ hg debugwalk 'relglob:Procyonidae/**'
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   $ hg debugwalk 'relglob:Procyonidae/**' fennel
   f  fennel                          fennel                          exact
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   $ hg debugwalk beans 'glob:beans/*'
   f  beans/black     beans/black
   f  beans/borlotti  beans/borlotti
   f  beans/kidney    beans/kidney
   f  beans/navy      beans/navy
   f  beans/pinto     beans/pinto
   f  beans/turtle    beans/turtle
   $ hg debugwalk 'glob:mamm**'
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   f  mammals/skunk                   mammals/skunk
   $ hg debugwalk 'glob:mamm**' fennel
   f  fennel                          fennel                          exact
   f  mammals/Procyonidae/cacomistle  mammals/Procyonidae/cacomistle
   f  mammals/Procyonidae/coatimundi  mammals/Procyonidae/coatimundi
   f  mammals/Procyonidae/raccoon     mammals/Procyonidae/raccoon
   f  mammals/skunk                   mammals/skunk
   $ hg debugwalk 'glob:j*'
   $ hg debugwalk NOEXIST
   NOEXIST: No such file or directory
 
   $ mkfifo fifo
   $ hg debugwalk fifo
   fifo: unsupported file type (type is fifo)
 
   $ rm fenugreek
   $ hg debugwalk fenugreek
   f  fenugreek  fenugreek  exact
   $ hg rm fenugreek
   $ hg debugwalk fenugreek
   f  fenugreek  fenugreek  exact
   $ touch new
   $ hg debugwalk new
   f  new  new  exact
 
   $ mkdir ignored
   $ touch ignored/file
   $ echo '^ignored$' > .hgignore
   $ hg debugwalk ignored
   $ hg debugwalk ignored/file
   f  ignored/file  ignored/file  exact
 
+Test listfile and listfile0
+
+  $ python -c "file('../listfile0', 'wb').write('fenugreek\0new\0')"
+  $ hg debugwalk -I 'listfile0:../listfile0'
+  f  fenugreek  fenugreek
+  f  new        new
+  $ python -c "file('../listfile', 'wb').write('fenugreek\nnew\r\nmammals/skunk\n')"
+  $ hg debugwalk -I 'listfile:../listfile'
+  f  fenugreek      fenugreek
+  f  mammals/skunk  mammals/skunk
+  f  new            new
+
   $ cd ..
   $ hg debugwalk -R t t/mammals/skunk
   f  mammals/skunk  t/mammals/skunk  exact
   $ mkdir t2
   $ cd t2
   $ hg debugwalk -R ../t ../t/mammals/skunk
   f  mammals/skunk  ../t/mammals/skunk  exact
   $ hg debugwalk --cwd ../t mammals/skunk
   f  mammals/skunk  mammals/skunk  exact