darwin: omit ignorable codepoints when normcase()ing a file path

This lets us avoid some nasty case collision problems in OS X with
invisible codepoints.

mercurial/posix.py

@@ -1,596 +1,599
 # posix.py - Posix utility function implementations for Mercurial
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from i18n import _
 import encoding
 import os, sys, errno, stat, getpass, pwd, grp, socket, tempfile, unicodedata
 import fcntl
 
 posixfile = open
 normpath = os.path.normpath
 samestat = os.path.samestat
 oslink = os.link
 unlink = os.unlink
 rename = os.rename
 expandglobs = False
 
 umask = os.umask(0)
 os.umask(umask)
 
 def split(p):
     '''Same as posixpath.split, but faster
 
     >>> import posixpath
     >>> for f in ['/absolute/path/to/file',
     ...           'relative/path/to/file',
     ...           'file_alone',
     ...           'path/to/directory/',
     ...           '/multiple/path//separators',
     ...           '/file_at_root',
     ...           '///multiple_leading_separators_at_root',
     ...           '']:
     ...     assert split(f) == posixpath.split(f), f
     '''
     ht = p.rsplit('/', 1)
     if len(ht) == 1:
         return '', p
     nh = ht[0].rstrip('/')
     if nh:
         return nh, ht[1]
     return ht[0] + '/', ht[1]
 
 def openhardlinks():
     '''return true if it is safe to hold open file handles to hardlinks'''
     return True
 
 def nlinks(name):
     '''return number of hardlinks for the given file'''
     return os.lstat(name).st_nlink
 
 def parsepatchoutput(output_line):
     """parses the output produced by patch and returns the filename"""
     pf = output_line[14:]
     if os.sys.platform == 'OpenVMS':
         if pf[0] == '`':
             pf = pf[1:-1] # Remove the quotes
     else:
         if pf.startswith("'") and pf.endswith("'") and " " in pf:
             pf = pf[1:-1] # Remove the quotes
     return pf
 
 def sshargs(sshcmd, host, user, port):
     '''Build argument list for ssh'''
     args = user and ("%s@%s" % (user, host)) or host
     return port and ("%s -p %s" % (args, port)) or args
 
 def isexec(f):
     """check whether a file is executable"""
     return (os.lstat(f).st_mode & 0100 != 0)
 
 def setflags(f, l, x):
     s = os.lstat(f).st_mode
     if l:
         if not stat.S_ISLNK(s):
             # switch file to link
             fp = open(f)
             data = fp.read()
             fp.close()
             os.unlink(f)
             try:
                 os.symlink(data, f)
             except OSError:
                 # failed to make a link, rewrite file
                 fp = open(f, "w")
                 fp.write(data)
                 fp.close()
         # no chmod needed at this point
         return
     if stat.S_ISLNK(s):
         # switch link to file
         data = os.readlink(f)
         os.unlink(f)
         fp = open(f, "w")
         fp.write(data)
         fp.close()
         s = 0666 & ~umask # avoid restatting for chmod
 
     sx = s & 0100
     if x and not sx:
         # Turn on +x for every +r bit when making a file executable
         # and obey umask.
         os.chmod(f, s | (s & 0444) >> 2 & ~umask)
     elif not x and sx:
         # Turn off all +x bits
         os.chmod(f, s & 0666)
 
 def copymode(src, dst, mode=None):
     '''Copy the file mode from the file at path src to dst.
     If src doesn't exist, we're using mode instead. If mode is None, we're
     using umask.'''
     try:
         st_mode = os.lstat(src).st_mode & 0777
     except OSError, inst:
         if inst.errno != errno.ENOENT:
             raise
         st_mode = mode
         if st_mode is None:
             st_mode = ~umask
         st_mode &= 0666
     os.chmod(dst, st_mode)
 
 def checkexec(path):
     """
     Check whether the given path is on a filesystem with UNIX-like exec flags
 
     Requires a directory (like /foo/.hg)
     """
 
     # VFAT on some Linux versions can flip mode but it doesn't persist
     # a FS remount. Frequently we can detect it if files are created
     # with exec bit on.
 
     try:
         EXECFLAGS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
         fh, fn = tempfile.mkstemp(dir=path, prefix='hg-checkexec-')
         try:
             os.close(fh)
             m = os.stat(fn).st_mode & 0777
             new_file_has_exec = m & EXECFLAGS
             os.chmod(fn, m ^ EXECFLAGS)
             exec_flags_cannot_flip = ((os.stat(fn).st_mode & 0777) == m)
         finally:
             os.unlink(fn)
     except (IOError, OSError):
         # we don't care, the user probably won't be able to commit anyway
         return False
     return not (new_file_has_exec or exec_flags_cannot_flip)
 
 def checklink(path):
     """check whether the given path is on a symlink-capable filesystem"""
     # mktemp is not racy because symlink creation will fail if the
     # file already exists
     name = tempfile.mktemp(dir=path, prefix='hg-checklink-')
     try:
         fd = tempfile.NamedTemporaryFile(dir=path, prefix='hg-checklink-')
         try:
             os.symlink(os.path.basename(fd.name), name)
             os.unlink(name)
             return True
         finally:
             fd.close()
     except AttributeError:
         return False
     except OSError, inst:
         # sshfs might report failure while successfully creating the link
         if inst[0] == errno.EIO and os.path.exists(name):
             os.unlink(name)
         return False
 
 def checkosfilename(path):
     '''Check that the base-relative path is a valid filename on this platform.
     Returns None if the path is ok, or a UI string describing the problem.'''
     pass # on posix platforms, every path is ok
 
 def setbinary(fd):
     pass
 
 def pconvert(path):
     return path
 
 def localpath(path):
     return path
 
 def samefile(fpath1, fpath2):
     """Returns whether path1 and path2 refer to the same file. This is only
     guaranteed to work for files, not directories."""
     return os.path.samefile(fpath1, fpath2)
 
 def samedevice(fpath1, fpath2):
     """Returns whether fpath1 and fpath2 are on the same device. This is only
     guaranteed to work for files, not directories."""
     st1 = os.lstat(fpath1)
     st2 = os.lstat(fpath2)
     return st1.st_dev == st2.st_dev
 
 # os.path.normcase is a no-op, which doesn't help us on non-native filesystems
 def normcase(path):
     return path.lower()
 
 if sys.platform == 'darwin':
 
     def normcase(path):
         '''
         Normalize a filename for OS X-compatible comparison:
         - escape-encode invalid characters
         - decompose to NFD
         - lowercase
+        - omit ignored characters [200c-200f, 202a-202e, 206a-206f,feff]
 
         >>> normcase('UPPER')
         'upper'
         >>> normcase('Caf\xc3\xa9')
         'cafe\\xcc\\x81'
         >>> normcase('\xc3\x89')
         'e\\xcc\\x81'
         >>> normcase('\xb8\xca\xc3\xca\xbe\xc8.JPG') # issue3918
         '%b8%ca%c3\\xca\\xbe%c8.jpg'
         '''
 
         try:
             return encoding.asciilower(path)  # exception for non-ASCII
         except UnicodeDecodeError:
             pass
         try:
             u = path.decode('utf-8')
         except UnicodeDecodeError:
             # OS X percent-encodes any bytes that aren't valid utf-8
             s = ''
             g = ''
             l = 0
             for c in path:
                 o = ord(c)
                 if l and o < 128 or o >= 192:
                     # we want a continuation byte, but didn't get one
                     s += ''.join(["%%%02X" % ord(x) for x in g])
                     g = ''
                     l = 0
                 if l == 0 and o < 128:
                     # ascii
                     s += c
                 elif l == 0 and 194 <= o < 245:
                     # valid leading bytes
                     if o < 224:
                         l = 1
                     elif o < 240:
                         l = 2
                     else:
                         l = 3
                     g = c
                 elif l > 0 and 128 <= o < 192:
                     # valid continuations
                     g += c
                     l -= 1
                     if not l:
                         s += g
                         g = ''
                 else:
                     # invalid
                     s += "%%%02X" % o
 
             # any remaining partial characters
             s += ''.join(["%%%02X" % ord(x) for x in g])
             u = s.decode('utf-8')
 
         # Decompose then lowercase (HFS+ technote specifies lower)
-        return unicodedata.normalize('NFD', u).lower().encode('utf-8')
+        enc = unicodedata.normalize('NFD', u).lower().encode('utf-8')
+        # drop HFS+ ignored characters
+        return encoding.hfsignoreclean(enc)
 
 if sys.platform == 'cygwin':
     # workaround for cygwin, in which mount point part of path is
     # treated as case sensitive, even though underlying NTFS is case
     # insensitive.
 
     # default mount points
     cygwinmountpoints = sorted([
             "/usr/bin",
             "/usr/lib",
             "/cygdrive",
             ], reverse=True)
 
     # use upper-ing as normcase as same as NTFS workaround
     def normcase(path):
         pathlen = len(path)
         if (pathlen == 0) or (path[0] != os.sep):
             # treat as relative
             return encoding.upper(path)
 
         # to preserve case of mountpoint part
         for mp in cygwinmountpoints:
             if not path.startswith(mp):
                 continue
 
             mplen = len(mp)
             if mplen == pathlen: # mount point itself
                 return mp
             if path[mplen] == os.sep:
                 return mp + encoding.upper(path[mplen:])
 
         return encoding.upper(path)
 
     # Cygwin translates native ACLs to POSIX permissions,
     # but these translations are not supported by native
     # tools, so the exec bit tends to be set erroneously.
     # Therefore, disable executable bit access on Cygwin.
     def checkexec(path):
         return False
 
     # Similarly, Cygwin's symlink emulation is likely to create
     # problems when Mercurial is used from both Cygwin and native
     # Windows, with other native tools, or on shared volumes
     def checklink(path):
         return False
 
 def shellquote(s):
     if os.sys.platform == 'OpenVMS':
         return '"%s"' % s
     else:
         return "'%s'" % s.replace("'", "'\\''")
 
 def quotecommand(cmd):
     return cmd
 
 def popen(command, mode='r'):
     return os.popen(command, mode)
 
 def testpid(pid):
     '''return False if pid dead, True if running or not sure'''
     if os.sys.platform == 'OpenVMS':
         return True
     try:
         os.kill(pid, 0)
         return True
     except OSError, inst:
         return inst.errno != errno.ESRCH
 
 def explainexit(code):
     """return a 2-tuple (desc, code) describing a subprocess status
     (codes from kill are negative - not os.system/wait encoding)"""
     if code >= 0:
         return _("exited with status %d") % code, code
     return _("killed by signal %d") % -code, -code
 
 def isowner(st):
     """Return True if the stat object st is from the current user."""
     return st.st_uid == os.getuid()
 
 def findexe(command):
     '''Find executable for command searching like which does.
     If command is a basename then PATH is searched for command.
     PATH isn't searched if command is an absolute or relative path.
     If command isn't found None is returned.'''
     if sys.platform == 'OpenVMS':
         return command
 
     def findexisting(executable):
         'Will return executable if existing file'
         if os.path.isfile(executable) and os.access(executable, os.X_OK):
             return executable
         return None
 
     if os.sep in command:
         return findexisting(command)
 
     if sys.platform == 'plan9':
         return findexisting(os.path.join('/bin', command))
 
     for path in os.environ.get('PATH', '').split(os.pathsep):
         executable = findexisting(os.path.join(path, command))
         if executable is not None:
             return executable
     return None
 
 def setsignalhandler():
     pass
 
 _wantedkinds = set([stat.S_IFREG, stat.S_IFLNK])
 
 def statfiles(files):
     '''Stat each file in files. Yield each stat, or None if a file does not
     exist or has a type we don't care about.'''
     lstat = os.lstat
     getkind = stat.S_IFMT
     for nf in files:
         try:
             st = lstat(nf)
             if getkind(st.st_mode) not in _wantedkinds:
                 st = None
         except OSError, err:
             if err.errno not in (errno.ENOENT, errno.ENOTDIR):
                 raise
             st = None
         yield st
 
 def getuser():
     '''return name of current user'''
     return getpass.getuser()
 
 def username(uid=None):
     """Return the name of the user with the given uid.
 
     If uid is None, return the name of the current user."""
 
     if uid is None:
         uid = os.getuid()
     try:
         return pwd.getpwuid(uid)[0]
     except KeyError:
         return str(uid)
 
 def groupname(gid=None):
     """Return the name of the group with the given gid.
 
     If gid is None, return the name of the current group."""
 
     if gid is None:
         gid = os.getgid()
     try:
         return grp.getgrgid(gid)[0]
     except KeyError:
         return str(gid)
 
 def groupmembers(name):
     """Return the list of members of the group with the given
     name, KeyError if the group does not exist.
     """
     return list(grp.getgrnam(name).gr_mem)
 
 def spawndetached(args):
     return os.spawnvp(os.P_NOWAIT | getattr(os, 'P_DETACH', 0),
                       args[0], args)
 
 def gethgcmd():
     return sys.argv[:1]
 
 def termwidth():
     try:
         import termios, array
         for dev in (sys.stderr, sys.stdout, sys.stdin):
             try:
                 try:
                     fd = dev.fileno()
                 except AttributeError:
                     continue
                 if not os.isatty(fd):
                     continue
                 try:
                     arri = fcntl.ioctl(fd, termios.TIOCGWINSZ, '\0' * 8)
                     width = array.array('h', arri)[1]
                     if width > 0:
                         return width
                 except AttributeError:
                     pass
             except ValueError:
                 pass
             except IOError, e:
                 if e[0] == errno.EINVAL:
                     pass
                 else:
                     raise
     except ImportError:
         pass
     return 80
 
 def makedir(path, notindexed):
     os.mkdir(path)
 
 def unlinkpath(f, ignoremissing=False):
     """unlink and remove the directory if it is empty"""
     try:
         os.unlink(f)
     except OSError, e:
         if not (ignoremissing and e.errno == errno.ENOENT):
             raise
     # try removing directories that might now be empty
     try:
         os.removedirs(os.path.dirname(f))
     except OSError:
         pass
 
 def lookupreg(key, name=None, scope=None):
     return None
 
 def hidewindow():
     """Hide current shell window.
 
     Used to hide the window opened when starting asynchronous
     child process under Windows, unneeded on other systems.
     """
     pass
 
 class cachestat(object):
     def __init__(self, path):
         self.stat = os.stat(path)
 
     def cacheable(self):
         return bool(self.stat.st_ino)
 
     __hash__ = object.__hash__
 
     def __eq__(self, other):
         try:
             # Only dev, ino, size, mtime and atime are likely to change. Out
             # of these, we shouldn't compare atime but should compare the
             # rest. However, one of the other fields changing indicates
             # something fishy going on, so return False if anything but atime
             # changes.
             return (self.stat.st_mode == other.stat.st_mode and
                     self.stat.st_ino == other.stat.st_ino and
                     self.stat.st_dev == other.stat.st_dev and
                     self.stat.st_nlink == other.stat.st_nlink and
                     self.stat.st_uid == other.stat.st_uid and
                     self.stat.st_gid == other.stat.st_gid and
                     self.stat.st_size == other.stat.st_size and
                     self.stat.st_mtime == other.stat.st_mtime and
                     self.stat.st_ctime == other.stat.st_ctime)
         except AttributeError:
             return False
 
     def __ne__(self, other):
         return not self == other
 
 def executablepath():
     return None # available on Windows only
 
 class unixdomainserver(socket.socket):
     def __init__(self, join, subsystem):
         '''Create a unix domain socket with the given prefix.'''
         super(unixdomainserver, self).__init__(socket.AF_UNIX)
         sockname = subsystem + '.sock'
         self.realpath = self.path = join(sockname)
         if os.path.islink(self.path):
             if os.path.exists(self.path):
                 self.realpath = os.readlink(self.path)
             else:
                 os.unlink(self.path)
         try:
             self.bind(self.realpath)
         except socket.error, err:
             if err.args[0] == 'AF_UNIX path too long':
                 tmpdir = tempfile.mkdtemp(prefix='hg-%s-' % subsystem)
                 self.realpath = os.path.join(tmpdir, sockname)
                 try:
                     self.bind(self.realpath)
                     os.symlink(self.realpath, self.path)
                 except (OSError, socket.error):
                     self.cleanup()
                     raise
             else:
                 raise
         self.listen(5)
 
     def cleanup(self):
         def okayifmissing(f, path):
             try:
                 f(path)
             except OSError, err:
                 if err.errno != errno.ENOENT:
                     raise
 
         okayifmissing(os.unlink, self.path)
         if self.realpath != self.path:
             okayifmissing(os.unlink, self.realpath)
             okayifmissing(os.rmdir, os.path.dirname(self.realpath))
 
 def statislink(st):
     '''check whether a stat result is a symlink'''
     return st and stat.S_ISLNK(st.st_mode)
 
 def statisexec(st):
     '''check whether a stat result is an executable file'''
     return st and (st.st_mode & 0100 != 0)
 
 def readpipe(pipe):
     """Read all available data from a pipe."""
     # We can't fstat() a pipe because Linux will always report 0.
     # So, we set the pipe to non-blocking mode and read everything
     # that's available.
     flags = fcntl.fcntl(pipe, fcntl.F_GETFL)
     flags |= os.O_NONBLOCK
     oldflags = fcntl.fcntl(pipe, fcntl.F_SETFL, flags)
 
     try:
         chunks = []
         while True:
             try:
                 s = pipe.read()
                 if not s:
                     break
                 chunks.append(s)
             except IOError:
                 break
 
         return ''.join(chunks)
     finally:
         fcntl.fcntl(pipe, fcntl.F_SETFL, oldflags)

tests/test-casefolding.t

@@ -1,211 +1,210
 #require icasefs
 
   $ hg debugfs | grep 'case-sensitive:'
   case-sensitive: no
 
 test file addition with bad case
 
   $ hg init repo1
   $ cd repo1
   $ echo a > a
   $ hg add A
   adding a
   $ hg st
   A a
   $ hg ci -m adda
   $ hg manifest
   a
   $ cd ..
 
 test case collision on rename (issue750)
 
   $ hg init repo2
   $ cd repo2
   $ echo a > a
   $ hg --debug ci -Am adda
   adding a
   a
   committed changeset 0:07f4944404050f47db2e5c5071e0e84e7a27bba9
 
 Case-changing renames should work:
 
   $ hg mv a A
   $ hg mv A a
   $ hg st
 
 test changing case of path components
 
   $ mkdir D
   $ echo b > D/b
   $ hg ci -Am addb D/b
   $ hg mv D/b d/b
   D/b: not overwriting - file exists
   $ hg mv D/b d/c
   $ hg st
   A D/c
   R D/b
   $ mv D temp
   $ mv temp d
   $ hg st
   A D/c
   R D/b
   $ hg revert -aq
   $ rm d/c
   $ echo c > D/c
   $ hg add D/c
   $ hg st
   A D/c
   $ hg ci -m addc D/c
   $ hg mv d/b d/e
   moving D/b to D/e (glob)
   $ hg st
   A D/e
   R D/b
   $ hg revert -aq
   $ rm d/e
   $ hg mv d/b D/B
   moving D/b to D/B (glob)
   $ hg st
   A D/B
   R D/b
   $ cd ..
 
 test case collision between revisions (issue912)
 
   $ hg init repo3
   $ cd repo3
   $ echo a > a
   $ hg ci -Am adda
   adding a
   $ hg rm a
   $ hg ci -Am removea
   $ echo A > A
 
 on linux hfs keeps the old case stored, force it
 
   $ mv a aa
   $ mv aa A
   $ hg ci -Am addA
   adding A
 
 used to fail under case insensitive fs
 
   $ hg up -C 0
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ hg up -C
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
 
 no clobbering of untracked files with wrong casing
 
   $ hg up -r null
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ echo gold > a
   $ hg up
   A: untracked file differs
   abort: untracked files in working directory differ from files in requested revision
   [255]
   $ cat a
   gold
   $ rm a
 
 test that normal file in different case on target context is not
 unlinked by largefiles extension.
 
   $ cat >> .hg/hgrc <<EOF
   > [extensions]
   > largefiles=
   > EOF
   $ hg update -q -C 1
   $ hg status -A
   $ echo 'A as largefiles' > A
   $ hg add --large A
   $ hg commit -m '#3'
   created new head
   $ hg manifest -r 3
   .hglf/A
   $ hg manifest -r 0
   a
   $ hg update -q -C 0
   $ hg status -A
   C a
   $ hg update -q -C 3
   $ hg update -q 0
 
   $ cd ..
 
 issue 3342: file in nested directory causes unexpected abort
 
   $ hg init issue3342
   $ cd issue3342
 
   $ mkdir -p a/B/c/D
   $ echo e > a/B/c/D/e
   $ hg add a/B/c/D/e
 
   $ cd ..
 
 issue 3340: mq does not handle case changes correctly
 
 in addition to reported case, 'hg qrefresh' is also tested against
 case changes.
 
   $ echo "[extensions]" >> $HGRCPATH
   $ echo "mq=" >> $HGRCPATH
 
   $ hg init issue3340
   $ cd issue3340
 
   $ echo a > mIxEdCaSe
   $ hg add mIxEdCaSe
   $ hg commit -m '#0'
   $ hg rename mIxEdCaSe tmp
   $ hg rename tmp MiXeDcAsE
   $ hg status -A
   A MiXeDcAsE
     mIxEdCaSe
   R mIxEdCaSe
   $ hg qnew changecase
   $ hg status -A
   C MiXeDcAsE
 
   $ hg qpop -a
   popping changecase
   patch queue now empty
   $ hg qnew refresh-casechange
   $ hg status -A
   C mIxEdCaSe
   $ hg rename mIxEdCaSe tmp
   $ hg rename tmp MiXeDcAsE
   $ hg status -A
   A MiXeDcAsE
     mIxEdCaSe
   R mIxEdCaSe
   $ hg qrefresh
   $ hg status -A
   C MiXeDcAsE
 
   $ hg qpop -a
   popping refresh-casechange
   patch queue now empty
   $ hg qnew refresh-pattern
   $ hg status
   $ echo A > A
   $ hg add
   adding A
   $ hg qrefresh a # issue 3271, qrefresh with file handled case wrong
   $ hg status # empty status means the qrefresh worked
 
 #if osx
 
 We assume anyone running the tests on a case-insensitive volume on OS
 X will be using HFS+. If that's not true, this test will fail.
 
-Bug: some codepoints are to be ignored on HFS+:
-
   $ rm A
   >>> open(u'a\u200c'.encode('utf-8'), 'w').write('unicode is fun')
   $ hg status
   M A
-  ? a\xe2\x80\x8c (esc)
+
 #endif
+
   $ cd ..