diff --git a/.hgsigs b/.hgsigs --- a/.hgsigs +++ b/.hgsigs @@ -98,3 +98,4 @@ 7f8d16af8cae246fa5a48e723d48d58b015aed94 ced632394371a36953ce4d394f86278ae51a2aae 0 iQIVAwUAVFWpfSBXgaxoKi1yAQLCQw//cvCi/Di3z/2ZEDQt4Ayyxv18gzewqrYyoElgnEzr5uTynD9Mf25hprstKla/Y5C6q+y0K6qCHPimGOkz3H+wZ2GVUgLKAwMABkfSb5IZiLTGaB2DjAJKZRwB6h43wG/DSFggE3dYszWuyHW88c72ZzVF5CSNc4J1ARLjDSgnNYJQ6XdPw3C9KgiLFDXzynPpZbPg0AK5bdPUKJruMeIKPn36Hx/Tv5GXUrbc2/lcnyRDFWisaDl0X/5eLdA+r3ID0cSmyPLYOeCgszRiW++KGw+PPDsWVeM3ZaZ9SgaBWU7MIn9A7yQMnnSzgDbN+9v/VMT3zbk1WJXlQQK8oA+CCdHH9EY33RfZ6ST/lr3pSQbUG1hdK6Sw+H6WMkOnnEk6HtLwa4xZ3HjDpoPkhVV+S0C7D5WWOovbubxuBiW5v8tK4sIOS6bAaKevTBKRbo4Rs6qmS/Ish5Q+z5bKst80cyEdi4QSoPZ/W+6kh1KfOprMxynwPQhtEcDYW2gfLpgPIM7RdXPKukLlkV2qX3eF/tqApGU4KNdP4I3N80Ri0h+6tVU/K4TMYzlRV3ziLBumJ4TnBrTHU3X6AfZUfTgslQzokX8/7a3tbctX6kZuJPggLGisdFSdirHbrUc+y5VKuJtPr+LxxgZKRFbs2VpJRem6FvwGNyndWLv32v0GMtQ= 643c58303fb0ec020907af28b9e486be299ba043 0 iQIVAwUAVGKawCBXgaxoKi1yAQL7zxAAjpXKNvzm/PKVlTfDjuVOYZ9H8w9QKUZ0vfrNJrN6Eo6hULIostbdRc25FcMWocegTqvKbz3IG+L2TKOIdZJS9M9QS4URybUd37URq4Jai8kMiJY31KixNNnjO2G1B39aIXUhY+EPx12aY31/OVy4laXIVtN6qpSncjo9baXSOMZmx6RyA1dbyfwXRjT/aODCGHZXgLJHS/kHlkCsThVlqYQ4rUCDkXIeMqIGF1CR0KjfmKpp1fS14OMgpLgdnt9+pnBZ+qcf1YdpOeQob1zwunjMYOyYC74FyOTdwaynU2iDsuBrmkE8kgEedIn7+WWe9fp/6TQJMVOeTQPZBNSRRSUYCw5Tg/0L/+jLtzjc2mY4444sDPbR7scrtU+/GtvlR5z0Y5pofwEdFME7PZNOp9a4kMiSa7ZERyGdN7U1pDu9JU6BZRz+nPzW217PVnTF7YFV/GGUzMTk9i7EZb5M4T9r9gfxFSMPeT5ct712CdBfyRlsSbSWk8XclTXwW385kLVYNDtOukWrvEiwxpA14Xb/ZUXbIDZVf5rP2HrZHMkghzeUYPjRn/IlgYUt7sDNmqFZNIc9mRFrZC9uFQ/Nul5InZodNODQDM+nHpxaztt4xl4qKep8SDEPAQjNr8biC6T9MtLKbWbSKDlqYYNv0pb2PuGub3y9rvkF1Y05mgM= 902554884335e5ca3661d63be9978eb4aec3f68a 0 iQIVAwUAVH0KMyBXgaxoKi1yAQLUKxAAjgyYpmqD0Ji5OQ3995yX0dmwHOaaSuYpq71VUsOMYBskjH4xE2UgcTrX8RWUf0E+Ya91Nw3veTf+IZlYLaWuOYuJPRzw+zD1sVY8xprwqBOXNaA7n8SsTqZPSh6qgw4S0pUm0xJUOZzUP1l9S7BtIdJP7KwZ7hs9YZev4r9M3G15xOIPn5qJqBAtIeE6f5+ezoyOpSPZFtLFc4qKQ/YWzOT5uuSaYogXgVByXRFaO84+1TD93LR0PyVWxhwU9JrDU5d7P/bUTW1BXdjsxTbBnigWswKHC71EHpgz/HCYxivVL30qNdOm4Fow1Ec2GdUzGunSqTPrq18ScZDYW1x87f3JuqPM+ce/lxRWBBqP1yE30/8l/Us67m6enWXdGER8aL1lYTGOIWAhvJpfzv9KebaUq1gMFLo6j+OfwR3rYPiCHgi20nTNBa+LOceWFjCGzFa3T9UQWHW/MBElfAxK65uecbGRRYY9V1/+wxtTUiS6ixpmzL8S7uUd5n6oMaeeMiD82NLgPIbMyUHQv6eFEcCj0U9NT2uKbFRmclMs5V+8D+RTCsLJ55R9PD5OoRw/6K/coqqPShYmJvgYsFQPzXVpQdCRae31xdfGFmd5KUetqyrT+4GUdJWzSm0giSgovpEJNxXglrvNdvSO7fX3R1oahhwOwtGqMwNilcK+iDw= +6dad422ecc5adb63d9fa649eeb8e05a5f9bc4900 0 iQIVAwUAVJNALCBXgaxoKi1yAQKgmw/+OFbHHOMmN2zs2lI2Y0SoMALPNQBInMBq2E6RMCMbfcS9Cn75iD29DnvBwAYNWaWsYEGyheJ7JjGBiuNKPOrLaHkdjG+5ypbhAfNDyHDiteMsXfH7D1L+cTOAB8yvhimZHOTTVF0zb/uRyVIPNowAyervUVRjDptzdfcvjUS+X+/Ufgwms6Y4CcuzFLFCxpmryJhLtOpwUPLlzIqeNkFOYWkHanCgtZX03PNIWhorH3AWOc9yztwWPQ+kcKl3FMlyuNMPhS/ElxSF6GHGtreRbtP+ZLoSIOMb2QBKpGDpZLgJ3JQEHDcZ0h5CLZWL9dDUJR3M8pg1qglqMFSWMgRPTzxPS4QntPgT/Ewd3+U5oCZUh052fG41OeCZ0CnVCpqi5PjUIDhzQkONxRCN2zbjQ2GZY7glbXoqytissihEIVP9m7RmBVq1rbjOKr+yUetJ9gOZcsMtZiCEq4Uj2cbA1x32MQv7rxwAgQP1kgQ62b0sN08HTjQpI7/IkNALLIDHoQWWr45H97i34qK1dd5uCOnYk7juvhGNX5XispxNnC01/CUVNnqChfDHpgnDjgT+1H618LiTgUAD3zo4IVAhCqF5XWsS4pQEENOB3Msffi62fYowvJx7f/htWeRLZ2OA+B85hhDiD4QBdHCRoz3spVp0asNqDxX4f4ndj8RlzfM= diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -111,3 +111,4 @@ 7f8d16af8cae246fa5a48e723d48d58b015aed94 ced632394371a36953ce4d394f86278ae51a2aae 3.2 643c58303fb0ec020907af28b9e486be299ba043 3.2.1 902554884335e5ca3661d63be9978eb4aec3f68a 3.2.2 +6dad422ecc5adb63d9fa649eeb8e05a5f9bc4900 3.2.3 diff --git a/hgext/largefiles/overrides.py b/hgext/largefiles/overrides.py --- a/hgext/largefiles/overrides.py +++ b/hgext/largefiles/overrides.py @@ -191,8 +191,15 @@ def removelargefiles(ui, repo, isaddremo # are removing the file. if isaddremove: ui.status(_('removing %s\n') % f) - util.unlinkpath(repo.wjoin(f), ignoremissing=True) - lfdirstate.remove(f) + + if not opts.get('dry_run'): + if not after: + util.unlinkpath(repo.wjoin(f), ignoremissing=True) + lfdirstate.remove(f) + + if opts.get('dry_run'): + return result + lfdirstate.write() remove = [lfutil.standin(f) for f in remove] # If this is being called by addremove, let the original addremove diff --git a/mercurial/context.py b/mercurial/context.py --- a/mercurial/context.py +++ b/mercurial/context.py @@ -17,6 +17,11 @@ import revlog propertycache = util.propertycache +# Phony node value to stand-in for new files in some uses of +# manifests. Manifests support 21-byte hashes for nodes which are +# dirty in the working copy. +_newnode = '!' * 21 + class basectx(object): """A basectx object represents the common logic for its children: changectx: read-only context that is already present in the repo, @@ -104,7 +109,7 @@ class basectx(object): if (fn not in deletedset and ((fn in withflags and mf1.flags(fn) != mf2.flags(fn)) or (mf1[fn] != mf2node and - (mf2node or self[fn].cmp(other[fn]))))): + (mf2node != _newnode or self[fn].cmp(other[fn]))))): modified.append(fn) elif listclean: clean.append(fn) @@ -1382,7 +1387,7 @@ class workingctx(committablectx): """ mf = self._repo['.']._manifestmatches(match, s) for f in s.modified + s.added: - mf[f] = None + mf[f] = _newnode mf.setflag(f, self.flags(f)) for f in s.removed: if f in mf: diff --git a/mercurial/encoding.py b/mercurial/encoding.py --- a/mercurial/encoding.py +++ b/mercurial/encoding.py @@ -8,6 +8,28 @@ import error import unicodedata, locale, os +# These unicode characters are ignored by HFS+ (Apple Technote 1150, +# "Unicode Subtleties"), so we need to ignore them in some places for +# sanity. +_ignore = [unichr(int(x, 16)).encode("utf-8") for x in + "200c 200d 200e 200f 202a 202b 202c 202d 202e " + "206a 206b 206c 206d 206e 206f feff".split()] +# verify the next function will work +assert set([i[0] for i in _ignore]) == set(["\xe2", "\xef"]) + +def hfsignoreclean(s): + """Remove codepoints ignored by HFS+ from s. + + >>> hfsignoreclean(u'.h\u200cg'.encode('utf-8')) + '.hg' + >>> hfsignoreclean(u'.h\ufeffg'.encode('utf-8')) + '.hg' + """ + if "\xe2" in s or "\xef" in s: + for c in _ignore: + s = s.replace(c, '') + return s + def _getpreferredencoding(): ''' On darwin, getpreferredencoding ignores the locale environment and diff --git a/mercurial/manifest.py b/mercurial/manifest.py --- a/mercurial/manifest.py +++ b/mercurial/manifest.py @@ -17,6 +17,9 @@ class manifestdict(dict): flags = {} dict.__init__(self, mapping) self._flags = flags + def __setitem__(self, k, v): + assert v is not None + dict.__setitem__(self, k, v) def flags(self, f): return self._flags.get(f, "") def withflags(self): diff --git a/mercurial/pathutil.py b/mercurial/pathutil.py --- a/mercurial/pathutil.py +++ b/mercurial/pathutil.py @@ -1,8 +1,12 @@ import os, errno, stat +import encoding import util from i18n import _ +def _lowerclean(s): + return encoding.hfsignoreclean(s.lower()) + class pathauditor(object): '''ensure that a filesystem path contains no banned components. the following properties of a path are checked: @@ -39,11 +43,18 @@ class pathauditor(object): raise util.Abort(_("path ends in directory separator: %s") % path) parts = util.splitpath(path) if (os.path.splitdrive(path)[0] - or parts[0].lower() in ('.hg', '.hg.', '') + or _lowerclean(parts[0]) in ('.hg', '.hg.', '') or os.pardir in parts): raise util.Abort(_("path contains illegal component: %s") % path) - if '.hg' in path.lower(): - lparts = [p.lower() for p in parts] + # Windows shortname aliases + for p in parts: + if "~" in p: + first, last = p.split("~", 1) + if last.isdigit() and first.upper() in ["HG", "HG8B6C"]: + raise util.Abort(_("path contains illegal component: %s") + % path) + if '.hg' in _lowerclean(path): + lparts = [_lowerclean(p.lower()) for p in parts] for p in '.hg', '.hg.': if p in lparts[1:]: pos = lparts.index(p) diff --git a/mercurial/posix.py b/mercurial/posix.py --- a/mercurial/posix.py +++ b/mercurial/posix.py @@ -208,6 +208,7 @@ if sys.platform == 'darwin': - escape-encode invalid characters - decompose to NFD - lowercase + - omit ignored characters [200c-200f, 202a-202e, 206a-206f,feff] >>> normcase('UPPER') 'upper' @@ -265,7 +266,9 @@ if sys.platform == 'darwin': u = s.decode('utf-8') # Decompose then lowercase (HFS+ technote specifies lower) - return unicodedata.normalize('NFD', u).lower().encode('utf-8') + enc = unicodedata.normalize('NFD', u).lower().encode('utf-8') + # drop HFS+ ignored characters + return encoding.hfsignoreclean(enc) if sys.platform == 'cygwin': # workaround for cygwin, in which mount point part of path is diff --git a/tests/test-casefolding.t b/tests/test-casefolding.t --- a/tests/test-casefolding.t +++ b/tests/test-casefolding.t @@ -195,4 +195,16 @@ case changes. $ hg qrefresh a # issue 3271, qrefresh with file handled case wrong $ hg status # empty status means the qrefresh worked +#if osx + +We assume anyone running the tests on a case-insensitive volume on OS +X will be using HFS+. If that's not true, this test will fail. + + $ rm A + >>> open(u'a\u200c'.encode('utf-8'), 'w').write('unicode is fun') + $ hg status + M A + +#endif + $ cd .. diff --git a/tests/test-commit.t b/tests/test-commit.t --- a/tests/test-commit.t +++ b/tests/test-commit.t @@ -518,4 +518,55 @@ commit copy 0 0 6 ..... 0 26d3ca0dfd18 000000000000 000000000000 (re) 1 6 7 ..... 1 d267bddd54f7 26d3ca0dfd18 000000000000 (re) - $ cd .. +verify pathauditor blocks evil filepaths + $ cat > evil-commit.py < from mercurial import ui, hg, context, node + > notrc = u".h\u200cg".encode('utf-8') + '/hgrc' + > u = ui.ui() + > r = hg.repository(u, '.') + > def filectxfn(repo, memctx, path): + > return context.memfilectx(repo, path, '[hooks]\nupdate = echo owned') + > c = context.memctx(r, [r['tip'].node(), node.nullid], + > 'evil', [notrc], filectxfn, 0) + > r.commitctx(c) + > EOF + $ $PYTHON evil-commit.py + $ hg co --clean tip + abort: path contains illegal component: .h\xe2\x80\x8cg/hgrc (esc) + [255] + + $ hg rollback -f + repository tip rolled back to revision 1 (undo commit) + $ cat > evil-commit.py < from mercurial import ui, hg, context, node + > notrc = "HG~1/hgrc" + > u = ui.ui() + > r = hg.repository(u, '.') + > def filectxfn(repo, memctx, path): + > return context.memfilectx(repo, path, '[hooks]\nupdate = echo owned') + > c = context.memctx(r, [r['tip'].node(), node.nullid], + > 'evil', [notrc], filectxfn, 0) + > r.commitctx(c) + > EOF + $ $PYTHON evil-commit.py + $ hg co --clean tip + abort: path contains illegal component: HG~1/hgrc + [255] + + $ hg rollback -f + repository tip rolled back to revision 1 (undo commit) + $ cat > evil-commit.py < from mercurial import ui, hg, context, node + > notrc = "HG8B6C~2/hgrc" + > u = ui.ui() + > r = hg.repository(u, '.') + > def filectxfn(repo, memctx, path): + > return context.memfilectx(repo, path, '[hooks]\nupdate = echo owned') + > c = context.memctx(r, [r['tip'].node(), node.nullid], + > 'evil', [notrc], filectxfn, 0) + > r.commitctx(c) + > EOF + $ $PYTHON evil-commit.py + $ hg co --clean tip + abort: path contains illegal component: HG8B6C~2/hgrc + [255] diff --git a/tests/test-largefiles-misc.t b/tests/test-largefiles-misc.t --- a/tests/test-largefiles-misc.t +++ b/tests/test-largefiles-misc.t @@ -255,6 +255,16 @@ verify that large files in subrepos hand Add a normal file to the subrepo, then test archiving $ echo 'normal file' > subrepo/normal.txt + $ mv subrepo/large.txt subrepo/renamed-large.txt + $ hg -R subrepo addremove --dry-run + removing large.txt + adding normal.txt + adding renamed-large.txt + $ hg status -S + ! subrepo/large.txt + ? subrepo/normal.txt + ? subrepo/renamed-large.txt + $ mv subrepo/renamed-large.txt subrepo/large.txt $ hg -R subrepo add subrepo/normal.txt Lock in subrepo, otherwise the change isn't archived