narrowspec: move module into core

Having support for parsing the narrow specification in core is
necessary for moving many other parts of narrow to core.

We do still want to harmonize the narrow spec with the sparse
spec. And the format needs to be documented. But this shouldn't
hold up the code moving to core.

Differential Revision: https://phab.mercurial-scm.org/D2201

hgext/narrow/narrowbundle2.py

@@ -1,494 +1,494 b''
 # narrowbundle2.py - bundle2 extensions for narrow repository support
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import errno
 import struct
 
 from mercurial.i18n import _
 from mercurial.node import (
     bin,
     nullid,
     nullrev,
 )
 from mercurial import (
     bundle2,
     changegroup,
     dagutil,
     error,
     exchange,
     extensions,
+    narrowspec,
     repair,
     util,
     wireproto,
 )
 
 from . import (
     narrowrepo,
-    narrowspec,
 )
 
 NARROWCAP = 'narrow'
 _NARROWACL_SECTION = 'narrowhgacl'
 _CHANGESPECPART = NARROWCAP + ':changespec'
 _SPECPART = NARROWCAP + ':spec'
 _SPECPART_INCLUDE = 'include'
 _SPECPART_EXCLUDE = 'exclude'
 _KILLNODESIGNAL = 'KILL'
 _DONESIGNAL = 'DONE'
 _ELIDEDCSHEADER = '>20s20s20sl' # cset id, p1, p2, len(text)
 _ELIDEDMFHEADER = '>20s20s20s20sl' # manifest id, p1, p2, link id, len(text)
 _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
 _MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
 
 # When advertising capabilities, always include narrow clone support.
 def getrepocaps_narrow(orig, repo, **kwargs):
     caps = orig(repo, **kwargs)
     caps[NARROWCAP] = ['v0']
     return caps
 
 def _computeellipsis(repo, common, heads, known, match, depth=None):
     """Compute the shape of a narrowed DAG.
 
     Args:
       repo: The repository we're transferring.
       common: The roots of the DAG range we're transferring.
               May be just [nullid], which means all ancestors of heads.
       heads: The heads of the DAG range we're transferring.
       match: The narrowmatcher that allows us to identify relevant changes.
       depth: If not None, only consider nodes to be full nodes if they are at
              most depth changesets away from one of heads.
 
     Returns:
       A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
 
         visitnodes: The list of nodes (either full or ellipsis) which
                     need to be sent to the client.
         relevant_nodes: The set of changelog nodes which change a file inside
                  the narrowspec. The client needs these as non-ellipsis nodes.
         ellipsisroots: A dict of {rev: parents} that is used in
                        narrowchangegroup to produce ellipsis nodes with the
                        correct parents.
     """
     cl = repo.changelog
     mfl = repo.manifestlog
 
     cldag = dagutil.revlogdag(cl)
     # dagutil does not like nullid/nullrev
     commonrevs = cldag.internalizeall(common - set([nullid])) | set([nullrev])
     headsrevs = cldag.internalizeall(heads)
     if depth:
         revdepth = {h: 0 for h in headsrevs}
 
     ellipsisheads = collections.defaultdict(set)
     ellipsisroots = collections.defaultdict(set)
 
     def addroot(head, curchange):
         """Add a root to an ellipsis head, splitting heads with 3 roots."""
         ellipsisroots[head].add(curchange)
         # Recursively split ellipsis heads with 3 roots by finding the
         # roots' youngest common descendant which is an elided merge commit.
         # That descendant takes 2 of the 3 roots as its own, and becomes a
         # root of the head.
         while len(ellipsisroots[head]) > 2:
             child, roots = splithead(head)
             splitroots(head, child, roots)
             head = child  # Recurse in case we just added a 3rd root
 
     def splitroots(head, child, roots):
         ellipsisroots[head].difference_update(roots)
         ellipsisroots[head].add(child)
         ellipsisroots[child].update(roots)
         ellipsisroots[child].discard(child)
 
     def splithead(head):
         r1, r2, r3 = sorted(ellipsisroots[head])
         for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
             mid = repo.revs('sort(merge() & %d::%d & %d::%d, -rev)',
                             nr1, head, nr2, head)
             for j in mid:
                 if j == nr2:
                     return nr2, (nr1, nr2)
                 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
                     return j, (nr1, nr2)
         raise error.Abort('Failed to split up ellipsis node! head: %d, '
                           'roots: %d %d %d' % (head, r1, r2, r3))
 
     missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
     visit = reversed(missing)
     relevant_nodes = set()
     visitnodes = map(cl.node, missing)
     required = set(headsrevs) | known
     for rev in visit:
         clrev = cl.changelogrevision(rev)
         ps = cldag.parents(rev)
         if depth is not None:
             curdepth = revdepth[rev]
             for p in ps:
                 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
         needed = False
         shallow_enough = depth is None or revdepth[rev] <= depth
         if shallow_enough:
             curmf = mfl[clrev.manifest].read()
             if ps:
                 # We choose to not trust the changed files list in
                 # changesets because it's not always correct. TODO: could
                 # we trust it for the non-merge case?
                 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
                 needed = any(match(f) for f in curmf.diff(p1mf).iterkeys())
                 if not needed and len(ps) > 1:
                     # For merge changes, the list of changed files is not
                     # helpful, since we need to emit the merge if a file
                     # in the narrow spec has changed on either side of the
                     # merge. As a result, we do a manifest diff to check.
                     p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
                     needed = any(match(f) for f in curmf.diff(p2mf).iterkeys())
             else:
                 # For a root node, we need to include the node if any
                 # files in the node match the narrowspec.
                 needed = any(match(f) for f in curmf)
 
         if needed:
             for head in ellipsisheads[rev]:
                 addroot(head, rev)
             for p in ps:
                 required.add(p)
             relevant_nodes.add(cl.node(rev))
         else:
             if not ps:
                 ps = [nullrev]
             if rev in required:
                 for head in ellipsisheads[rev]:
                     addroot(head, rev)
                 for p in ps:
                     ellipsisheads[p].add(rev)
             else:
                 for p in ps:
                     ellipsisheads[p] |= ellipsisheads[rev]
 
     # add common changesets as roots of their reachable ellipsis heads
     for c in commonrevs:
         for head in ellipsisheads[c]:
             addroot(head, c)
     return visitnodes, relevant_nodes, ellipsisroots
 
 def _packellipsischangegroup(repo, common, match, relevant_nodes,
                              ellipsisroots, visitnodes, depth, source, version):
     if version in ('01', '02'):
         raise error.Abort(
             'ellipsis nodes require at least cg3 on client and server, '
             'but negotiated version %s' % version)
     # We wrap cg1packer.revchunk, using a side channel to pass
     # relevant_nodes into that area. Then if linknode isn't in the
     # set, we know we have an ellipsis node and we should defer
     # sending that node's data. We override close() to detect
     # pending ellipsis nodes and flush them.
     packer = changegroup.getbundler(version, repo)
     # Let the packer have access to the narrow matcher so it can
     # omit filelogs and dirlogs as needed
     packer._narrow_matcher = lambda : match
     # Give the packer the list of nodes which should not be
     # ellipsis nodes. We store this rather than the set of nodes
     # that should be an ellipsis because for very large histories
     # we expect this to be significantly smaller.
     packer.full_nodes = relevant_nodes
     # Maps ellipsis revs to their roots at the changelog level.
     packer.precomputed_ellipsis = ellipsisroots
     # Maps CL revs to per-revlog revisions. Cleared in close() at
     # the end of each group.
     packer.clrev_to_localrev = {}
     packer.next_clrev_to_localrev = {}
     # Maps changelog nodes to changelog revs. Filled in once
     # during changelog stage and then left unmodified.
     packer.clnode_to_rev = {}
     packer.changelog_done = False
     # If true, informs the packer that it is serving shallow content and might
     # need to pack file contents not introduced by the changes being packed.
     packer.is_shallow = depth is not None
 
     return packer.generate(common, visitnodes, False, source)
 
 # Serve a changegroup for a client with a narrow clone.
 def getbundlechangegrouppart_narrow(bundler, repo, source,
                                     bundlecaps=None, b2caps=None, heads=None,
                                     common=None, **kwargs):
     cgversions = b2caps.get('changegroup')
     getcgkwargs = {}
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [v for v in cgversions
                       if v in changegroup.supportedoutgoingversions(repo)]
         if not cgversions:
             raise ValueError(_('no common changegroup version'))
         version = getcgkwargs['version'] = max(cgversions)
     else:
         raise ValueError(_("server does not advertise changegroup version,"
                            " can't negotiate support for ellipsis nodes"))
 
     include = sorted(filter(bool, kwargs.get('includepats', [])))
     exclude = sorted(filter(bool, kwargs.get('excludepats', [])))
     newmatch = narrowspec.match(repo.root, include=include, exclude=exclude)
     if not repo.ui.configbool("experimental", "narrowservebrokenellipses"):
         outgoing = exchange._computeoutgoing(repo, heads, common)
         if not outgoing.missing:
             return
         def wrappedgetbundler(orig, *args, **kwargs):
             bundler = orig(*args, **kwargs)
             bundler._narrow_matcher = lambda : newmatch
             return bundler
         with extensions.wrappedfunction(changegroup, 'getbundler',
                                         wrappedgetbundler):
             cg = changegroup.makestream(repo, outgoing, version, source)
         part = bundler.newpart('changegroup', data=cg)
         part.addparam('version', version)
         if 'treemanifest' in repo.requirements:
             part.addparam('treemanifest', '1')
 
         if include or exclude:
             narrowspecpart = bundler.newpart(_SPECPART)
             if include:
                 narrowspecpart.addparam(
                     _SPECPART_INCLUDE, '\n'.join(include), mandatory=True)
             if exclude:
                 narrowspecpart.addparam(
                     _SPECPART_EXCLUDE, '\n'.join(exclude), mandatory=True)
 
         return
 
     depth = kwargs.get('depth', None)
     if depth is not None:
         depth = int(depth)
         if depth < 1:
             raise error.Abort(_('depth must be positive, got %d') % depth)
 
     heads = set(heads or repo.heads())
     common = set(common or [nullid])
     oldinclude = sorted(filter(bool, kwargs.get('oldincludepats', [])))
     oldexclude = sorted(filter(bool, kwargs.get('oldexcludepats', [])))
     known = {bin(n) for n in kwargs.get('known', [])}
     if known and (oldinclude != include or oldexclude != exclude):
         # Steps:
         # 1. Send kill for "$known & ::common"
         #
         # 2. Send changegroup for ::common
         #
         # 3. Proceed.
         #
         # In the future, we can send kills for only the specific
         # nodes we know should go away or change shape, and then
         # send a data stream that tells the client something like this:
         #
         # a) apply this changegroup
         # b) apply nodes XXX, YYY, ZZZ that you already have
         # c) goto a
         #
         # until they've built up the full new state.
         # Convert to revnums and intersect with "common". The client should
         # have made it a subset of "common" already, but let's be safe.
         known = set(repo.revs("%ln & ::%ln", known, common))
         # TODO: we could send only roots() of this set, and the
         # list of nodes in common, and the client could work out
         # what to strip, instead of us explicitly sending every
         # single node.
         deadrevs = known
         def genkills():
             for r in deadrevs:
                 yield _KILLNODESIGNAL
                 yield repo.changelog.node(r)
             yield _DONESIGNAL
         bundler.newpart(_CHANGESPECPART, data=genkills())
         newvisit, newfull, newellipsis = _computeellipsis(
             repo, set(), common, known, newmatch)
         if newvisit:
             cg = _packellipsischangegroup(
                 repo, common, newmatch, newfull, newellipsis,
                 newvisit, depth, source, version)
             part = bundler.newpart('changegroup', data=cg)
             part.addparam('version', version)
             if 'treemanifest' in repo.requirements:
                 part.addparam('treemanifest', '1')
 
     visitnodes, relevant_nodes, ellipsisroots = _computeellipsis(
         repo, common, heads, set(), newmatch, depth=depth)
 
     repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
     if visitnodes:
         cg = _packellipsischangegroup(
             repo, common, newmatch, relevant_nodes, ellipsisroots,
             visitnodes, depth, source, version)
         part = bundler.newpart('changegroup', data=cg)
         part.addparam('version', version)
         if 'treemanifest' in repo.requirements:
             part.addparam('treemanifest', '1')
 
 def applyacl_narrow(repo, kwargs):
     username = repo.ui.shortuser(repo.ui.username())
     user_includes = repo.ui.configlist(
         _NARROWACL_SECTION, username + '.includes',
         repo.ui.configlist(_NARROWACL_SECTION, 'default.includes'))
     user_excludes = repo.ui.configlist(
         _NARROWACL_SECTION, username + '.excludes',
         repo.ui.configlist(_NARROWACL_SECTION, 'default.excludes'))
     if not user_includes:
         raise error.Abort(_("{} configuration for user {} is empty")
                           .format(_NARROWACL_SECTION, username))
 
     user_includes = [
         'path:.' if p == '*' else 'path:' + p for p in user_includes]
     user_excludes = [
         'path:.' if p == '*' else 'path:' + p for p in user_excludes]
 
     req_includes = set(kwargs.get('includepats', []))
     req_excludes = set(kwargs.get('excludepats', []))
 
     req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
         req_includes, req_excludes, user_includes, user_excludes)
 
     if invalid_includes:
         raise error.Abort(
             _("The following includes are not accessible for {}: {}")
             .format(username, invalid_includes))
 
     new_args = {}
     new_args.update(kwargs)
     new_args['includepats'] = req_includes
     if req_excludes:
         new_args['excludepats'] = req_excludes
     return new_args
 
 @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
 def _handlechangespec_2(op, inpart):
     includepats = set(inpart.params.get(_SPECPART_INCLUDE, '').splitlines())
     excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, '').splitlines())
     narrowspec.save(op.repo, includepats, excludepats)
     if not narrowrepo.REQUIREMENT in op.repo.requirements:
         op.repo.requirements.add(narrowrepo.REQUIREMENT)
         op.repo._writerequirements()
     op.repo.invalidate(clearfilecache=True)
 
 @bundle2.parthandler(_CHANGESPECPART)
 def _handlechangespec(op, inpart):
     repo = op.repo
     cl = repo.changelog
 
     # changesets which need to be stripped entirely. either they're no longer
     # needed in the new narrow spec, or the server is sending a replacement
     # in the changegroup part.
     clkills = set()
 
     # A changespec part contains all the updates to ellipsis nodes
     # that will happen as a result of widening or narrowing a
     # repo. All the changes that this block encounters are ellipsis
     # nodes or flags to kill an existing ellipsis.
     chunksignal = changegroup.readexactly(inpart, 4)
     while chunksignal != _DONESIGNAL:
         if chunksignal == _KILLNODESIGNAL:
             # a node used to be an ellipsis but isn't anymore
             ck = changegroup.readexactly(inpart, 20)
             if cl.hasnode(ck):
                 clkills.add(ck)
         else:
             raise error.Abort(
                 _('unexpected changespec node chunk type: %s') % chunksignal)
         chunksignal = changegroup.readexactly(inpart, 4)
 
     if clkills:
         # preserve bookmarks that repair.strip() would otherwise strip
         bmstore = repo._bookmarks
         class dummybmstore(dict):
             def applychanges(self, repo, tr, changes):
                 pass
             def recordchange(self, tr): # legacy version
                 pass
         repo._bookmarks = dummybmstore()
         chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
                                  topic='widen')
         repo._bookmarks = bmstore
         if chgrpfile:
             # presence of _widen_bundle attribute activates widen handler later
             op._widen_bundle = chgrpfile
     # Set the new narrowspec if we're widening. The setnewnarrowpats() method
     # will currently always be there when using the core+narrowhg server, but
     # other servers may include a changespec part even when not widening (e.g.
     # because we're deepening a shallow repo).
     if util.safehasattr(repo, 'setnewnarrowpats'):
         repo.setnewnarrowpats()
 
 def handlechangegroup_widen(op, inpart):
     """Changegroup exchange handler which restores temporarily-stripped nodes"""
     # We saved a bundle with stripped node data we must now restore.
     # This approach is based on mercurial/repair.py@6ee26a53c111.
     repo = op.repo
     ui = op.ui
 
     chgrpfile = op._widen_bundle
     del op._widen_bundle
     vfs = repo.vfs
 
     ui.note(_("adding branch\n"))
     f = vfs.open(chgrpfile, "rb")
     try:
         gen = exchange.readbundle(ui, f, chgrpfile, vfs)
         if not ui.verbose:
             # silence internal shuffling chatter
             ui.pushbuffer()
         if isinstance(gen, bundle2.unbundle20):
             with repo.transaction('strip') as tr:
                 bundle2.processbundle(repo, gen, lambda: tr)
         else:
             gen.apply(repo, 'strip', 'bundle:' + vfs.join(chgrpfile), True)
         if not ui.verbose:
             ui.popbuffer()
     finally:
         f.close()
 
     # remove undo files
     for undovfs, undofile in repo.undofiles():
         try:
             undovfs.unlink(undofile)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 ui.warn(_('error removing %s: %s\n') %
                         (undovfs.join(undofile), str(e)))
 
     # Remove partial backup only if there were no exceptions
     vfs.unlink(chgrpfile)
 
 def setup():
     """Enable narrow repo support in bundle2-related extension points."""
     extensions.wrapfunction(bundle2, 'getrepocaps', getrepocaps_narrow)
 
     wireproto.gboptsmap['narrow'] = 'boolean'
     wireproto.gboptsmap['depth'] = 'plain'
     wireproto.gboptsmap['oldincludepats'] = 'csv'
     wireproto.gboptsmap['oldexcludepats'] = 'csv'
     wireproto.gboptsmap['includepats'] = 'csv'
     wireproto.gboptsmap['excludepats'] = 'csv'
     wireproto.gboptsmap['known'] = 'csv'
 
     # Extend changegroup serving to handle requests from narrow clients.
     origcgfn = exchange.getbundle2partsmapping['changegroup']
     def wrappedcgfn(*args, **kwargs):
         repo = args[1]
         if repo.ui.has_section(_NARROWACL_SECTION):
             getbundlechangegrouppart_narrow(
                 *args, **applyacl_narrow(repo, kwargs))
         elif kwargs.get('narrow', False):
             getbundlechangegrouppart_narrow(*args, **kwargs)
         else:
             origcgfn(*args, **kwargs)
     exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
 
     # Extend changegroup receiver so client can fixup after widen requests.
     origcghandler = bundle2.parthandlermapping['changegroup']
     def wrappedcghandler(op, inpart):
         origcghandler(op, inpart)
         if util.safehasattr(op, '_widen_bundle'):
             handlechangegroup_widen(op, inpart)
     wrappedcghandler.params = origcghandler.params
     bundle2.parthandlermapping['changegroup'] = wrappedcghandler

hgext/narrow/narrowcommands.py

@@ -1,404 +1,404 b''
 # narrowcommands.py - command modifications for narrowhg extension
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import itertools
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     discovery,
     error,
     exchange,
     extensions,
     hg,
     merge,
+    narrowspec,
     node,
     pycompat,
     registrar,
     repair,
     repoview,
     util,
 )
 
 from . import (
     narrowbundle2,
     narrowrepo,
-    narrowspec,
 )
 
 table = {}
 command = registrar.command(table)
 
 def setup():
     """Wraps user-facing mercurial commands with narrow-aware versions."""
 
     entry = extensions.wrapcommand(commands.table, 'clone', clonenarrowcmd)
     entry[1].append(('', 'narrow', None,
                      _("create a narrow clone of select files")))
     entry[1].append(('', 'depth', '',
                      _("limit the history fetched by distance from heads")))
     # TODO(durin42): unify sparse/narrow --include/--exclude logic a bit
     if 'sparse' not in extensions.enabled():
         entry[1].append(('', 'include', [],
                          _("specifically fetch this file/directory")))
         entry[1].append(
             ('', 'exclude', [],
              _("do not fetch this file/directory, even if included")))
 
     entry = extensions.wrapcommand(commands.table, 'pull', pullnarrowcmd)
     entry[1].append(('', 'depth', '',
                      _("limit the history fetched by distance from heads")))
 
     extensions.wrapcommand(commands.table, 'archive', archivenarrowcmd)
 
 def expandpull(pullop, includepats, excludepats):
     if not narrowspec.needsexpansion(includepats):
         return includepats, excludepats
 
     heads = pullop.heads or pullop.rheads
     includepats, excludepats = pullop.remote.expandnarrow(
         includepats, excludepats, heads)
     pullop.repo.ui.debug('Expanded narrowspec to inc=%s, exc=%s\n' % (
         includepats, excludepats))
     return set(includepats), set(excludepats)
 
 def clonenarrowcmd(orig, ui, repo, *args, **opts):
     """Wraps clone command, so 'hg clone' first wraps localrepo.clone()."""
     opts = pycompat.byteskwargs(opts)
     wrappedextraprepare = util.nullcontextmanager()
     opts_narrow = opts['narrow']
     if opts_narrow:
         def pullbundle2extraprepare_widen(orig, pullop, kwargs):
             # Create narrow spec patterns from clone flags
             includepats = narrowspec.parsepatterns(opts['include'])
             excludepats = narrowspec.parsepatterns(opts['exclude'])
 
             # If necessary, ask the server to expand the narrowspec.
             includepats, excludepats = expandpull(
                 pullop, includepats, excludepats)
 
             if not includepats and excludepats:
                 # If nothing was included, we assume the user meant to include
                 # everything, except what they asked to exclude.
                 includepats = {'path:.'}
 
             narrowspec.save(pullop.repo, includepats, excludepats)
 
             # This will populate 'includepats' etc with the values from the
             # narrowspec we just saved.
             orig(pullop, kwargs)
 
             if opts.get('depth'):
                 kwargs['depth'] = opts['depth']
         wrappedextraprepare = extensions.wrappedfunction(exchange,
             '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
 
     def pullnarrow(orig, repo, *args, **kwargs):
         narrowrepo.wraprepo(repo.unfiltered(), opts_narrow)
         if isinstance(repo, repoview.repoview):
             repo.__class__.__bases__ = (repo.__class__.__bases__[0],
                                         repo.unfiltered().__class__)
         if opts_narrow:
             repo.requirements.add(narrowrepo.REQUIREMENT)
             repo._writerequirements()
 
         return orig(repo, *args, **kwargs)
 
     wrappedpull = extensions.wrappedfunction(exchange, 'pull', pullnarrow)
 
     with wrappedextraprepare, wrappedpull:
         return orig(ui, repo, *args, **pycompat.strkwargs(opts))
 
 def pullnarrowcmd(orig, ui, repo, *args, **opts):
     """Wraps pull command to allow modifying narrow spec."""
     wrappedextraprepare = util.nullcontextmanager()
     if narrowrepo.REQUIREMENT in repo.requirements:
 
         def pullbundle2extraprepare_widen(orig, pullop, kwargs):
             orig(pullop, kwargs)
             if opts.get('depth'):
                 kwargs['depth'] = opts['depth']
         wrappedextraprepare = extensions.wrappedfunction(exchange,
             '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
 
     with wrappedextraprepare:
         return orig(ui, repo, *args, **opts)
 
 def archivenarrowcmd(orig, ui, repo, *args, **opts):
     """Wraps archive command to narrow the default includes."""
     if narrowrepo.REQUIREMENT in repo.requirements:
         repo_includes, repo_excludes = repo.narrowpats
         includes = set(opts.get('include', []))
         excludes = set(opts.get('exclude', []))
         includes, excludes, unused_invalid = narrowspec.restrictpatterns(
             includes, excludes, repo_includes, repo_excludes)
         if includes:
             opts['include'] = includes
         if excludes:
             opts['exclude'] = excludes
     return orig(ui, repo, *args, **opts)
 
 def pullbundle2extraprepare(orig, pullop, kwargs):
     repo = pullop.repo
     if narrowrepo.REQUIREMENT not in repo.requirements:
         return orig(pullop, kwargs)
 
     if narrowbundle2.NARROWCAP not in pullop.remotebundle2caps:
         raise error.Abort(_("server doesn't support narrow clones"))
     orig(pullop, kwargs)
     kwargs['narrow'] = True
     include, exclude = repo.narrowpats
     kwargs['oldincludepats'] = include
     kwargs['oldexcludepats'] = exclude
     kwargs['includepats'] = include
     kwargs['excludepats'] = exclude
     kwargs['known'] = [node.hex(ctx.node()) for ctx in
                        repo.set('::%ln', pullop.common)
                        if ctx.node() != node.nullid]
     if not kwargs['known']:
         # Mercurial serialized an empty list as '' and deserializes it as
         # [''], so delete it instead to avoid handling the empty string on the
         # server.
         del kwargs['known']
 
 extensions.wrapfunction(exchange,'_pullbundle2extraprepare',
                         pullbundle2extraprepare)
 
 def _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
             newincludes, newexcludes, force):
     oldmatch = narrowspec.match(repo.root, oldincludes, oldexcludes)
     newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
 
     # This is essentially doing "hg outgoing" to find all local-only
     # commits. We will then check that the local-only commits don't
     # have any changes to files that will be untracked.
     unfi = repo.unfiltered()
     outgoing = discovery.findcommonoutgoing(unfi, remote,
                                             commoninc=commoninc)
     ui.status(_('looking for local changes to affected paths\n'))
     localnodes = []
     for n in itertools.chain(outgoing.missing, outgoing.excluded):
         if any(oldmatch(f) and not newmatch(f) for f in unfi[n].files()):
             localnodes.append(n)
     revstostrip = unfi.revs('descendants(%ln)', localnodes)
     hiddenrevs = repoview.filterrevs(repo, 'visible')
     visibletostrip = list(repo.changelog.node(r)
                           for r in (revstostrip - hiddenrevs))
     if visibletostrip:
         ui.status(_('The following changeset(s) or their ancestors have '
                     'local changes not on the remote:\n'))
         maxnodes = 10
         if ui.verbose or len(visibletostrip) <= maxnodes:
             for n in visibletostrip:
                 ui.status('%s\n' % node.short(n))
         else:
             for n in visibletostrip[:maxnodes]:
                 ui.status('%s\n' % node.short(n))
             ui.status(_('...and %d more, use --verbose to list all\n') %
                       (len(visibletostrip) - maxnodes))
         if not force:
             raise error.Abort(_('local changes found'),
                               hint=_('use --force-delete-local-changes to '
                                      'ignore'))
 
     if revstostrip:
         tostrip = [unfi.changelog.node(r) for r in revstostrip]
         if repo['.'].node() in tostrip:
             # stripping working copy, so move to a different commit first
             urev = max(repo.revs('(::%n) - %ln + null',
                                  repo['.'].node(), visibletostrip))
             hg.clean(repo, urev)
         repair.strip(ui, unfi, tostrip, topic='narrow')
 
     todelete = []
     for f, f2, size in repo.store.datafiles():
         if f.startswith('data/'):
             file = f[5:-2]
             if not newmatch(file):
                 todelete.append(f)
         elif f.startswith('meta/'):
             dir = f[5:-13]
             dirs = ['.'] + sorted(util.dirs({dir})) + [dir]
             include = True
             for d in dirs:
                 visit = newmatch.visitdir(d)
                 if not visit:
                     include = False
                     break
                 if visit == 'all':
                     break
             if not include:
                 todelete.append(f)
 
     repo.destroying()
 
     with repo.transaction("narrowing"):
         for f in todelete:
             ui.status(_('deleting %s\n') % f)
             util.unlinkpath(repo.svfs.join(f))
             repo.store.markremoved(f)
 
         for f in repo.dirstate:
             if not newmatch(f):
                 repo.dirstate.drop(f)
                 repo.wvfs.unlinkpath(f)
         repo.setnarrowpats(newincludes, newexcludes)
 
     repo.destroyed()
 
 def _widen(ui, repo, remote, commoninc, newincludes, newexcludes):
     newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
 
     # TODO(martinvonz): Get expansion working with widening/narrowing.
     if narrowspec.needsexpansion(newincludes):
         raise error.Abort('Expansion not yet supported on pull')
 
     def pullbundle2extraprepare_widen(orig, pullop, kwargs):
         orig(pullop, kwargs)
         # The old{in,ex}cludepats have already been set by orig()
         kwargs['includepats'] = newincludes
         kwargs['excludepats'] = newexcludes
     wrappedextraprepare = extensions.wrappedfunction(exchange,
         '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
 
     # define a function that narrowbundle2 can call after creating the
     # backup bundle, but before applying the bundle from the server
     def setnewnarrowpats():
         repo.setnarrowpats(newincludes, newexcludes)
     repo.setnewnarrowpats = setnewnarrowpats
 
     ds = repo.dirstate
     p1, p2 = ds.p1(), ds.p2()
     with ds.parentchange():
         ds.setparents(node.nullid, node.nullid)
     common = commoninc[0]
     with wrappedextraprepare:
         exchange.pull(repo, remote, heads=common)
     with ds.parentchange():
         ds.setparents(p1, p2)
 
     actions = {k: [] for k in 'a am f g cd dc r dm dg m e k p pr'.split()}
     addgaction = actions['g'].append
 
     mf = repo['.'].manifest().matches(newmatch)
     for f, fn in mf.iteritems():
         if f not in repo.dirstate:
             addgaction((f, (mf.flags(f), False),
                         "add from widened narrow clone"))
 
     merge.applyupdates(repo, actions, wctx=repo[None],
                        mctx=repo['.'], overwrite=False)
     merge.recordupdates(repo, actions, branchmerge=False)
 
 # TODO(rdamazio): Make new matcher format and update description
 @command('tracked',
     [('', 'addinclude', [], _('new paths to include')),
      ('', 'removeinclude', [], _('old paths to no longer include')),
      ('', 'addexclude', [], _('new paths to exclude')),
      ('', 'removeexclude', [], _('old paths to no longer exclude')),
      ('', 'clear', False, _('whether to replace the existing narrowspec')),
      ('', 'force-delete-local-changes', False,
        _('forces deletion of local changes when narrowing')),
     ] + commands.remoteopts,
     _('[OPTIONS]... [REMOTE]'),
     inferrepo=True)
 def trackedcmd(ui, repo, remotepath=None, *pats, **opts):
     """show or change the current narrowspec
 
     With no argument, shows the current narrowspec entries, one per line. Each
     line will be prefixed with 'I' or 'X' for included or excluded patterns,
     respectively.
 
     The narrowspec is comprised of expressions to match remote files and/or
     directories that should be pulled into your client.
     The narrowspec has *include* and *exclude* expressions, with excludes always
     trumping includes: that is, if a file matches an exclude expression, it will
     be excluded even if it also matches an include expression.
     Excluding files that were never included has no effect.
 
     Each included or excluded entry is in the format described by
     'hg help patterns'.
 
     The options allow you to add or remove included and excluded expressions.
 
     If --clear is specified, then all previous includes and excludes are DROPPED
     and replaced by the new ones specified to --addinclude and --addexclude.
     If --clear is specified without any further options, the narrowspec will be
     empty and will not match any files.
     """
     if narrowrepo.REQUIREMENT not in repo.requirements:
         ui.warn(_('The narrow command is only supported on respositories cloned'
                   ' with --narrow.\n'))
         return 1
 
     # Before supporting, decide whether it "hg tracked --clear" should mean
     # tracking no paths or all paths.
     if opts['clear']:
         ui.warn(_('The --clear option is not yet supported.\n'))
         return 1
 
     if narrowspec.needsexpansion(opts['addinclude'] + opts['addexclude']):
         raise error.Abort('Expansion not yet supported on widen/narrow')
 
     addedincludes = narrowspec.parsepatterns(opts['addinclude'])
     removedincludes = narrowspec.parsepatterns(opts['removeinclude'])
     addedexcludes = narrowspec.parsepatterns(opts['addexclude'])
     removedexcludes = narrowspec.parsepatterns(opts['removeexclude'])
     widening = addedincludes or removedexcludes
     narrowing = removedincludes or addedexcludes
     only_show = not widening and not narrowing
 
     # Only print the current narrowspec.
     if only_show:
         include, exclude = repo.narrowpats
 
         ui.pager('tracked')
         fm = ui.formatter('narrow', opts)
         for i in sorted(include):
             fm.startitem()
             fm.write('status', '%s ', 'I', label='narrow.included')
             fm.write('pat', '%s\n', i, label='narrow.included')
         for i in sorted(exclude):
             fm.startitem()
             fm.write('status', '%s ', 'X', label='narrow.excluded')
             fm.write('pat', '%s\n', i, label='narrow.excluded')
         fm.end()
         return 0
 
     with repo.wlock(), repo.lock():
         cmdutil.bailifchanged(repo)
 
         # Find the revisions we have in common with the remote. These will
         # be used for finding local-only changes for narrowing. They will
         # also define the set of revisions to update for widening.
         remotepath = ui.expandpath(remotepath or 'default')
         url, branches = hg.parseurl(remotepath)
         ui.status(_('comparing with %s\n') % util.hidepassword(url))
         remote = hg.peer(repo, opts, url)
         commoninc = discovery.findcommonincoming(repo, remote)
 
         oldincludes, oldexcludes = repo.narrowpats
         if narrowing:
             newincludes = oldincludes - removedincludes
             newexcludes = oldexcludes | addedexcludes
             _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
                     newincludes, newexcludes,
                     opts['force_delete_local_changes'])
             # _narrow() updated the narrowspec and _widen() below needs to
             # use the updated values as its base (otherwise removed includes
             # and addedexcludes will be lost in the resulting narrowspec)
             oldincludes = newincludes
             oldexcludes = newexcludes
 
         if widening:
             newincludes = oldincludes | addedincludes
             newexcludes = oldexcludes - removedexcludes
             _widen(ui, repo, remote, commoninc, newincludes, newexcludes)
 
     return 0

hgext/narrow/narrowdirstate.py

@@ -1,80 +1,79 b''
 # narrowdirstate.py - extensions to mercurial dirstate to support narrow clones
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     dirstate,
     error,
     extensions,
     match as matchmod,
+    narrowspec,
     util as hgutil,
 )
 
-from . import narrowspec
-
 def setup(repo):
     """Add narrow spec dirstate ignore, block changes outside narrow spec."""
 
     def walk(orig, self, match, subrepos, unknown, ignored, full=True,
              narrowonly=True):
         if narrowonly:
             narrowmatch = repo.narrowmatch()
             match = matchmod.intersectmatchers(match, narrowmatch)
         return orig(self, match, subrepos, unknown, ignored, full)
 
     extensions.wrapfunction(dirstate.dirstate, 'walk', walk)
 
     # Prevent adding files that are outside the sparse checkout
     editfuncs = ['normal', 'add', 'normallookup', 'copy', 'remove', 'merge']
     for func in editfuncs:
         def _wrapper(orig, self, *args):
             dirstate = repo.dirstate
             narrowmatch = repo.narrowmatch()
             for f in args:
                 if f is not None and not narrowmatch(f) and f not in dirstate:
                     raise error.Abort(_("cannot track '%s' - it is outside " +
                         "the narrow clone") % f)
             return orig(self, *args)
         extensions.wrapfunction(dirstate.dirstate, func, _wrapper)
 
     def filterrebuild(orig, self, parent, allfiles, changedfiles=None):
         if changedfiles is None:
             # Rebuilding entire dirstate, let's filter allfiles to match the
             # narrowspec.
             allfiles = [f for f in allfiles if repo.narrowmatch()(f)]
         orig(self, parent, allfiles, changedfiles)
 
     extensions.wrapfunction(dirstate.dirstate, 'rebuild', filterrebuild)
 
     def _narrowbackupname(backupname):
         assert 'dirstate' in backupname
         return backupname.replace('dirstate', narrowspec.FILENAME)
 
     def restorebackup(orig, self, tr, backupname):
         self._opener.rename(_narrowbackupname(backupname), narrowspec.FILENAME,
                             checkambig=True)
         orig(self, tr, backupname)
 
     extensions.wrapfunction(dirstate.dirstate, 'restorebackup', restorebackup)
 
     def savebackup(orig, self, tr, backupname):
         orig(self, tr, backupname)
 
         narrowbackupname = _narrowbackupname(backupname)
         self._opener.tryunlink(narrowbackupname)
         hgutil.copyfile(self._opener.join(narrowspec.FILENAME),
                         self._opener.join(narrowbackupname), hardlink=True)
 
     extensions.wrapfunction(dirstate.dirstate, 'savebackup', savebackup)
 
     def clearbackup(orig, self, tr, backupname):
         orig(self, tr, backupname)
         self._opener.unlink(_narrowbackupname(backupname))
 
     extensions.wrapfunction(dirstate.dirstate, 'clearbackup', clearbackup)

hgext/narrow/narrowrepo.py

@@ -1,113 +1,113 b''
 # narrowrepo.py - repository which supports narrow revlogs, lazy loading
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from mercurial import (
     bundlerepo,
     hg,
     localrepo,
     match as matchmod,
+    narrowspec,
     scmutil,
 )
 
 from . import (
     narrowrevlog,
-    narrowspec,
 )
 
 # When narrowing is finalized and no longer subject to format changes,
 # we should move this to just "narrow" or similar.
 REQUIREMENT = 'narrowhg-experimental'
 
 def wrappostshare(orig, sourcerepo, destrepo, **kwargs):
     orig(sourcerepo, destrepo, **kwargs)
     if REQUIREMENT in sourcerepo.requirements:
         with destrepo.wlock():
             with destrepo.vfs('shared', 'a') as fp:
                 fp.write(narrowspec.FILENAME + '\n')
 
 def unsharenarrowspec(orig, ui, repo, repopath):
     if (REQUIREMENT in repo.requirements
         and repo.path == repopath and repo.shared()):
         srcrepo = hg.sharedreposource(repo)
         with srcrepo.vfs(narrowspec.FILENAME) as f:
             spec = f.read()
         with repo.vfs(narrowspec.FILENAME, 'w') as f:
             f.write(spec)
     return orig(ui, repo, repopath)
 
 def wraprepo(repo, opts_narrow):
     """Enables narrow clone functionality on a single local repository."""
 
     cacheprop = localrepo.storecache
     if isinstance(repo, bundlerepo.bundlerepository):
         # We have to use a different caching property decorator for
         # bundlerepo because storecache blows up in strange ways on a
         # bundlerepo. Fortunately, there's no risk of data changing in
         # a bundlerepo.
         cacheprop = lambda name: localrepo.unfilteredpropertycache
 
     class narrowrepository(repo.__class__):
 
         def _constructmanifest(self):
             manifest = super(narrowrepository, self)._constructmanifest()
             narrowrevlog.makenarrowmanifestrevlog(manifest, repo)
             return manifest
 
         @cacheprop('00manifest.i')
         def manifestlog(self):
             mfl = super(narrowrepository, self).manifestlog
             narrowrevlog.makenarrowmanifestlog(mfl, self)
             return mfl
 
         def file(self, f):
             fl = super(narrowrepository, self).file(f)
             narrowrevlog.makenarrowfilelog(fl, self.narrowmatch())
             return fl
 
         @localrepo.repofilecache(narrowspec.FILENAME)
         def narrowpats(self):
             """matcher patterns for this repository's narrowspec
 
             A tuple of (includes, excludes).
             """
             return narrowspec.load(self)
 
         @localrepo.repofilecache(narrowspec.FILENAME)
         def _narrowmatch(self):
             include, exclude = self.narrowpats
             if not opts_narrow and not include and not exclude:
                 return matchmod.always(self.root, '')
             return narrowspec.match(self.root, include=include, exclude=exclude)
 
         # TODO(martinvonz): make this property-like instead?
         def narrowmatch(self):
             return self._narrowmatch
 
         def setnarrowpats(self, newincludes, newexcludes):
             narrowspec.save(self, newincludes, newexcludes)
             self.invalidate(clearfilecache=True)
 
         # I'm not sure this is the right place to do this filter.
         # context._manifestmatches() would probably be better, or perhaps
         # move it to a later place, in case some of the callers do want to know
         # which directories changed. This seems to work for now, though.
         def status(self, *args, **kwargs):
             s = super(narrowrepository, self).status(*args, **kwargs)
             narrowmatch = self.narrowmatch()
             modified = list(filter(narrowmatch, s.modified))
             added = list(filter(narrowmatch, s.added))
             removed = list(filter(narrowmatch, s.removed))
             deleted = list(filter(narrowmatch, s.deleted))
             unknown = list(filter(narrowmatch, s.unknown))
             ignored = list(filter(narrowmatch, s.ignored))
             clean = list(filter(narrowmatch, s.clean))
             return scmutil.status(modified, added, removed, deleted, unknown,
                                   ignored, clean)
 
     repo.__class__ = narrowrepository

hgext/narrow/narrowwirepeer.py

@@ -1,53 +1,52 b''
 # narrowwirepeer.py - passes narrow spec with unbundle command
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     extensions,
     hg,
+    narrowspec,
     node,
 )
 
-from . import narrowspec
-
 def uisetup():
     def peersetup(ui, peer):
         # We must set up the expansion before reposetup below, since it's used
         # at clone time before we have a repo.
         class expandingpeer(peer.__class__):
             def expandnarrow(self, narrow_include, narrow_exclude, nodes):
                 ui.status(_("expanding narrowspec\n"))
                 if not self.capable('exp-expandnarrow'):
                     raise error.Abort(
                         'peer does not support expanding narrowspecs')
 
                 hex_nodes = (node.hex(n) for n in nodes)
                 new_narrowspec = self._call(
                     'expandnarrow',
                     includepats=','.join(narrow_include),
                     excludepats=','.join(narrow_exclude),
                     nodes=','.join(hex_nodes))
 
                 return narrowspec.parseserverpatterns(new_narrowspec)
         peer.__class__ = expandingpeer
     hg.wirepeersetupfuncs.append(peersetup)
 
 def reposetup(repo):
     def wirereposetup(ui, peer):
         def wrapped(orig, cmd, *args, **kwargs):
             if cmd == 'unbundle':
                 # TODO: don't blindly add include/exclude wireproto
                 # arguments to unbundle.
                 include, exclude = repo.narrowpats
                 kwargs["includepats"] = ','.join(include)
                 kwargs["excludepats"] = ','.join(exclude)
             return orig(cmd, *args, **kwargs)
         extensions.wrapfunction(peer, '_calltwowaystream', wrapped)
     hg.wirepeersetupfuncs.append(wirereposetup)

mercurial/narrowspec.py

@@ -1,204 +1,204 b''
 # narrowspec.py - methods for working with a narrow view of a repository
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 
-from mercurial.i18n import _
-from mercurial import (
+from .i18n import _
+from . import (
     error,
     hg,
     match as matchmod,
     util,
 )
 
 FILENAME = 'narrowspec'
 
 def _parsestoredpatterns(text):
     """Parses the narrowspec format that's stored on disk."""
     patlist = None
     includepats = []
     excludepats = []
     for l in text.splitlines():
         if l == '[includes]':
             if patlist is None:
                 patlist = includepats
             else:
                 raise error.Abort(_('narrowspec includes section must appear '
                                     'at most once, before excludes'))
         elif l == '[excludes]':
             if patlist is not excludepats:
                 patlist = excludepats
             else:
                 raise error.Abort(_('narrowspec excludes section must appear '
                                     'at most once'))
         else:
             patlist.append(l)
 
     return set(includepats), set(excludepats)
 
 def parseserverpatterns(text):
     """Parses the narrowspec format that's returned by the server."""
     includepats = set()
     excludepats = set()
 
     # We get one entry per line, in the format "<key> <value>".
     # It's OK for value to contain other spaces.
     for kp in (l.split(' ', 1) for l in text.splitlines()):
         if len(kp) != 2:
             raise error.Abort(_('Invalid narrowspec pattern line: "%s"') % kp)
         key = kp[0]
         pat = kp[1]
         if key == 'include':
             includepats.add(pat)
         elif key == 'exclude':
             excludepats.add(pat)
         else:
             raise error.Abort(_('Invalid key "%s" in server response') % key)
 
     return includepats, excludepats
 
 def normalizesplitpattern(kind, pat):
     """Returns the normalized version of a pattern and kind.
 
     Returns a tuple with the normalized kind and normalized pattern.
     """
     pat = pat.rstrip('/')
     _validatepattern(pat)
     return kind, pat
 
 def _numlines(s):
     """Returns the number of lines in s, including ending empty lines."""
     # We use splitlines because it is Unicode-friendly and thus Python 3
     # compatible. However, it does not count empty lines at the end, so trick
     # it by adding a character at the end.
     return len((s + 'x').splitlines())
 
 def _validatepattern(pat):
     """Validates the pattern and aborts if it is invalid.
 
     Patterns are stored in the narrowspec as newline-separated
     POSIX-style bytestring paths. There's no escaping.
     """
 
     # We use newlines as separators in the narrowspec file, so don't allow them
     # in patterns.
     if _numlines(pat) > 1:
-        raise error.Abort('newlines are not allowed in narrowspec paths')
+        raise error.Abort(_('newlines are not allowed in narrowspec paths'))
 
     components = pat.split('/')
     if '.' in components or '..' in components:
         raise error.Abort(_('"." and ".." are not allowed in narrowspec paths'))
 
 def normalizepattern(pattern, defaultkind='path'):
     """Returns the normalized version of a text-format pattern.
 
     If the pattern has no kind, the default will be added.
     """
     kind, pat = matchmod._patsplit(pattern, defaultkind)
     return '%s:%s' % normalizesplitpattern(kind, pat)
 
 def parsepatterns(pats):
     """Parses a list of patterns into a typed pattern set."""
     return set(normalizepattern(p) for p in pats)
 
 def format(includes, excludes):
     output = '[includes]\n'
     for i in sorted(includes - excludes):
         output += i + '\n'
     output += '[excludes]\n'
     for e in sorted(excludes):
         output += e + '\n'
     return output
 
 def match(root, include=None, exclude=None):
     if not include:
         # Passing empty include and empty exclude to matchmod.match()
         # gives a matcher that matches everything, so explicitly use
         # the nevermatcher.
         return matchmod.never(root, '')
     return matchmod.match(root, '', [], include=include or [],
                           exclude=exclude or [])
 
 def needsexpansion(includes):
     return [i for i in includes if i.startswith('include:')]
 
 def load(repo):
     if repo.shared():
         repo = hg.sharedreposource(repo)
     try:
         spec = repo.vfs.read(FILENAME)
     except IOError as e:
         # Treat "narrowspec does not exist" the same as "narrowspec file exists
         # and is empty".
         if e.errno == errno.ENOENT:
             # Without this the next call to load will use the cached
             # non-existence of the file, which can cause some odd issues.
             repo.invalidate(clearfilecache=True)
             return set(), set()
         raise
     return _parsestoredpatterns(spec)
 
 def save(repo, includepats, excludepats):
     spec = format(includepats, excludepats)
     if repo.shared():
         repo = hg.sharedreposource(repo)
     repo.vfs.write(FILENAME, spec)
 
 def restrictpatterns(req_includes, req_excludes, repo_includes, repo_excludes):
     r""" Restricts the patterns according to repo settings,
     results in a logical AND operation
 
     :param req_includes: requested includes
     :param req_excludes: requested excludes
     :param repo_includes: repo includes
     :param repo_excludes: repo excludes
     :return: include patterns, exclude patterns, and invalid include patterns.
 
     >>> restrictpatterns({'f1','f2'}, {}, ['f1'], [])
     (set(['f1']), {}, [])
     >>> restrictpatterns({'f1'}, {}, ['f1','f2'], [])
     (set(['f1']), {}, [])
     >>> restrictpatterns({'f1/fc1', 'f3/fc3'}, {}, ['f1','f2'], [])
     (set(['f1/fc1']), {}, [])
     >>> restrictpatterns({'f1_fc1'}, {}, ['f1','f2'], [])
     ([], set(['path:.']), [])
     >>> restrictpatterns({'f1/../f2/fc2'}, {}, ['f1','f2'], [])
     (set(['f2/fc2']), {}, [])
     >>> restrictpatterns({'f1/../f3/fc3'}, {}, ['f1','f2'], [])
     ([], set(['path:.']), [])
     >>> restrictpatterns({'f1/$non_exitent_var'}, {}, ['f1','f2'], [])
     (set(['f1/$non_exitent_var']), {}, [])
     """
     res_excludes = set(req_excludes)
     res_excludes.update(repo_excludes)
     invalid_includes = []
     if not req_includes:
         res_includes = set(repo_includes)
     elif 'path:.' not in repo_includes:
         res_includes = []
         for req_include in req_includes:
             req_include = util.expandpath(util.normpath(req_include))
             if req_include in repo_includes:
                 res_includes.append(req_include)
                 continue
             valid = False
             for repo_include in repo_includes:
                 if req_include.startswith(repo_include + '/'):
                     valid = True
                     res_includes.append(req_include)
                     break
             if not valid:
                 invalid_includes.append(req_include)
         if len(res_includes) == 0:
             res_excludes = {'path:.'}
         else:
             res_includes = set(res_includes)
     else:
         res_includes = set(req_includes)
     return res_includes, res_excludes, invalid_includes

tests/test-narrow-expanddirstate.t

@@ -1,170 +1,170 b''
   $ . "$TESTDIR/narrow-library.sh"
 
   $ hg init master
   $ cd master
 
   $ mkdir inside
   $ echo inside > inside/f1
   $ mkdir outside
   $ echo outside > outside/f2
   $ mkdir patchdir
   $ echo patch_this > patchdir/f3
   $ hg ci -Aqm 'initial'
 
   $ cd ..
 
   $ hg clone --narrow ssh://user@dummy/master narrow --include inside
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   new changesets dff6a2a6d433
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
   $ cd narrow
 
   $ mkdir outside
   $ echo other_contents > outside/f2
   $ grep outside .hg/narrowspec
   [1]
   $ grep outside .hg/dirstate
   [1]
   $ hg status
 
 `hg status` did not add outside.
   $ grep outside .hg/narrowspec
   [1]
   $ grep outside .hg/dirstate
   [1]
 
 Unfortunately this is not really a candidate for adding to narrowhg proper,
 since it depends on some other source for providing the manifests (when using
 treemanifests) and file contents. Something like a virtual filesystem and/or
 remotefilelog. We want to be useful when not using those systems, so we do not
 have this method available in narrowhg proper at the moment.
   $ cat > "$TESTTMP/expand_extension.py" <<EOF
   > import os
   > import sys
   > 
   > from mercurial import extensions
   > from mercurial import localrepo
   > from mercurial import match as matchmod
+  > from mercurial import narrowspec
   > from mercurial import patch
   > from mercurial import util as hgutil
   > 
   > def expandnarrowspec(ui, repo, newincludes=None):
   >   if not newincludes:
   >     return
   >   import sys
   >   newincludes = set([newincludes])
-  >   narrowhg = extensions.find('narrow')
   >   includes, excludes = repo.narrowpats
-  >   currentmatcher = narrowhg.narrowspec.match(repo.root, includes, excludes)
+  >   currentmatcher = narrowspec.match(repo.root, includes, excludes)
   >   includes = includes | newincludes
   >   if not repo.currenttransaction():
   >     ui.develwarn('expandnarrowspec called outside of transaction!')
   >   repo.setnarrowpats(includes, excludes)
-  >   newmatcher = narrowhg.narrowspec.match(repo.root, includes, excludes)
+  >   newmatcher = narrowspec.match(repo.root, includes, excludes)
   >   added = matchmod.differencematcher(newmatcher, currentmatcher)
   >   for f in repo['.'].manifest().walk(added):
   >     repo.dirstate.normallookup(f)
   > 
   > def makeds(ui, repo):
   >   def wrapds(orig, self):
   >     ds = orig(self)
   >     class expandingdirstate(ds.__class__):
   >       # Mercurial 4.4 uses this version.
   >       @hgutil.propertycache
   >       def _map(self):
   >         ret = super(expandingdirstate, self)._map
   >         with repo.wlock(), repo.lock(), repo.transaction(
   >             'expandnarrowspec'):
   >           expandnarrowspec(ui, repo, os.environ.get('DIRSTATEINCLUDES'))
   >         return ret
   >       # Mercurial 4.3.3 and earlier uses this version. It seems that
   >       # narrowhg does not currently support this version, but we include
   >       # it just in case backwards compatibility is restored.
   >       def _read(self):
   >         ret = super(expandingdirstate, self)._read()
   >         with repo.wlock(), repo.lock(), repo.transaction(
   >             'expandnarrowspec'):
   >           expandnarrowspec(ui, repo, os.environ.get('DIRSTATEINCLUDES'))
   >         return ret
   >     ds.__class__ = expandingdirstate
   >     return ds
   >   return wrapds
   > 
   > def reposetup(ui, repo):
   >   extensions.wrapfilecache(localrepo.localrepository, 'dirstate',
   >                            makeds(ui, repo))
   >   def overridepatch(orig, *args, **kwargs):
   >     with repo.wlock():
   >       expandnarrowspec(ui, repo, os.environ.get('PATCHINCLUDES'))
   >       return orig(*args, **kwargs)
   > 
   >   extensions.wrapfunction(patch, 'patch', overridepatch)
   > EOF
   $ cat >> ".hg/hgrc" <<EOF
   > [extensions]
   > expand_extension = $TESTTMP/expand_extension.py
   > EOF
 
 Since we do not have the ability to rely on a virtual filesystem or
 remotefilelog in the test, we just fake it by copying the data from the 'master'
 repo.
   $ cp -a ../master/.hg/store/data/* .hg/store/data
 Do that for patchdir as well.
   $ cp -a ../master/patchdir .
 
 `hg status` will now add outside, but not patchdir.
   $ DIRSTATEINCLUDES=path:outside hg status
   M outside/f2
   $ grep outside .hg/narrowspec
   path:outside
   $ grep outside .hg/dirstate > /dev/null
   $ grep patchdir .hg/narrowspec
   [1]
   $ grep patchdir .hg/dirstate
   [1]
 
 Get rid of the modification to outside/f2.
   $ hg update -C .
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 This patch will not apply cleanly at the moment, so `hg import` will break
   $ cat > "$TESTTMP/foo.patch" <<EOF
   > --- patchdir/f3
   > +++ patchdir/f3
   > @@ -1,1 +1,1 @@
   > -this should be "patch_this", but its not, so patch fails
   > +this text is irrelevant
   > EOF
   $ PATCHINCLUDES=path:patchdir hg import -p0 -e "$TESTTMP/foo.patch" -m ignored
   applying $TESTTMP/foo.patch
   patching file patchdir/f3
   Hunk #1 FAILED at 0
   1 out of 1 hunks FAILED -- saving rejects to file patchdir/f3.rej
   abort: patch failed to apply
   [255]
   $ grep patchdir .hg/narrowspec
   [1]
   $ grep patchdir .hg/dirstate > /dev/null
   [1]
 
 Let's make it apply cleanly and see that it *did* expand properly
   $ cat > "$TESTTMP/foo.patch" <<EOF
   > --- patchdir/f3
   > +++ patchdir/f3
   > @@ -1,1 +1,1 @@
   > -patch_this
   > +patched_this
   > EOF
   $ PATCHINCLUDES=path:patchdir hg import -p0 -e "$TESTTMP/foo.patch" -m message
   applying $TESTTMP/foo.patch
   $ cat patchdir/f3
   patched_this
   $ grep patchdir .hg/narrowspec
   path:patchdir
   $ grep patchdir .hg/dirstate > /dev/null