sparse.py
439 lines
| 14.4 KiB
| text/x-python
|
PythonLexer
/ hgext / sparse.py
|
r33289 | # sparse.py - allow sparse checkouts of the working directory | ||
# | ||||
# Copyright 2014 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
"""allow sparse checkouts of the working directory (EXPERIMENTAL) | ||||
|
r33290 | |||
(This extension is not yet protected by backwards compatibility | ||||
guarantees. Any aspect may break in future releases until this | ||||
notice is removed.) | ||||
This extension allows the working directory to only consist of a | ||||
subset of files for the revision. This allows specific files or | ||||
directories to be explicitly included or excluded. Many repository | ||||
operations have performance proportional to the number of files in | ||||
the working directory. So only realizing a subset of files in the | ||||
working directory can improve performance. | ||||
|
r33294 | |||
Sparse Config Files | ||||
------------------- | ||||
The set of files that are part of a sparse checkout are defined by | ||||
a sparse config file. The file defines 3 things: includes (files to | ||||
include in the sparse checkout), excludes (files to exclude from the | ||||
sparse checkout), and profiles (links to other config files). | ||||
The file format is newline delimited. Empty lines and lines beginning | ||||
with ``#`` are ignored. | ||||
Lines beginning with ``%include `` denote another sparse config file | ||||
to include. e.g. ``%include tests.sparse``. The filename is relative | ||||
to the repository root. | ||||
The special lines ``[include]`` and ``[exclude]`` denote the section | ||||
for includes and excludes that follow, respectively. It is illegal to | ||||
|
r33551 | have ``[include]`` after ``[exclude]``. | ||
|
r33294 | |||
Non-special lines resemble file patterns to be added to either includes | ||||
or excludes. The syntax of these lines is documented by :hg:`help patterns`. | ||||
Patterns are interpreted as ``glob:`` by default and match against the | ||||
root of the repository. | ||||
Exclusion patterns take precedence over inclusion patterns. So even | ||||
if a file is explicitly included, an ``[exclude]`` entry can remove it. | ||||
For example, say you have a repository with 3 directories, ``frontend/``, | ||||
``backend/``, and ``tools/``. ``frontend/`` and ``backend/`` correspond | ||||
to different projects and it is uncommon for someone working on one | ||||
to need the files for the other. But ``tools/`` contains files shared | ||||
between both projects. Your sparse config files may resemble:: | ||||
# frontend.sparse | ||||
frontend/** | ||||
tools/** | ||||
# backend.sparse | ||||
backend/** | ||||
tools/** | ||||
Say the backend grows in size. Or there's a directory with thousands | ||||
of files you wish to exclude. You can modify the profile to exclude | ||||
certain files:: | ||||
[include] | ||||
backend/** | ||||
tools/** | ||||
[exclude] | ||||
tools/tests/** | ||||
|
r33289 | """ | ||
from __future__ import absolute_import | ||||
from mercurial.i18n import _ | ||||
from mercurial import ( | ||||
commands, | ||||
dirstate, | ||||
error, | ||||
extensions, | ||||
hg, | ||||
|
r35903 | logcmdutil, | ||
|
r33289 | match as matchmod, | ||
|
r35193 | pycompat, | ||
|
r33289 | registrar, | ||
|
r33297 | sparse, | ||
|
r33289 | util, | ||
) | ||||
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||||
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
|
r43347 | testedwith = b'ships-with-hg-core' | ||
|
r33289 | |||
cmdtable = {} | ||||
command = registrar.command(cmdtable) | ||||
|
r43346 | |||
|
r33289 | def extsetup(ui): | ||
|
r33299 | sparse.enabled = True | ||
|
r33289 | _setupclone(ui) | ||
_setuplog(ui) | ||||
_setupadd(ui) | ||||
_setupdirstate(ui) | ||||
|
r43346 | |||
|
r33289 | def replacefilecache(cls, propname, replacement): | ||
"""Replace a filecache property with a new class. This allows changing the | ||||
cache invalidation condition.""" | ||||
origcls = cls | ||||
assert callable(replacement) | ||||
while cls is not object: | ||||
if propname in cls.__dict__: | ||||
orig = cls.__dict__[propname] | ||||
setattr(cls, propname, replacement(orig)) | ||||
break | ||||
cls = cls.__bases__[0] | ||||
if cls is object: | ||||
|
r43346 | raise AttributeError( | ||
|
r43347 | _(b"type '%s' has no property '%s'") % (origcls, propname) | ||
|
r43346 | ) | ||
|
r33289 | |||
def _setuplog(ui): | ||||
|
r43347 | entry = commands.table[b'log|history'] | ||
|
r43346 | entry[1].append( | ||
( | ||||
|
r43347 | b'', | ||
b'sparse', | ||||
|
r43346 | None, | ||
|
r43347 | b"limit to changesets affecting the sparse checkout", | ||
|
r43346 | ) | ||
) | ||||
|
r33289 | |||
|
r35905 | def _initialrevs(orig, repo, opts): | ||
|
r33289 | revs = orig(repo, opts) | ||
|
r43347 | if opts.get(b'sparse'): | ||
|
r33320 | sparsematch = sparse.matcher(repo) | ||
|
r43346 | |||
|
r33289 | def ctxmatch(rev): | ||
ctx = repo[rev] | ||||
return any(f for f in ctx.files() if sparsematch(f)) | ||||
|
r43346 | |||
|
r33289 | revs = revs.filter(ctxmatch) | ||
return revs | ||||
|
r43346 | |||
|
r43347 | extensions.wrapfunction(logcmdutil, b'_initialrevs', _initialrevs) | ||
|
r33289 | |||
|
r43346 | |||
|
r33289 | def _clonesparsecmd(orig, ui, repo, *args, **opts): | ||
|
r38124 | include_pat = opts.get(r'include') | ||
exclude_pat = opts.get(r'exclude') | ||||
enableprofile_pat = opts.get(r'enable_profile') | ||||
|
r41183 | narrow_pat = opts.get(r'narrow') | ||
|
r33289 | include = exclude = enableprofile = False | ||
if include_pat: | ||||
pat = include_pat | ||||
include = True | ||||
if exclude_pat: | ||||
pat = exclude_pat | ||||
exclude = True | ||||
if enableprofile_pat: | ||||
pat = enableprofile_pat | ||||
enableprofile = True | ||||
if sum([include, exclude, enableprofile]) > 1: | ||||
|
r43347 | raise error.Abort(_(b"too many flags specified.")) | ||
|
r41183 | # if --narrow is passed, it means they are includes and excludes for narrow | ||
# clone | ||||
if not narrow_pat and (include or exclude or enableprofile): | ||||
|
r43346 | |||
|
r33289 | def clonesparse(orig, self, node, overwrite, *args, **kwargs): | ||
|
r43346 | sparse.updateconfig( | ||
self.unfiltered(), | ||||
pat, | ||||
{}, | ||||
include=include, | ||||
exclude=exclude, | ||||
enableprofile=enableprofile, | ||||
usereporootpaths=True, | ||||
) | ||||
|
r33289 | return orig(self, node, overwrite, *args, **kwargs) | ||
|
r43346 | |||
|
r43347 | extensions.wrapfunction(hg, b'updaterepo', clonesparse) | ||
|
r33289 | return orig(ui, repo, *args, **opts) | ||
|
r43346 | |||
|
r33289 | def _setupclone(ui): | ||
|
r43347 | entry = commands.table[b'clone'] | ||
entry[1].append((b'', b'enable-profile', [], b'enable a sparse profile')) | ||||
entry[1].append((b'', b'include', [], b'include sparse pattern')) | ||||
entry[1].append((b'', b'exclude', [], b'exclude sparse pattern')) | ||||
extensions.wrapcommand(commands.table, b'clone', _clonesparsecmd) | ||||
|
r33289 | |||
|
r43346 | |||
|
r33289 | def _setupadd(ui): | ||
|
r43347 | entry = commands.table[b'add'] | ||
|
r43346 | entry[1].append( | ||
( | ||||
|
r43347 | b's', | ||
b'sparse', | ||||
|
r43346 | None, | ||
|
r43347 | b'also include directories of added files in sparse config', | ||
|
r43346 | ) | ||
) | ||||
|
r33289 | |||
def _add(orig, ui, repo, *pats, **opts): | ||||
|
r38124 | if opts.get(r'sparse'): | ||
|
r33289 | dirs = set() | ||
for pat in pats: | ||||
dirname, basename = util.split(pat) | ||||
dirs.add(dirname) | ||||
|
r33374 | sparse.updateconfig(repo, list(dirs), opts, include=True) | ||
|
r33289 | return orig(ui, repo, *pats, **opts) | ||
|
r43347 | extensions.wrapcommand(commands.table, b'add', _add) | ||
|
r33289 | |||
|
r43346 | |||
|
r33289 | def _setupdirstate(ui): | ||
"""Modify the dirstate to prevent stat'ing excluded files, | ||||
and to prevent modifications to files outside the checkout. | ||||
""" | ||||
|
r33496 | def walk(orig, self, match, subrepos, unknown, ignored, full=True): | ||
|
r36218 | # hack to not exclude explicitly-specified paths so that they can | ||
# be warned later on e.g. dirstate.add() | ||||
|
r41825 | em = matchmod.exact(match.files()) | ||
|
r36218 | sm = matchmod.unionmatcher([self._sparsematcher, em]) | ||
match = matchmod.intersectmatchers(match, sm) | ||||
|
r33496 | return orig(self, match, subrepos, unknown, ignored, full) | ||
|
r33320 | |||
|
r43347 | extensions.wrapfunction(dirstate.dirstate, b'walk', walk) | ||
|
r33289 | |||
# dirstate.rebuild should not add non-matching files | ||||
def _rebuild(orig, self, parent, allfiles, changedfiles=None): | ||||
|
r33373 | matcher = self._sparsematcher | ||
|
r33320 | if not matcher.always(): | ||
|
r41185 | allfiles = [f for f in allfiles if matcher(f)] | ||
|
r33289 | if changedfiles: | ||
changedfiles = [f for f in changedfiles if matcher(f)] | ||||
if changedfiles is not None: | ||||
# In _rebuild, these files will be deleted from the dirstate | ||||
# when they are not found to be in allfiles | ||||
dirstatefilestoremove = set(f for f in self if not matcher(f)) | ||||
changedfiles = dirstatefilestoremove.union(changedfiles) | ||||
return orig(self, parent, allfiles, changedfiles) | ||||
|
r43346 | |||
|
r43347 | extensions.wrapfunction(dirstate.dirstate, b'rebuild', _rebuild) | ||
|
r33289 | |||
# Prevent adding files that are outside the sparse checkout | ||||
|
r43347 | editfuncs = [ | ||
b'normal', | ||||
b'add', | ||||
b'normallookup', | ||||
b'copy', | ||||
b'remove', | ||||
b'merge', | ||||
] | ||||
|
r43346 | hint = _( | ||
|
r43347 | b'include file with `hg debugsparse --include <pattern>` or use ' | ||
+ b'`hg add -s <file>` to include file directory while adding' | ||||
|
r43346 | ) | ||
|
r33289 | for func in editfuncs: | ||
|
r43346 | |||
|
r42656 | def _wrapper(orig, self, *args, **kwargs): | ||
|
r33373 | sparsematch = self._sparsematcher | ||
|
r33320 | if not sparsematch.always(): | ||
|
r33289 | for f in args: | ||
|
r43346 | if f is not None and not sparsematch(f) and f not in self: | ||
raise error.Abort( | ||||
_( | ||||
|
r43347 | b"cannot add '%s' - it is outside " | ||
b"the sparse checkout" | ||||
|
r43346 | ) | ||
% f, | ||||
hint=hint, | ||||
) | ||||
|
r42656 | return orig(self, *args, **kwargs) | ||
|
r43346 | |||
|
r33289 | extensions.wrapfunction(dirstate.dirstate, func, _wrapper) | ||
|
r43346 | |||
@command( | ||||
|
r43347 | b'debugsparse', | ||
|
r43346 | [ | ||
|
r43347 | (b'I', b'include', False, _(b'include files in the sparse checkout')), | ||
(b'X', b'exclude', False, _(b'exclude files in the sparse checkout')), | ||||
(b'd', b'delete', False, _(b'delete an include/exclude rule')), | ||||
|
r43346 | ( | ||
|
r43347 | b'f', | ||
b'force', | ||||
|
r43346 | False, | ||
|
r43347 | _(b'allow changing rules even with pending changes'), | ||
|
r43346 | ), | ||
|
r43347 | (b'', b'enable-profile', False, _(b'enables the specified profile')), | ||
(b'', b'disable-profile', False, _(b'disables the specified profile')), | ||||
(b'', b'import-rules', False, _(b'imports rules from a file')), | ||||
(b'', b'clear-rules', False, _(b'clears local include/exclude rules')), | ||||
|
r43346 | ( | ||
|
r43347 | b'', | ||
b'refresh', | ||||
|
r43346 | False, | ||
|
r43347 | _(b'updates the working after sparseness changes'), | ||
|
r43346 | ), | ||
|
r43347 | (b'', b'reset', False, _(b'makes the repo full again')), | ||
|
r43346 | ] | ||
+ commands.templateopts, | ||||
|
r43347 | _(b'[--OPTION] PATTERN...'), | ||
|
r43346 | helpbasic=True, | ||
) | ||||
|
r33293 | def debugsparse(ui, repo, *pats, **opts): | ||
|
r33289 | """make the current checkout sparse, or edit the existing checkout | ||
The sparse command is used to make the current checkout sparse. | ||||
This means files that don't meet the sparse condition will not be | ||||
written to disk, or show up in any working copy operations. It does | ||||
not affect files in history in any way. | ||||
Passing no arguments prints the currently applied sparse rules. | ||||
--include and --exclude are used to add and remove files from the sparse | ||||
checkout. The effects of adding an include or exclude rule are applied | ||||
immediately. If applying the new rule would cause a file with pending | ||||
changes to be added or removed, the command will fail. Pass --force to | ||||
force a rule change even with pending changes (the changes on disk will | ||||
be preserved). | ||||
--delete removes an existing include/exclude rule. The effects are | ||||
immediate. | ||||
--refresh refreshes the files on disk based on the sparse rules. This is | ||||
only necessary if .hg/sparse was changed by hand. | ||||
--enable-profile and --disable-profile accept a path to a .hgsparse file. | ||||
This allows defining sparse checkouts and tracking them inside the | ||||
repository. This is useful for defining commonly used sparse checkouts for | ||||
many people to use. As the profile definition changes over time, the sparse | ||||
checkout will automatically be updated appropriately, depending on which | ||||
changeset is checked out. Changes to .hgsparse are not applied until they | ||||
have been committed. | ||||
--import-rules accepts a path to a file containing rules in the .hgsparse | ||||
format, allowing you to add --include, --exclude and --enable-profile rules | ||||
in bulk. Like the --include, --exclude and --enable-profile switches, the | ||||
changes are applied immediately. | ||||
--clear-rules removes all local include and exclude rules, while leaving | ||||
any enabled profiles in place. | ||||
Returns 0 if editing the sparse checkout succeeds. | ||||
""" | ||||
|
r35193 | opts = pycompat.byteskwargs(opts) | ||
|
r43347 | include = opts.get(b'include') | ||
exclude = opts.get(b'exclude') | ||||
force = opts.get(b'force') | ||||
enableprofile = opts.get(b'enable_profile') | ||||
disableprofile = opts.get(b'disable_profile') | ||||
importrules = opts.get(b'import_rules') | ||||
clearrules = opts.get(b'clear_rules') | ||||
delete = opts.get(b'delete') | ||||
refresh = opts.get(b'refresh') | ||||
reset = opts.get(b'reset') | ||||
|
r43346 | count = sum( | ||
[ | ||||
include, | ||||
exclude, | ||||
enableprofile, | ||||
disableprofile, | ||||
delete, | ||||
importrules, | ||||
refresh, | ||||
clearrules, | ||||
reset, | ||||
] | ||||
) | ||||
|
r33289 | if count > 1: | ||
|
r43347 | raise error.Abort(_(b"too many flags specified")) | ||
|
r33289 | |||
if count == 0: | ||||
|
r43347 | if repo.vfs.exists(b'sparse'): | ||
ui.status(repo.vfs.read(b"sparse") + b"\n") | ||||
|
r33304 | temporaryincludes = sparse.readtemporaryincludes(repo) | ||
|
r33289 | if temporaryincludes: | ||
|
r43347 | ui.status( | ||
_(b"Temporarily Included Files (for merge/rebase):\n") | ||||
) | ||||
ui.status((b"\n".join(temporaryincludes) + b"\n")) | ||||
|
r42155 | return | ||
|
r33289 | else: | ||
|
r43346 | raise error.Abort( | ||
_( | ||||
|
r43347 | b'the debugsparse command is only supported on' | ||
b' sparse repositories' | ||||
|
r43346 | ) | ||
) | ||||
|
r33289 | |||
if include or exclude or delete or reset or enableprofile or disableprofile: | ||||
|
r43346 | sparse.updateconfig( | ||
repo, | ||||
pats, | ||||
opts, | ||||
include=include, | ||||
exclude=exclude, | ||||
reset=reset, | ||||
delete=delete, | ||||
enableprofile=enableprofile, | ||||
disableprofile=disableprofile, | ||||
force=force, | ||||
) | ||||
|
r33289 | |||
if importrules: | ||||
|
r33371 | sparse.importfromfiles(repo, opts, pats, force=force) | ||
|
r33289 | |||
if clearrules: | ||||
|
r33354 | sparse.clearrules(repo, force=force) | ||
|
r33289 | |||
if refresh: | ||||
try: | ||||
wlock = repo.wlock() | ||||
fcounts = map( | ||||
len, | ||||
|
r43346 | sparse.refreshwdir( | ||
repo, repo.status(), sparse.matcher(repo), force=force | ||||
), | ||||
) | ||||
sparse.printchanges( | ||||
ui, | ||||
opts, | ||||
added=fcounts[0], | ||||
dropped=fcounts[1], | ||||
conflicting=fcounts[2], | ||||
) | ||||
|
r33289 | finally: | ||
wlock.release() | ||||