eol.py
468 lines
| 15.8 KiB
| text/x-python
|
PythonLexer
/ hgext / eol.py
|
r11249 | """automatically manage newlines in repository files | ||
This extension allows you to manage the type of line endings (CRLF or | ||||
LF) that are used in the repository and in the local working | ||||
directory. That way you can get CRLF line endings on Windows and LF on | ||||
Unix/Mac, thereby letting everybody use their OS native line endings. | ||||
The extension reads its configuration from a versioned ``.hgeol`` | ||||
|
r24367 | configuration file found in the root of the working directory. The | ||
|
r11249 | ``.hgeol`` file use the same syntax as all other Mercurial | ||
configuration files. It uses two sections, ``[patterns]`` and | ||||
``[repository]``. | ||||
|
r13124 | The ``[patterns]`` section specifies how line endings should be | ||
|
r24367 | converted between the working directory and the repository. The format is | ||
|
r13124 | specified by a file pattern. The first match is used, so put more | ||
specific patterns first. The available line endings are ``LF``, | ||||
``CRLF``, and ``BIN``. | ||||
|
r11249 | |||
Files with the declared format of ``CRLF`` or ``LF`` are always | ||||
|
r13124 | checked out and stored in the repository in that format and files | ||
declared to be binary (``BIN``) are left unchanged. Additionally, | ||||
``native`` is an alias for checking out in the platform's default line | ||||
ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on | ||||
Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's | ||||
|
r26098 | default behavior; it is only needed if you need to override a later, | ||
|
r13124 | more general pattern. | ||
|
r11249 | |||
The optional ``[repository]`` section specifies the line endings to | ||||
use for files stored in the repository. It has a single setting, | ||||
``native``, which determines the storage line endings for files | ||||
declared as ``native`` in the ``[patterns]`` section. It can be set to | ||||
``LF`` or ``CRLF``. The default is ``LF``. For example, this means | ||||
that on Windows, files configured as ``native`` (``CRLF`` by default) | ||||
will be converted to ``LF`` when stored in the repository. Files | ||||
declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section | ||||
are always stored as-is in the repository. | ||||
Example versioned ``.hgeol`` file:: | ||||
[patterns] | ||||
**.py = native | ||||
**.vcproj = CRLF | ||||
**.txt = native | ||||
Makefile = LF | ||||
**.jpg = BIN | ||||
[repository] | ||||
native = LF | ||||
|
r13124 | .. note:: | ||
|
r19997 | |||
|
r13124 | The rules will first apply when files are touched in the working | ||
|
r24367 | directory, e.g. by updating to null and back to tip to touch all files. | ||
|
r13124 | |||
|
r14856 | The extension uses an optional ``[eol]`` section read from both the | ||
normal Mercurial configuration files and the ``.hgeol`` file, with the | ||||
latter overriding the former. You can use that section to control the | ||||
overall behavior. There are three settings: | ||||
|
r11249 | |||
- ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or | ||||
|
r12802 | ``CRLF`` to override the default interpretation of ``native`` for | ||
|
r11249 | checkout. This can be used with :hg:`archive` on Unix, say, to | ||
generate an archive where files have line endings for Windows. | ||||
- ``eol.only-consistent`` (default True) can be set to False to make | ||||
the extension convert files with inconsistent EOLs. Inconsistent | ||||
means that there is both ``CRLF`` and ``LF`` present in the file. | ||||
Such files are normally not touched under the assumption that they | ||||
have mixed EOLs on purpose. | ||||
|
r14856 | - ``eol.fix-trailing-newline`` (default False) can be set to True to | ||
|
r14857 | ensure that converted files end with a EOL character (either ``\\n`` | ||
or ``\\r\\n`` as per the configured patterns). | ||||
|
r14856 | |||
|
r12979 | The extension provides ``cleverencode:`` and ``cleverdecode:`` filters | ||
like the deprecated win32text extension does. This means that you can | ||||
disable win32text and enable eol and your filters will still work. You | ||||
only need to these filters until you have prepared a ``.hgeol`` file. | ||||
|
r12980 | The ``win32text.forbid*`` hooks provided by the win32text extension | ||
|
r13617 | have been unified into a single hook named ``eol.checkheadshook``. The | ||
hook will lookup the expected line endings from the ``.hgeol`` file, | ||||
which means you must migrate to a ``.hgeol`` file first before using | ||||
the hook. ``eol.checkheadshook`` only checks heads, intermediate | ||||
invalid revisions will be pushed. To forbid them completely, use the | ||||
``eol.checkallhook`` hook. These hooks are best used as | ||||
``pretxnchangegroup`` hooks. | ||||
|
r12980 | |||
|
r11249 | See :hg:`help patterns` for more information about the glob patterns | ||
used. | ||||
""" | ||||
|
r28969 | from __future__ import absolute_import | ||
import os | ||||
import re | ||||
|
r11249 | from mercurial.i18n import _ | ||
|
r28969 | from mercurial import ( | ||
config, | ||||
r32995 | error as errormod, | |||
|
r28969 | extensions, | ||
match, | ||||
|
r31775 | pycompat, | ||
|
r34119 | registrar, | ||
|
r37525 | scmutil, | ||
|
r28969 | util, | ||
) | ||||
|
r43346 | from mercurial.utils import stringutil | ||
|
r11249 | |||
|
r29841 | # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||
|
r25186 | # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
|
r43347 | testedwith = b'ships-with-hg-core' | ||
|
r16743 | |||
|
r34119 | configtable = {} | ||
configitem = registrar.configitem(configtable) | ||||
|
r43346 | configitem( | ||
|
r43347 | b'eol', b'fix-trailing-newline', default=False, | ||
|
r34119 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'eol', b'native', default=pycompat.oslinesep, | ||
|
r34120 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'eol', b'only-consistent', default=True, | ||
|
r34121 | ) | ||
|
r34119 | |||
|
r11249 | # Matches a lone LF, i.e., one that is not part of CRLF. | ||
|
r43347 | singlelf = re.compile(b'(^|[^\r])\n') | ||
|
r11249 | |||
|
r43346 | |||
|
r11249 | def inconsistenteol(data): | ||
|
r43347 | return b'\r\n' in data and singlelf.search(data) | ||
|
r11249 | |||
|
r43346 | |||
|
r11249 | def tolf(s, params, ui, **kwargs): | ||
"""Filter to convert to LF EOLs.""" | ||||
|
r37102 | if stringutil.binary(s): | ||
|
r11249 | return s | ||
|
r43347 | if ui.configbool(b'eol', b'only-consistent') and inconsistenteol(s): | ||
|
r11249 | return s | ||
|
r43346 | if ( | ||
|
r43347 | ui.configbool(b'eol', b'fix-trailing-newline') | ||
|
r43346 | and s | ||
|
r43347 | and not s.endswith(b'\n') | ||
|
r43346 | ): | ||
|
r43347 | s = s + b'\n' | ||
|
r31776 | return util.tolf(s) | ||
|
r11249 | |||
|
r43346 | |||
|
r11249 | def tocrlf(s, params, ui, **kwargs): | ||
"""Filter to convert to CRLF EOLs.""" | ||||
|
r37102 | if stringutil.binary(s): | ||
|
r11249 | return s | ||
|
r43347 | if ui.configbool(b'eol', b'only-consistent') and inconsistenteol(s): | ||
|
r11249 | return s | ||
|
r43346 | if ( | ||
|
r43347 | ui.configbool(b'eol', b'fix-trailing-newline') | ||
|
r43346 | and s | ||
|
r43347 | and not s.endswith(b'\n') | ||
|
r43346 | ): | ||
|
r43347 | s = s + b'\n' | ||
|
r31776 | return util.tocrlf(s) | ||
|
r11249 | |||
|
r43346 | |||
|
r11249 | def isbinary(s, params): | ||
"""Filter to do nothing with the file.""" | ||||
return s | ||||
|
r43346 | |||
|
r11249 | filters = { | ||
|
r43347 | b'to-lf': tolf, | ||
b'to-crlf': tocrlf, | ||||
b'is-binary': isbinary, | ||||
|
r12975 | # The following provide backwards compatibility with win32text | ||
|
r43347 | b'cleverencode:': tolf, | ||
b'cleverdecode:': tocrlf, | ||||
|
r11249 | } | ||
|
r43346 | |||
|
r13613 | class eolfile(object): | ||
def __init__(self, ui, root, data): | ||||
|
r43347 | self._decode = { | ||
b'LF': b'to-lf', | ||||
b'CRLF': b'to-crlf', | ||||
b'BIN': b'is-binary', | ||||
} | ||||
self._encode = { | ||||
b'LF': b'to-lf', | ||||
b'CRLF': b'to-crlf', | ||||
b'BIN': b'is-binary', | ||||
} | ||||
|
r11249 | |||
|
r13613 | self.cfg = config.config() | ||
# Our files should not be touched. The pattern must be | ||||
# inserted first override a '** = native' pattern. | ||||
|
r43347 | self.cfg.set(b'patterns', b'.hg*', b'BIN', b'eol') | ||
|
r13613 | # We can then parse the user's patterns. | ||
|
r43347 | self.cfg.parse(b'.hgeol', data) | ||
|
r13613 | |||
|
r43347 | isrepolf = self.cfg.get(b'repository', b'native') != b'CRLF' | ||
self._encode[b'NATIVE'] = isrepolf and b'to-lf' or b'to-crlf' | ||||
iswdlf = ui.config(b'eol', b'native') in (b'LF', b'\n') | ||||
self._decode[b'NATIVE'] = iswdlf and b'to-lf' or b'to-crlf' | ||||
|
r13613 | |||
include = [] | ||||
exclude = [] | ||||
|
r30114 | self.patterns = [] | ||
|
r43347 | for pattern, style in self.cfg.items(b'patterns'): | ||
|
r13613 | key = style.upper() | ||
|
r43347 | if key == b'BIN': | ||
|
r13613 | exclude.append(pattern) | ||
else: | ||||
include.append(pattern) | ||||
|
r43347 | m = match.match(root, b'', [pattern]) | ||
|
r30114 | self.patterns.append((pattern, key, m)) | ||
|
r13613 | # This will match the files for which we need to care | ||
# about inconsistent newlines. | ||||
|
r43347 | self.match = match.match(root, b'', [], include, exclude) | ||
|
r13613 | |||
|
r14854 | def copytoui(self, ui): | ||
|
r30114 | for pattern, key, m in self.patterns: | ||
|
r13613 | try: | ||
|
r43347 | ui.setconfig(b'decode', pattern, self._decode[key], b'eol') | ||
ui.setconfig(b'encode', pattern, self._encode[key], b'eol') | ||||
|
r13613 | except KeyError: | ||
|
r43346 | ui.warn( | ||
|
r43347 | _(b"ignoring unknown EOL style '%s' from %s\n") | ||
% (key, self.cfg.source(b'patterns', pattern)) | ||||
|
r43346 | ) | ||
|
r14854 | # eol.only-consistent can be specified in ~/.hgrc or .hgeol | ||
|
r43347 | for k, v in self.cfg.items(b'eol'): | ||
ui.setconfig(b'eol', k, v, b'eol') | ||||
|
r13613 | |||
|
r13615 | def checkrev(self, repo, ctx, files): | ||
|
r13649 | failed = [] | ||
|
r43346 | for f in files or ctx.files(): | ||
|
r13615 | if f not in ctx: | ||
continue | ||||
|
r30114 | for pattern, key, m in self.patterns: | ||
if not m(f): | ||||
|
r13615 | continue | ||
|
r30114 | target = self._encode[key] | ||
|
r13615 | data = ctx[f].data() | ||
|
r43346 | if ( | ||
|
r43347 | target == b"to-lf" | ||
and b"\r\n" in data | ||||
or target == b"to-crlf" | ||||
|
r43346 | and singlelf.search(data) | ||
): | ||||
|
r36685 | failed.append((f, target, bytes(ctx))) | ||
|
r13501 | break | ||
|
r13649 | return failed | ||
|
r11249 | |||
|
r43346 | |||
|
r13614 | def parseeol(ui, repo, nodes): | ||
|
r13613 | try: | ||
|
r13614 | for node in nodes: | ||
try: | ||||
if node is None: | ||||
# Cannot use workingctx.data() since it would load | ||||
# and cache the filters before we configure them. | ||||
|
r43347 | data = repo.wvfs(b'.hgeol').read() | ||
|
r13614 | else: | ||
|
r43347 | data = repo[node][b'.hgeol'].data() | ||
|
r13614 | return eolfile(ui, repo.root, data) | ||
except (IOError, LookupError): | ||||
pass | ||||
r32995 | except errormod.ParseError as inst: | |||
|
r43346 | ui.warn( | ||
|
r43347 | _( | ||
b"warning: ignoring .hgeol file due to parse error " | ||||
b"at %s: %s\n" | ||||
) | ||||
|
r43346 | % (inst.args[1], inst.args[0]) | ||
) | ||||
|
r13614 | return None | ||
|
r11249 | |||
|
r43346 | |||
|
r34831 | def ensureenabled(ui): | ||
"""make sure the extension is enabled when used as hook | ||||
When eol is used through hooks, the extension is never formally loaded and | ||||
enabled. This has some side effect, for example the config declaration is | ||||
never loaded. This function ensure the extension is enabled when running | ||||
hooks. | ||||
""" | ||||
|
r43347 | if b'eol' in ui._knownconfig: | ||
|
r34831 | return | ||
|
r43347 | ui.setconfig(b'extensions', b'eol', b'', source=b'internal') | ||
extensions.loadall(ui, [b'eol']) | ||||
|
r34831 | |||
|
r43346 | |||
|
r13617 | def _checkhook(ui, repo, node, headsonly): | ||
# Get revisions to check and touched files at the same time | ||||
|
r34831 | ensureenabled(ui) | ||
|
r11249 | files = set() | ||
|
r13617 | revs = set() | ||
|
r38806 | for rev in pycompat.xrange(repo[node].rev(), len(repo)): | ||
|
r13617 | revs.add(rev) | ||
if headsonly: | ||||
|
r13650 | ctx = repo[rev] | ||
files.update(ctx.files()) | ||||
|
r13617 | for pctx in ctx.parents(): | ||
revs.discard(pctx.rev()) | ||||
|
r13649 | failed = [] | ||
|
r13617 | for rev in revs: | ||
|
r13616 | ctx = repo[rev] | ||
eol = parseeol(ui, repo, [ctx.node()]) | ||||
if eol: | ||||
|
r13649 | failed.extend(eol.checkrev(repo, ctx, files)) | ||
if failed: | ||||
|
r43347 | eols = {b'to-lf': b'CRLF', b'to-crlf': b'LF'} | ||
|
r13649 | msgs = [] | ||
|
r27524 | for f, target, node in sorted(failed): | ||
|
r43346 | msgs.append( | ||
|
r43347 | _(b" %s in %s should not have %s line endings") | ||
|
r43346 | % (f, node, eols[target]) | ||
) | ||||
|
r43347 | raise errormod.Abort( | ||
_(b"end-of-line check failed:\n") + b"\n".join(msgs) | ||||
) | ||||
|
r11249 | |||
|
r43346 | |||
|
r13617 | def checkallhook(ui, repo, node, hooktype, **kwargs): | ||
"""verify that files have expected EOLs""" | ||||
_checkhook(ui, repo, node, False) | ||||
|
r43346 | |||
|
r13617 | def checkheadshook(ui, repo, node, hooktype, **kwargs): | ||
"""verify that files have expected EOLs""" | ||||
_checkhook(ui, repo, node, True) | ||||
|
r43346 | |||
|
r13617 | # "checkheadshook" used to be called "hook" | ||
hook = checkheadshook | ||||
|
r11249 | |||
|
r43346 | |||
|
r11249 | def preupdate(ui, repo, hooktype, parent1, parent2): | ||
|
r37696 | p1node = scmutil.resolvehexnodeidprefix(repo, parent1) | ||
|
r37525 | repo.loadeol([p1node]) | ||
|
r11249 | return False | ||
|
r43346 | |||
|
r11249 | def uisetup(ui): | ||
|
r43347 | ui.setconfig(b'hooks', b'preupdate.eol', preupdate, b'eol') | ||
|
r11249 | |||
|
r43346 | |||
|
r11249 | def extsetup(ui): | ||
try: | ||||
|
r43347 | extensions.find(b'win32text') | ||
|
r43346 | ui.warn( | ||
_( | ||||
|
r43347 | b"the eol extension is incompatible with the " | ||
b"win32text extension\n" | ||||
|
r43346 | ) | ||
) | ||||
|
r11249 | except KeyError: | ||
pass | ||||
def reposetup(ui, repo): | ||||
|
r12307 | uisetup(repo.ui) | ||
|
r11249 | |||
if not repo.local(): | ||||
return | ||||
for name, fn in filters.iteritems(): | ||||
repo.adddatafilter(name, fn) | ||||
|
r43347 | ui.setconfig(b'patch', b'eol', b'auto', b'eol') | ||
|
r11249 | |||
class eolrepo(repo.__class__): | ||||
|
r13614 | def loadeol(self, nodes): | ||
eol = parseeol(self.ui, self, nodes) | ||||
|
r13613 | if eol is None: | ||
|
r13612 | return None | ||
|
r14854 | eol.copytoui(self.ui) | ||
|
r13613 | return eol.match | ||
|
r11249 | |||
def _hgcleardirstate(self): | ||||
|
r43347 | self._eolmatch = self.loadeol([None, b'tip']) | ||
|
r30113 | if not self._eolmatch: | ||
self._eolmatch = util.never | ||||
|
r11249 | return | ||
|
r30140 | oldeol = None | ||
|
r11249 | try: | ||
|
r43347 | cachemtime = os.path.getmtime(self.vfs.join(b"eol.cache")) | ||
|
r11249 | except OSError: | ||
cachemtime = 0 | ||||
|
r30140 | else: | ||
|
r43347 | olddata = self.vfs.read(b"eol.cache") | ||
|
r30140 | if olddata: | ||
oldeol = eolfile(self.ui, self.root, olddata) | ||||
|
r11249 | |||
try: | ||||
|
r43347 | eolmtime = os.path.getmtime(self.wjoin(b".hgeol")) | ||
|
r11249 | except OSError: | ||
eolmtime = 0 | ||||
if eolmtime > cachemtime: | ||||
|
r43347 | self.ui.debug(b"eol: detected change in .hgeol\n") | ||
|
r30140 | |||
|
r43347 | hgeoldata = self.wvfs.read(b'.hgeol') | ||
|
r30140 | neweol = eolfile(self.ui, self.root, hgeoldata) | ||
|
r11249 | wlock = None | ||
try: | ||||
wlock = self.wlock() | ||||
|
r13581 | for f in self.dirstate: | ||
|
r43347 | if self.dirstate[f] != b'n': | ||
|
r30140 | continue | ||
if oldeol is not None: | ||||
if not oldeol.match(f) and not neweol.match(f): | ||||
continue | ||||
oldkey = None | ||||
for pattern, key, m in oldeol.patterns: | ||||
if m(f): | ||||
oldkey = key | ||||
break | ||||
newkey = None | ||||
for pattern, key, m in neweol.patterns: | ||||
if m(f): | ||||
newkey = key | ||||
break | ||||
if oldkey == newkey: | ||||
continue | ||||
# all normal files need to be looked at again since | ||||
# the new .hgeol file specify a different filter | ||||
self.dirstate.normallookup(f) | ||||
# Write the cache to update mtime and cache .hgeol | ||||
|
r43347 | with self.vfs(b"eol.cache", b"w") as f: | ||
|
r30140 | f.write(hgeoldata) | ||
r32995 | except errormod.LockUnavailable: | |||
|
r13475 | # If we cannot lock the repository and clear the | ||
# dirstate, then a commit might not see all files | ||||
# as modified. But if we cannot lock the | ||||
# repository, then we can also not make a commit, | ||||
# so ignore the error. | ||||
pass | ||||
|
r30164 | finally: | ||
if wlock is not None: | ||||
wlock.release() | ||||
|
r11249 | |||
|
r42839 | def commitctx(self, ctx, error=False, origctx=None): | ||
|
r11249 | for f in sorted(ctx.added() + ctx.modified()): | ||
|
r30113 | if not self._eolmatch(f): | ||
|
r11249 | continue | ||
|
r23068 | fctx = ctx[f] | ||
if fctx is None: | ||||
|
r14862 | continue | ||
|
r23068 | data = fctx.data() | ||
|
r37102 | if stringutil.binary(data): | ||
|
r11249 | # We should not abort here, since the user should | ||
# be able to say "** = native" to automatically | ||||
# have all non-binary files taken care of. | ||||
continue | ||||
if inconsistenteol(data): | ||||
|
r43346 | raise errormod.Abort( | ||
|
r43347 | _(b"inconsistent newline style " b"in %s\n") % f | ||
|
r43346 | ) | ||
|
r42839 | return super(eolrepo, self).commitctx(ctx, error, origctx) | ||
|
r43346 | |||
|
r11249 | repo.__class__ = eolrepo | ||
repo._hgcleardirstate() | ||||