eol.py
388 lines
| 14.3 KiB
| text/x-python
|
PythonLexer
/ hgext / eol.py
|
r11249 | """automatically manage newlines in repository files | ||
This extension allows you to manage the type of line endings (CRLF or | ||||
LF) that are used in the repository and in the local working | ||||
directory. That way you can get CRLF line endings on Windows and LF on | ||||
Unix/Mac, thereby letting everybody use their OS native line endings. | ||||
The extension reads its configuration from a versioned ``.hgeol`` | ||||
|
r24367 | configuration file found in the root of the working directory. The | ||
|
r11249 | ``.hgeol`` file use the same syntax as all other Mercurial | ||
configuration files. It uses two sections, ``[patterns]`` and | ||||
``[repository]``. | ||||
|
r13124 | The ``[patterns]`` section specifies how line endings should be | ||
|
r24367 | converted between the working directory and the repository. The format is | ||
|
r13124 | specified by a file pattern. The first match is used, so put more | ||
specific patterns first. The available line endings are ``LF``, | ||||
``CRLF``, and ``BIN``. | ||||
|
r11249 | |||
Files with the declared format of ``CRLF`` or ``LF`` are always | ||||
|
r13124 | checked out and stored in the repository in that format and files | ||
declared to be binary (``BIN``) are left unchanged. Additionally, | ||||
``native`` is an alias for checking out in the platform's default line | ||||
ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on | ||||
Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's | ||||
|
r26098 | default behavior; it is only needed if you need to override a later, | ||
|
r13124 | more general pattern. | ||
|
r11249 | |||
The optional ``[repository]`` section specifies the line endings to | ||||
use for files stored in the repository. It has a single setting, | ||||
``native``, which determines the storage line endings for files | ||||
declared as ``native`` in the ``[patterns]`` section. It can be set to | ||||
``LF`` or ``CRLF``. The default is ``LF``. For example, this means | ||||
that on Windows, files configured as ``native`` (``CRLF`` by default) | ||||
will be converted to ``LF`` when stored in the repository. Files | ||||
declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section | ||||
are always stored as-is in the repository. | ||||
Example versioned ``.hgeol`` file:: | ||||
[patterns] | ||||
**.py = native | ||||
**.vcproj = CRLF | ||||
**.txt = native | ||||
Makefile = LF | ||||
**.jpg = BIN | ||||
[repository] | ||||
native = LF | ||||
|
r13124 | .. note:: | ||
|
r19997 | |||
|
r13124 | The rules will first apply when files are touched in the working | ||
|
r24367 | directory, e.g. by updating to null and back to tip to touch all files. | ||
|
r13124 | |||
|
r14856 | The extension uses an optional ``[eol]`` section read from both the | ||
normal Mercurial configuration files and the ``.hgeol`` file, with the | ||||
latter overriding the former. You can use that section to control the | ||||
overall behavior. There are three settings: | ||||
|
r11249 | |||
- ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or | ||||
|
r12802 | ``CRLF`` to override the default interpretation of ``native`` for | ||
|
r11249 | checkout. This can be used with :hg:`archive` on Unix, say, to | ||
generate an archive where files have line endings for Windows. | ||||
- ``eol.only-consistent`` (default True) can be set to False to make | ||||
the extension convert files with inconsistent EOLs. Inconsistent | ||||
means that there is both ``CRLF`` and ``LF`` present in the file. | ||||
Such files are normally not touched under the assumption that they | ||||
have mixed EOLs on purpose. | ||||
|
r14856 | - ``eol.fix-trailing-newline`` (default False) can be set to True to | ||
|
r14857 | ensure that converted files end with a EOL character (either ``\\n`` | ||
or ``\\r\\n`` as per the configured patterns). | ||||
|
r14856 | |||
|
r12979 | The extension provides ``cleverencode:`` and ``cleverdecode:`` filters | ||
like the deprecated win32text extension does. This means that you can | ||||
disable win32text and enable eol and your filters will still work. You | ||||
only need to these filters until you have prepared a ``.hgeol`` file. | ||||
|
r12980 | The ``win32text.forbid*`` hooks provided by the win32text extension | ||
|
r13617 | have been unified into a single hook named ``eol.checkheadshook``. The | ||
hook will lookup the expected line endings from the ``.hgeol`` file, | ||||
which means you must migrate to a ``.hgeol`` file first before using | ||||
the hook. ``eol.checkheadshook`` only checks heads, intermediate | ||||
invalid revisions will be pushed. To forbid them completely, use the | ||||
``eol.checkallhook`` hook. These hooks are best used as | ||||
``pretxnchangegroup`` hooks. | ||||
|
r12980 | |||
|
r11249 | See :hg:`help patterns` for more information about the glob patterns | ||
used. | ||||
""" | ||||
|
r28969 | from __future__ import absolute_import | ||
import os | ||||
import re | ||||
|
r11249 | from mercurial.i18n import _ | ||
|
r28969 | from mercurial import ( | ||
config, | ||||
error, | ||||
extensions, | ||||
match, | ||||
|
r31775 | pycompat, | ||
|
r28969 | util, | ||
) | ||||
|
r11249 | |||
|
r29841 | # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||
|
r25186 | # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
|
r29841 | testedwith = 'ships-with-hg-core' | ||
|
r16743 | |||
|
r11249 | # Matches a lone LF, i.e., one that is not part of CRLF. | ||
singlelf = re.compile('(^|[^\r])\n') | ||||
def inconsistenteol(data): | ||||
return '\r\n' in data and singlelf.search(data) | ||||
def tolf(s, params, ui, **kwargs): | ||||
"""Filter to convert to LF EOLs.""" | ||||
if util.binary(s): | ||||
return s | ||||
if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s): | ||||
return s | ||||
|
r16683 | if (ui.configbool('eol', 'fix-trailing-newline', False) | ||
and s and s[-1] != '\n'): | ||||
|
r14855 | s = s + '\n' | ||
|
r31776 | return util.tolf(s) | ||
|
r11249 | |||
def tocrlf(s, params, ui, **kwargs): | ||||
"""Filter to convert to CRLF EOLs.""" | ||||
if util.binary(s): | ||||
return s | ||||
if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s): | ||||
return s | ||||
|
r16683 | if (ui.configbool('eol', 'fix-trailing-newline', False) | ||
and s and s[-1] != '\n'): | ||||
|
r14855 | s = s + '\n' | ||
|
r31776 | return util.tocrlf(s) | ||
|
r11249 | |||
def isbinary(s, params): | ||||
"""Filter to do nothing with the file.""" | ||||
return s | ||||
filters = { | ||||
'to-lf': tolf, | ||||
'to-crlf': tocrlf, | ||||
'is-binary': isbinary, | ||||
|
r12975 | # The following provide backwards compatibility with win32text | ||
|
r12979 | 'cleverencode:': tolf, | ||
'cleverdecode:': tocrlf | ||||
|
r11249 | } | ||
|
r13613 | class eolfile(object): | ||
def __init__(self, ui, root, data): | ||||
self._decode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'} | ||||
self._encode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'} | ||||
|
r11249 | |||
|
r13613 | self.cfg = config.config() | ||
# Our files should not be touched. The pattern must be | ||||
# inserted first override a '** = native' pattern. | ||||
|
r20790 | self.cfg.set('patterns', '.hg*', 'BIN', 'eol') | ||
|
r13613 | # We can then parse the user's patterns. | ||
self.cfg.parse('.hgeol', data) | ||||
isrepolf = self.cfg.get('repository', 'native') != 'CRLF' | ||||
self._encode['NATIVE'] = isrepolf and 'to-lf' or 'to-crlf' | ||||
|
r31775 | iswdlf = ui.config('eol', 'native', pycompat.oslinesep) in ('LF', '\n') | ||
|
r13613 | self._decode['NATIVE'] = iswdlf and 'to-lf' or 'to-crlf' | ||
include = [] | ||||
exclude = [] | ||||
|
r30114 | self.patterns = [] | ||
|
r13613 | for pattern, style in self.cfg.items('patterns'): | ||
key = style.upper() | ||||
if key == 'BIN': | ||||
exclude.append(pattern) | ||||
else: | ||||
include.append(pattern) | ||||
|
r30114 | m = match.match(root, '', [pattern]) | ||
self.patterns.append((pattern, key, m)) | ||||
|
r13613 | # This will match the files for which we need to care | ||
# about inconsistent newlines. | ||||
self.match = match.match(root, '', [], include, exclude) | ||||
|
r14854 | def copytoui(self, ui): | ||
|
r30114 | for pattern, key, m in self.patterns: | ||
|
r13613 | try: | ||
|
r20790 | ui.setconfig('decode', pattern, self._decode[key], 'eol') | ||
ui.setconfig('encode', pattern, self._encode[key], 'eol') | ||||
|
r13613 | except KeyError: | ||
ui.warn(_("ignoring unknown EOL style '%s' from %s\n") | ||||
|
r30114 | % (key, self.cfg.source('patterns', pattern))) | ||
|
r14854 | # eol.only-consistent can be specified in ~/.hgrc or .hgeol | ||
for k, v in self.cfg.items('eol'): | ||||
|
r20790 | ui.setconfig('eol', k, v, 'eol') | ||
|
r13613 | |||
|
r13615 | def checkrev(self, repo, ctx, files): | ||
|
r13649 | failed = [] | ||
|
r13650 | for f in (files or ctx.files()): | ||
|
r13615 | if f not in ctx: | ||
continue | ||||
|
r30114 | for pattern, key, m in self.patterns: | ||
if not m(f): | ||||
|
r13615 | continue | ||
|
r30114 | target = self._encode[key] | ||
|
r13615 | data = ctx[f].data() | ||
|
r13649 | if (target == "to-lf" and "\r\n" in data | ||
or target == "to-crlf" and singlelf.search(data)): | ||||
|
r27524 | failed.append((f, target, str(ctx))) | ||
|
r13501 | break | ||
|
r13649 | return failed | ||
|
r11249 | |||
|
r13614 | def parseeol(ui, repo, nodes): | ||
|
r13613 | try: | ||
|
r13614 | for node in nodes: | ||
try: | ||||
if node is None: | ||||
# Cannot use workingctx.data() since it would load | ||||
# and cache the filters before we configure them. | ||||
|
r31417 | data = repo.wvfs('.hgeol').read() | ||
|
r13614 | else: | ||
data = repo[node]['.hgeol'].data() | ||||
return eolfile(ui, repo.root, data) | ||||
except (IOError, LookupError): | ||||
pass | ||||
|
r25660 | except error.ParseError as inst: | ||
|
r13614 | ui.warn(_("warning: ignoring .hgeol file due to parse error " | ||
"at %s: %s\n") % (inst.args[1], inst.args[0])) | ||||
return None | ||||
|
r11249 | |||
|
r13617 | def _checkhook(ui, repo, node, headsonly): | ||
# Get revisions to check and touched files at the same time | ||||
|
r11249 | files = set() | ||
|
r13617 | revs = set() | ||
|
r11249 | for rev in xrange(repo[node].rev(), len(repo)): | ||
|
r13617 | revs.add(rev) | ||
if headsonly: | ||||
|
r13650 | ctx = repo[rev] | ||
files.update(ctx.files()) | ||||
|
r13617 | for pctx in ctx.parents(): | ||
revs.discard(pctx.rev()) | ||||
|
r13649 | failed = [] | ||
|
r13617 | for rev in revs: | ||
|
r13616 | ctx = repo[rev] | ||
eol = parseeol(ui, repo, [ctx.node()]) | ||||
if eol: | ||||
|
r13649 | failed.extend(eol.checkrev(repo, ctx, files)) | ||
if failed: | ||||
eols = {'to-lf': 'CRLF', 'to-crlf': 'LF'} | ||||
msgs = [] | ||||
|
r27524 | for f, target, node in sorted(failed): | ||
|
r13649 | msgs.append(_(" %s in %s should not have %s line endings") % | ||
(f, node, eols[target])) | ||||
|
r26587 | raise error.Abort(_("end-of-line check failed:\n") + "\n".join(msgs)) | ||
|
r11249 | |||
|
r13617 | def checkallhook(ui, repo, node, hooktype, **kwargs): | ||
"""verify that files have expected EOLs""" | ||||
_checkhook(ui, repo, node, False) | ||||
def checkheadshook(ui, repo, node, hooktype, **kwargs): | ||||
"""verify that files have expected EOLs""" | ||||
_checkhook(ui, repo, node, True) | ||||
# "checkheadshook" used to be called "hook" | ||||
hook = checkheadshook | ||||
|
r11249 | |||
def preupdate(ui, repo, hooktype, parent1, parent2): | ||||
|
r13614 | repo.loadeol([parent1]) | ||
|
r11249 | return False | ||
def uisetup(ui): | ||||
|
r20790 | ui.setconfig('hooks', 'preupdate.eol', preupdate, 'eol') | ||
|
r11249 | |||
def extsetup(ui): | ||||
try: | ||||
extensions.find('win32text') | ||||
|
r13624 | ui.warn(_("the eol extension is incompatible with the " | ||
"win32text extension\n")) | ||||
|
r11249 | except KeyError: | ||
pass | ||||
def reposetup(ui, repo): | ||||
|
r12307 | uisetup(repo.ui) | ||
|
r11249 | |||
if not repo.local(): | ||||
return | ||||
for name, fn in filters.iteritems(): | ||||
repo.adddatafilter(name, fn) | ||||
|
r20790 | ui.setconfig('patch', 'eol', 'auto', 'eol') | ||
|
r11249 | |||
class eolrepo(repo.__class__): | ||||
|
r13614 | def loadeol(self, nodes): | ||
eol = parseeol(self.ui, self, nodes) | ||||
|
r13613 | if eol is None: | ||
|
r13612 | return None | ||
|
r14854 | eol.copytoui(self.ui) | ||
|
r13613 | return eol.match | ||
|
r11249 | |||
def _hgcleardirstate(self): | ||||
|
r30113 | self._eolmatch = self.loadeol([None, 'tip']) | ||
if not self._eolmatch: | ||||
self._eolmatch = util.never | ||||
|
r11249 | return | ||
|
r30140 | oldeol = None | ||
|
r11249 | try: | ||
|
r31328 | cachemtime = os.path.getmtime(self.vfs.join("eol.cache")) | ||
|
r11249 | except OSError: | ||
cachemtime = 0 | ||||
|
r30140 | else: | ||
olddata = self.vfs.read("eol.cache") | ||||
if olddata: | ||||
oldeol = eolfile(self.ui, self.root, olddata) | ||||
|
r11249 | |||
try: | ||||
eolmtime = os.path.getmtime(self.wjoin(".hgeol")) | ||||
except OSError: | ||||
eolmtime = 0 | ||||
if eolmtime > cachemtime: | ||||
|
r17955 | self.ui.debug("eol: detected change in .hgeol\n") | ||
|
r30140 | |||
hgeoldata = self.wvfs.read('.hgeol') | ||||
neweol = eolfile(self.ui, self.root, hgeoldata) | ||||
|
r11249 | wlock = None | ||
try: | ||||
wlock = self.wlock() | ||||
|
r13581 | for f in self.dirstate: | ||
|
r30140 | if self.dirstate[f] != 'n': | ||
continue | ||||
if oldeol is not None: | ||||
if not oldeol.match(f) and not neweol.match(f): | ||||
continue | ||||
oldkey = None | ||||
for pattern, key, m in oldeol.patterns: | ||||
if m(f): | ||||
oldkey = key | ||||
break | ||||
newkey = None | ||||
for pattern, key, m in neweol.patterns: | ||||
if m(f): | ||||
newkey = key | ||||
break | ||||
if oldkey == newkey: | ||||
continue | ||||
# all normal files need to be looked at again since | ||||
# the new .hgeol file specify a different filter | ||||
self.dirstate.normallookup(f) | ||||
# Write the cache to update mtime and cache .hgeol | ||||
with self.vfs("eol.cache", "w") as f: | ||||
f.write(hgeoldata) | ||||
|
r13475 | except error.LockUnavailable: | ||
# If we cannot lock the repository and clear the | ||||
# dirstate, then a commit might not see all files | ||||
# as modified. But if we cannot lock the | ||||
# repository, then we can also not make a commit, | ||||
# so ignore the error. | ||||
pass | ||||
|
r30164 | finally: | ||
if wlock is not None: | ||||
wlock.release() | ||||
|
r11249 | |||
|
r26586 | def commitctx(self, ctx, haserror=False): | ||
|
r11249 | for f in sorted(ctx.added() + ctx.modified()): | ||
|
r30113 | if not self._eolmatch(f): | ||
|
r11249 | continue | ||
|
r23068 | fctx = ctx[f] | ||
if fctx is None: | ||||
|
r14862 | continue | ||
|
r23068 | data = fctx.data() | ||
|
r11249 | if util.binary(data): | ||
# We should not abort here, since the user should | ||||
# be able to say "** = native" to automatically | ||||
# have all non-binary files taken care of. | ||||
continue | ||||
if inconsistenteol(data): | ||||
|
r26587 | raise error.Abort(_("inconsistent newline style " | ||
|
r20869 | "in %s\n") % f) | ||
|
r26586 | return super(eolrepo, self).commitctx(ctx, haserror) | ||
|
r11249 | repo.__class__ = eolrepo | ||
repo._hgcleardirstate() | ||||