safehasattr: drop usage in favor of hasattr

The two functions should now be equivalent at least in their usage in core.

contrib/benchmarks/__init__.py

@@ -1,125 +1,124 b''
 # __init__.py - asv benchmark suite
 #
 # Copyright 2016 Logilab SA <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # "historical portability" policy of contrib/benchmarks:
 #
 # We have to make this code work correctly with current mercurial stable branch
 # and if possible with reasonable cost with early Mercurial versions.
 
 '''ASV (https://asv.readthedocs.io) benchmark suite
 
 Benchmark are parameterized against reference repositories found in the
 directory pointed by the REPOS_DIR environment variable.
 
 Invocation example:
 
     $ export REPOS_DIR=~/hgperf/repos
     # run suite on given revision
     $ asv --config contrib/asv.conf.json run REV
     # run suite on new changesets found in stable and default branch
     $ asv --config contrib/asv.conf.json run NEW
     # display a comparative result table of benchmark results between two given
     # revisions
     $ asv --config contrib/asv.conf.json compare REV1 REV2
     # compute regression detection and generate ASV static website
     $ asv --config contrib/asv.conf.json publish
     # serve the static website
     $ asv --config contrib/asv.conf.json preview
 '''
 
 
 import functools
 import os
 import re
 
 from mercurial import (
     extensions,
     hg,
     ui as uimod,
-    util,
 )
 
 basedir = os.path.abspath(
     os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
 )
 reposdir = os.environ['REPOS_DIR']
 reposnames = [
     name
     for name in os.listdir(reposdir)
     if os.path.isdir(os.path.join(reposdir, name, ".hg"))
 ]
 if not reposnames:
     raise ValueError("No repositories found in $REPO_DIR")
 outputre = re.compile(
     (
         r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
         r'\d+.\d+ \(best of \d+\)'
     )
 )
 
 
 def runperfcommand(reponame, command, *args, **kwargs):
     os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "")
     # for "historical portability"
     # ui.load() has been available since d83ca85
-    if util.safehasattr(uimod.ui, "load"):
+    if hasattr(uimod.ui, "load"):
         ui = uimod.ui.load()
     else:
         ui = uimod.ui()
     repo = hg.repository(ui, os.path.join(reposdir, reponame))
     perfext = extensions.load(
         ui, 'perfext', os.path.join(basedir, 'contrib', 'perf.py')
     )
     cmd = getattr(perfext, command)
     ui.pushbuffer()
     cmd(ui, repo, *args, **kwargs)
     output = ui.popbuffer()
     match = outputre.search(output)
     if not match:
         raise ValueError("Invalid output {}".format(output))
     return float(match.group(1))
 
 
 def perfbench(repos=reposnames, name=None, params=None):
     """decorator to declare ASV benchmark based on contrib/perf.py extension
 
     An ASV benchmark is a python function with the given attributes:
 
     __name__: should start with track_, time_ or mem_ to be collected by ASV
     params and param_name: parameter matrix to display multiple graphs on the
     same page.
     pretty_name: If defined it's displayed in web-ui instead of __name__
     (useful for revsets)
     the module name is prepended to the benchmark name and displayed as
     "category" in webui.
 
     Benchmarks are automatically parameterized with repositories found in the
     REPOS_DIR environment variable.
 
     `params` is the param matrix in the form of a list of tuple
     (param_name, [value0, value1])
 
     For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
     (a, c), (a, d), (b, c) and (b, d).
     """
     params = list(params or [])
     params.insert(0, ("repo", repos))
 
     def decorator(func):
         @functools.wraps(func)
         def wrapped(repo, *args):
             def perf(command, *a, **kw):
                 return runperfcommand(repo, command, *a, **kw)
 
             return func(perf, *args)
 
         wrapped.params = [p[1] for p in params]
         wrapped.param_names = [p[0] for p in params]
         wrapped.pretty_name = name
         return wrapped
 
     return decorator

hgext/absorb.py

@@ -1,1162 +1,1162 b''
 # absorb.py
 #
 # Copyright 2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """apply working directory changes to changesets (EXPERIMENTAL)
 
 The absorb extension provides a command to use annotate information to
 amend modified chunks into the corresponding non-public changesets.
 
 ::
 
     [absorb]
     # only check 50 recent non-public changesets at most
     max-stack-size = 50
     # whether to add noise to new commits to avoid obsolescence cycle
     add-noise = 1
     # make `amend --correlated` a shortcut to the main command
     amend-flag = correlated
 
     [color]
     absorb.description = yellow
     absorb.node = blue bold
     absorb.path = bold
 """
 
 # TODO:
 #  * Rename config items to [commands] namespace
 #  * Converge getdraftstack() with other code in core
 #  * move many attributes on fixupstate to be private
 
 
 import collections
 
 from mercurial.i18n import _
 from mercurial.node import (
     hex,
     short,
 )
 from mercurial import (
     cmdutil,
     commands,
     context,
     crecord,
     error,
     linelog,
     mdiff,
     obsolete,
     patch,
     phases,
     pycompat,
     registrar,
     rewriteutil,
     scmutil,
     util,
 )
 from mercurial.utils import stringutil
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(b'absorb', b'add-noise', default=True)
 configitem(b'absorb', b'amend-flag', default=None)
 configitem(b'absorb', b'max-stack-size', default=50)
 
 colortable = {
     b'absorb.description': b'yellow',
     b'absorb.node': b'blue bold',
     b'absorb.path': b'bold',
 }
 
 defaultdict = collections.defaultdict
 
 
 class nullui:
     """blank ui object doing nothing"""
 
     debugflag = False
     verbose = False
     quiet = True
 
     def __getitem__(name):
         def nullfunc(*args, **kwds):
             return
 
         return nullfunc
 
 
 class emptyfilecontext:
     """minimal filecontext representing an empty file"""
 
     def __init__(self, repo):
         self._repo = repo
 
     def data(self):
         return b''
 
     def node(self):
         return self._repo.nullid
 
 
 def uniq(lst):
     """list -> list. remove duplicated items without changing the order"""
     seen = set()
     result = []
     for x in lst:
         if x not in seen:
             seen.add(x)
             result.append(x)
     return result
 
 
 def getdraftstack(headctx, limit=None):
     """(ctx, int?) -> [ctx]. get a linear stack of non-public changesets.
 
     changesets are sorted in topo order, oldest first.
     return at most limit items, if limit is a positive number.
 
     merges are considered as non-draft as well. i.e. every commit
     returned has and only has 1 parent.
     """
     ctx = headctx
     result = []
     while ctx.phase() != phases.public:
         if limit and len(result) >= limit:
             break
         parents = ctx.parents()
         if len(parents) != 1:
             break
         result.append(ctx)
         ctx = parents[0]
     result.reverse()
     return result
 
 
 def getfilestack(stack, path, seenfctxs=None):
     """([ctx], str, set) -> [fctx], {ctx: fctx}
 
     stack is a list of contexts, from old to new. usually they are what
     "getdraftstack" returns.
 
     follows renames, but not copies.
 
     seenfctxs is a set of filecontexts that will be considered "immutable".
     they are usually what this function returned in earlier calls, useful
     to avoid issues that a file was "moved" to multiple places and was then
     modified differently, like: "a" was copied to "b", "a" was also copied to
     "c" and then "a" was deleted, then both "b" and "c" were "moved" from "a"
     and we enforce only one of them to be able to affect "a"'s content.
 
     return an empty list and an empty dict, if the specified path does not
     exist in stack[-1] (the top of the stack).
 
     otherwise, return a list of de-duplicated filecontexts, and the map to
     convert ctx in the stack to fctx, for possible mutable fctxs. the first item
     of the list would be outside the stack and should be considered immutable.
     the remaining items are within the stack.
 
     for example, given the following changelog and corresponding filelog
     revisions:
 
       changelog: 3----4----5----6----7
       filelog:   x    0----1----1----2 (x: no such file yet)
 
     - if stack = [5, 6, 7], returns ([0, 1, 2], {5: 1, 6: 1, 7: 2})
     - if stack = [3, 4, 5], returns ([e, 0, 1], {4: 0, 5: 1}), where "e" is a
       dummy empty filecontext.
     - if stack = [2], returns ([], {})
     - if stack = [7], returns ([1, 2], {7: 2})
     - if stack = [6, 7], returns ([1, 2], {6: 1, 7: 2}), although {6: 1} can be
       removed, since 1 is immutable.
     """
     if seenfctxs is None:
         seenfctxs = set()
     assert stack
 
     if path not in stack[-1]:
         return [], {}
 
     fctxs = []
     fctxmap = {}
 
     pctx = stack[0].p1()  # the public (immutable) ctx we stop at
     for ctx in reversed(stack):
         if path not in ctx:  # the file is added in the next commit
             pctx = ctx
             break
         fctx = ctx[path]
         fctxs.append(fctx)
         if fctx in seenfctxs:  # treat fctx as the immutable one
             pctx = None  # do not add another immutable fctx
             break
         fctxmap[ctx] = fctx  # only for mutable fctxs
         copy = fctx.copysource()
         if copy:
             path = copy  # follow rename
             if path in ctx:  # but do not follow copy
                 pctx = ctx.p1()
                 break
 
     if pctx is not None:  # need an extra immutable fctx
         if path in pctx:
             fctxs.append(pctx[path])
         else:
             fctxs.append(emptyfilecontext(pctx.repo()))
 
     fctxs.reverse()
     # note: we rely on a property of hg: filerev is not reused for linear
     # history. i.e. it's impossible to have:
     #   changelog:  4----5----6 (linear, no merges)
     #   filelog:    1----2----1
     #                         ^ reuse filerev (impossible)
     # because parents are part of the hash. if that's not true, we need to
     # remove uniq and find a different way to identify fctxs.
     return uniq(fctxs), fctxmap
 
 
 class overlaystore(patch.filestore):
     """read-only, hybrid store based on a dict and ctx.
     memworkingcopy: {path: content}, overrides file contents.
     """
 
     def __init__(self, basectx, memworkingcopy):
         self.basectx = basectx
         self.memworkingcopy = memworkingcopy
 
     def getfile(self, path):
         """comply with mercurial.patch.filestore.getfile"""
         if path not in self.basectx:
             return None, None, None
         fctx = self.basectx[path]
         if path in self.memworkingcopy:
             content = self.memworkingcopy[path]
         else:
             content = fctx.data()
         mode = (fctx.islink(), fctx.isexec())
         copy = fctx.copysource()
         return content, mode, copy
 
 
 def overlaycontext(memworkingcopy, ctx, parents=None, extra=None, desc=None):
     """({path: content}, ctx, (p1node, p2node)?, {}?) -> memctx
     memworkingcopy overrides file contents.
     """
     # parents must contain 2 items: (node1, node2)
     if parents is None:
         parents = ctx.repo().changelog.parents(ctx.node())
     if extra is None:
         extra = ctx.extra()
     if desc is None:
         desc = ctx.description()
     date = ctx.date()
     user = ctx.user()
     files = set(ctx.files()).union(memworkingcopy)
     store = overlaystore(ctx, memworkingcopy)
     return context.memctx(
         repo=ctx.repo(),
         parents=parents,
         text=desc,
         files=files,
         filectxfn=store,
         user=user,
         date=date,
         branch=None,
         extra=extra,
     )
 
 
 class filefixupstate:
     """state needed to apply fixups to a single file
 
     internally, it keeps file contents of several revisions and a linelog.
 
     the linelog uses odd revision numbers for original contents (fctxs passed
     to __init__), and even revision numbers for fixups, like:
 
         linelog rev 1: self.fctxs[0] (from an immutable "public" changeset)
         linelog rev 2: fixups made to self.fctxs[0]
         linelog rev 3: self.fctxs[1] (a child of fctxs[0])
         linelog rev 4: fixups made to self.fctxs[1]
         ...
 
     a typical use is like:
 
         1. call diffwith, to calculate self.fixups
         2. (optionally), present self.fixups to the user, or change it
         3. call apply, to apply changes
         4. read results from "finalcontents", or call getfinalcontent
     """
 
     def __init__(self, fctxs, path, ui=None, **opts):
         """([fctx], ui or None) -> None
 
         fctxs should be linear, and sorted by topo order - oldest first.
         fctxs[0] will be considered as "immutable" and will not be changed.
         """
         self.fctxs = fctxs
         self.path = path
         self.ui = ui or nullui()
         self.opts = opts
 
         # following fields are built from fctxs. they exist for perf reason
         self.contents = [f.data() for f in fctxs]
         self.contentlines = pycompat.maplist(mdiff.splitnewlines, self.contents)
         self.linelog = self._buildlinelog()
         if self.ui.debugflag:
             assert self._checkoutlinelog() == self.contents
 
         # following fields will be filled later
         self.chunkstats = [0, 0]  # [adopted, total : int]
         self.targetlines = []  # [str]
         self.fixups = []  # [(linelog rev, a1, a2, b1, b2)]
         self.finalcontents = []  # [str]
         self.ctxaffected = set()
 
     def diffwith(self, targetfctx, fm=None):
         """calculate fixups needed by examining the differences between
         self.fctxs[-1] and targetfctx, chunk by chunk.
 
         targetfctx is the target state we move towards. we may or may not be
         able to get there because not all modified chunks can be amended into
         a non-public fctx unambiguously.
 
         call this only once, before apply().
 
         update self.fixups, self.chunkstats, and self.targetlines.
         """
         a = self.contents[-1]
         alines = self.contentlines[-1]
         b = targetfctx.data()
         blines = mdiff.splitnewlines(b)
         self.targetlines = blines
 
         self.linelog.annotate(self.linelog.maxrev)
         annotated = self.linelog.annotateresult  # [(linelog rev, linenum)]
         assert len(annotated) == len(alines)
         # add a dummy end line to make insertion at the end easier
         if annotated:
             dummyendline = (annotated[-1][0], annotated[-1][1] + 1)
             annotated.append(dummyendline)
 
         # analyse diff blocks
         for chunk in self._alldiffchunks(a, b, alines, blines):
             newfixups = self._analysediffchunk(chunk, annotated)
             self.chunkstats[0] += bool(newfixups)  # 1 or 0
             self.chunkstats[1] += 1
             self.fixups += newfixups
             if fm is not None:
                 self._showchanges(fm, alines, blines, chunk, newfixups)
 
     def apply(self):
         """apply self.fixups. update self.linelog, self.finalcontents.
 
         call this only once, before getfinalcontent(), after diffwith().
         """
         # the following is unnecessary, as it's done by "diffwith":
         #   self.linelog.annotate(self.linelog.maxrev)
         for rev, a1, a2, b1, b2 in reversed(self.fixups):
             blines = self.targetlines[b1:b2]
             if self.ui.debugflag:
                 idx = (max(rev - 1, 0)) // 2
                 self.ui.write(
                     _(b'%s: chunk %d:%d -> %d lines\n')
                     % (short(self.fctxs[idx].node()), a1, a2, len(blines))
                 )
             self.linelog.replacelines(rev, a1, a2, b1, b2)
         if self.opts.get('edit_lines', False):
             self.finalcontents = self._checkoutlinelogwithedits()
         else:
             self.finalcontents = self._checkoutlinelog()
 
     def getfinalcontent(self, fctx):
         """(fctx) -> str. get modified file content for a given filecontext"""
         idx = self.fctxs.index(fctx)
         return self.finalcontents[idx]
 
     def _analysediffchunk(self, chunk, annotated):
         """analyse a different chunk and return new fixups found
 
         return [] if no lines from the chunk can be safely applied.
 
         the chunk (or lines) cannot be safely applied, if, for example:
           - the modified (deleted) lines belong to a public changeset
             (self.fctxs[0])
           - the chunk is a pure insertion and the adjacent lines (at most 2
             lines) belong to different non-public changesets, or do not belong
             to any non-public changesets.
           - the chunk is modifying lines from different changesets.
             in this case, if the number of lines deleted equals to the number
             of lines added, assume it's a simple 1:1 map (could be wrong).
             otherwise, give up.
           - the chunk is modifying lines from a single non-public changeset,
             but other revisions touch the area as well. i.e. the lines are
             not continuous as seen from the linelog.
         """
         a1, a2, b1, b2 = chunk
         # find involved indexes from annotate result
         involved = annotated[a1:a2]
         if not involved and annotated:  # a1 == a2 and a is not empty
             # pure insertion, check nearby lines. ignore lines belong
             # to the public (first) changeset (i.e. annotated[i][0] == 1)
             nearbylinenums = {a2, max(0, a1 - 1)}
             involved = [
                 annotated[i] for i in nearbylinenums if annotated[i][0] != 1
             ]
         involvedrevs = list({r for r, l in involved})
         newfixups = []
         if len(involvedrevs) == 1 and self._iscontinuous(a1, a2 - 1, True):
             # chunk belongs to a single revision
             rev = involvedrevs[0]
             if rev > 1:
                 fixuprev = rev + 1
                 newfixups.append((fixuprev, a1, a2, b1, b2))
         elif a2 - a1 == b2 - b1 or b1 == b2:
             # 1:1 line mapping, or chunk was deleted
             for i in range(a1, a2):
                 rev, linenum = annotated[i]
                 if rev > 1:
                     if b1 == b2:  # deletion, simply remove that single line
                         nb1 = nb2 = 0
                     else:  # 1:1 line mapping, change the corresponding rev
                         nb1 = b1 + i - a1
                         nb2 = nb1 + 1
                     fixuprev = rev + 1
                     newfixups.append((fixuprev, i, i + 1, nb1, nb2))
         return self._optimizefixups(newfixups)
 
     @staticmethod
     def _alldiffchunks(a, b, alines, blines):
         """like mdiff.allblocks, but only care about differences"""
         blocks = mdiff.allblocks(a, b, lines1=alines, lines2=blines)
         for chunk, btype in blocks:
             if btype != b'!':
                 continue
             yield chunk
 
     def _buildlinelog(self):
         """calculate the initial linelog based on self.content{,line}s.
         this is similar to running a partial "annotate".
         """
         llog = linelog.linelog()
         a, alines = b'', []
         for i in range(len(self.contents)):
             b, blines = self.contents[i], self.contentlines[i]
             llrev = i * 2 + 1
             chunks = self._alldiffchunks(a, b, alines, blines)
             for a1, a2, b1, b2 in reversed(list(chunks)):
                 llog.replacelines(llrev, a1, a2, b1, b2)
             a, alines = b, blines
         return llog
 
     def _checkoutlinelog(self):
         """() -> [str]. check out file contents from linelog"""
         contents = []
         for i in range(len(self.contents)):
             rev = (i + 1) * 2
             self.linelog.annotate(rev)
             content = b''.join(map(self._getline, self.linelog.annotateresult))
             contents.append(content)
         return contents
 
     def _checkoutlinelogwithedits(self):
         """() -> [str]. prompt all lines for edit"""
         alllines = self.linelog.getalllines()
         # header
         editortext = (
             _(
                 b'HG: editing %s\nHG: "y" means the line to the right '
                 b'exists in the changeset to the top\nHG:\n'
             )
             % self.fctxs[-1].path()
         )
         # [(idx, fctx)]. hide the dummy emptyfilecontext
         visiblefctxs = [
             (i, f)
             for i, f in enumerate(self.fctxs)
             if not isinstance(f, emptyfilecontext)
         ]
         for i, (j, f) in enumerate(visiblefctxs):
             editortext += _(b'HG: %s/%s %s %s\n') % (
                 b'|' * i,
                 b'-' * (len(visiblefctxs) - i + 1),
                 short(f.node()),
                 f.description().split(b'\n', 1)[0],
             )
         editortext += _(b'HG: %s\n') % (b'|' * len(visiblefctxs))
         # figure out the lifetime of a line, this is relatively inefficient,
         # but probably fine
         lineset = defaultdict(lambda: set())  # {(llrev, linenum): {llrev}}
         for i, f in visiblefctxs:
             self.linelog.annotate((i + 1) * 2)
             for l in self.linelog.annotateresult:
                 lineset[l].add(i)
         # append lines
         for l in alllines:
             editortext += b'    %s : %s' % (
                 b''.join(
                     [
                         (b'y' if i in lineset[l] else b' ')
                         for i, _f in visiblefctxs
                     ]
                 ),
                 self._getline(l),
             )
         # run editor
         editedtext = self.ui.edit(editortext, b'', action=b'absorb')
         if not editedtext:
             raise error.InputError(_(b'empty editor text'))
         # parse edited result
         contents = [b''] * len(self.fctxs)
         leftpadpos = 4
         colonpos = leftpadpos + len(visiblefctxs) + 1
         for l in mdiff.splitnewlines(editedtext):
             if l.startswith(b'HG:'):
                 continue
             if l[colonpos - 1 : colonpos + 2] != b' : ':
                 raise error.InputError(_(b'malformed line: %s') % l)
             linecontent = l[colonpos + 2 :]
             for i, ch in enumerate(
                 pycompat.bytestr(l[leftpadpos : colonpos - 1])
             ):
                 if ch == b'y':
                     contents[visiblefctxs[i][0]] += linecontent
         # chunkstats is hard to calculate if anything changes, therefore
         # set them to just a simple value (1, 1).
         if editedtext != editortext:
             self.chunkstats = [1, 1]
         return contents
 
     def _getline(self, lineinfo):
         """((rev, linenum)) -> str. convert rev+line number to line content"""
         rev, linenum = lineinfo
         if rev & 1:  # odd: original line taken from fctxs
             return self.contentlines[rev // 2][linenum]
         else:  # even: fixup line from targetfctx
             return self.targetlines[linenum]
 
     def _iscontinuous(self, a1, a2, closedinterval=False):
         """(a1, a2 : int) -> bool
 
         check if these lines are continuous. i.e. no other insertions or
         deletions (from other revisions) among these lines.
 
         closedinterval decides whether a2 should be included or not. i.e. is
         it [a1, a2), or [a1, a2] ?
         """
         if a1 >= a2:
             return True
         llog = self.linelog
         offset1 = llog.getoffset(a1)
         offset2 = llog.getoffset(a2) + int(closedinterval)
         linesinbetween = llog.getalllines(offset1, offset2)
         return len(linesinbetween) == a2 - a1 + int(closedinterval)
 
     def _optimizefixups(self, fixups):
         """[(rev, a1, a2, b1, b2)] -> [(rev, a1, a2, b1, b2)].
         merge adjacent fixups to make them less fragmented.
         """
         result = []
         pcurrentchunk = [[-1, -1, -1, -1, -1]]
 
         def pushchunk():
             if pcurrentchunk[0][0] != -1:
                 result.append(tuple(pcurrentchunk[0]))
 
         for i, chunk in enumerate(fixups):
             rev, a1, a2, b1, b2 = chunk
             lastrev = pcurrentchunk[0][0]
             lasta2 = pcurrentchunk[0][2]
             lastb2 = pcurrentchunk[0][4]
             if (
                 a1 == lasta2
                 and b1 == lastb2
                 and rev == lastrev
                 and self._iscontinuous(max(a1 - 1, 0), a1)
             ):
                 # merge into currentchunk
                 pcurrentchunk[0][2] = a2
                 pcurrentchunk[0][4] = b2
             else:
                 pushchunk()
                 pcurrentchunk[0] = list(chunk)
         pushchunk()
         return result
 
     def _showchanges(self, fm, alines, blines, chunk, fixups):
         def trim(line):
             if line.endswith(b'\n'):
                 line = line[:-1]
             return line
 
         # this is not optimized for perf but _showchanges only gets executed
         # with an extra command-line flag.
         a1, a2, b1, b2 = chunk
         aidxs, bidxs = [0] * (a2 - a1), [0] * (b2 - b1)
         for idx, fa1, fa2, fb1, fb2 in fixups:
             for i in range(fa1, fa2):
                 aidxs[i - a1] = (max(idx, 1) - 1) // 2
             for i in range(fb1, fb2):
                 bidxs[i - b1] = (max(idx, 1) - 1) // 2
 
         fm.startitem()
         fm.write(
             b'hunk',
             b'        %s\n',
             b'@@ -%d,%d +%d,%d @@' % (a1, a2 - a1, b1, b2 - b1),
             label=b'diff.hunk',
         )
         fm.data(path=self.path, linetype=b'hunk')
 
         def writeline(idx, diffchar, line, linetype, linelabel):
             fm.startitem()
             node = b''
             if idx:
                 ctx = self.fctxs[idx]
                 fm.context(fctx=ctx)
                 node = ctx.hex()
                 self.ctxaffected.add(ctx.changectx())
             fm.write(b'node', b'%-7.7s ', node, label=b'absorb.node')
             fm.write(
                 b'diffchar ' + linetype,
                 b'%s%s\n',
                 diffchar,
                 line,
                 label=linelabel,
             )
             fm.data(path=self.path, linetype=linetype)
 
         for i in range(a1, a2):
             writeline(
                 aidxs[i - a1],
                 b'-',
                 trim(alines[i]),
                 b'deleted',
                 b'diff.deleted',
             )
         for i in range(b1, b2):
             writeline(
                 bidxs[i - b1],
                 b'+',
                 trim(blines[i]),
                 b'inserted',
                 b'diff.inserted',
             )
 
 
 class fixupstate:
     """state needed to run absorb
 
     internally, it keeps paths and filefixupstates.
 
     a typical use is like filefixupstates:
 
         1. call diffwith, to calculate fixups
         2. (optionally), present fixups to the user, or edit fixups
         3. call apply, to apply changes to memory
         4. call commit, to commit changes to hg database
     """
 
     def __init__(self, stack, ui=None, **opts):
         """([ctx], ui or None) -> None
 
         stack: should be linear, and sorted by topo order - oldest first.
         all commits in stack are considered mutable.
         """
         assert stack
         self.ui = ui or nullui()
         self.opts = opts
         self.stack = stack
         self.repo = stack[-1].repo().unfiltered()
 
         # following fields will be filled later
         self.paths = []  # [str]
         self.status = None  # ctx.status output
         self.fctxmap = {}  # {path: {ctx: fctx}}
         self.fixupmap = {}  # {path: filefixupstate}
         self.replacemap = {}  # {oldnode: newnode or None}
         self.finalnode = None  # head after all fixups
         self.ctxaffected = set()  # ctx that will be absorbed into
 
     def diffwith(self, targetctx, match=None, fm=None):
         """diff and prepare fixups. update self.fixupmap, self.paths"""
         # only care about modified files
         self.status = self.stack[-1].status(targetctx, match)
         self.paths = []
         # but if --edit-lines is used, the user may want to edit files
         # even if they are not modified
         editopt = self.opts.get('edit_lines')
         if not self.status.modified and editopt and match:
             interestingpaths = match.files()
         else:
             interestingpaths = self.status.modified
         # prepare the filefixupstate
         seenfctxs = set()
         # sorting is necessary to eliminate ambiguity for the "double move"
         # case: "hg cp A B; hg cp A C; hg rm A", then only "B" can affect "A".
         for path in sorted(interestingpaths):
             self.ui.debug(b'calculating fixups for %s\n' % path)
             targetfctx = targetctx[path]
             fctxs, ctx2fctx = getfilestack(self.stack, path, seenfctxs)
             # ignore symbolic links or binary, or unchanged files
             if any(
                 f.islink() or stringutil.binary(f.data())
                 for f in [targetfctx] + fctxs
                 if not isinstance(f, emptyfilecontext)
             ):
                 continue
             if targetfctx.data() == fctxs[-1].data() and not editopt:
                 continue
             seenfctxs.update(fctxs[1:])
             self.fctxmap[path] = ctx2fctx
             fstate = filefixupstate(fctxs, path, ui=self.ui, **self.opts)
             if fm is not None:
                 fm.startitem()
                 fm.plain(b'showing changes for ')
                 fm.write(b'path', b'%s\n', path, label=b'absorb.path')
                 fm.data(linetype=b'path')
             fstate.diffwith(targetfctx, fm)
             self.fixupmap[path] = fstate
             self.paths.append(path)
             self.ctxaffected.update(fstate.ctxaffected)
 
     def apply(self):
         """apply fixups to individual filefixupstates"""
         for path, state in self.fixupmap.items():
             if self.ui.debugflag:
                 self.ui.write(_(b'applying fixups to %s\n') % path)
             state.apply()
 
     @property
     def chunkstats(self):
         """-> {path: chunkstats}. collect chunkstats from filefixupstates"""
         return {path: state.chunkstats for path, state in self.fixupmap.items()}
 
     def commit(self):
         """commit changes. update self.finalnode, self.replacemap"""
         with self.repo.transaction(b'absorb') as tr:
             self._commitstack()
             self._movebookmarks(tr)
             if self.repo[b'.'].node() in self.replacemap:
                 self._moveworkingdirectoryparent()
             self._cleanupoldcommits()
         return self.finalnode
 
     def printchunkstats(self):
         """print things like '1 of 2 chunk(s) applied'"""
         ui = self.ui
         chunkstats = self.chunkstats
         if ui.verbose:
             # chunkstats for each file
             for path, stat in chunkstats.items():
                 if stat[0]:
                     ui.write(
                         _(b'%s: %d of %d chunk(s) applied\n')
                         % (path, stat[0], stat[1])
                     )
         elif not ui.quiet:
             # a summary for all files
             stats = chunkstats.values()
             applied, total = (sum(s[i] for s in stats) for i in (0, 1))
             ui.write(_(b'%d of %d chunk(s) applied\n') % (applied, total))
 
     def _commitstack(self):
         """make new commits. update self.finalnode, self.replacemap.
         it is splitted from "commit" to avoid too much indentation.
         """
         # last node (20-char) committed by us
         lastcommitted = None
         # p1 which overrides the parent of the next commit, "None" means use
         # the original parent unchanged
         nextp1 = None
         for ctx in self.stack:
             memworkingcopy = self._getnewfilecontents(ctx)
             if not memworkingcopy and not lastcommitted:
                 # nothing changed, nothing commited
                 nextp1 = ctx
                 continue
             willbecomenoop = ctx.files() and self._willbecomenoop(
                 memworkingcopy, ctx, nextp1
             )
             if self.skip_empty_successor and willbecomenoop:
                 # changeset is no longer necessary
                 self.replacemap[ctx.node()] = None
                 msg = _(b'became empty and was dropped')
             else:
                 # changeset needs re-commit
                 nodestr = self._commitsingle(memworkingcopy, ctx, p1=nextp1)
                 lastcommitted = self.repo[nodestr]
                 nextp1 = lastcommitted
                 self.replacemap[ctx.node()] = lastcommitted.node()
                 if memworkingcopy:
                     if willbecomenoop:
                         msg = _(b'%d file(s) changed, became empty as %s')
                     else:
                         msg = _(b'%d file(s) changed, became %s')
                     msg = msg % (
                         len(memworkingcopy),
                         self._ctx2str(lastcommitted),
                     )
                 else:
                     msg = _(b'became %s') % self._ctx2str(lastcommitted)
             if self.ui.verbose and msg:
                 self.ui.write(_(b'%s: %s\n') % (self._ctx2str(ctx), msg))
         self.finalnode = lastcommitted and lastcommitted.node()
 
     def _ctx2str(self, ctx):
         if self.ui.debugflag:
             return b'%d:%s' % (ctx.rev(), ctx.hex())
         else:
             return b'%d:%s' % (ctx.rev(), short(ctx.node()))
 
     def _getnewfilecontents(self, ctx):
         """(ctx) -> {path: str}
 
         fetch file contents from filefixupstates.
         return the working copy overrides - files different from ctx.
         """
         result = {}
         for path in self.paths:
             ctx2fctx = self.fctxmap[path]  # {ctx: fctx}
             if ctx not in ctx2fctx:
                 continue
             fctx = ctx2fctx[ctx]
             content = fctx.data()
             newcontent = self.fixupmap[path].getfinalcontent(fctx)
             if content != newcontent:
                 result[fctx.path()] = newcontent
         return result
 
     def _movebookmarks(self, tr):
         repo = self.repo
         needupdate = [
             (name, self.replacemap[hsh])
             for name, hsh in repo._bookmarks.items()
             if hsh in self.replacemap
         ]
         changes = []
         for name, hsh in needupdate:
             if hsh:
                 changes.append((name, hsh))
                 if self.ui.verbose:
                     self.ui.write(
                         _(b'moving bookmark %s to %s\n') % (name, hex(hsh))
                     )
             else:
                 changes.append((name, None))
                 if self.ui.verbose:
                     self.ui.write(_(b'deleting bookmark %s\n') % name)
         repo._bookmarks.applychanges(repo, tr, changes)
 
     def _moveworkingdirectoryparent(self):
         if not self.finalnode:
             # Find the latest not-{obsoleted,stripped} parent.
             revs = self.repo.revs(b'max(::. - %ln)', self.replacemap.keys())
             ctx = self.repo[revs.first()]
             self.finalnode = ctx.node()
         else:
             ctx = self.repo[self.finalnode]
 
         dirstate = self.repo.dirstate
         # dirstate.rebuild invalidates fsmonitorstate, causing "hg status" to
         # be slow. in absorb's case, no need to invalidate fsmonitorstate.
         noop = lambda: 0
         restore = noop
-        if util.safehasattr(dirstate, '_fsmonitorstate'):
+        if hasattr(dirstate, '_fsmonitorstate'):
             bak = dirstate._fsmonitorstate.invalidate
 
             def restore():
                 dirstate._fsmonitorstate.invalidate = bak
 
             dirstate._fsmonitorstate.invalidate = noop
         try:
             with dirstate.changing_parents(self.repo):
                 dirstate.rebuild(ctx.node(), ctx.manifest(), self.paths)
         finally:
             restore()
 
     @staticmethod
     def _willbecomenoop(memworkingcopy, ctx, pctx=None):
         """({path: content}, ctx, ctx) -> bool. test if a commit will be noop
 
         if it will become an empty commit (does not change anything, after the
         memworkingcopy overrides), return True. otherwise return False.
         """
         if not pctx:
             parents = ctx.parents()
             if len(parents) != 1:
                 return False
             pctx = parents[0]
         if ctx.branch() != pctx.branch():
             return False
         if ctx.extra().get(b'close'):
             return False
         # ctx changes more files (not a subset of memworkingcopy)
         if not set(ctx.files()).issubset(set(memworkingcopy)):
             return False
         for path, content in memworkingcopy.items():
             if path not in pctx or path not in ctx:
                 return False
             fctx = ctx[path]
             pfctx = pctx[path]
             if pfctx.flags() != fctx.flags():
                 return False
             if pfctx.data() != content:
                 return False
         return True
 
     def _commitsingle(self, memworkingcopy, ctx, p1=None):
         """(ctx, {path: content}, node) -> node. make a single commit
 
         the commit is a clone from ctx, with a (optionally) different p1, and
         different file contents replaced by memworkingcopy.
         """
         parents = p1 and (p1, self.repo.nullid)
         extra = ctx.extra()
         if self._useobsolete and self.ui.configbool(b'absorb', b'add-noise'):
             extra[b'absorb_source'] = ctx.hex()
 
         desc = rewriteutil.update_hash_refs(
             ctx.repo(),
             ctx.description(),
             {
                 oldnode: [newnode]
                 for oldnode, newnode in self.replacemap.items()
             },
         )
         mctx = overlaycontext(
             memworkingcopy, ctx, parents, extra=extra, desc=desc
         )
         return mctx.commit()
 
     @util.propertycache
     def _useobsolete(self):
         """() -> bool"""
         return obsolete.isenabled(self.repo, obsolete.createmarkersopt)
 
     def _cleanupoldcommits(self):
         replacements = {
             k: ([v] if v is not None else [])
             for k, v in self.replacemap.items()
         }
         if replacements:
             scmutil.cleanupnodes(
                 self.repo, replacements, operation=b'absorb', fixphase=True
             )
 
     @util.propertycache
     def skip_empty_successor(self):
         return rewriteutil.skip_empty_successor(self.ui, b'absorb')
 
 
 def _parsechunk(hunk):
     """(crecord.uihunk or patch.recordhunk) -> (path, (a1, a2, [bline]))"""
     if type(hunk) not in (crecord.uihunk, patch.recordhunk):
         return None, None
     path = hunk.header.filename()
     a1 = hunk.fromline + len(hunk.before) - 1
     # remove before and after context
     hunk.before = hunk.after = []
     buf = util.stringio()
     hunk.write(buf)
     patchlines = mdiff.splitnewlines(buf.getvalue())
     # hunk.prettystr() will update hunk.removed
     a2 = a1 + hunk.removed
     blines = [l[1:] for l in patchlines[1:] if not l.startswith(b'-')]
     return path, (a1, a2, blines)
 
 
 def overlaydiffcontext(ctx, chunks):
     """(ctx, [crecord.uihunk]) -> memctx
 
     return a memctx with some [1] patches (chunks) applied to ctx.
     [1]: modifications are handled. renames, mode changes, etc. are ignored.
     """
     # sadly the applying-patch logic is hardly reusable, and messy:
     # 1. the core logic "_applydiff" is too heavy - it writes .rej files, it
     #    needs a file stream of a patch and will re-parse it, while we have
     #    structured hunk objects at hand.
     # 2. a lot of different implementations about "chunk" (patch.hunk,
     #    patch.recordhunk, crecord.uihunk)
     # as we only care about applying changes to modified files, no mode
     # change, no binary diff, and no renames, it's probably okay to
     # re-invent the logic using much simpler code here.
     memworkingcopy = {}  # {path: content}
     patchmap = defaultdict(lambda: [])  # {path: [(a1, a2, [bline])]}
     for path, info in map(_parsechunk, chunks):
         if not path or not info:
             continue
         patchmap[path].append(info)
     for path, patches in patchmap.items():
         if path not in ctx or not patches:
             continue
         patches.sort(reverse=True)
         lines = mdiff.splitnewlines(ctx[path].data())
         for a1, a2, blines in patches:
             lines[a1:a2] = blines
         memworkingcopy[path] = b''.join(lines)
     return overlaycontext(memworkingcopy, ctx)
 
 
 def absorb(ui, repo, stack=None, targetctx=None, pats=None, **opts):
     """pick fixup chunks from targetctx, apply them to stack.
 
     if targetctx is None, the working copy context will be used.
     if stack is None, the current draft stack will be used.
     return fixupstate.
     """
     if stack is None:
         limit = ui.configint(b'absorb', b'max-stack-size')
         headctx = repo[b'.']
         if len(headctx.parents()) > 1:
             raise error.InputError(_(b'cannot absorb into a merge'))
         stack = getdraftstack(headctx, limit)
         if limit and len(stack) >= limit:
             ui.warn(
                 _(
                     b'absorb: only the recent %d changesets will '
                     b'be analysed\n'
                 )
                 % limit
             )
     if not stack:
         raise error.InputError(_(b'no mutable changeset to change'))
     if targetctx is None:  # default to working copy
         targetctx = repo[None]
     if pats is None:
         pats = ()
 
     state = fixupstate(stack, ui=ui, **opts)
     matcher = scmutil.match(targetctx, pats, pycompat.byteskwargs(opts))
     if opts.get('interactive'):
         diff = patch.diff(repo, stack[-1].node(), targetctx.node(), matcher)
         origchunks = patch.parsepatch(diff)
         chunks = cmdutil.recordfilter(ui, origchunks, matcher)[0]
         targetctx = overlaydiffcontext(stack[-1], chunks)
     if opts.get('edit_lines'):
         # If we're going to open the editor, don't ask the user to confirm
         # first
         opts['apply_changes'] = True
     fm = None
     if opts.get('print_changes') or not opts.get('apply_changes'):
         fm = ui.formatter(b'absorb', pycompat.byteskwargs(opts))
     state.diffwith(targetctx, matcher, fm)
     if fm is not None:
         fm.startitem()
         fm.write(
             b"count", b"\n%d changesets affected\n", len(state.ctxaffected)
         )
         fm.data(linetype=b'summary')
         for ctx in reversed(stack):
             if ctx not in state.ctxaffected:
                 continue
             fm.startitem()
             fm.context(ctx=ctx)
             fm.data(linetype=b'changeset')
             fm.write(b'node', b'%-7.7s ', ctx.hex(), label=b'absorb.node')
             descfirstline = stringutil.firstline(ctx.description())
             fm.write(
                 b'descfirstline',
                 b'%s\n',
                 descfirstline,
                 label=b'absorb.description',
             )
         fm.end()
     if not opts.get('dry_run'):
         if (
             not opts.get('apply_changes')
             and state.ctxaffected
             and ui.promptchoice(
                 b"apply changes (y/N)? $$ &Yes $$ &No", default=1
             )
         ):
             raise error.CanceledError(_(b'absorb cancelled\n'))
 
         state.apply()
         if state.commit():
             state.printchunkstats()
         elif not ui.quiet:
             ui.write(_(b'nothing applied\n'))
     return state
 
 
 @command(
     b'absorb',
     [
         (
             b'a',
             b'apply-changes',
             None,
             _(b'apply changes without prompting for confirmation'),
         ),
         (
             b'p',
             b'print-changes',
             None,
             _(b'always print which changesets are modified by which changes'),
         ),
         (
             b'i',
             b'interactive',
             None,
             _(b'interactively select which chunks to apply'),
         ),
         (
             b'e',
             b'edit-lines',
             None,
             _(
                 b'edit what lines belong to which changesets before commit '
                 b'(EXPERIMENTAL)'
             ),
         ),
     ]
     + commands.dryrunopts
     + commands.templateopts
     + commands.walkopts,
     _(b'hg absorb [OPTION] [FILE]...'),
     helpcategory=command.CATEGORY_COMMITTING,
     helpbasic=True,
 )
 def absorbcmd(ui, repo, *pats, **opts):
     """incorporate corrections into the stack of draft changesets
 
     absorb analyzes each change in your working directory and attempts to
     amend the changed lines into the changesets in your stack that first
     introduced those lines.
 
     If absorb cannot find an unambiguous changeset to amend for a change,
     that change will be left in the working directory, untouched. They can be
     observed by :hg:`status` or :hg:`diff` afterwards. In other words,
     absorb does not write to the working directory.
 
     Changesets outside the revset `::. and not public() and not merge()` will
     not be changed.
 
     Changesets that become empty after applying the changes will be deleted.
 
     By default, absorb will show what it plans to do and prompt for
     confirmation.  If you are confident that the changes will be absorbed
     to the correct place, run :hg:`absorb -a` to apply the changes
     immediately.
 
     Returns 0 on success, 1 if all chunks were ignored and nothing amended.
     """
     with repo.wlock(), repo.lock():
         if not opts['dry_run']:
             cmdutil.checkunfinished(repo)
 
         state = absorb(ui, repo, pats=pats, **opts)
         if sum(s[0] for s in state.chunkstats.values()) == 0:
             return 1

hgext/bugzilla.py

@@ -1,1251 +1,1251 b''
 # bugzilla.py - bugzilla integration for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 # Copyright 2011-4 Jim Hague <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''hooks for integrating with the Bugzilla bug tracker
 
 This hook extension adds comments on bugs in Bugzilla when changesets
 that refer to bugs by Bugzilla ID are seen. The comment is formatted using
 the Mercurial template mechanism.
 
 The bug references can optionally include an update for Bugzilla of the
 hours spent working on the bug. Bugs can also be marked fixed.
 
 Four basic modes of access to Bugzilla are provided:
 
 1. Access via the Bugzilla REST-API. Requires bugzilla 5.0 or later.
 
 2. Access via the Bugzilla XMLRPC interface. Requires Bugzilla 3.4 or later.
 
 3. Check data via the Bugzilla XMLRPC interface and submit bug change
    via email to Bugzilla email interface. Requires Bugzilla 3.4 or later.
 
 4. Writing directly to the Bugzilla database. Only Bugzilla installations
    using MySQL are supported. Requires Python MySQLdb.
 
 Writing directly to the database is susceptible to schema changes, and
 relies on a Bugzilla contrib script to send out bug change
 notification emails. This script runs as the user running Mercurial,
 must be run on the host with the Bugzilla install, and requires
 permission to read Bugzilla configuration details and the necessary
 MySQL user and password to have full access rights to the Bugzilla
 database. For these reasons this access mode is now considered
 deprecated, and will not be updated for new Bugzilla versions going
 forward. Only adding comments is supported in this access mode.
 
 Access via XMLRPC needs a Bugzilla username and password to be specified
 in the configuration. Comments are added under that username. Since the
 configuration must be readable by all Mercurial users, it is recommended
 that the rights of that user are restricted in Bugzilla to the minimum
 necessary to add comments. Marking bugs fixed requires Bugzilla 4.0 and later.
 
 Access via XMLRPC/email uses XMLRPC to query Bugzilla, but sends
 email to the Bugzilla email interface to submit comments to bugs.
 The From: address in the email is set to the email address of the Mercurial
 user, so the comment appears to come from the Mercurial user. In the event
 that the Mercurial user email is not recognized by Bugzilla as a Bugzilla
 user, the email associated with the Bugzilla username used to log into
 Bugzilla is used instead as the source of the comment. Marking bugs fixed
 works on all supported Bugzilla versions.
 
 Access via the REST-API needs either a Bugzilla username and password
 or an apikey specified in the configuration. Comments are made under
 the given username or the user associated with the apikey in Bugzilla.
 
 Configuration items common to all access modes:
 
 bugzilla.version
   The access type to use. Values recognized are:
 
   :``restapi``:      Bugzilla REST-API, Bugzilla 5.0 and later.
   :``xmlrpc``:       Bugzilla XMLRPC interface.
   :``xmlrpc+email``: Bugzilla XMLRPC and email interfaces.
   :``3.0``:          MySQL access, Bugzilla 3.0 and later.
   :``2.18``:         MySQL access, Bugzilla 2.18 and up to but not
                      including 3.0.
   :``2.16``:         MySQL access, Bugzilla 2.16 and up to but not
                      including 2.18.
 
 bugzilla.regexp
   Regular expression to match bug IDs for update in changeset commit message.
   It must contain one "()" named group ``<ids>`` containing the bug
   IDs separated by non-digit characters. It may also contain
   a named group ``<hours>`` with a floating-point number giving the
   hours worked on the bug. If no named groups are present, the first
   "()" group is assumed to contain the bug IDs, and work time is not
   updated. The default expression matches ``Bug 1234``, ``Bug no. 1234``,
   ``Bug number 1234``, ``Bugs 1234,5678``, ``Bug 1234 and 5678`` and
   variations thereof, followed by an hours number prefixed by ``h`` or
   ``hours``, e.g. ``hours 1.5``. Matching is case insensitive.
 
 bugzilla.fixregexp
   Regular expression to match bug IDs for marking fixed in changeset
   commit message. This must contain a "()" named group ``<ids>` containing
   the bug IDs separated by non-digit characters. It may also contain
   a named group ``<hours>`` with a floating-point number giving the
   hours worked on the bug. If no named groups are present, the first
   "()" group is assumed to contain the bug IDs, and work time is not
   updated. The default expression matches ``Fixes 1234``, ``Fixes bug 1234``,
   ``Fixes bugs 1234,5678``, ``Fixes 1234 and 5678`` and
   variations thereof, followed by an hours number prefixed by ``h`` or
   ``hours``, e.g. ``hours 1.5``. Matching is case insensitive.
 
 bugzilla.fixstatus
   The status to set a bug to when marking fixed. Default ``RESOLVED``.
 
 bugzilla.fixresolution
   The resolution to set a bug to when marking fixed. Default ``FIXED``.
 
 bugzilla.style
   The style file to use when formatting comments.
 
 bugzilla.template
   Template to use when formatting comments. Overrides style if
   specified. In addition to the usual Mercurial keywords, the
   extension specifies:
 
   :``{bug}``:     The Bugzilla bug ID.
   :``{root}``:    The full pathname of the Mercurial repository.
   :``{webroot}``: Stripped pathname of the Mercurial repository.
   :``{hgweb}``:   Base URL for browsing Mercurial repositories.
 
   Default ``changeset {node|short} in repo {root} refers to bug
   {bug}.\\ndetails:\\n\\t{desc|tabindent}``
 
 bugzilla.strip
   The number of path separator characters to strip from the front of
   the Mercurial repository path (``{root}`` in templates) to produce
   ``{webroot}``. For example, a repository with ``{root}``
   ``/var/local/my-project`` with a strip of 2 gives a value for
   ``{webroot}`` of ``my-project``. Default 0.
 
 web.baseurl
   Base URL for browsing Mercurial repositories. Referenced from
   templates as ``{hgweb}``.
 
 Configuration items common to XMLRPC+email and MySQL access modes:
 
 bugzilla.usermap
   Path of file containing Mercurial committer email to Bugzilla user email
   mappings. If specified, the file should contain one mapping per
   line::
 
     committer = Bugzilla user
 
   See also the ``[usermap]`` section.
 
 The ``[usermap]`` section is used to specify mappings of Mercurial
 committer email to Bugzilla user email. See also ``bugzilla.usermap``.
 Contains entries of the form ``committer = Bugzilla user``.
 
 XMLRPC and REST-API access mode configuration:
 
 bugzilla.bzurl
   The base URL for the Bugzilla installation.
   Default ``http://localhost/bugzilla``.
 
 bugzilla.user
   The username to use to log into Bugzilla via XMLRPC. Default
   ``bugs``.
 
 bugzilla.password
   The password for Bugzilla login.
 
 REST-API access mode uses the options listed above as well as:
 
 bugzilla.apikey
   An apikey generated on the Bugzilla instance for api access.
   Using an apikey removes the need to store the user and password
   options.
 
 XMLRPC+email access mode uses the XMLRPC access mode configuration items,
 and also:
 
 bugzilla.bzemail
   The Bugzilla email address.
 
 In addition, the Mercurial email settings must be configured. See the
 documentation in hgrc(5), sections ``[email]`` and ``[smtp]``.
 
 MySQL access mode configuration:
 
 bugzilla.host
   Hostname of the MySQL server holding the Bugzilla database.
   Default ``localhost``.
 
 bugzilla.db
   Name of the Bugzilla database in MySQL. Default ``bugs``.
 
 bugzilla.user
   Username to use to access MySQL server. Default ``bugs``.
 
 bugzilla.password
   Password to use to access MySQL server.
 
 bugzilla.timeout
   Database connection timeout (seconds). Default 5.
 
 bugzilla.bzuser
   Fallback Bugzilla user name to record comments with, if changeset
   committer cannot be found as a Bugzilla user.
 
 bugzilla.bzdir
    Bugzilla install directory. Used by default notify. Default
    ``/var/www/html/bugzilla``.
 
 bugzilla.notify
   The command to run to get Bugzilla to send bug change notification
   emails. Substitutes from a map with 3 keys, ``bzdir``, ``id`` (bug
   id) and ``user`` (committer bugzilla email). Default depends on
   version; from 2.18 it is "cd %(bzdir)s && perl -T
   contrib/sendbugmail.pl %(id)s %(user)s".
 
 Activating the extension::
 
     [extensions]
     bugzilla =
 
     [hooks]
     # run bugzilla hook on every change pulled or pushed in here
     incoming.bugzilla = python:hgext.bugzilla.hook
 
 Example configurations:
 
 XMLRPC example configuration. This uses the Bugzilla at
 ``http://my-project.org/bugzilla``, logging in as user
 ``[e-mail]`` with password ``plugh``. It is used with a
 collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. ::
 
     [bugzilla]
     bzurl=http://my-project.org/bugzilla
     user=[e-mail]
     password=plugh
     version=xmlrpc
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
 XMLRPC+email example configuration. This uses the Bugzilla at
 ``http://my-project.org/bugzilla``, logging in as user
 ``[e-mail]`` with password ``plugh``. It is used with a
 collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. Bug comments
 are sent to the Bugzilla email address
 ``[e-mail]``. ::
 
     [bugzilla]
     bzurl=http://my-project.org/bugzilla
     user=[e-mail]
     password=plugh
     version=xmlrpc+email
     bzemail=[e-mail]
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
     [usermap]
     [e-mail]=[e-mail]
 
 MySQL example configuration. This has a local Bugzilla 3.2 installation
 in ``/opt/bugzilla-3.2``. The MySQL database is on ``localhost``,
 the Bugzilla database name is ``bugs`` and MySQL is
 accessed with MySQL username ``bugs`` password ``XYZZY``. It is used
 with a collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. ::
 
     [bugzilla]
     host=localhost
     password=XYZZY
     version=3.0
     bzuser=[e-mail]
     bzdir=/opt/bugzilla-3.2
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
     [usermap]
     [e-mail]=[e-mail]
 
 All the above add a comment to the Bugzilla bug record of the form::
 
     Changeset 3b16791d6642 in repository-name.
     http://my-project.org/hg/repository-name/rev/3b16791d6642
 
     Changeset commit comment. Bug 1234.
 '''
 
 
 import json
 import re
 import time
 
 from mercurial.i18n import _
 from mercurial.node import short
 from mercurial import (
     error,
     logcmdutil,
     mail,
     pycompat,
     registrar,
     url,
     util,
 )
 from mercurial.utils import (
     procutil,
     stringutil,
 )
 
 xmlrpclib = util.xmlrpclib
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'bugzilla',
     b'apikey',
     default=b'',
 )
 configitem(
     b'bugzilla',
     b'bzdir',
     default=b'/var/www/html/bugzilla',
 )
 configitem(
     b'bugzilla',
     b'bzemail',
     default=None,
 )
 configitem(
     b'bugzilla',
     b'bzurl',
     default=b'http://localhost/bugzilla/',
 )
 configitem(
     b'bugzilla',
     b'bzuser',
     default=None,
 )
 configitem(
     b'bugzilla',
     b'db',
     default=b'bugs',
 )
 configitem(
     b'bugzilla',
     b'fixregexp',
     default=(
         br'fix(?:es)?\s*(?:bugs?\s*)?,?\s*'
         br'(?:nos?\.?|num(?:ber)?s?)?\s*'
         br'(?P<ids>(?:#?\d+\s*(?:,?\s*(?:and)?)?\s*)+)'
         br'\.?\s*(?:h(?:ours?)?\s*(?P<hours>\d*(?:\.\d+)?))?'
     ),
 )
 configitem(
     b'bugzilla',
     b'fixresolution',
     default=b'FIXED',
 )
 configitem(
     b'bugzilla',
     b'fixstatus',
     default=b'RESOLVED',
 )
 configitem(
     b'bugzilla',
     b'host',
     default=b'localhost',
 )
 configitem(
     b'bugzilla',
     b'notify',
     default=configitem.dynamicdefault,
 )
 configitem(
     b'bugzilla',
     b'password',
     default=None,
 )
 configitem(
     b'bugzilla',
     b'regexp',
     default=(
         br'bugs?\s*,?\s*(?:#|nos?\.?|num(?:ber)?s?)?\s*'
         br'(?P<ids>(?:\d+\s*(?:,?\s*(?:and)?)?\s*)+)'
         br'\.?\s*(?:h(?:ours?)?\s*(?P<hours>\d*(?:\.\d+)?))?'
     ),
 )
 configitem(
     b'bugzilla',
     b'strip',
     default=0,
 )
 configitem(
     b'bugzilla',
     b'style',
     default=None,
 )
 configitem(
     b'bugzilla',
     b'template',
     default=None,
 )
 configitem(
     b'bugzilla',
     b'timeout',
     default=5,
 )
 configitem(
     b'bugzilla',
     b'user',
     default=b'bugs',
 )
 configitem(
     b'bugzilla',
     b'usermap',
     default=None,
 )
 configitem(
     b'bugzilla',
     b'version',
     default=None,
 )
 
 
 class bzaccess:
     '''Base class for access to Bugzilla.'''
 
     def __init__(self, ui):
         self.ui = ui
         usermap = self.ui.config(b'bugzilla', b'usermap')
         if usermap:
             self.ui.readconfig(usermap, sections=[b'usermap'])
 
     def map_committer(self, user):
         '''map name of committer to Bugzilla user name.'''
         for committer, bzuser in self.ui.configitems(b'usermap'):
             if committer.lower() == user.lower():
                 return bzuser
         return user
 
     # Methods to be implemented by access classes.
     #
     # 'bugs' is a dict keyed on bug id, where values are a dict holding
     # updates to bug state. Recognized dict keys are:
     #
     # 'hours': Value, float containing work hours to be updated.
     # 'fix':   If key present, bug is to be marked fixed. Value ignored.
 
     def filter_real_bug_ids(self, bugs):
         '''remove bug IDs that do not exist in Bugzilla from bugs.'''
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''remove bug IDs where node occurs in comment text from bugs.'''
 
     def updatebug(self, bugid, newstate, text, committer):
         """update the specified bug. Add comment text and set new states.
 
         If possible add the comment as being from the committer of
         the changeset. Otherwise use the default Bugzilla user.
         """
 
     def notify(self, bugs, committer):
         """Force sending of Bugzilla notification emails.
 
         Only required if the access method does not trigger notification
         emails automatically.
         """
 
 
 # Bugzilla via direct access to MySQL database.
 class bzmysql(bzaccess):
     """Support for direct MySQL access to Bugzilla.
 
     The earliest Bugzilla version this is tested with is version 2.16.
 
     If your Bugzilla is version 3.4 or above, you are strongly
     recommended to use the XMLRPC access method instead.
     """
 
     @staticmethod
     def sql_buglist(ids):
         '''return SQL-friendly list of bug ids'''
         return b'(' + b','.join(map(str, ids)) + b')'
 
     _MySQLdb = None
 
     def __init__(self, ui):
         try:
             import MySQLdb as mysql
 
             bzmysql._MySQLdb = mysql
         except ImportError as err:
             raise error.Abort(
                 _(b'python mysql support not available: %s') % err
             )
 
         bzaccess.__init__(self, ui)
 
         host = self.ui.config(b'bugzilla', b'host')
         user = self.ui.config(b'bugzilla', b'user')
         passwd = self.ui.config(b'bugzilla', b'password')
         db = self.ui.config(b'bugzilla', b'db')
         timeout = int(self.ui.config(b'bugzilla', b'timeout'))
         self.ui.note(
             _(b'connecting to %s:%s as %s, password %s\n')
             % (host, db, user, b'*' * len(passwd))
         )
         self.conn = bzmysql._MySQLdb.connect(
             host=host, user=user, passwd=passwd, db=db, connect_timeout=timeout
         )
         self.cursor = self.conn.cursor()
         self.longdesc_id = self.get_longdesc_id()
         self.user_ids = {}
         self.default_notify = b"cd %(bzdir)s && ./processmail %(id)s %(user)s"
 
     def run(self, *args, **kwargs):
         '''run a query.'''
         self.ui.note(_(b'query: %s %s\n') % (args, kwargs))
         try:
             self.cursor.execute(*args, **kwargs)
         except bzmysql._MySQLdb.MySQLError:
             self.ui.note(_(b'failed query: %s %s\n') % (args, kwargs))
             raise
 
     def get_longdesc_id(self):
         '''get identity of longdesc field'''
         self.run(b'select fieldid from fielddefs where name = "longdesc"')
         ids = self.cursor.fetchall()
         if len(ids) != 1:
             raise error.Abort(_(b'unknown database schema'))
         return ids[0][0]
 
     def filter_real_bug_ids(self, bugs):
         '''filter not-existing bugs from set.'''
         self.run(
             b'select bug_id from bugs where bug_id in %s'
             % bzmysql.sql_buglist(bugs.keys())
         )
         existing = [id for (id,) in self.cursor.fetchall()]
         for id in bugs.keys():
             if id not in existing:
                 self.ui.status(_(b'bug %d does not exist\n') % id)
                 del bugs[id]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''filter bug ids that already refer to this changeset from set.'''
         self.run(
             '''select bug_id from longdescs where
                     bug_id in %s and thetext like "%%%s%%"'''
             % (bzmysql.sql_buglist(bugs.keys()), short(node))
         )
         for (id,) in self.cursor.fetchall():
             self.ui.status(
                 _(b'bug %d already knows about changeset %s\n')
                 % (id, short(node))
             )
             del bugs[id]
 
     def notify(self, bugs, committer):
         '''tell bugzilla to send mail.'''
         self.ui.status(_(b'telling bugzilla to send mail:\n'))
         (user, userid) = self.get_bugzilla_user(committer)
         for id in bugs.keys():
             self.ui.status(_(b'  bug %s\n') % id)
             cmdfmt = self.ui.config(b'bugzilla', b'notify', self.default_notify)
             bzdir = self.ui.config(b'bugzilla', b'bzdir')
             try:
                 # Backwards-compatible with old notify string, which
                 # took one string. This will throw with a new format
                 # string.
                 cmd = cmdfmt % id
             except TypeError:
                 cmd = cmdfmt % {b'bzdir': bzdir, b'id': id, b'user': user}
             self.ui.note(_(b'running notify command %s\n') % cmd)
             fp = procutil.popen(b'(%s) 2>&1' % cmd, b'rb')
             out = util.fromnativeeol(fp.read())
             ret = fp.close()
             if ret:
                 self.ui.warn(out)
                 raise error.Abort(
                     _(b'bugzilla notify command %s') % procutil.explainexit(ret)
                 )
         self.ui.status(_(b'done\n'))
 
     def get_user_id(self, user):
         '''look up numeric bugzilla user id.'''
         try:
             return self.user_ids[user]
         except KeyError:
             try:
                 userid = int(user)
             except ValueError:
                 self.ui.note(_(b'looking up user %s\n') % user)
                 self.run(
                     '''select userid from profiles
                             where login_name like %s''',
                     user,
                 )
                 all = self.cursor.fetchall()
                 if len(all) != 1:
                     raise KeyError(user)
                 userid = int(all[0][0])
             self.user_ids[user] = userid
             return userid
 
     def get_bugzilla_user(self, committer):
         """See if committer is a registered bugzilla user. Return
         bugzilla username and userid if so. If not, return default
         bugzilla username and userid."""
         user = self.map_committer(committer)
         try:
             userid = self.get_user_id(user)
         except KeyError:
             try:
                 defaultuser = self.ui.config(b'bugzilla', b'bzuser')
                 if not defaultuser:
                     raise error.Abort(
                         _(b'cannot find bugzilla user id for %s') % user
                     )
                 userid = self.get_user_id(defaultuser)
                 user = defaultuser
             except KeyError:
                 raise error.Abort(
                     _(b'cannot find bugzilla user id for %s or %s')
                     % (user, defaultuser)
                 )
         return (user, userid)
 
     def updatebug(self, bugid, newstate, text, committer):
         """update bug state with comment text.
 
         Try adding comment as committer of changeset, otherwise as
         default bugzilla user."""
         if len(newstate) > 0:
             self.ui.warn(_(b"Bugzilla/MySQL cannot update bug state\n"))
 
         (user, userid) = self.get_bugzilla_user(committer)
         now = time.strftime('%Y-%m-%d %H:%M:%S')
         self.run(
             '''insert into longdescs
                     (bug_id, who, bug_when, thetext)
                     values (%s, %s, %s, %s)''',
             (bugid, userid, now, text),
         )
         self.run(
             '''insert into bugs_activity (bug_id, who, bug_when, fieldid)
                     values (%s, %s, %s, %s)''',
             (bugid, userid, now, self.longdesc_id),
         )
         self.conn.commit()
 
 
 class bzmysql_2_18(bzmysql):
     '''support for bugzilla 2.18 series.'''
 
     def __init__(self, ui):
         bzmysql.__init__(self, ui)
         self.default_notify = (
             b"cd %(bzdir)s && perl -T contrib/sendbugmail.pl %(id)s %(user)s"
         )
 
 
 class bzmysql_3_0(bzmysql_2_18):
     '''support for bugzilla 3.0 series.'''
 
     def __init__(self, ui):
         bzmysql_2_18.__init__(self, ui)
 
     def get_longdesc_id(self):
         '''get identity of longdesc field'''
         self.run(b'select id from fielddefs where name = "longdesc"')
         ids = self.cursor.fetchall()
         if len(ids) != 1:
             raise error.Abort(_(b'unknown database schema'))
         return ids[0][0]
 
 
 # Bugzilla via XMLRPC interface.
 
 
 class cookietransportrequest:
     """A Transport request method that retains cookies over its lifetime.
 
     The regular xmlrpclib transports ignore cookies. Which causes
     a bit of a problem when you need a cookie-based login, as with
     the Bugzilla XMLRPC interface prior to 4.4.3.
 
     So this is a helper for defining a Transport which looks for
     cookies being set in responses and saves them to add to all future
     requests.
     """
 
     # Inspiration drawn from
     # http://blog.godson.in/2010/09/how-to-make-python-xmlrpclib-client.html
     # http://www.itkovian.net/base/transport-class-for-pythons-xml-rpc-lib/
 
     cookies = []
 
     def send_cookies(self, connection):
         if self.cookies:
             for cookie in self.cookies:
                 connection.putheader(b"Cookie", cookie)
 
     def request(self, host, handler, request_body, verbose=0):
         self.verbose = verbose
         self.accept_gzip_encoding = False
 
         # issue XML-RPC request
         h = self.make_connection(host)
         if verbose:
             h.set_debuglevel(1)
 
         self.send_request(h, handler, request_body)
         self.send_host(h, host)
         self.send_cookies(h)
         self.send_user_agent(h)
         self.send_content(h, request_body)
 
         # Deal with differences between Python 2.6 and 2.7.
         # In the former h is a HTTP(S). In the latter it's a
         # HTTP(S)Connection. Luckily, the 2.6 implementation of
         # HTTP(S) has an underlying HTTP(S)Connection, so extract
         # that and use it.
         try:
             response = h.getresponse()
         except AttributeError:
             response = h._conn.getresponse()
 
         # Add any cookie definitions to our list.
         for header in response.msg.getallmatchingheaders(b"Set-Cookie"):
             val = header.split(b": ", 1)[1]
             cookie = val.split(b";", 1)[0]
             self.cookies.append(cookie)
 
         if response.status != 200:
             raise xmlrpclib.ProtocolError(
                 host + handler,
                 response.status,
                 response.reason,
                 response.msg.headers,
             )
 
         payload = response.read()
         parser, unmarshaller = self.getparser()
         parser.feed(payload)
         parser.close()
 
         return unmarshaller.close()
 
 
 # The explicit calls to the underlying xmlrpclib __init__() methods are
 # necessary. The xmlrpclib.Transport classes are old-style classes, and
 # it turns out their __init__() doesn't get called when doing multiple
 # inheritance with a new-style class.
 class cookietransport(cookietransportrequest, xmlrpclib.Transport):
     def __init__(self, use_datetime=0):
-        if util.safehasattr(xmlrpclib.Transport, "__init__"):
+        if hasattr(xmlrpclib.Transport, "__init__"):
             xmlrpclib.Transport.__init__(self, use_datetime)
 
 
 class cookiesafetransport(cookietransportrequest, xmlrpclib.SafeTransport):
     def __init__(self, use_datetime=0):
-        if util.safehasattr(xmlrpclib.Transport, "__init__"):
+        if hasattr(xmlrpclib.Transport, "__init__"):
             xmlrpclib.SafeTransport.__init__(self, use_datetime)
 
 
 class bzxmlrpc(bzaccess):
     """Support for access to Bugzilla via the Bugzilla XMLRPC API.
 
     Requires a minimum Bugzilla version 3.4.
     """
 
     def __init__(self, ui):
         bzaccess.__init__(self, ui)
 
         bzweb = self.ui.config(b'bugzilla', b'bzurl')
         bzweb = bzweb.rstrip(b"/") + b"/xmlrpc.cgi"
 
         user = self.ui.config(b'bugzilla', b'user')
         passwd = self.ui.config(b'bugzilla', b'password')
 
         self.fixstatus = self.ui.config(b'bugzilla', b'fixstatus')
         self.fixresolution = self.ui.config(b'bugzilla', b'fixresolution')
 
         self.bzproxy = xmlrpclib.ServerProxy(
             pycompat.strurl(bzweb), self.transport(bzweb)
         )
         ver = self.bzproxy.Bugzilla.version()[b'version'].split(b'.')
         self.bzvermajor = int(ver[0])
         self.bzverminor = int(ver[1])
         login = self.bzproxy.User.login(
             {b'login': user, b'password': passwd, b'restrict_login': True}
         )
         self.bztoken = login.get(b'token', b'')
 
     def transport(self, uri):
         if util.urlreq.urlparse(uri, b"http")[0] == b"https":
             return cookiesafetransport()
         else:
             return cookietransport()
 
     def get_bug_comments(self, id):
         """Return a string with all comment text for a bug."""
         c = self.bzproxy.Bug.comments(
             {b'ids': [id], b'include_fields': [b'text'], b'token': self.bztoken}
         )
         return b''.join(
             [t[b'text'] for t in c[b'bugs'][b'%d' % id][b'comments']]
         )
 
     def filter_real_bug_ids(self, bugs):
         probe = self.bzproxy.Bug.get(
             {
                 b'ids': sorted(bugs.keys()),
                 b'include_fields': [],
                 b'permissive': True,
                 b'token': self.bztoken,
             }
         )
         for badbug in probe[b'faults']:
             id = badbug[b'id']
             self.ui.status(_(b'bug %d does not exist\n') % id)
             del bugs[id]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         for id in sorted(bugs.keys()):
             if self.get_bug_comments(id).find(short(node)) != -1:
                 self.ui.status(
                     _(b'bug %d already knows about changeset %s\n')
                     % (id, short(node))
                 )
                 del bugs[id]
 
     def updatebug(self, bugid, newstate, text, committer):
         args = {}
         if b'hours' in newstate:
             args[b'work_time'] = newstate[b'hours']
 
         if self.bzvermajor >= 4:
             args[b'ids'] = [bugid]
             args[b'comment'] = {b'body': text}
             if b'fix' in newstate:
                 args[b'status'] = self.fixstatus
                 args[b'resolution'] = self.fixresolution
             args[b'token'] = self.bztoken
             self.bzproxy.Bug.update(args)
         else:
             if b'fix' in newstate:
                 self.ui.warn(
                     _(
                         b"Bugzilla/XMLRPC needs Bugzilla 4.0 or later "
                         b"to mark bugs fixed\n"
                     )
                 )
             args[b'id'] = bugid
             args[b'comment'] = text
             self.bzproxy.Bug.add_comment(args)
 
 
 class bzxmlrpcemail(bzxmlrpc):
     """Read data from Bugzilla via XMLRPC, send updates via email.
 
     Advantages of sending updates via email:
       1. Comments can be added as any user, not just logged in user.
       2. Bug statuses or other fields not accessible via XMLRPC can
          potentially be updated.
 
     There is no XMLRPC function to change bug status before Bugzilla
     4.0, so bugs cannot be marked fixed via XMLRPC before Bugzilla 4.0.
     But bugs can be marked fixed via email from 3.4 onwards.
     """
 
     # The email interface changes subtly between 3.4 and 3.6. In 3.4,
     # in-email fields are specified as '@<fieldname> = <value>'. In
     # 3.6 this becomes '@<fieldname> <value>'. And fieldname @bug_id
     # in 3.4 becomes @id in 3.6. 3.6 and 4.0 both maintain backwards
     # compatibility, but rather than rely on this use the new format for
     # 4.0 onwards.
 
     def __init__(self, ui):
         bzxmlrpc.__init__(self, ui)
 
         self.bzemail = self.ui.config(b'bugzilla', b'bzemail')
         if not self.bzemail:
             raise error.Abort(_(b"configuration 'bzemail' missing"))
         mail.validateconfig(self.ui)
 
     def makecommandline(self, fieldname, value):
         if self.bzvermajor >= 4:
             return b"@%s %s" % (fieldname, pycompat.bytestr(value))
         else:
             if fieldname == b"id":
                 fieldname = b"bug_id"
             return b"@%s = %s" % (fieldname, pycompat.bytestr(value))
 
     def send_bug_modify_email(self, bugid, commands, comment, committer):
         """send modification message to Bugzilla bug via email.
 
         The message format is documented in the Bugzilla email_in.pl
         specification. commands is a list of command lines, comment is the
         comment text.
 
         To stop users from crafting commit comments with
         Bugzilla commands, specify the bug ID via the message body, rather
         than the subject line, and leave a blank line after it.
         """
         user = self.map_committer(committer)
         matches = self.bzproxy.User.get(
             {b'match': [user], b'token': self.bztoken}
         )
         if not matches[b'users']:
             user = self.ui.config(b'bugzilla', b'user')
             matches = self.bzproxy.User.get(
                 {b'match': [user], b'token': self.bztoken}
             )
             if not matches[b'users']:
                 raise error.Abort(
                     _(b"default bugzilla user %s email not found") % user
                 )
         user = matches[b'users'][0][b'email']
         commands.append(self.makecommandline(b"id", bugid))
 
         text = b"\n".join(commands) + b"\n\n" + comment
 
         _charsets = mail._charsets(self.ui)
         user = mail.addressencode(self.ui, user, _charsets)
         bzemail = mail.addressencode(self.ui, self.bzemail, _charsets)
         msg = mail.mimeencode(self.ui, text, _charsets)
         msg[b'From'] = user
         msg[b'To'] = bzemail
         msg[b'Subject'] = mail.headencode(
             self.ui, b"Bug modification", _charsets
         )
         sendmail = mail.connect(self.ui)
         sendmail(user, bzemail, msg.as_string())
 
     def updatebug(self, bugid, newstate, text, committer):
         cmds = []
         if b'hours' in newstate:
             cmds.append(self.makecommandline(b"work_time", newstate[b'hours']))
         if b'fix' in newstate:
             cmds.append(self.makecommandline(b"bug_status", self.fixstatus))
             cmds.append(self.makecommandline(b"resolution", self.fixresolution))
         self.send_bug_modify_email(bugid, cmds, text, committer)
 
 
 class NotFound(LookupError):
     pass
 
 
 class bzrestapi(bzaccess):
     """Read and write bugzilla data using the REST API available since
     Bugzilla 5.0.
     """
 
     def __init__(self, ui):
         bzaccess.__init__(self, ui)
         bz = self.ui.config(b'bugzilla', b'bzurl')
         self.bzroot = b'/'.join([bz, b'rest'])
         self.apikey = self.ui.config(b'bugzilla', b'apikey')
         self.user = self.ui.config(b'bugzilla', b'user')
         self.passwd = self.ui.config(b'bugzilla', b'password')
         self.fixstatus = self.ui.config(b'bugzilla', b'fixstatus')
         self.fixresolution = self.ui.config(b'bugzilla', b'fixresolution')
 
     def apiurl(self, targets, include_fields=None):
         url = b'/'.join([self.bzroot] + [pycompat.bytestr(t) for t in targets])
         qv = {}
         if self.apikey:
             qv[b'api_key'] = self.apikey
         elif self.user and self.passwd:
             qv[b'login'] = self.user
             qv[b'password'] = self.passwd
         if include_fields:
             qv[b'include_fields'] = include_fields
         if qv:
             url = b'%s?%s' % (url, util.urlreq.urlencode(qv))
         return url
 
     def _fetch(self, burl):
         try:
             resp = url.open(self.ui, burl)
             return pycompat.json_loads(resp.read())
         except util.urlerr.httperror as inst:
             if inst.code == 401:
                 raise error.Abort(_(b'authorization failed'))
             if inst.code == 404:
                 raise NotFound()
             else:
                 raise
 
     def _submit(self, burl, data, method=b'POST'):
         data = json.dumps(data)
         if method == b'PUT':
 
             class putrequest(util.urlreq.request):
                 def get_method(self):
                     return b'PUT'
 
             request_type = putrequest
         else:
             request_type = util.urlreq.request
         req = request_type(burl, data, {b'Content-Type': b'application/json'})
         try:
             resp = url.opener(self.ui).open(req)
             return pycompat.json_loads(resp.read())
         except util.urlerr.httperror as inst:
             if inst.code == 401:
                 raise error.Abort(_(b'authorization failed'))
             if inst.code == 404:
                 raise NotFound()
             else:
                 raise
 
     def filter_real_bug_ids(self, bugs):
         '''remove bug IDs that do not exist in Bugzilla from bugs.'''
         badbugs = set()
         for bugid in bugs:
             burl = self.apiurl((b'bug', bugid), include_fields=b'status')
             try:
                 self._fetch(burl)
             except NotFound:
                 badbugs.add(bugid)
         for bugid in badbugs:
             del bugs[bugid]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''remove bug IDs where node occurs in comment text from bugs.'''
         sn = short(node)
         for bugid in bugs.keys():
             burl = self.apiurl(
                 (b'bug', bugid, b'comment'), include_fields=b'text'
             )
             result = self._fetch(burl)
             comments = result[b'bugs'][pycompat.bytestr(bugid)][b'comments']
             if any(sn in c[b'text'] for c in comments):
                 self.ui.status(
                     _(b'bug %d already knows about changeset %s\n')
                     % (bugid, sn)
                 )
                 del bugs[bugid]
 
     def updatebug(self, bugid, newstate, text, committer):
         """update the specified bug. Add comment text and set new states.
 
         If possible add the comment as being from the committer of
         the changeset. Otherwise use the default Bugzilla user.
         """
         bugmod = {}
         if b'hours' in newstate:
             bugmod[b'work_time'] = newstate[b'hours']
         if b'fix' in newstate:
             bugmod[b'status'] = self.fixstatus
             bugmod[b'resolution'] = self.fixresolution
         if bugmod:
             # if we have to change the bugs state do it here
             bugmod[b'comment'] = {
                 b'comment': text,
                 b'is_private': False,
                 b'is_markdown': False,
             }
             burl = self.apiurl((b'bug', bugid))
             self._submit(burl, bugmod, method=b'PUT')
             self.ui.debug(b'updated bug %s\n' % bugid)
         else:
             burl = self.apiurl((b'bug', bugid, b'comment'))
             self._submit(
                 burl,
                 {
                     b'comment': text,
                     b'is_private': False,
                     b'is_markdown': False,
                 },
             )
             self.ui.debug(b'added comment to bug %s\n' % bugid)
 
     def notify(self, bugs, committer):
         """Force sending of Bugzilla notification emails.
 
         Only required if the access method does not trigger notification
         emails automatically.
         """
         pass
 
 
 class bugzilla:
     # supported versions of bugzilla. different versions have
     # different schemas.
     _versions = {
         b'2.16': bzmysql,
         b'2.18': bzmysql_2_18,
         b'3.0': bzmysql_3_0,
         b'xmlrpc': bzxmlrpc,
         b'xmlrpc+email': bzxmlrpcemail,
         b'restapi': bzrestapi,
     }
 
     def __init__(self, ui, repo):
         self.ui = ui
         self.repo = repo
 
         bzversion = self.ui.config(b'bugzilla', b'version')
         try:
             bzclass = bugzilla._versions[bzversion]
         except KeyError:
             raise error.Abort(
                 _(b'bugzilla version %s not supported') % bzversion
             )
         self.bzdriver = bzclass(self.ui)
 
         self.bug_re = re.compile(
             self.ui.config(b'bugzilla', b'regexp'), re.IGNORECASE
         )
         self.fix_re = re.compile(
             self.ui.config(b'bugzilla', b'fixregexp'), re.IGNORECASE
         )
         self.split_re = re.compile(br'\D+')
 
     def find_bugs(self, ctx):
         """return bugs dictionary created from commit comment.
 
         Extract bug info from changeset comments. Filter out any that are
         not known to Bugzilla, and any that already have a reference to
         the given changeset in their comments.
         """
         start = 0
         bugs = {}
         bugmatch = self.bug_re.search(ctx.description(), start)
         fixmatch = self.fix_re.search(ctx.description(), start)
         while True:
             bugattribs = {}
             if not bugmatch and not fixmatch:
                 break
             if not bugmatch:
                 m = fixmatch
             elif not fixmatch:
                 m = bugmatch
             else:
                 if bugmatch.start() < fixmatch.start():
                     m = bugmatch
                 else:
                     m = fixmatch
             start = m.end()
             if m is bugmatch:
                 bugmatch = self.bug_re.search(ctx.description(), start)
                 if b'fix' in bugattribs:
                     del bugattribs[b'fix']
             else:
                 fixmatch = self.fix_re.search(ctx.description(), start)
                 bugattribs[b'fix'] = None
 
             try:
                 ids = m.group(b'ids')
             except IndexError:
                 ids = m.group(1)
             try:
                 hours = float(m.group(b'hours'))
                 bugattribs[b'hours'] = hours
             except IndexError:
                 pass
             except TypeError:
                 pass
             except ValueError:
                 self.ui.status(_(b"%s: invalid hours\n") % m.group(b'hours'))
 
             for id in self.split_re.split(ids):
                 if not id:
                     continue
                 bugs[int(id)] = bugattribs
         if bugs:
             self.bzdriver.filter_real_bug_ids(bugs)
         if bugs:
             self.bzdriver.filter_cset_known_bug_ids(ctx.node(), bugs)
         return bugs
 
     def update(self, bugid, newstate, ctx):
         '''update bugzilla bug with reference to changeset.'''
 
         def webroot(root):
             """strip leading prefix of repo root and turn into
             url-safe path."""
             count = int(self.ui.config(b'bugzilla', b'strip'))
             root = util.pconvert(root)
             while count > 0:
                 c = root.find(b'/')
                 if c == -1:
                     break
                 root = root[c + 1 :]
                 count -= 1
             return root
 
         mapfile = None
         tmpl = self.ui.config(b'bugzilla', b'template')
         if not tmpl:
             mapfile = self.ui.config(b'bugzilla', b'style')
         if not mapfile and not tmpl:
             tmpl = _(
                 b'changeset {node|short} in repo {root} refers '
                 b'to bug {bug}.\ndetails:\n\t{desc|tabindent}'
             )
         spec = logcmdutil.templatespec(tmpl, mapfile)
         t = logcmdutil.changesettemplater(self.ui, self.repo, spec)
         self.ui.pushbuffer()
         t.show(
             ctx,
             changes=ctx.changeset(),
             bug=pycompat.bytestr(bugid),
             hgweb=self.ui.config(b'web', b'baseurl'),
             root=self.repo.root,
             webroot=webroot(self.repo.root),
         )
         data = self.ui.popbuffer()
         self.bzdriver.updatebug(
             bugid, newstate, data, stringutil.email(ctx.user())
         )
 
     def notify(self, bugs, committer):
         '''ensure Bugzilla users are notified of bug change.'''
         self.bzdriver.notify(bugs, committer)
 
 
 def hook(ui, repo, hooktype, node=None, **kwargs):
     """add comment to bugzilla for each changeset that refers to a
     bugzilla bug id. only add a comment once per bug, so same change
     seen multiple times does not fill bug with duplicate data."""
     if node is None:
         raise error.Abort(
             _(b'hook type %s does not pass a changeset id') % hooktype
         )
     try:
         bz = bugzilla(ui, repo)
         ctx = repo[node]
         bugs = bz.find_bugs(ctx)
         if bugs:
             for bug in bugs:
                 bz.update(bug, bugs[bug], ctx)
             bz.notify(bugs, stringutil.email(ctx.user()))
     except Exception as e:
         raise error.Abort(_(b'Bugzilla error: %s') % stringutil.forcebytestr(e))

hgext/clonebundles.py

@@ -1,1090 +1,1090 b''
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """advertise pre-generated bundles to seed clones
 
 "clonebundles" is a server-side extension used to advertise the existence
 of pre-generated, externally hosted bundle files to clients that are
 cloning so that cloning can be faster, more reliable, and require less
 resources on the server. "pullbundles" is a related feature for sending
 pre-generated bundle files to clients as part of pull operations.
 
 Cloning can be a CPU and I/O intensive operation on servers. Traditionally,
 the server, in response to a client's request to clone, dynamically generates
 a bundle containing the entire repository content and sends it to the client.
 There is no caching on the server and the server will have to redundantly
 generate the same outgoing bundle in response to each clone request. For
 servers with large repositories or with high clone volume, the load from
 clones can make scaling the server challenging and costly.
 
 This extension provides server operators the ability to offload
 potentially expensive clone load to an external service. Pre-generated
 bundles also allow using more CPU intensive compression, reducing the
 effective bandwidth requirements.
 
 Here's how clone bundles work:
 
 1. A server operator establishes a mechanism for making bundle files available
    on a hosting service where Mercurial clients can fetch them.
 2. A manifest file listing available bundle URLs and some optional metadata
    is added to the Mercurial repository on the server.
 3. A client initiates a clone against a clone bundles aware server.
 4. The client sees the server is advertising clone bundles and fetches the
    manifest listing available bundles.
 5. The client filters and sorts the available bundles based on what it
    supports and prefers.
 6. The client downloads and applies an available bundle from the
    server-specified URL.
 7. The client reconnects to the original server and performs the equivalent
    of :hg:`pull` to retrieve all repository data not in the bundle. (The
    repository could have been updated between when the bundle was created
    and when the client started the clone.) This may use "pullbundles".
 
 Instead of the server generating full repository bundles for every clone
 request, it generates full bundles once and they are subsequently reused to
 bootstrap new clones. The server may still transfer data at clone time.
 However, this is only data that has been added/changed since the bundle was
 created. For large, established repositories, this can reduce server load for
 clones to less than 1% of original.
 
 Here's how pullbundles work:
 
 1. A manifest file listing available bundles and describing the revisions
    is added to the Mercurial repository on the server.
 2. A new-enough client informs the server that it supports partial pulls
    and initiates a pull.
 3. If the server has pull bundles enabled and sees the client advertising
    partial pulls, it checks for a matching pull bundle in the manifest.
    A bundle matches if the format is supported by the client, the client
    has the required revisions already and needs something from the bundle.
 4. If there is at least one matching bundle, the server sends it to the client.
 5. The client applies the bundle and notices that the server reply was
    incomplete. It initiates another pull.
 
 To work, this extension requires the following of server operators:
 
 * Generating bundle files of repository content (typically periodically,
   such as once per day).
 * Clone bundles: A file server that clients have network access to and that
   Python knows how to talk to through its normal URL handling facility
   (typically an HTTP/HTTPS server).
 * A process for keeping the bundles manifest in sync with available bundle
   files.
 
 Strictly speaking, using a static file hosting server isn't required: a server
 operator could use a dynamic service for retrieving bundle data. However,
 static file hosting services are simple and scalable and should be sufficient
 for most needs.
 
 Bundle files can be generated with the :hg:`bundle` command. Typically
 :hg:`bundle --all` is used to produce a bundle of the entire repository.
 
 The bundlespec option `stream` (see :hg:`help bundlespec`)
 can be used to produce a special *streaming clonebundle*, typically using
 :hg:`bundle --all --type="none-streamv2"`.
 These are bundle files that are extremely efficient
 to produce and consume (read: fast). However, they are larger than
 traditional bundle formats and require that clients support the exact set
 of repository data store formats in use by the repository that created them.
 Typically, a newer server can serve data that is compatible with older clients.
 However, *streaming clone bundles* don't have this guarantee. **Server
 operators need to be aware that newer versions of Mercurial may produce
 streaming clone bundles incompatible with older Mercurial versions.**
 
 A server operator is responsible for creating a ``.hg/clonebundles.manifest``
 file containing the list of available bundle files suitable for seeding
 clones. If this file does not exist, the repository will not advertise the
 existence of clone bundles when clients connect. For pull bundles,
 ``.hg/pullbundles.manifest`` is used.
 
 The manifest file contains a newline (\\n) delimited list of entries.
 
 Each line in this file defines an available bundle. Lines have the format:
 
     <URL> [<key>=<value>[ <key>=<value>]]
 
 That is, a URL followed by an optional, space-delimited list of key=value
 pairs describing additional properties of this bundle. Both keys and values
 are URI encoded.
 
 For pull bundles, the URL is a path under the ``.hg`` directory of the
 repository.
 
 Keys in UPPERCASE are reserved for use by Mercurial and are defined below.
 All non-uppercase keys can be used by site installations. An example use
 for custom properties is to use the *datacenter* attribute to define which
 data center a file is hosted in. Clients could then prefer a server in the
 data center closest to them.
 
 The following reserved keys are currently defined:
 
 BUNDLESPEC
    A "bundle specification" string that describes the type of the bundle.
 
    These are string values that are accepted by the "--type" argument of
    :hg:`bundle`.
 
    The values are parsed in strict mode, which means they must be of the
    "<compression>-<type>" form. See
    mercurial.exchange.parsebundlespec() for more details.
 
    :hg:`debugbundle --spec` can be used to print the bundle specification
    string for a bundle file. The output of this command can be used verbatim
    for the value of ``BUNDLESPEC`` (it is already escaped).
 
    Clients will automatically filter out specifications that are unknown or
    unsupported so they won't attempt to download something that likely won't
    apply.
 
    The actual value doesn't impact client behavior beyond filtering:
    clients will still sniff the bundle type from the header of downloaded
    files.
 
    **Use of this key is highly recommended**, as it allows clients to
    easily skip unsupported bundles. If this key is not defined, an old
    client may attempt to apply a bundle that it is incapable of reading.
 
 REQUIRESNI
    Whether Server Name Indication (SNI) is required to connect to the URL.
    SNI allows servers to use multiple certificates on the same IP. It is
    somewhat common in CDNs and other hosting providers. Older Python
    versions do not support SNI. Defining this attribute enables clients
    with older Python versions to filter this entry without experiencing
    an opaque SSL failure at connection time.
 
    If this is defined, it is important to advertise a non-SNI fallback
    URL or clients running old Python releases may not be able to clone
    with the clonebundles facility.
 
    Value should be "true".
 
 REQUIREDRAM
    Value specifies expected memory requirements to decode the payload.
    Values can have suffixes for common bytes sizes. e.g. "64MB".
 
    This key is often used with zstd-compressed bundles using a high
    compression level / window size, which can require 100+ MB of memory
    to decode.
 
 heads
    Used for pull bundles. This contains the ``;`` separated changeset
    hashes of the heads of the bundle content.
 
 bases
    Used for pull bundles. This contains the ``;`` separated changeset
    hashes of the roots of the bundle content. This can be skipped if
    the bundle was created without ``--base``.
 
 Manifests can contain multiple entries. Assuming metadata is defined, clients
 will filter entries from the manifest that they don't support. The remaining
 entries are optionally sorted by client preferences
 (``ui.clonebundleprefers`` config option). The client then attempts
 to fetch the bundle at the first URL in the remaining list.
 
 **Errors when downloading a bundle will fail the entire clone operation:
 clients do not automatically fall back to a traditional clone.** The reason
 for this is that if a server is using clone bundles, it is probably doing so
 because the feature is necessary to help it scale. In other words, there
 is an assumption that clone load will be offloaded to another service and
 that the Mercurial server isn't responsible for serving this clone load.
 If that other service experiences issues and clients start mass falling back to
 the original Mercurial server, the added clone load could overwhelm the server
 due to unexpected load and effectively take it offline. Not having clients
 automatically fall back to cloning from the original server mitigates this
 scenario.
 
 Because there is no automatic Mercurial server fallback on failure of the
 bundle hosting service, it is important for server operators to view the bundle
 hosting service as an extension of the Mercurial server in terms of
 availability and service level agreements: if the bundle hosting service goes
 down, so does the ability for clients to clone. Note: clients will see a
 message informing them how to bypass the clone bundles facility when a failure
 occurs. So server operators should prepare for some people to follow these
 instructions when a failure occurs, thus driving more load to the original
 Mercurial server when the bundle hosting service fails.
 
 
 inline clonebundles
 -------------------
 
 It is possible to transmit clonebundles inline in case repositories are
 accessed over SSH. This avoids having to setup an external HTTPS server
 and results in the same access control as already present for the SSH setup.
 
 Inline clonebundles should be placed into the `.hg/bundle-cache` directory.
 A clonebundle at `.hg/bundle-cache/mybundle.bundle` is referred to
 in the `clonebundles.manifest` file as `peer-bundle-cache://mybundle.bundle`.
 
 
 auto-generation of clone bundles
 --------------------------------
 
 It is possible to set Mercurial to automatically re-generate clone bundles when
 enough new content is available.
 
 Mercurial will take care of the process asynchronously. The defined list of
 bundle-type will be generated, uploaded, and advertised. Older bundles will get
 decommissioned as newer ones replace them.
 
 Bundles Generation:
 ...................
 
 The extension can generate multiple variants of the clone bundle. Each
 different variant will be defined by the "bundle-spec" they use::
 
     [clone-bundles]
     auto-generate.formats= zstd-v2, gzip-v2
 
 See `hg help bundlespec` for details about available options.
 
 By default, new bundles are generated when 5% of the repository contents or at
 least 1000 revisions are not contained in the cached bundles. This option can
 be controlled by the `clone-bundles.trigger.below-bundled-ratio` option
 (default 0.95) and the `clone-bundles.trigger.revs` option (default 1000)::
 
     [clone-bundles]
     trigger.below-bundled-ratio=0.95
     trigger.revs=1000
 
 This logic can be manually triggered using the `admin::clone-bundles-refresh`
 command, or automatically on each repository change if
 `clone-bundles.auto-generate.on-change` is set to `yes`::
 
     [clone-bundles]
     auto-generate.on-change=yes
     auto-generate.formats= zstd-v2, gzip-v2
 
 Automatic Inline serving
 ........................
 
 The simplest way to serve the generated bundle is through the Mercurial
 protocol. However it is not the most efficient as request will still be served
 by that main server. It is useful in case where authentication is complexe or
 when an efficient mirror system is already in use anyway. See the `inline
 clonebundles` section above for details about inline clonebundles
 
 To automatically serve generated bundle through inline clonebundle, simply set
 the following option::
 
     auto-generate.serve-inline=yes
 
 Enabling this option disable the managed upload and serving explained below.
 
 Bundles Upload and Serving:
 ...........................
 
 This is the most efficient way to serve automatically generated clone bundles,
 but requires some setup.
 
 The generated bundles need to be made available to users through a "public" URL.
 This should be donne through `clone-bundles.upload-command` configuration. The
 value of this command should be a shell command. It will have access to the
 bundle file path through the `$HGCB_BUNDLE_PATH` variable. And the expected
 basename in the "public" URL is accessible at::
 
   [clone-bundles]
   upload-command=sftp put $HGCB_BUNDLE_PATH \
       sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME
 
 If the file was already uploaded, the command must still succeed.
 
 After upload, the file should be available at an url defined by
 `clone-bundles.url-template`.
 
   [clone-bundles]
   url-template=https://bundles.host/cache/clone-bundles/{basename}
 
 Old bundles cleanup:
 ....................
 
 When new bundles are generated, the older ones are no longer necessary and can
 be removed from storage. This is done through the `clone-bundles.delete-command`
 configuration. The command is given the url of the artifact to delete through
 the `$HGCB_BUNDLE_URL` environment variable.
 
   [clone-bundles]
   delete-command=sftp rm sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME
 
 If the file was already deleted, the command must still succeed.
 """
 
 
 import os
 import weakref
 
 from mercurial.i18n import _
 
 from mercurial import (
     bundlecaches,
     commands,
     error,
     extensions,
     localrepo,
     lock,
     node,
     registrar,
     util,
     wireprotov1server,
 )
 
 
 from mercurial.utils import (
     procutil,
 )
 
 testedwith = b'ships-with-hg-core'
 
 
 def capabilities(orig, repo, proto):
     caps = orig(repo, proto)
 
     # Only advertise if a manifest exists. This does add some I/O to requests.
     # But this should be cheaper than a wasted network round trip due to
     # missing file.
     if repo.vfs.exists(bundlecaches.CB_MANIFEST_FILE):
         caps.append(b'clonebundles')
         caps.append(b'clonebundles_manifest')
 
     return caps
 
 
 def extsetup(ui):
     extensions.wrapfunction(wireprotov1server, '_capabilities', capabilities)
 
 
 # logic for bundle auto-generation
 
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 configitem(b'clone-bundles', b'auto-generate.on-change', default=False)
 configitem(b'clone-bundles', b'auto-generate.formats', default=list)
 configitem(b'clone-bundles', b'auto-generate.serve-inline', default=False)
 configitem(b'clone-bundles', b'trigger.below-bundled-ratio', default=0.95)
 configitem(b'clone-bundles', b'trigger.revs', default=1000)
 
 configitem(b'clone-bundles', b'upload-command', default=None)
 
 configitem(b'clone-bundles', b'delete-command', default=None)
 
 configitem(b'clone-bundles', b'url-template', default=None)
 
 configitem(b'devel', b'debug.clonebundles', default=False)
 
 
 # category for the post-close transaction hooks
 CAT_POSTCLOSE = b"clonebundles-autobundles"
 
 # template for bundle file names
 BUNDLE_MASK = (
     b"full-%(bundle_type)s-%(revs)d_revs-%(tip_short)s_tip-%(op_id)s.hg"
 )
 
 
 # file in .hg/ use to track clonebundles being auto-generated
 AUTO_GEN_FILE = b'clonebundles.auto-gen'
 
 
 class BundleBase(object):
     """represents the core of properties that matters for us in a bundle
 
     :bundle_type: the bundlespec (see hg help bundlespec)
     :revs:        the number of revisions in the repo at bundle creation time
     :tip_rev:     the rev-num of the tip revision
     :tip_node:    the node id of the tip-most revision in the bundle
 
     :ready:       True if the bundle is ready to be served
     """
 
     ready = False
 
     def __init__(self, bundle_type, revs, tip_rev, tip_node):
         self.bundle_type = bundle_type
         self.revs = revs
         self.tip_rev = tip_rev
         self.tip_node = tip_node
 
     def valid_for(self, repo):
         """is this bundle applicable to the current repository
 
         This is useful for detecting bundles made irrelevant by stripping.
         """
         tip_node = node.bin(self.tip_node)
         return repo.changelog.index.get_rev(tip_node) == self.tip_rev
 
     def __eq__(self, other):
         left = (self.ready, self.bundle_type, self.tip_rev, self.tip_node)
         right = (other.ready, other.bundle_type, other.tip_rev, other.tip_node)
         return left == right
 
     def __neq__(self, other):
         return not self == other
 
     def __cmp__(self, other):
         if self == other:
             return 0
         return -1
 
 
 class RequestedBundle(BundleBase):
     """A bundle that should be generated.
 
     Additional attributes compared to BundleBase
     :heads:       list of head revisions (as rev-num)
     :op_id:       a "unique" identifier for the operation triggering the change
     """
 
     def __init__(self, bundle_type, revs, tip_rev, tip_node, head_revs, op_id):
         self.head_revs = head_revs
         self.op_id = op_id
         super(RequestedBundle, self).__init__(
             bundle_type,
             revs,
             tip_rev,
             tip_node,
         )
 
     @property
     def suggested_filename(self):
         """A filename that can be used for the generated bundle"""
         data = {
             b'bundle_type': self.bundle_type,
             b'revs': self.revs,
             b'heads': self.head_revs,
             b'tip_rev': self.tip_rev,
             b'tip_node': self.tip_node,
             b'tip_short': self.tip_node[:12],
             b'op_id': self.op_id,
         }
         return BUNDLE_MASK % data
 
     def generate_bundle(self, repo, file_path):
         """generate the bundle at `filepath`"""
         commands.bundle(
             repo.ui,
             repo,
             file_path,
             base=[b"null"],
             rev=self.head_revs,
             type=self.bundle_type,
             quiet=True,
         )
 
     def generating(self, file_path, hostname=None, pid=None):
         """return a GeneratingBundle object from this object"""
         if pid is None:
             pid = os.getpid()
         if hostname is None:
             hostname = lock._getlockprefix()
         return GeneratingBundle(
             self.bundle_type,
             self.revs,
             self.tip_rev,
             self.tip_node,
             hostname,
             pid,
             file_path,
         )
 
 
 class GeneratingBundle(BundleBase):
     """A bundle being generated
 
     extra attributes compared to BundleBase:
 
     :hostname: the hostname of the machine generating the bundle
     :pid:      the pid of the process generating the bundle
     :filepath: the target filename of the bundle
 
     These attributes exist to help detect stalled generation processes.
     """
 
     ready = False
 
     def __init__(
         self, bundle_type, revs, tip_rev, tip_node, hostname, pid, filepath
     ):
         self.hostname = hostname
         self.pid = pid
         self.filepath = filepath
         super(GeneratingBundle, self).__init__(
             bundle_type, revs, tip_rev, tip_node
         )
 
     @classmethod
     def from_line(cls, line):
         """create an object by deserializing a line from AUTO_GEN_FILE"""
         assert line.startswith(b'PENDING-v1 ')
         (
             __,
             bundle_type,
             revs,
             tip_rev,
             tip_node,
             hostname,
             pid,
             filepath,
         ) = line.split()
         hostname = util.urlreq.unquote(hostname)
         filepath = util.urlreq.unquote(filepath)
         revs = int(revs)
         tip_rev = int(tip_rev)
         pid = int(pid)
         return cls(
             bundle_type, revs, tip_rev, tip_node, hostname, pid, filepath
         )
 
     def to_line(self):
         """serialize the object to include as a line in AUTO_GEN_FILE"""
         templ = b"PENDING-v1 %s %d %d %s %s %d %s"
         data = (
             self.bundle_type,
             self.revs,
             self.tip_rev,
             self.tip_node,
             util.urlreq.quote(self.hostname),
             self.pid,
             util.urlreq.quote(self.filepath),
         )
         return templ % data
 
     def __eq__(self, other):
         if not super(GeneratingBundle, self).__eq__(other):
             return False
         left = (self.hostname, self.pid, self.filepath)
         right = (other.hostname, other.pid, other.filepath)
         return left == right
 
     def uploaded(self, url, basename):
         """return a GeneratedBundle from this object"""
         return GeneratedBundle(
             self.bundle_type,
             self.revs,
             self.tip_rev,
             self.tip_node,
             url,
             basename,
         )
 
 
 class GeneratedBundle(BundleBase):
     """A bundle that is done being generated and can be served
 
     extra attributes compared to BundleBase:
 
     :file_url: the url where the bundle is available.
     :basename: the "basename" used to upload (useful for deletion)
 
     These attributes exist to generate a bundle manifest
     (.hg/pullbundles.manifest)
     """
 
     ready = True
 
     def __init__(
         self, bundle_type, revs, tip_rev, tip_node, file_url, basename
     ):
         self.file_url = file_url
         self.basename = basename
         super(GeneratedBundle, self).__init__(
             bundle_type, revs, tip_rev, tip_node
         )
 
     @classmethod
     def from_line(cls, line):
         """create an object by deserializing a line from AUTO_GEN_FILE"""
         assert line.startswith(b'DONE-v1 ')
         (
             __,
             bundle_type,
             revs,
             tip_rev,
             tip_node,
             file_url,
             basename,
         ) = line.split()
         revs = int(revs)
         tip_rev = int(tip_rev)
         file_url = util.urlreq.unquote(file_url)
         return cls(bundle_type, revs, tip_rev, tip_node, file_url, basename)
 
     def to_line(self):
         """serialize the object to include as a line in AUTO_GEN_FILE"""
         templ = b"DONE-v1 %s %d %d %s %s %s"
         data = (
             self.bundle_type,
             self.revs,
             self.tip_rev,
             self.tip_node,
             util.urlreq.quote(self.file_url),
             self.basename,
         )
         return templ % data
 
     def manifest_line(self):
         """serialize the object to include as a line in pullbundles.manifest"""
         templ = b"%s BUNDLESPEC=%s"
         if self.file_url.startswith(b'http'):
             templ += b" REQUIRESNI=true"
         return templ % (self.file_url, self.bundle_type)
 
     def __eq__(self, other):
         if not super(GeneratedBundle, self).__eq__(other):
             return False
         return self.file_url == other.file_url
 
 
 def parse_auto_gen(content):
     """parse the AUTO_GEN_FILE to return a list of Bundle object"""
     bundles = []
     for line in content.splitlines():
         if line.startswith(b'PENDING-v1 '):
             bundles.append(GeneratingBundle.from_line(line))
         elif line.startswith(b'DONE-v1 '):
             bundles.append(GeneratedBundle.from_line(line))
     return bundles
 
 
 def dumps_auto_gen(bundles):
     """serialize a list of Bundle as a AUTO_GEN_FILE content"""
     lines = []
     for b in bundles:
         lines.append(b"%s\n" % b.to_line())
     lines.sort()
     return b"".join(lines)
 
 
 def read_auto_gen(repo):
     """read the AUTO_GEN_FILE for the <repo> a list of Bundle object"""
     data = repo.vfs.tryread(AUTO_GEN_FILE)
     if not data:
         return []
     return parse_auto_gen(data)
 
 
 def write_auto_gen(repo, bundles):
     """write a list of Bundle objects into the repo's AUTO_GEN_FILE"""
     assert repo._cb_lock_ref is not None
     data = dumps_auto_gen(bundles)
     with repo.vfs(AUTO_GEN_FILE, mode=b'wb', atomictemp=True) as f:
         f.write(data)
 
 
 def generate_manifest(bundles):
     """write a list of Bundle objects into the repo's AUTO_GEN_FILE"""
     bundles = list(bundles)
     bundles.sort(key=lambda b: b.bundle_type)
     lines = []
     for b in bundles:
         lines.append(b"%s\n" % b.manifest_line())
     return b"".join(lines)
 
 
 def update_ondisk_manifest(repo):
     """update the clonebundle manifest with latest url"""
     with repo.clonebundles_lock():
         bundles = read_auto_gen(repo)
 
         per_types = {}
         for b in bundles:
             if not (b.ready and b.valid_for(repo)):
                 continue
             current = per_types.get(b.bundle_type)
             if current is not None and current.revs >= b.revs:
                 continue
             per_types[b.bundle_type] = b
         manifest = generate_manifest(per_types.values())
         with repo.vfs(
             bundlecaches.CB_MANIFEST_FILE, mode=b"wb", atomictemp=True
         ) as f:
             f.write(manifest)
 
 
 def update_bundle_list(repo, new_bundles=(), del_bundles=()):
     """modify the repo's AUTO_GEN_FILE
 
     This method also regenerates the clone bundle manifest when needed"""
     with repo.clonebundles_lock():
         bundles = read_auto_gen(repo)
         if del_bundles:
             bundles = [b for b in bundles if b not in del_bundles]
         new_bundles = [b for b in new_bundles if b not in bundles]
         bundles.extend(new_bundles)
         write_auto_gen(repo, bundles)
         all_changed = []
         all_changed.extend(new_bundles)
         all_changed.extend(del_bundles)
         if any(b.ready for b in all_changed):
             update_ondisk_manifest(repo)
 
 
 def cleanup_tmp_bundle(repo, target):
     """remove a GeneratingBundle file and entry"""
     assert not target.ready
     with repo.clonebundles_lock():
         repo.vfs.tryunlink(target.filepath)
         update_bundle_list(repo, del_bundles=[target])
 
 
 def finalize_one_bundle(repo, target):
     """upload a generated bundle and advertise it in the clonebundles.manifest"""
     with repo.clonebundles_lock():
         bundles = read_auto_gen(repo)
         if target in bundles and target.valid_for(repo):
             result = upload_bundle(repo, target)
             update_bundle_list(repo, new_bundles=[result])
     cleanup_tmp_bundle(repo, target)
 
 
 def find_outdated_bundles(repo, bundles):
     """finds outdated bundles"""
     olds = []
     per_types = {}
     for b in bundles:
         if not b.valid_for(repo):
             olds.append(b)
             continue
         l = per_types.setdefault(b.bundle_type, [])
         l.append(b)
     for key in sorted(per_types):
         all = per_types[key]
         if len(all) > 1:
             all.sort(key=lambda b: b.revs, reverse=True)
             olds.extend(all[1:])
     return olds
 
 
 def collect_garbage(repo):
     """finds outdated bundles and get them deleted"""
     with repo.clonebundles_lock():
         bundles = read_auto_gen(repo)
         olds = find_outdated_bundles(repo, bundles)
         for o in olds:
             delete_bundle(repo, o)
         update_bundle_list(repo, del_bundles=olds)
 
 
 def upload_bundle(repo, bundle):
     """upload the result of a GeneratingBundle and return a GeneratedBundle
 
     The upload is done using the `clone-bundles.upload-command`
     """
     inline = repo.ui.config(b'clone-bundles', b'auto-generate.serve-inline')
     basename = repo.vfs.basename(bundle.filepath)
     if inline:
         dest_dir = repo.vfs.join(bundlecaches.BUNDLE_CACHE_DIR)
         repo.vfs.makedirs(dest_dir)
         dest = repo.vfs.join(dest_dir, basename)
         util.copyfiles(bundle.filepath, dest, hardlink=True)
         url = bundlecaches.CLONEBUNDLESCHEME + basename
         return bundle.uploaded(url, basename)
     else:
         cmd = repo.ui.config(b'clone-bundles', b'upload-command')
         url = repo.ui.config(b'clone-bundles', b'url-template')
         filepath = procutil.shellquote(bundle.filepath)
         variables = {
             b'HGCB_BUNDLE_PATH': filepath,
             b'HGCB_BUNDLE_BASENAME': basename,
         }
         env = procutil.shellenviron(environ=variables)
         ret = repo.ui.system(cmd, environ=env)
         if ret:
             raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
         url = (
             url.decode('utf8')
             .format(basename=basename.decode('utf8'))
             .encode('utf8')
         )
         return bundle.uploaded(url, basename)
 
 
 def delete_bundle(repo, bundle):
     """delete a bundle from storage"""
     assert bundle.ready
 
     inline = bundle.file_url.startswith(bundlecaches.CLONEBUNDLESCHEME)
 
     if inline:
         msg = b'clone-bundles: deleting inline bundle %s\n'
     else:
         msg = b'clone-bundles: deleting bundle %s\n'
     msg %= bundle.basename
     if repo.ui.configbool(b'devel', b'debug.clonebundles'):
         repo.ui.write(msg)
     else:
         repo.ui.debug(msg)
 
     if inline:
         inline_path = repo.vfs.join(
             bundlecaches.BUNDLE_CACHE_DIR,
             bundle.basename,
         )
         util.tryunlink(inline_path)
     else:
         cmd = repo.ui.config(b'clone-bundles', b'delete-command')
         variables = {
             b'HGCB_BUNDLE_URL': bundle.file_url,
             b'HGCB_BASENAME': bundle.basename,
         }
         env = procutil.shellenviron(environ=variables)
         ret = repo.ui.system(cmd, environ=env)
         if ret:
             raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
 
 
 def auto_bundle_needed_actions(repo, bundles, op_id):
     """find the list of bundles that need action
 
     returns a list of RequestedBundle objects that need to be generated and
     uploaded."""
     create_bundles = []
     delete_bundles = []
     repo = repo.filtered(b"immutable")
     targets = repo.ui.configlist(b'clone-bundles', b'auto-generate.formats')
     ratio = float(
         repo.ui.config(b'clone-bundles', b'trigger.below-bundled-ratio')
     )
     abs_revs = repo.ui.configint(b'clone-bundles', b'trigger.revs')
     revs = len(repo.changelog)
     generic_data = {
         'revs': revs,
         'head_revs': repo.changelog.headrevs(),
         'tip_rev': repo.changelog.tiprev(),
         'tip_node': node.hex(repo.changelog.tip()),
         'op_id': op_id,
     }
     for t in targets:
         t = bundlecaches.parsebundlespec(repo, t, strict=False).as_spec()
         if new_bundle_needed(repo, bundles, ratio, abs_revs, t, revs):
             data = generic_data.copy()
             data['bundle_type'] = t
             b = RequestedBundle(**data)
             create_bundles.append(b)
     delete_bundles.extend(find_outdated_bundles(repo, bundles))
     return create_bundles, delete_bundles
 
 
 def new_bundle_needed(repo, bundles, ratio, abs_revs, bundle_type, revs):
     """consider the current cached content and trigger new bundles if needed"""
     threshold = max((revs * ratio), (revs - abs_revs))
     for b in bundles:
         if not b.valid_for(repo) or b.bundle_type != bundle_type:
             continue
         if b.revs > threshold:
             return False
     return True
 
 
 def start_one_bundle(repo, bundle):
     """start the generation of a single bundle file
 
     the `bundle` argument should be a RequestedBundle object.
 
     This data is passed to the `debugmakeclonebundles` "as is".
     """
     data = util.pickle.dumps(bundle)
     cmd = [procutil.hgexecutable(), b'--cwd', repo.path, INTERNAL_CMD]
     env = procutil.shellenviron()
     msg = b'clone-bundles: starting bundle generation: %s\n'
     stdout = None
     stderr = None
     waits = []
     record_wait = None
     if repo.ui.configbool(b'devel', b'debug.clonebundles'):
         stdout = procutil.stdout
         stderr = procutil.stderr
         repo.ui.write(msg % bundle.bundle_type)
         record_wait = waits.append
     else:
         repo.ui.debug(msg % bundle.bundle_type)
     bg = procutil.runbgcommand
     bg(
         cmd,
         env,
         stdin_bytes=data,
         stdout=stdout,
         stderr=stderr,
         record_wait=record_wait,
     )
     for f in waits:
         f()
 
 
 INTERNAL_CMD = b'debug::internal-make-clone-bundles'
 
 
 @command(INTERNAL_CMD, [], b'')
 def debugmakeclonebundles(ui, repo):
     """Internal command to auto-generate debug bundles"""
     requested_bundle = util.pickle.load(procutil.stdin)
     procutil.stdin.close()
 
     collect_garbage(repo)
 
     fname = requested_bundle.suggested_filename
     fpath = repo.vfs.makedirs(b'tmp-bundles')
     fpath = repo.vfs.join(b'tmp-bundles', fname)
     bundle = requested_bundle.generating(fpath)
     update_bundle_list(repo, new_bundles=[bundle])
 
     requested_bundle.generate_bundle(repo, fpath)
 
     repo.invalidate()
     finalize_one_bundle(repo, bundle)
 
 
 def make_auto_bundler(source_repo):
     reporef = weakref.ref(source_repo)
 
     def autobundle(tr):
         repo = reporef()
         assert repo is not None
         bundles = read_auto_gen(repo)
         new, __ = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr))
         for data in new:
             start_one_bundle(repo, data)
         return None
 
     return autobundle
 
 
 def reposetup(ui, repo):
     """install the two pieces needed for automatic clonebundle generation
 
     - add a "post-close" hook that fires bundling when needed
     - introduce a clone-bundle lock to let multiple processes meddle with the
       state files.
     """
     if not repo.local():
         return
 
     class autobundlesrepo(repo.__class__):
         def transaction(self, *args, **kwargs):
             tr = super(autobundlesrepo, self).transaction(*args, **kwargs)
             enabled = repo.ui.configbool(
                 b'clone-bundles',
                 b'auto-generate.on-change',
             )
             targets = repo.ui.configlist(
                 b'clone-bundles', b'auto-generate.formats'
             )
             if enabled:
                 if not targets:
                     repo.ui.warn(
                         _(
                             b'clone-bundle auto-generate enabled, '
                             b'but no formats specified: disabling generation\n'
                         )
                     )
                 else:
                     tr.addpostclose(CAT_POSTCLOSE, make_auto_bundler(self))
             return tr
 
         @localrepo.unfilteredmethod
         def clonebundles_lock(self, wait=True):
             '''Lock the repository file related to clone bundles'''
-            if not util.safehasattr(self, '_cb_lock_ref'):
+            if not hasattr(self, '_cb_lock_ref'):
                 self._cb_lock_ref = None
             l = self._currentlock(self._cb_lock_ref)
             if l is not None:
                 l.lock()
                 return l
 
             l = self._lock(
                 vfs=self.vfs,
                 lockname=b"clonebundleslock",
                 wait=wait,
                 releasefn=None,
                 acquirefn=None,
                 desc=_(b'repository %s') % self.origroot,
             )
             self._cb_lock_ref = weakref.ref(l)
             return l
 
     repo._wlockfreeprefix.add(AUTO_GEN_FILE)
     repo._wlockfreeprefix.add(bundlecaches.CB_MANIFEST_FILE)
     repo.__class__ = autobundlesrepo
 
 
 @command(
     b'admin::clone-bundles-refresh',
     [
         (
             b'',
             b'background',
             False,
             _(b'start bundle generation in the background'),
         ),
     ],
     b'',
 )
 def cmd_admin_clone_bundles_refresh(
     ui,
     repo: localrepo.localrepository,
     background=False,
 ):
     """generate clone bundles according to the configuration
 
     This runs the logic for automatic generation, removing outdated bundles and
     generating new ones if necessary. See :hg:`help -e clone-bundles` for
     details about how to configure this feature.
     """
     debug = repo.ui.configbool(b'devel', b'debug.clonebundles')
     bundles = read_auto_gen(repo)
     op_id = b"%d_acbr" % os.getpid()
     create, delete = auto_bundle_needed_actions(repo, bundles, op_id)
 
     # if some bundles are scheduled for creation in the background, they will
     # deal with garbage collection too, so no need to synchroniously do it.
     #
     # However if no bundles are scheduled for creation, we need to explicitly do
     # it here.
     if not (background and create):
         # we clean up outdated bundles before generating new ones to keep the
         # last two versions of the bundle around for a while and avoid having to
         # deal with clients that just got served a manifest.
         for o in delete:
             delete_bundle(repo, o)
         update_bundle_list(repo, del_bundles=delete)
 
     if create:
         fpath = repo.vfs.makedirs(b'tmp-bundles')
 
     if background:
         for requested_bundle in create:
             start_one_bundle(repo, requested_bundle)
     else:
         for requested_bundle in create:
             if debug:
                 msg = b'clone-bundles: starting bundle generation: %s\n'
                 repo.ui.write(msg % requested_bundle.bundle_type)
             fname = requested_bundle.suggested_filename
             fpath = repo.vfs.join(b'tmp-bundles', fname)
             generating_bundle = requested_bundle.generating(fpath)
             update_bundle_list(repo, new_bundles=[generating_bundle])
             requested_bundle.generate_bundle(repo, fpath)
             result = upload_bundle(repo, generating_bundle)
             update_bundle_list(repo, new_bundles=[result])
             update_ondisk_manifest(repo)
             cleanup_tmp_bundle(repo, generating_bundle)
 
 
 @command(b'admin::clone-bundles-clear', [], b'')
 def cmd_admin_clone_bundles_clear(ui, repo: localrepo.localrepository):
     """remove existing clone bundle caches
 
     See `hg help admin::clone-bundles-refresh` for details on how to regenerate
     them.
 
     This command will only affect bundles currently available, it will not
     affect bundles being asynchronously generated.
     """
     bundles = read_auto_gen(repo)
     delete = [b for b in bundles if b.ready]
     for o in delete:
         delete_bundle(repo, o)
     update_bundle_list(repo, del_bundles=delete)

hgext/commitextras.py

@@ -1,88 +1,87 b''
 # commitextras.py
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''adds a new flag extras to commit (ADVANCED)'''
 
 
 import re
 
 from mercurial.i18n import _
 from mercurial import (
     commands,
     error,
     extensions,
     registrar,
-    util,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 testedwith = b'ships-with-hg-core'
 
 usedinternally = {
     b'amend_source',
     b'branch',
     b'close',
     b'histedit_source',
     b'topic',
     b'rebase_source',
     b'intermediate-source',
     b'__touch-noise__',
     b'source',
     b'transplant_source',
 }
 
 
 def extsetup(ui):
     entry = extensions.wrapcommand(commands.table, b'commit', _commit)
     options = entry[1]
     options.append(
         (
             b'',
             b'extra',
             [],
             _(b'set a changeset\'s extra values'),
             _(b"KEY=VALUE"),
         )
     )
 
 
 def _commit(orig, ui, repo, *pats, **opts):
-    if util.safehasattr(repo, 'unfiltered'):
+    if hasattr(repo, 'unfiltered'):
         repo = repo.unfiltered()
 
     class repoextra(repo.__class__):
         def commit(self, *innerpats, **inneropts):
             extras = opts.get('extra')
             for raw in extras:
                 if b'=' not in raw:
                     msg = _(
                         b"unable to parse '%s', should follow "
                         b"KEY=VALUE format"
                     )
                     raise error.InputError(msg % raw)
                 k, v = raw.split(b'=', 1)
                 if not k:
                     msg = _(b"unable to parse '%s', keys can't be empty")
                     raise error.InputError(msg % raw)
                 if re.search(br'[^\w-]', k):
                     msg = _(
                         b"keys can only contain ascii letters, digits,"
                         b" '_' and '-'"
                     )
                     raise error.InputError(msg)
                 if k in usedinternally:
                     msg = _(
                         b"key '%s' is used internally, can't be set "
                         b"manually"
                     )
                     raise error.InputError(msg % k)
                 inneropts['extra'][k] = v
             return super(repoextra, self).commit(*innerpats, **inneropts)
 
     repo.__class__ = repoextra
     return orig(ui, repo, *pats, **opts)

hgext/convert/cvsps.py

@@ -1,1071 +1,1071 b''
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import functools
 import os
 import pickle
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     encoding,
     error,
     hook,
     pycompat,
     util,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
 
 class logentry:
     """Class logentry has the following attributes:
     .author    - author name as CVS knows it
     .branch    - name of branch this revision is on
     .branches  - revision tuple of branches starting at this revision
     .comment   - commit message
     .commitid  - CVS commitid or None
     .date      - the commit date as a (time, tz) tuple
     .dead      - true if file revision is dead
     .file      - Name of file
     .lines     - a tuple (+lines, -lines) or None
     .parent    - Previous revision of this entry
     .rcs       - name of file as returned from CVS
     .revision  - revision number as tuple
     .tags      - list of tags on the file
     .synthetic - is this a synthetic "file ... added on ..." revision?
     .mergepoint - the branch that has been merged from (if present in
                   rlog output) or None
     .branchpoints - the branches that start at the current entry or empty
     """
 
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r" % (k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)" % (type(self).__name__, ", ".join(items))
 
 
 class logerror(Exception):
     pass
 
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath(b'/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b':pserver:[e-mail]:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(b'user@server/path/to/repository')
     '/path/to/repository'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # CVSpath is splitted into parts and then position of the first occurrence
     # of the '/' char after the '@' is located. The solution is the rest of the
     # string after that '/' sign including it
 
     parts = cvspath.split(b':')
     atposition = parts[-1].find(b'@')
     start = 0
 
     if atposition != -1:
         start = atposition
 
     repopath = parts[-1][parts[-1].find(b'/', start) :]
     return repopath
 
 
 def createlog(ui, directory=None, root=b"", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
 
     def scache(s):
         """return a shared version of a string"""
         return _scache.setdefault(s, s)
 
     ui.status(_(b'collecting CVS rlog\n'))
 
     log = []  # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile(b'RCS file: (.+)$')
     re_01 = re.compile(b'cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile(b'cvs (r?log|server): (.+)\n$')
     re_03 = re.compile(
         b"(Cannot access.+CVSROOT)|(can't create temporary directory.+)$"
     )
     re_10 = re.compile(b'Working file: (.+)$')
     re_20 = re.compile(b'symbolic names:')
     re_30 = re.compile(b'\t(.+): ([\\d.]+)$')
     re_31 = re.compile(b'----------------------------$')
     re_32 = re.compile(
         b'======================================='
         b'======================================$'
     )
     re_50 = re.compile(br'revision ([\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(
         br'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
         br'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
         br'(\s+commitid:\s+([^;]+);)?'
         br'(.*mergepoint:\s+([^;]+);)?'
     )
     re_70 = re.compile(b'branches: (.+);$')
 
     file_added_re = re.compile(br'file [^/]+ was (initially )?added on branch')
 
     prefix = b''  # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             with open(os.path.join(b'CVS', b'Repository'), b'rb') as f:
                 prefix = f.read().strip()
             directory = prefix
             if prefix == b".":
                 prefix = b""
         except IOError:
             raise logerror(_(b'not a CVS sandbox'))
 
         if prefix and not prefix.endswith(pycompat.ossep):
             prefix += pycompat.ossep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join(b'CVS', b'Root'), b'rb').read().strip()
         except IOError:
             pass
 
     if not root:
         root = encoding.environ.get(b'CVSROOT', b'')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser(b'~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumeric characters, concatenated in a way that does not
         # mix up the various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(b":") + [directory, b"cache"]
         cachefile = [b'-'.join(re.findall(br'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(
             cachedir, b'.'.join([s for s in cachefile if s])
         )
 
     if cache == b'update':
         try:
             ui.note(_(b'reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile, b'rb'))
             for e in oldlog:
                 if not (
-                    util.safehasattr(e, b'branchpoints')
-                    and util.safehasattr(e, b'commitid')
-                    and util.safehasattr(e, b'mergepoint')
+                    hasattr(e, b'branchpoints')
+                    and hasattr(e, b'commitid')
+                    and hasattr(e, b'mergepoint')
                 ):
                     ui.status(_(b'ignoring old cache\n'))
                     oldlog = []
                     break
 
             ui.note(_(b'cache has %d log entries\n') % len(oldlog))
         except Exception as e:
             ui.note(_(b'error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date  # last commit date as a (time,tz) tuple
             date = dateutil.datestr(date, b'%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = [b'cvs', b'-q']
     if root:
         cmd.append(b'-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith(b'/'):
             p += b'/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append([b'log', b'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append(b'-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}  # dictionary of revisions on current file with their tags
     branchmap = {}  # mapping between branch names and revision numbers
     rcsmap = {}
     state = 0
     store = False  # set when a new record can be appended
 
     cmd = [procutil.shellquote(arg) for arg in cmd]
     ui.note(_(b"running %s\n") % (b' '.join(cmd)))
     ui.debug(b"prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = procutil.popen(b' '.join(cmd), b'rb')
     peek = util.fromnativeeol(pfp.readline())
     while True:
         line = peek
         if line == b'':
             break
         peek = util.fromnativeeol(pfp.readline())
         if line.endswith(b'\n'):
             line = line[:-1]
         # ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix) :]
                     if filename.startswith(b'/'):
                         filename = filename[1:]
                     if filename.startswith(b'Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace(b'/Attic/', b'/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _(b'RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split(b'.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _(
                     b'must have at least some revisions'
                 )
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _(b'expected revision number')
             e = logentry(
                 rcs=scache(rcs),
                 file=scache(filename),
                 revision=tuple([int(x) for x in match.group(1).split(b'.')]),
                 branches=[],
                 parent=None,
                 commitid=None,
                 mergepoint=None,
                 branchpoints=set(),
             )
 
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _(b'revision must be followed by date line')
             d = match.group(1)
             if d[2] == b'/':
                 # Y2K
                 d = b'19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + b' UTC'
             e.date = dateutil.parsedate(
                 d,
                 [
                     b'%y/%m/%d %H:%M:%S',
                     b'%Y/%m/%d %H:%M:%S',
                     b'%Y-%m-%d %H:%M:%S',
                 ],
             )
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == b'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7):  # cvs 1.12 commitid
                 e.commitid = match.group(8)
 
             if match.group(9):  # cvsnt mergepoint
                 myrev = match.group(10).split(b'.')
                 if len(myrev) == 2:  # head
                     e.mergepoint = b'HEAD'
                 else:
                     myrev = b'.'.join(myrev[:-2] + [b'0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, (
                         b'unknown branch: %s' % e.mergepoint
                     )
                     e.mergepoint = branches[0]
 
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [
                     tuple([int(y) for y in x.strip().split(b'.')])
                     for x in m.group(1).split(b';')
                 ]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
                 cpeek = peek
                 if cpeek.endswith(b'\n'):
                     cpeek = cpeek[:-1]
                 if re_50.match(cpeek):
                     state = 5
                     store = True
                 else:
                     e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (
             store
             and e.dead
             and e.revision[-1] == 1
             and len(e.comment) == 1  # 1.1 or 1.1.x.1
             and file_added_re.match(e.comment[0])
         ):
             ui.debug(
                 b'found synthetic revision in %s: %r\n' % (e.rcs, e.comment[0])
             )
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache(b'\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
             for branch, revision in branchmap.items():
                 revparts = tuple([int(i) for i in revision.split(b'.')])
                 if len(revparts) < 2:  # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1):  # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             rcsmap[e.rcs.replace(b'/Attic/', b'/')] = e.rcs
 
             if len(log) % 100 == 0:
                 ui.status(
                     stringutil.ellipsis(b'%d %s' % (len(log), e.file), 80)
                     + b'\n'
                 )
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in sorted(oldlog, key=lambda x: (x.rcs, x.revision)):
         rcs = e.rcs.replace(b'/Attic/', b'/')
         if rcs in rcsmap:
             e.rcs = rcsmap[rcs]
         branch = e.revision[:-1]
         versions[(e.rcs, branch)] = e.revision
 
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(
                     _(
                         b'log cache overlaps with new log entries,'
                         b' re-run without cache.'
                     )
                 )
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_(b'writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, b'wb'))
         else:
             log = oldlog
 
     ui.status(_(b'%d log entries\n') % len(log))
 
     encodings = ui.configlist(b'convert', b'cvsps.logencoding')
     if encodings:
 
         def revstr(r):
             # this is needed, because logentry.revision is a tuple of "int"
             # (e.g. (1, 2) for "1.2")
             return b'.'.join(pycompat.maplist(pycompat.bytestr, r))
 
         for entry in log:
             comment = entry.comment
             for e in encodings:
                 try:
                     entry.comment = comment.decode(pycompat.sysstr(e)).encode(
                         'utf-8'
                     )
                     if ui.debugflag:
                         ui.debug(
                             b"transcoding by %s: %s of %s\n"
                             % (e, revstr(entry.revision), entry.file)
                         )
                     break
                 except UnicodeDecodeError:
                     pass  # try next encoding
                 except LookupError as inst:  # unknown encoding, maybe
                     raise error.Abort(
                         pycompat.bytestr(inst),
                         hint=_(
                             b'check convert.cvsps.logencoding configuration'
                         ),
                     )
             else:
                 raise error.Abort(
                     _(
                         b"no encoding can transcode"
                         b" CVS log message for %s of %s"
                     )
                     % (revstr(entry.revision), entry.file),
                     hint=_(b'check convert.cvsps.logencoding configuration'),
                 )
 
     hook.hook(ui, None, b"cvslog", True, log=log)
 
     return log
 
 
 class changeset:
     """Class changeset has the following attributes:
     .id        - integer identifying this changeset (list index)
     .author    - author name as CVS knows it
     .branch    - name of branch this changeset is on, or None
     .comment   - commit message
     .commitid  - CVS commitid or None
     .date      - the commit date as a (time,tz) tuple
     .entries   - list of logentry objects in this changeset
     .parents   - list of one or two parent changesets
     .tags      - list of tags on this changeset
     .synthetic - from synthetic revision "file ... added on branch ..."
     .mergepoint- the branch that has been merged from or None
     .branchpoints- the branches that start at the current entry or empty
     """
 
     def __init__(self, **entries):
         self.id = None
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = (
             b"%s=%r" % (k, self.__dict__[k]) for k in sorted(self.__dict__)
         )
         return b"%s(%s)" % (type(self).__name__, b", ".join(items))
 
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_(b'creating changesets\n'))
 
     # try to order commitids by date
     mindate = {}
     for e in log:
         if e.commitid:
             if e.commitid not in mindate:
                 mindate[e.commitid] = e.date
             else:
                 mindate[e.commitid] = min(e.date, mindate[e.commitid])
 
     # Merge changesets
     log.sort(
         key=lambda x: (
             mindate.get(x.commitid, (-1, 0)),
             x.commitid or b'',
             x.comment,
             x.author,
             x.branch or b'',
             x.date,
             x.branchpoints,
         )
     )
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file-centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (
             c
             and e.branchpoints == c.branchpoints
             and (  # cvs commitids
                 (e.commitid is not None and e.commitid == c.commitid)
                 or (  # no commitids, use fuzzy commit detection
                     (e.commitid is None or c.commitid is None)
                     and e.comment == c.comment
                     and e.author == c.author
                     and e.branch == c.branch
                     and (
                         (c.date[0] + c.date[1])
                         <= (e.date[0] + e.date[1])
                         <= (c.date[0] + c.date[1]) + fuzz
                     )
                     and e.file not in files
                 )
             )
         ):
             c = changeset(
                 comment=e.comment,
                 author=e.author,
                 branch=e.branch,
                 date=e.date,
                 entries=[],
                 mergepoint=e.mergepoint,
                 branchpoints=e.branchpoints,
                 commitid=e.commitid,
             )
             changesets.append(c)
 
             files = set()
             if len(changesets) % 100 == 0:
                 t = b'%d %s' % (
                     len(changesets),
                     pycompat.byterepr(e.comment)[2:-1],
                 )
                 ui.status(stringutil.ellipsis(t, 80) + b'\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date  # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     def entitycompare(l, r):
         """Mimic cvsps sorting order"""
         l = l.file.split(b'/')
         r = r.file.split(b'/')
         nl = len(l)
         nr = len(r)
         n = min(nl, nr)
         for i in range(n):
             if i + 1 == nl and nl < nr:
                 return -1
             elif i + 1 == nr and nl > nr:
                 return +1
             elif l[i] < r[i]:
                 return -1
             elif l[i] > r[i]:
                 return +1
         return 0
 
     for c in changesets:
         c.entries.sort(key=functools.cmp_to_key(entitycompare))
 
     # Sort changesets by date
 
     odd = set()
 
     def cscmp(l, r):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 if d:
                     odd.add((l, r))
                 d = -1
                 break
         # By this point, the changesets are sufficiently compared that
         # we don't really care about ordering. However, this leaves
         # some race conditions in the tests, so we compare on the
         # number of files modified, the files contained in each
         # changeset, and the branchpoints in the change to ensure test
         # output remains stable.
 
         # recommended replacement for cmp from
         # https://docs.python.org/3.0/whatsnew/3.0.html
         c = lambda x, y: (x > y) - (x < y)
         # Sort bigger changes first.
         if not d:
             d = c(len(l.entries), len(r.entries))
         # Try sorting by filename in the change.
         if not d:
             d = c([e.file for e in l.entries], [e.file for e in r.entries])
         # Try and put changes without a branch point before ones with
         # a branch point.
         if not d:
             d = c(len(l.branchpoints), len(r.branchpoints))
         return d
 
     changesets.sort(key=functools.cmp_to_key(cscmp))
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = br'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = br'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}  # changeset index where we saw any particular file version
     branches = {}  # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in range(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, _(
                     b'synthetic changeset cannot have multiple parents'
                 )
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == b'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == b'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(
                         _(
                             b"warning: CVS commit message references "
                             b"non-existent branch %r:\n%s\n"
                         )
                         % (pycompat.bytestr(m), c.comment)
                     )
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 if m.groups():
                     m = m.group(1)
                     if m == b'HEAD':
                         m = None
                 else:
                     m = None  # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author,
                         branch=m,
                         date=c.date,
                         comment=b'convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[],
                         tags=[],
                         parents=[changesets[branches[m]], c],
                     )
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     if odd:
         for l, r in odd:
             if l.id is not None and r.id is not None:
                 ui.warn(
                     _(b'changeset %d is both before and after %d\n')
                     % (l.id, r.id)
                 )
 
     ui.status(_(b'%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, b"cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     """Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     """
     opts = pycompat.byteskwargs(opts)
     if opts[b"new_cache"]:
         cache = b"write"
     elif opts[b"update_cache"]:
         cache = b"update"
     else:
         cache = None
 
     revisions = opts[b"revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts[b"root"], cache=cache)
         else:
             log = createlog(ui, root=opts[b"root"], cache=cache)
     except logerror as e:
         ui.write(b"%r\n" % e)
         return
 
     changesets = createchangeset(ui, log, opts[b"fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}  # latest version number in each branch
     ancestors = {}  # parent branch
     for cs in changesets:
 
         if opts[b"ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (
                     changesets[cs.parents[0].id - 1].branch,
                     cs.parents[0].id,
                 )
             branches[cs.branch] = cs.id
 
         # limit by branches
         if (
             opts[b"branches"]
             and (cs.branch or b'HEAD') not in opts[b"branches"]
         ):
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write(b'---------------------\n')
             ui.write((b'PatchSet %d \n' % cs.id))
             ui.write(
                 (
                     b'Date: %s\n'
                     % dateutil.datestr(cs.date, b'%Y/%m/%d %H:%M:%S %1%2')
                 )
             )
             ui.write((b'Author: %s\n' % cs.author))
             ui.write((b'Branch: %s\n' % (cs.branch or b'HEAD')))
             ui.write(
                 (
                     b'Tag%s: %s \n'
                     % (
                         [b'', b's'][len(cs.tags) > 1],
                         b','.join(cs.tags) or b'(none)',
                     )
                 )
             )
             if cs.branchpoints:
                 ui.writenoi18n(
                     b'Branchpoints: %s \n' % b', '.join(sorted(cs.branchpoints))
                 )
             if opts[b"parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write(
                         (
                             b'Parents: %s\n'
                             % (b','.join([(b"%d" % p.id) for p in cs.parents]))
                         )
                     )
                 else:
                     ui.write((b'Parent: %d\n' % cs.parents[0].id))
 
             if opts[b"ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append(b'%s:%d:%d' % (b or b"HEAD", c, branches[b]))
                 if r:
                     ui.write((b'Ancestors: %s\n' % (b','.join(r))))
 
             ui.writenoi18n(b'Log:\n')
             ui.write(b'%s\n\n' % cs.comment)
             ui.writenoi18n(b'Members: \n')
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts[b"prefix"]):
                     fn = fn[len(opts[b"prefix"]) :]
                 ui.write(
                     b'\t%s:%s->%s%s \n'
                     % (
                         fn,
                         b'.'.join([b"%d" % x for x in f.parent]) or b'INITIAL',
                         b'.'.join([(b"%d" % x) for x in f.revision]),
                         [b'', b'(DEAD)'][f.dead],
                     )
                 )
             ui.write(b'\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == (b"%d" % cs.id) or revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == (b"%d" % cs.id) or revisions[1] in cs.tags:
                 break

hgext/convert/transport.py

@@ -1,160 +1,159 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2007 Daniel Holth <[e-mail]>
 # This is a stripped-down version of the original bzr-svn transport.py,
 # Copyright (C) 2006 Jelmer Vernooij <[e-mail]>
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 # pytype: disable=import-error
 import svn.client
 import svn.core
 import svn.ra
 
 # pytype: enable=import-error
 
 Pool = svn.core.Pool
 SubversionException = svn.core.SubversionException
 
 from mercurial.pycompat import getattr
-from mercurial import util
 
 # Some older versions of the Python bindings need to be
 # explicitly initialized. But what we want to do probably
 # won't work worth a darn against those libraries anyway!
 svn.ra.initialize()
 
 svn_config = None
 
 
 def _create_auth_baton(pool):
     """Create a Subversion authentication baton."""
     import svn.client  # pytype: disable=import-error
 
     # Give the client context baton a suite of authentication
     # providers.h
     providers = [
         svn.client.get_simple_provider(pool),
         svn.client.get_username_provider(pool),
         svn.client.get_ssl_client_cert_file_provider(pool),
         svn.client.get_ssl_client_cert_pw_file_provider(pool),
         svn.client.get_ssl_server_trust_file_provider(pool),
     ]
     # Platform-dependent authentication methods
     getprovider = getattr(
         svn.core, 'svn_auth_get_platform_specific_provider', None
     )
     if getprovider:
         # Available in svn >= 1.6
         for name in (b'gnome_keyring', b'keychain', b'kwallet', b'windows'):
             for type in (b'simple', b'ssl_client_cert_pw', b'ssl_server_trust'):
                 p = getprovider(name, type, pool)
                 if p:
                     providers.append(p)
     else:
-        if util.safehasattr(svn.client, 'get_windows_simple_provider'):
+        if hasattr(svn.client, 'get_windows_simple_provider'):
             providers.append(svn.client.get_windows_simple_provider(pool))
 
     return svn.core.svn_auth_open(providers, pool)
 
 
 class NotBranchError(SubversionException):
     pass
 
 
 class SvnRaTransport:
     """
     Open an ra connection to a Subversion repository.
     """
 
     def __init__(self, url=b"", ra=None):
         self.pool = Pool()
         self.svn_url = url
         self.username = b''
         self.password = b''
 
         # Only Subversion 1.4 has reparent()
-        if ra is None or not util.safehasattr(svn.ra, 'reparent'):
+        if ra is None or not hasattr(svn.ra, 'reparent'):
             self.client = svn.client.create_context(self.pool)
             ab = _create_auth_baton(self.pool)
             self.client.auth_baton = ab
             global svn_config
             if svn_config is None:
                 svn_config = svn.core.svn_config_get_config(None)
             self.client.config = svn_config
             try:
                 self.ra = svn.client.open_ra_session(
                     self.svn_url, self.client, self.pool
                 )
             except SubversionException as xxx_todo_changeme:
                 (inst, num) = xxx_todo_changeme.args
                 if num in (
                     svn.core.SVN_ERR_RA_ILLEGAL_URL,
                     svn.core.SVN_ERR_RA_LOCAL_REPOS_OPEN_FAILED,
                     svn.core.SVN_ERR_BAD_URL,
                 ):
                     raise NotBranchError(url)
                 raise
         else:
             self.ra = ra
             svn.ra.reparent(self.ra, self.svn_url.encode('utf8'))
 
     class Reporter:
         def __init__(self, reporter_data):
             self._reporter, self._baton = reporter_data
 
         def set_path(self, path, revnum, start_empty, lock_token, pool=None):
             svn.ra.reporter2_invoke_set_path(
                 self._reporter,
                 self._baton,
                 path,
                 revnum,
                 start_empty,
                 lock_token,
                 pool,
             )
 
         def delete_path(self, path, pool=None):
             svn.ra.reporter2_invoke_delete_path(
                 self._reporter, self._baton, path, pool
             )
 
         def link_path(
             self, path, url, revision, start_empty, lock_token, pool=None
         ):
             svn.ra.reporter2_invoke_link_path(
                 self._reporter,
                 self._baton,
                 path,
                 url,
                 revision,
                 start_empty,
                 lock_token,
                 pool,
             )
 
         def finish_report(self, pool=None):
             svn.ra.reporter2_invoke_finish_report(
                 self._reporter, self._baton, pool
             )
 
         def abort_report(self, pool=None):
             svn.ra.reporter2_invoke_abort_report(
                 self._reporter, self._baton, pool
             )
 
     def do_update(self, revnum, path, *args, **kwargs):
         return self.Reporter(
             svn.ra.do_update(self.ra, revnum, path, *args, **kwargs)
         )

hgext/fastannotate/commands.py

@@ -1,357 +1,356 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # commands: fastannotate commands
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     commands,
     encoding,
     error,
     extensions,
     logcmdutil,
     patch,
     pycompat,
     registrar,
     scmutil,
-    util,
 )
 
 from . import (
     context as facontext,
     error as faerror,
     formatter as faformatter,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 
 def _matchpaths(repo, rev, pats, opts, aopts=facontext.defaultopts):
     """generate paths matching given patterns"""
     perfhack = repo.ui.configbool(b'fastannotate', b'perfhack')
 
     # disable perfhack if:
     # a) any walkopt is used
     # b) if we treat pats as plain file names, some of them do not have
     #    corresponding linelog files
     if perfhack:
         # cwd related to reporoot
         reporoot = os.path.dirname(repo.path)
         reldir = os.path.relpath(encoding.getcwd(), reporoot)
         if reldir == b'.':
             reldir = b''
         if any(opts.get(o[1]) for o in commands.walkopts):  # a)
             perfhack = False
         else:  # b)
             relpats = [
                 os.path.relpath(p, reporoot) if os.path.isabs(p) else p
                 for p in pats
             ]
             # disable perfhack on '..' since it allows escaping from the repo
             if any(
                 (
                     b'..' in f
                     or not os.path.isfile(
                         facontext.pathhelper(repo, f, aopts).linelogpath
                     )
                 )
                 for f in relpats
             ):
                 perfhack = False
 
     # perfhack: emit paths directory without checking with manifest
     # this can be incorrect if the rev dos not have file.
     if perfhack:
         for p in relpats:
             yield os.path.join(reldir, p)
     else:
 
         def bad(x, y):
             raise error.Abort(b"%s: %s" % (x, y))
 
         ctx = logcmdutil.revsingle(repo, rev)
         m = scmutil.match(ctx, pats, opts, badfn=bad)
         for p in ctx.walk(m):
             yield p
 
 
 fastannotatecommandargs = {
     'options': [
         (b'r', b'rev', b'.', _(b'annotate the specified revision'), _(b'REV')),
         (b'u', b'user', None, _(b'list the author (long with -v)')),
         (b'f', b'file', None, _(b'list the filename')),
         (b'd', b'date', None, _(b'list the date (short with -q)')),
         (b'n', b'number', None, _(b'list the revision number (default)')),
         (b'c', b'changeset', None, _(b'list the changeset')),
         (
             b'l',
             b'line-number',
             None,
             _(b'show line number at the first appearance'),
         ),
         (
             b'e',
             b'deleted',
             None,
             _(b'show deleted lines (slow) (EXPERIMENTAL)'),
         ),
         (
             b'',
             b'no-content',
             None,
             _(b'do not show file content (EXPERIMENTAL)'),
         ),
         (b'', b'no-follow', None, _(b"don't follow copies and renames")),
         (
             b'',
             b'linear',
             None,
             _(
                 b'enforce linear history, ignore second parent '
                 b'of merges (EXPERIMENTAL)'
             ),
         ),
         (
             b'',
             b'long-hash',
             None,
             _(b'show long changeset hash (EXPERIMENTAL)'),
         ),
         (
             b'',
             b'rebuild',
             None,
             _(b'rebuild cache even if it exists (EXPERIMENTAL)'),
         ),
     ]
     + commands.diffwsopts
     + commands.walkopts
     + commands.formatteropts,
     'synopsis': _(b'[-r REV] [-f] [-a] [-u] [-d] [-n] [-c] [-l] FILE...'),
     'inferrepo': True,
 }
 
 
 def fastannotate(ui, repo, *pats, **opts):
     """show changeset information by line for each file
 
     List changes in files, showing the revision id responsible for each line.
 
     This command is useful for discovering when a change was made and by whom.
 
     By default this command prints revision numbers. If you include --file,
     --user, or --date, the revision number is suppressed unless you also
     include --number. The default format can also be customized by setting
     fastannotate.defaultformat.
 
     Returns 0 on success.
 
     .. container:: verbose
 
         This command uses an implementation different from the vanilla annotate
         command, which may produce slightly different (while still reasonable)
         outputs for some cases.
 
         Unlike the vanilla anootate, fastannotate follows rename regardless of
         the existence of --file.
 
         For the best performance when running on a full repo, use -c, -l,
         avoid -u, -d, -n. Use --linear and --no-content to make it even faster.
 
         For the best performance when running on a shallow (remotefilelog)
         repo, avoid --linear, --no-follow, or any diff options. As the server
         won't be able to populate annotate cache when non-default options
         affecting results are used.
     """
     if not pats:
         raise error.Abort(_(b'at least one filename or pattern is required'))
 
     # performance hack: filtered repo can be slow. unfilter by default.
     if ui.configbool(b'fastannotate', b'unfilteredrepo'):
         repo = repo.unfiltered()
 
     opts = pycompat.byteskwargs(opts)
 
     rev = opts.get(b'rev', b'.')
     rebuild = opts.get(b'rebuild', False)
 
     diffopts = patch.difffeatureopts(
         ui, opts, section=b'annotate', whitespace=True
     )
     aopts = facontext.annotateopts(
         diffopts=diffopts,
         followmerge=not opts.get(b'linear', False),
         followrename=not opts.get(b'no_follow', False),
     )
 
     if not any(
         opts.get(s)
         for s in [b'user', b'date', b'file', b'number', b'changeset']
     ):
         # default 'number' for compatibility. but fastannotate is more
         # efficient with "changeset", "line-number" and "no-content".
         for name in ui.configlist(
             b'fastannotate', b'defaultformat', [b'number']
         ):
             opts[name] = True
 
     ui.pager(b'fastannotate')
     template = opts.get(b'template')
     if template == b'json':
         formatter = faformatter.jsonformatter(ui, repo, opts)
     else:
         formatter = faformatter.defaultformatter(ui, repo, opts)
     showdeleted = opts.get(b'deleted', False)
     showlines = not bool(opts.get(b'no_content'))
     showpath = opts.get(b'file', False)
 
     # find the head of the main (master) branch
     master = ui.config(b'fastannotate', b'mainbranch') or rev
 
     # paths will be used for prefetching and the real annotating
     paths = list(_matchpaths(repo, rev, pats, opts, aopts))
 
     # for client, prefetch from the server
-    if util.safehasattr(repo, 'prefetchfastannotate'):
+    if hasattr(repo, 'prefetchfastannotate'):
         repo.prefetchfastannotate(paths)
 
     for path in paths:
         result = lines = existinglines = None
         while True:
             try:
                 with facontext.annotatecontext(repo, path, aopts, rebuild) as a:
                     result = a.annotate(
                         rev,
                         master=master,
                         showpath=showpath,
                         showlines=(showlines and not showdeleted),
                     )
                     if showdeleted:
                         existinglines = {(l[0], l[1]) for l in result}
                         result = a.annotatealllines(
                             rev, showpath=showpath, showlines=showlines
                         )
                 break
             except (faerror.CannotReuseError, faerror.CorruptedFileError):
                 # happens if master moves backwards, or the file was deleted
                 # and readded, or renamed to an existing name, or corrupted.
                 if rebuild:  # give up since we have tried rebuild already
                     raise
                 else:  # try a second time rebuilding the cache (slow)
                     rebuild = True
                     continue
 
         if showlines:
             result, lines = result
 
         formatter.write(result, lines, existinglines=existinglines)
     formatter.end()
 
 
 _newopts = set()
 _knownopts = {
     opt[1].replace(b'-', b'_')
     for opt in (fastannotatecommandargs['options'] + commands.globalopts)
 }
 
 
 def _annotatewrapper(orig, ui, repo, *pats, **opts):
     """used by wrapdefault"""
     # we need this hack until the obsstore has 0.0 seconds perf impact
     if ui.configbool(b'fastannotate', b'unfilteredrepo'):
         repo = repo.unfiltered()
 
     # treat the file as text (skip the isbinary check)
     if ui.configbool(b'fastannotate', b'forcetext'):
         opts['text'] = True
 
     # check if we need to do prefetch (client-side)
     rev = opts.get('rev')
-    if util.safehasattr(repo, 'prefetchfastannotate') and rev is not None:
+    if hasattr(repo, 'prefetchfastannotate') and rev is not None:
         paths = list(_matchpaths(repo, rev, pats, pycompat.byteskwargs(opts)))
         repo.prefetchfastannotate(paths)
 
     return orig(ui, repo, *pats, **opts)
 
 
 def registercommand():
     """register the fastannotate command"""
     name = b'fastannotate|fastblame|fa'
     command(name, helpbasic=True, **fastannotatecommandargs)(fastannotate)
 
 
 def wrapdefault():
     """wrap the default annotate command, to be aware of the protocol"""
     extensions.wrapcommand(commands.table, b'annotate', _annotatewrapper)
 
 
 @command(
     b'debugbuildannotatecache',
     [(b'r', b'rev', b'', _(b'build up to the specific revision'), _(b'REV'))]
     + commands.walkopts,
     _(b'[-r REV] FILE...'),
 )
 def debugbuildannotatecache(ui, repo, *pats, **opts):
     """incrementally build fastannotate cache up to REV for specified files
 
     If REV is not specified, use the config 'fastannotate.mainbranch'.
 
     If fastannotate.client is True, download the annotate cache from the
     server. Otherwise, build the annotate cache locally.
 
     The annotate cache will be built using the default diff and follow
     options and lives in '.hg/fastannotate/default'.
     """
     opts = pycompat.byteskwargs(opts)
     rev = opts.get(b'REV') or ui.config(b'fastannotate', b'mainbranch')
     if not rev:
         raise error.Abort(
             _(b'you need to provide a revision'),
             hint=_(b'set fastannotate.mainbranch or use --rev'),
         )
     if ui.configbool(b'fastannotate', b'unfilteredrepo'):
         repo = repo.unfiltered()
     ctx = logcmdutil.revsingle(repo, rev)
     m = scmutil.match(ctx, pats, opts)
     paths = list(ctx.walk(m))
-    if util.safehasattr(repo, 'prefetchfastannotate'):
+    if hasattr(repo, 'prefetchfastannotate'):
         # client
         if opts.get(b'REV'):
             raise error.Abort(_(b'--rev cannot be used for client'))
         repo.prefetchfastannotate(paths)
     else:
         # server, or full repo
         progress = ui.makeprogress(_(b'building'), total=len(paths))
         for i, path in enumerate(paths):
             progress.update(i)
             with facontext.annotatecontext(repo, path) as actx:
                 try:
                     if actx.isuptodate(rev):
                         continue
                     actx.annotate(rev, rev)
                 except (faerror.CannotReuseError, faerror.CorruptedFileError):
                     # the cache is broken (could happen with renaming so the
                     # file history gets invalidated). rebuild and try again.
                     ui.debug(
                         b'fastannotate: %s: rebuilding broken cache\n' % path
                     )
                     actx.rebuild()
                     try:
                         actx.annotate(rev, rev)
                     except Exception as ex:
                         # possibly a bug, but should not stop us from building
                         # cache for other files.
                         ui.warn(
                             _(
                                 b'fastannotate: %s: failed to '
                                 b'build cache: %r\n'
                             )
                             % (path, ex)
                         )
         progress.complete()

hgext/fastannotate/context.py

@@ -1,860 +1,860 b''
 # Copyright 2016-present Facebook. All Rights Reserved.
 #
 # context: context needed to annotate a file
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import collections
 import contextlib
 import os
 
 from mercurial.i18n import _
 from mercurial.pycompat import (
     getattr,
     open,
     setattr,
 )
 from mercurial.node import (
     bin,
     hex,
     short,
 )
 from mercurial import (
     error,
     linelog as linelogmod,
     lock as lockmod,
     mdiff,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.utils import (
     hashutil,
     stringutil,
 )
 
 from . import (
     error as faerror,
     revmap as revmapmod,
 )
 
 # given path, get filelog, cached
 @util.lrucachefunc
 def _getflog(repo, path):
     return repo.file(path)
 
 
 # extracted from mercurial.context.basefilectx.annotate
 def _parents(f, follow=True):
     # Cut _descendantrev here to mitigate the penalty of lazy linkrev
     # adjustment. Otherwise, p._adjustlinkrev() would walk changelog
     # from the topmost introrev (= srcrev) down to p.linkrev() if it
     # isn't an ancestor of the srcrev.
     f._changeid
     pl = f.parents()
 
     # Don't return renamed parents if we aren't following.
     if not follow:
         pl = [p for p in pl if p.path() == f.path()]
 
     # renamed filectx won't have a filelog yet, so set it
     # from the cache to save time
     for p in pl:
         if not '_filelog' in p.__dict__:
             p._filelog = _getflog(f._repo, p.path())
 
     return pl
 
 
 # extracted from mercurial.context.basefilectx.annotate. slightly modified
 # so it takes a fctx instead of a pair of text and fctx.
 def _decorate(fctx):
     text = fctx.data()
     linecount = text.count(b'\n')
     if text and not text.endswith(b'\n'):
         linecount += 1
     return ([(fctx, i) for i in range(linecount)], text)
 
 
 # extracted from mercurial.context.basefilectx.annotate. slightly modified
 # so it takes an extra "blocks" parameter calculated elsewhere, instead of
 # calculating diff here.
 def _pair(parent, child, blocks):
     for (a1, a2, b1, b2), t in blocks:
         # Changed blocks ('!') or blocks made only of blank lines ('~')
         # belong to the child.
         if t == b'=':
             child[0][b1:b2] = parent[0][a1:a2]
     return child
 
 
 # like scmutil.revsingle, but with lru cache, so their states (like manifests)
 # could be reused
 _revsingle = util.lrucachefunc(scmutil.revsingle)
 
 
 def resolvefctx(repo, rev, path, resolverev=False, adjustctx=None):
     """(repo, str, str) -> fctx
 
     get the filectx object from repo, rev, path, in an efficient way.
 
     if resolverev is True, "rev" is a revision specified by the revset
     language, otherwise "rev" is a nodeid, or a revision number that can
     be consumed by repo.__getitem__.
 
     if adjustctx is not None, the returned fctx will point to a changeset
     that introduces the change (last modified the file). if adjustctx
     is 'linkrev', trust the linkrev and do not adjust it. this is noticeably
     faster for big repos but is incorrect for some cases.
     """
     if resolverev and not isinstance(rev, int) and rev is not None:
         ctx = _revsingle(repo, rev)
     else:
         ctx = repo[rev]
 
     # If we don't need to adjust the linkrev, create the filectx using the
     # changectx instead of using ctx[path]. This means it already has the
     # changectx information, so blame -u will be able to look directly at the
     # commitctx object instead of having to resolve it by going through the
     # manifest. In a lazy-manifest world this can prevent us from downloading a
     # lot of data.
     if adjustctx is None:
         # ctx.rev() is None means it's the working copy, which is a special
         # case.
         if ctx.rev() is None:
             fctx = ctx[path]
         else:
             fctx = repo.filectx(path, changeid=ctx.rev())
     else:
         fctx = ctx[path]
         if adjustctx == b'linkrev':
             introrev = fctx.linkrev()
         else:
             introrev = fctx.introrev()
         if introrev != ctx.rev():
             fctx._changeid = introrev
             fctx._changectx = repo[introrev]
     return fctx
 
 
 # like mercurial.store.encodedir, but use linelog suffixes: .m, .l, .lock
 def encodedir(path):
     return (
         path.replace(b'.hg/', b'.hg.hg/')
         .replace(b'.l/', b'.l.hg/')
         .replace(b'.m/', b'.m.hg/')
         .replace(b'.lock/', b'.lock.hg/')
     )
 
 
 def hashdiffopts(diffopts):
     diffoptstr = stringutil.pprint(
         sorted(
             (k, getattr(diffopts, pycompat.sysstr(k)))
             for k in mdiff.diffopts.defaults
         )
     )
     return hex(hashutil.sha1(diffoptstr).digest())[:6]
 
 
 _defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
 
 
 class annotateopts:
     """like mercurial.mdiff.diffopts, but is for annotate
 
     followrename: follow renames, like "hg annotate -f"
     followmerge: follow p2 of a merge changeset, otherwise p2 is ignored
     """
 
     defaults = {
         'diffopts': None,
         'followrename': True,
         'followmerge': True,
     }
 
     def __init__(self, **opts):
         for k, v in self.defaults.items():
             setattr(self, k, opts.get(k, v))
 
     @util.propertycache
     def shortstr(self):
         """represent opts in a short string, suitable for a directory name"""
         result = b''
         if not self.followrename:
             result += b'r0'
         if not self.followmerge:
             result += b'm0'
         if self.diffopts is not None:
             assert isinstance(self.diffopts, mdiff.diffopts)
             diffopthash = hashdiffopts(self.diffopts)
             if diffopthash != _defaultdiffopthash:
                 result += b'i' + diffopthash
         return result or b'default'
 
 
 defaultopts = annotateopts()
 
 
 class _annotatecontext:
     """do not use this class directly as it does not use lock to protect
     writes. use "with annotatecontext(...)" instead.
     """
 
     def __init__(self, repo, path, linelogpath, revmappath, opts):
         self.repo = repo
         self.ui = repo.ui
         self.path = path
         self.opts = opts
         self.linelogpath = linelogpath
         self.revmappath = revmappath
         self._linelog = None
         self._revmap = None
         self._node2path = {}  # {str: str}
 
     @property
     def linelog(self):
         if self._linelog is None:
             if os.path.exists(self.linelogpath):
                 with open(self.linelogpath, b'rb') as f:
                     try:
                         self._linelog = linelogmod.linelog.fromdata(f.read())
                     except linelogmod.LineLogError:
                         self._linelog = linelogmod.linelog()
             else:
                 self._linelog = linelogmod.linelog()
         return self._linelog
 
     @property
     def revmap(self):
         if self._revmap is None:
             self._revmap = revmapmod.revmap(self.revmappath)
         return self._revmap
 
     def close(self):
         if self._revmap is not None:
             self._revmap.flush()
             self._revmap = None
         if self._linelog is not None:
             with open(self.linelogpath, b'wb') as f:
                 f.write(self._linelog.encode())
             self._linelog = None
 
     __del__ = close
 
     def rebuild(self):
         """delete linelog and revmap, useful for rebuilding"""
         self.close()
         self._node2path.clear()
         _unlinkpaths([self.revmappath, self.linelogpath])
 
     @property
     def lastnode(self):
         """return last node in revmap, or None if revmap is empty"""
         if self._revmap is None:
             # fast path, read revmap without loading its full content
             return revmapmod.getlastnode(self.revmappath)
         else:
             return self._revmap.rev2hsh(self._revmap.maxrev)
 
     def isuptodate(self, master, strict=True):
         """return True if the revmap / linelog is up-to-date, or the file
         does not exist in the master revision. False otherwise.
 
         it tries to be fast and could return false negatives, because of the
         use of linkrev instead of introrev.
 
         useful for both server and client to decide whether to update
         fastannotate cache or not.
 
         if strict is True, even if fctx exists in the revmap, but is not the
         last node, isuptodate will return False. it's good for performance - no
         expensive check was done.
 
         if strict is False, if fctx exists in the revmap, this function may
         return True. this is useful for the client to skip downloading the
         cache if the client's master is behind the server's.
         """
         lastnode = self.lastnode
         try:
             f = self._resolvefctx(master, resolverev=True)
             # choose linkrev instead of introrev as the check is meant to be
             # *fast*.
             linknode = self.repo.changelog.node(f.linkrev())
             if not strict and lastnode and linknode != lastnode:
                 # check if f.node() is in the revmap. note: this loads the
                 # revmap and can be slow.
                 return self.revmap.hsh2rev(linknode) is not None
             # avoid resolving old manifest, or slow adjustlinkrev to be fast,
             # false negatives are acceptable in this case.
             return linknode == lastnode
         except LookupError:
             # master does not have the file, or the revmap is ahead
             return True
 
     def annotate(self, rev, master=None, showpath=False, showlines=False):
         """incrementally update the cache so it includes revisions in the main
         branch till 'master'. and run annotate on 'rev', which may or may not be
         included in the main branch.
 
         if master is None, do not update linelog.
 
         the first value returned is the annotate result, it is [(node, linenum)]
         by default. [(node, linenum, path)] if showpath is True.
 
         if showlines is True, a second value will be returned, it is a list of
         corresponding line contents.
         """
 
         # the fast path test requires commit hash, convert rev number to hash,
         # so it may hit the fast path. note: in the "fctx" mode, the "annotate"
         # command could give us a revision number even if the user passes a
         # commit hash.
         if isinstance(rev, int):
             rev = hex(self.repo.changelog.node(rev))
 
         # fast path: if rev is in the main branch already
         directly, revfctx = self.canannotatedirectly(rev)
         if directly:
             if self.ui.debugflag:
                 self.ui.debug(
                     b'fastannotate: %s: using fast path '
                     b'(resolved fctx: %s)\n'
                     % (
                         self.path,
-                        stringutil.pprint(util.safehasattr(revfctx, 'node')),
+                        stringutil.pprint(hasattr(revfctx, 'node')),
                     )
                 )
             return self.annotatedirectly(revfctx, showpath, showlines)
 
         # resolve master
         masterfctx = None
         if master:
             try:
                 masterfctx = self._resolvefctx(
                     master, resolverev=True, adjustctx=True
                 )
             except LookupError:  # master does not have the file
                 pass
             else:
                 if masterfctx in self.revmap:  # no need to update linelog
                     masterfctx = None
 
         #                  ... - @ <- rev (can be an arbitrary changeset,
         #                 /                not necessarily a descendant
         #      master -> o                 of master)
         #                |
         #     a merge -> o         'o': new changesets in the main branch
         #                |\        '#': revisions in the main branch that
         #                o *            exist in linelog / revmap
         #                | .       '*': changesets in side branches, or
         # last master -> # .            descendants of master
         #                | .
         #                # *       joint: '#', and is a parent of a '*'
         #                |/
         #     a joint -> # ^^^^ --- side branches
         #                |
         #                ^ --- main branch (in linelog)
 
         # these DFSes are similar to the traditional annotate algorithm.
         # we cannot really reuse the code for perf reason.
 
         # 1st DFS calculates merges, joint points, and needed.
         # "needed" is a simple reference counting dict to free items in
         # "hist", reducing its memory usage otherwise could be huge.
         initvisit = [revfctx]
         if masterfctx:
             if masterfctx.rev() is None:
                 raise error.Abort(
                     _(b'cannot update linelog to wdir()'),
                     hint=_(b'set fastannotate.mainbranch'),
                 )
             initvisit.append(masterfctx)
         visit = initvisit[:]
         pcache = {}
         needed = {revfctx: 1}
         hist = {}  # {fctx: ([(llrev or fctx, linenum)], text)}
         while visit:
             f = visit.pop()
             if f in pcache or f in hist:
                 continue
             if f in self.revmap:  # in the old main branch, it's a joint
                 llrev = self.revmap.hsh2rev(f.node())
                 self.linelog.annotate(llrev)
                 result = self.linelog.annotateresult
                 hist[f] = (result, f.data())
                 continue
             pl = self._parentfunc(f)
             pcache[f] = pl
             for p in pl:
                 needed[p] = needed.get(p, 0) + 1
                 if p not in pcache:
                     visit.append(p)
 
         # 2nd (simple) DFS calculates new changesets in the main branch
         # ('o' nodes in # the above graph), so we know when to update linelog.
         newmainbranch = set()
         f = masterfctx
         while f and f not in self.revmap:
             newmainbranch.add(f)
             pl = pcache[f]
             if pl:
                 f = pl[0]
             else:
                 f = None
                 break
 
         # f, if present, is the position where the last build stopped at, and
         # should be the "master" last time. check to see if we can continue
         # building the linelog incrementally. (we cannot if diverged)
         if masterfctx is not None:
             self._checklastmasterhead(f)
 
         if self.ui.debugflag:
             if newmainbranch:
                 self.ui.debug(
                     b'fastannotate: %s: %d new changesets in the main'
                     b' branch\n' % (self.path, len(newmainbranch))
                 )
             elif not hist:  # no joints, no updates
                 self.ui.debug(
                     b'fastannotate: %s: linelog cannot help in '
                     b'annotating this revision\n' % self.path
                 )
 
         # prepare annotateresult so we can update linelog incrementally
         self.linelog.annotate(self.linelog.maxrev)
 
         # 3rd DFS does the actual annotate
         visit = initvisit[:]
         progress = self.ui.makeprogress(
             b'building cache', total=len(newmainbranch)
         )
         while visit:
             f = visit[-1]
             if f in hist:
                 visit.pop()
                 continue
 
             ready = True
             pl = pcache[f]
             for p in pl:
                 if p not in hist:
                     ready = False
                     visit.append(p)
             if not ready:
                 continue
 
             visit.pop()
             blocks = None  # mdiff blocks, used for appending linelog
             ismainbranch = f in newmainbranch
             # curr is the same as the traditional annotate algorithm,
             # if we only care about linear history (do not follow merge),
             # then curr is not actually used.
             assert f not in hist
             curr = _decorate(f)
             for i, p in enumerate(pl):
                 bs = list(self._diffblocks(hist[p][1], curr[1]))
                 if i == 0 and ismainbranch:
                     blocks = bs
                 curr = _pair(hist[p], curr, bs)
                 if needed[p] == 1:
                     del hist[p]
                     del needed[p]
                 else:
                     needed[p] -= 1
 
             hist[f] = curr
             del pcache[f]
 
             if ismainbranch:  # need to write to linelog
                 progress.increment()
                 bannotated = None
                 if len(pl) == 2 and self.opts.followmerge:  # merge
                     bannotated = curr[0]
                 if blocks is None:  # no parents, add an empty one
                     blocks = list(self._diffblocks(b'', curr[1]))
                 self._appendrev(f, blocks, bannotated)
             elif showpath:  # not append linelog, but we need to record path
                 self._node2path[f.node()] = f.path()
 
         progress.complete()
 
         result = [
             ((self.revmap.rev2hsh(fr) if isinstance(fr, int) else fr.node()), l)
             for fr, l in hist[revfctx][0]
         ]  # [(node, linenumber)]
         return self._refineannotateresult(result, revfctx, showpath, showlines)
 
     def canannotatedirectly(self, rev):
         """(str) -> bool, fctx or node.
         return (True, f) if we can annotate without updating the linelog, pass
         f to annotatedirectly.
         return (False, f) if we need extra calculation. f is the fctx resolved
         from rev.
         """
         result = True
         f = None
         if not isinstance(rev, int) and rev is not None:
             hsh = {20: bytes, 40: bin}.get(len(rev), lambda x: None)(rev)
             if hsh is not None and (hsh, self.path) in self.revmap:
                 f = hsh
         if f is None:
             adjustctx = b'linkrev' if self._perfhack else True
             f = self._resolvefctx(rev, adjustctx=adjustctx, resolverev=True)
             result = f in self.revmap
             if not result and self._perfhack:
                 # redo the resolution without perfhack - as we are going to
                 # do write operations, we need a correct fctx.
                 f = self._resolvefctx(rev, adjustctx=True, resolverev=True)
         return result, f
 
     def annotatealllines(self, rev, showpath=False, showlines=False):
         """(rev : str) -> [(node : str, linenum : int, path : str)]
 
         the result has the same format with annotate, but include all (including
         deleted) lines up to rev. call this after calling annotate(rev, ...) for
         better performance and accuracy.
         """
         revfctx = self._resolvefctx(rev, resolverev=True, adjustctx=True)
 
         # find a chain from rev to anything in the mainbranch
         if revfctx not in self.revmap:
             chain = [revfctx]
             a = b''
             while True:
                 f = chain[-1]
                 pl = self._parentfunc(f)
                 if not pl:
                     break
                 if pl[0] in self.revmap:
                     a = pl[0].data()
                     break
                 chain.append(pl[0])
 
             # both self.linelog and self.revmap is backed by filesystem. now
             # we want to modify them but do not want to write changes back to
             # files. so we create in-memory objects and copy them. it's like
             # a "fork".
             linelog = linelogmod.linelog()
             linelog.copyfrom(self.linelog)
             linelog.annotate(linelog.maxrev)
             revmap = revmapmod.revmap()
             revmap.copyfrom(self.revmap)
 
             for f in reversed(chain):
                 b = f.data()
                 blocks = list(self._diffblocks(a, b))
                 self._doappendrev(linelog, revmap, f, blocks)
                 a = b
         else:
             # fastpath: use existing linelog, revmap as we don't write to them
             linelog = self.linelog
             revmap = self.revmap
 
         lines = linelog.getalllines()
         hsh = revfctx.node()
         llrev = revmap.hsh2rev(hsh)
         result = [(revmap.rev2hsh(r), l) for r, l in lines if r <= llrev]
         # cannot use _refineannotateresult since we need custom logic for
         # resolving line contents
         if showpath:
             result = self._addpathtoresult(result, revmap)
         if showlines:
             linecontents = self._resolvelines(result, revmap, linelog)
             result = (result, linecontents)
         return result
 
     def _resolvelines(self, annotateresult, revmap, linelog):
         """(annotateresult) -> [line]. designed for annotatealllines.
         this is probably the most inefficient code in the whole fastannotate
         directory. but we have made a decision that the linelog does not
         store line contents. so getting them requires random accesses to
         the revlog data, since they can be many, it can be very slow.
         """
         # [llrev]
         revs = [revmap.hsh2rev(l[0]) for l in annotateresult]
         result = [None] * len(annotateresult)
         # {(rev, linenum): [lineindex]}
         key2idxs = collections.defaultdict(list)
         for i in range(len(result)):
             key2idxs[(revs[i], annotateresult[i][1])].append(i)
         while key2idxs:
             # find an unresolved line and its linelog rev to annotate
             hsh = None
             try:
                 for (rev, _linenum), idxs in key2idxs.items():
                     if revmap.rev2flag(rev) & revmapmod.sidebranchflag:
                         continue
                     hsh = annotateresult[idxs[0]][0]
                     break
             except StopIteration:  # no more unresolved lines
                 return result
             if hsh is None:
                 # the remaining key2idxs are not in main branch, resolving them
                 # using the hard way...
                 revlines = {}
                 for (rev, linenum), idxs in key2idxs.items():
                     if rev not in revlines:
                         hsh = annotateresult[idxs[0]][0]
                         if self.ui.debugflag:
                             self.ui.debug(
                                 b'fastannotate: reading %s line #%d '
                                 b'to resolve lines %r\n'
                                 % (short(hsh), linenum, idxs)
                             )
                         fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
                         lines = mdiff.splitnewlines(fctx.data())
                         revlines[rev] = lines
                     for idx in idxs:
                         result[idx] = revlines[rev][linenum]
                 assert all(x is not None for x in result)
                 return result
 
             # run the annotate and the lines should match to the file content
             self.ui.debug(
                 b'fastannotate: annotate %s to resolve lines\n' % short(hsh)
             )
             linelog.annotate(rev)
             fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
             annotated = linelog.annotateresult
             lines = mdiff.splitnewlines(fctx.data())
             if len(lines) != len(annotated):
                 raise faerror.CorruptedFileError(b'unexpected annotated lines')
             # resolve lines from the annotate result
             for i, line in enumerate(lines):
                 k = annotated[i]
                 if k in key2idxs:
                     for idx in key2idxs[k]:
                         result[idx] = line
                     del key2idxs[k]
         return result
 
     def annotatedirectly(self, f, showpath, showlines):
         """like annotate, but when we know that f is in linelog.
         f can be either a 20-char str (node) or a fctx. this is for perf - in
         the best case, the user provides a node and we don't need to read the
         filelog or construct any filecontext.
         """
         if isinstance(f, bytes):
             hsh = f
         else:
             hsh = f.node()
         llrev = self.revmap.hsh2rev(hsh)
         if not llrev:
             raise faerror.CorruptedFileError(b'%s is not in revmap' % hex(hsh))
         if (self.revmap.rev2flag(llrev) & revmapmod.sidebranchflag) != 0:
             raise faerror.CorruptedFileError(
                 b'%s is not in revmap mainbranch' % hex(hsh)
             )
         self.linelog.annotate(llrev)
         result = [
             (self.revmap.rev2hsh(r), l) for r, l in self.linelog.annotateresult
         ]
         return self._refineannotateresult(result, f, showpath, showlines)
 
     def _refineannotateresult(self, result, f, showpath, showlines):
         """add the missing path or line contents, they can be expensive.
         f could be either node or fctx.
         """
         if showpath:
             result = self._addpathtoresult(result)
         if showlines:
             if isinstance(f, bytes):  # f: node or fctx
                 llrev = self.revmap.hsh2rev(f)
                 fctx = self._resolvefctx(f, self.revmap.rev2path(llrev))
             else:
                 fctx = f
             lines = mdiff.splitnewlines(fctx.data())
             if len(lines) != len(result):  # linelog is probably corrupted
                 raise faerror.CorruptedFileError()
             result = (result, lines)
         return result
 
     def _appendrev(self, fctx, blocks, bannotated=None):
         self._doappendrev(self.linelog, self.revmap, fctx, blocks, bannotated)
 
     def _diffblocks(self, a, b):
         return mdiff.allblocks(a, b, self.opts.diffopts)
 
     @staticmethod
     def _doappendrev(linelog, revmap, fctx, blocks, bannotated=None):
         """append a revision to linelog and revmap"""
 
         def getllrev(f):
             """(fctx) -> int"""
             # f should not be a linelog revision
             if isinstance(f, int):
                 raise error.ProgrammingError(b'f should not be an int')
             # f is a fctx, allocate linelog rev on demand
             hsh = f.node()
             rev = revmap.hsh2rev(hsh)
             if rev is None:
                 rev = revmap.append(hsh, sidebranch=True, path=f.path())
             return rev
 
         # append sidebranch revisions to revmap
         siderevs = []
         siderevmap = {}  # node: int
         if bannotated is not None:
             for (a1, a2, b1, b2), op in blocks:
                 if op != b'=':
                     # f could be either linelong rev, or fctx.
                     siderevs += [
                         f
                         for f, l in bannotated[b1:b2]
                         if not isinstance(f, int)
                     ]
         siderevs = set(siderevs)
         if fctx in siderevs:  # mainnode must be appended seperately
             siderevs.remove(fctx)
         for f in siderevs:
             siderevmap[f] = getllrev(f)
 
         # the changeset in the main branch, could be a merge
         llrev = revmap.append(fctx.node(), path=fctx.path())
         siderevmap[fctx] = llrev
 
         for (a1, a2, b1, b2), op in reversed(blocks):
             if op == b'=':
                 continue
             if bannotated is None:
                 linelog.replacelines(llrev, a1, a2, b1, b2)
             else:
                 blines = [
                     ((r if isinstance(r, int) else siderevmap[r]), l)
                     for r, l in bannotated[b1:b2]
                 ]
                 linelog.replacelines_vec(llrev, a1, a2, blines)
 
     def _addpathtoresult(self, annotateresult, revmap=None):
         """(revmap, [(node, linenum)]) -> [(node, linenum, path)]"""
         if revmap is None:
             revmap = self.revmap
 
         def _getpath(nodeid):
             path = self._node2path.get(nodeid)
             if path is None:
                 path = revmap.rev2path(revmap.hsh2rev(nodeid))
                 self._node2path[nodeid] = path
             return path
 
         return [(n, l, _getpath(n)) for n, l in annotateresult]
 
     def _checklastmasterhead(self, fctx):
         """check if fctx is the master's head last time, raise if not"""
         if fctx is None:
             llrev = 0
         else:
             llrev = self.revmap.hsh2rev(fctx.node())
             if not llrev:
                 raise faerror.CannotReuseError()
         if self.linelog.maxrev != llrev:
             raise faerror.CannotReuseError()
 
     @util.propertycache
     def _parentfunc(self):
         """-> (fctx) -> [fctx]"""
         followrename = self.opts.followrename
         followmerge = self.opts.followmerge
 
         def parents(f):
             pl = _parents(f, follow=followrename)
             if not followmerge:
                 pl = pl[:1]
             return pl
 
         return parents
 
     @util.propertycache
     def _perfhack(self):
         return self.ui.configbool(b'fastannotate', b'perfhack')
 
     def _resolvefctx(self, rev, path=None, **kwds):
         return resolvefctx(self.repo, rev, (path or self.path), **kwds)
 
 
 def _unlinkpaths(paths):
     """silent, best-effort unlink"""
     for path in paths:
         try:
             util.unlink(path)
         except OSError:
             pass
 
 
 class pathhelper:
     """helper for getting paths for lockfile, linelog and revmap"""
 
     def __init__(self, repo, path, opts=defaultopts):
         # different options use different directories
         self._vfspath = os.path.join(
             b'fastannotate', opts.shortstr, encodedir(path)
         )
         self._repo = repo
 
     @property
     def dirname(self):
         return os.path.dirname(self._repo.vfs.join(self._vfspath))
 
     @property
     def linelogpath(self):
         return self._repo.vfs.join(self._vfspath + b'.l')
 
     def lock(self):
         return lockmod.lock(self._repo.vfs, self._vfspath + b'.lock')
 
     @property
     def revmappath(self):
         return self._repo.vfs.join(self._vfspath + b'.m')
 
 
 @contextlib.contextmanager
 def annotatecontext(repo, path, opts=defaultopts, rebuild=False):
     """context needed to perform (fast) annotate on a file
 
     an annotatecontext of a single file consists of two structures: the
     linelog and the revmap. this function takes care of locking. only 1
     process is allowed to write that file's linelog and revmap at a time.
 
     when something goes wrong, this function will assume the linelog and the
     revmap are in a bad state, and remove them from disk.
 
     use this function in the following way:
 
         with annotatecontext(...) as actx:
             actx. ....
     """
     helper = pathhelper(repo, path, opts)
     util.makedirs(helper.dirname)
     revmappath = helper.revmappath
     linelogpath = helper.linelogpath
     actx = None
     try:
         with helper.lock():
             actx = _annotatecontext(repo, path, linelogpath, revmappath, opts)
             if rebuild:
                 actx.rebuild()
             yield actx
     except Exception:
         if actx is not None:
             actx.rebuild()
         repo.ui.debug(b'fastannotate: %s: cache broken and deleted\n' % path)
         raise
     finally:
         if actx is not None:
             actx.close()
 
 
 def fctxannotatecontext(fctx, follow=True, diffopts=None, rebuild=False):
     """like annotatecontext but get the context from a fctx. convenient when
     used in fctx.annotate
     """
     repo = fctx._repo
     path = fctx._path
     if repo.ui.configbool(b'fastannotate', b'forcefollow', True):
         follow = True
     aopts = annotateopts(diffopts=diffopts, followrename=follow)
     return annotatecontext(repo, path, aopts, rebuild)

hgext/fsmonitor/__init__.py

@@ -1,1016 +1,1016 b''
 # __init__.py - fsmonitor initialization and overrides
 #
 # Copyright 2013-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
 
 Integrates the file-watching program Watchman with Mercurial to produce faster
 status results.
 
 On a particular Linux system, for a real-world repository with over 400,000
 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
 system, with fsmonitor it takes about 0.3 seconds.
 
 fsmonitor requires no configuration -- it will tell Watchman about your
 repository as necessary. You'll need to install Watchman from
 https://facebook.github.io/watchman/ and make sure it is in your PATH.
 
 fsmonitor is incompatible with the largefiles and eol extensions, and
 will disable itself if any of those are active.
 
 The following configuration options exist:
 
 ::
 
     [fsmonitor]
     mode = {off, on, paranoid}
 
 When `mode = off`, fsmonitor will disable itself (similar to not loading the
 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
 and ensure that the results are consistent.
 
 ::
 
     [fsmonitor]
     timeout = (float)
 
 A value, in seconds, that determines how long fsmonitor will wait for Watchman
 to return results. Defaults to `2.0`.
 
 ::
 
     [fsmonitor]
     blacklistusers = (list of userids)
 
 A list of usernames for which fsmonitor will disable itself altogether.
 
 ::
 
     [fsmonitor]
     walk_on_invalidate = (boolean)
 
 Whether or not to walk the whole repo ourselves when our cached state has been
 invalidated, for example when Watchman has been restarted or .hgignore rules
 have been changed. Walking the repo in that case can result in competing for
 I/O with Watchman. For large repos it is recommended to set this value to
 false. You may wish to set this to true if you have a very fast filesystem
 that can outpace the IPC overhead of getting the result data for the full repo
 from Watchman. Defaults to false.
 
 ::
 
     [fsmonitor]
     warn_when_unused = (boolean)
 
 Whether to print a warning during certain operations when fsmonitor would be
 beneficial to performance but isn't enabled.
 
 ::
 
     [fsmonitor]
     warn_update_file_count = (integer)
     # or when mercurial is built with rust support
     warn_update_file_count_rust = (integer)
 
 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
 be printed during working directory updates if this many files will be
 created.
 '''
 
 # Platforms Supported
 # ===================
 #
 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
 #   even under severe loads.
 #
 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
 #   turned on, on case-insensitive HFS+. There has been a reasonable amount of
 #   user testing under normal loads.
 #
 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
 #   very little testing has been done.
 #
 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
 #
 # Known Issues
 # ============
 #
 # * fsmonitor will disable itself if any of the following extensions are
 #   enabled: largefiles, inotify, eol; or if the repository has subrepos.
 # * fsmonitor will produce incorrect results if nested repos that are not
 #   subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
 #
 # The issues related to nested repos and subrepos are probably not fundamental
 # ones. Patches to fix them are welcome.
 
 
 import codecs
 import os
 import stat
 import sys
 import tempfile
 import weakref
 
 from mercurial.i18n import _
 from mercurial.node import hex
 
 from mercurial.pycompat import open
 from mercurial import (
     context,
     encoding,
     error,
     extensions,
     localrepo,
     merge,
     pathutil,
     pycompat,
     registrar,
     scmutil,
     util,
 )
 
 # no-check-code because we're accessing private information only public in pure
 from mercurial.pure import parsers
 from mercurial import match as matchmod
 from mercurial.utils import (
     hashutil,
     stringutil,
 )
 
 from . import (
     pywatchman,
     state,
     watchmanclient,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 configitem(
     b'fsmonitor',
     b'mode',
     default=b'on',
 )
 configitem(
     b'fsmonitor',
     b'walk_on_invalidate',
     default=False,
 )
 configitem(
     b'fsmonitor',
     b'timeout',
     default=b'2',
 )
 configitem(
     b'fsmonitor',
     b'blacklistusers',
     default=list,
 )
 configitem(
     b'fsmonitor',
     b'watchman_exe',
     default=b'watchman',
 )
 configitem(
     b'fsmonitor',
     b'verbose',
     default=True,
     experimental=True,
 )
 configitem(
     b'experimental',
     b'fsmonitor.transaction_notify',
     default=False,
 )
 
 # This extension is incompatible with the following blacklisted extensions
 # and will disable itself when encountering one of these:
 _blacklist = [b'largefiles', b'eol']
 
 
 def debuginstall(ui, fm):
     fm.write(
         b"fsmonitor-watchman",
         _(b"fsmonitor checking for watchman binary... (%s)\n"),
         ui.configpath(b"fsmonitor", b"watchman_exe"),
     )
     root = tempfile.mkdtemp()
     c = watchmanclient.client(ui, root)
     err = None
     try:
         v = c.command(b"version")
         fm.write(
             b"fsmonitor-watchman-version",
             _(b" watchman binary version %s\n"),
             pycompat.bytestr(v["version"]),
         )
     except watchmanclient.Unavailable as e:
         err = stringutil.forcebytestr(e)
     fm.condwrite(
         err,
         b"fsmonitor-watchman-error",
         _(b" watchman binary missing or broken: %s\n"),
         err,
     )
     return 1 if err else 0
 
 
 def _handleunavailable(ui, state, ex):
     """Exception handler for Watchman interaction exceptions"""
     if isinstance(ex, watchmanclient.Unavailable):
         # experimental config: fsmonitor.verbose
         if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
             if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
                 ui.warn(stringutil.forcebytestr(ex) + b'\n')
         if ex.invalidate:
             state.invalidate()
         # experimental config: fsmonitor.verbose
         if ui.configbool(b'fsmonitor', b'verbose'):
             ui.log(
                 b'fsmonitor',
                 b'Watchman unavailable: %s\n',
                 stringutil.forcebytestr(ex.msg),
             )
     else:
         ui.log(
             b'fsmonitor',
             b'Watchman exception: %s\n',
             stringutil.forcebytestr(ex),
         )
 
 
 def _hashignore(ignore):
     """Calculate hash for ignore patterns and filenames
 
     If this information changes between Mercurial invocations, we can't
     rely on Watchman information anymore and have to re-scan the working
     copy.
 
     """
     sha1 = hashutil.sha1()
     sha1.update(pycompat.byterepr(ignore))
     return pycompat.sysbytes(sha1.hexdigest())
 
 
 _watchmanencoding = pywatchman.encoding.get_local_encoding()
 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
 
 
 def _watchmantofsencoding(path):
     """Fix path to match watchman and local filesystem encoding
 
     watchman's paths encoding can differ from filesystem encoding. For example,
     on Windows, it's always utf-8.
     """
     try:
         decoded = path.decode(_watchmanencoding)
     except UnicodeDecodeError as e:
         raise error.Abort(
             stringutil.forcebytestr(e), hint=b'watchman encoding error'
         )
 
     try:
         encoded = decoded.encode(_fsencoding, 'strict')
     except UnicodeEncodeError as e:
         raise error.Abort(stringutil.forcebytestr(e))
 
     return encoded
 
 
 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
     """Replacement for dirstate.walk, hooking into Watchman.
 
     Whenever full is False, ignored is False, and the Watchman client is
     available, use Watchman combined with saved state to possibly return only a
     subset of files."""
 
     def bail(reason):
         self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
         return orig(match, subrepos, unknown, ignored, full=True)
 
     if full:
         return bail(b'full rewalk requested')
     if ignored:
         return bail(b'listing ignored files')
     if not self._watchmanclient.available():
         return bail(b'client unavailable')
     state = self._fsmonitorstate
     clock, ignorehash, notefiles = state.get()
     if not clock:
         if state.walk_on_invalidate:
             return bail(b'no clock')
         # Initial NULL clock value, see
         # https://facebook.github.io/watchman/docs/clockspec.html
         clock = b'c:0:0'
         notefiles = []
 
     ignore = self._ignore
     dirignore = self._dirignore
     if unknown:
         if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
             # ignore list changed -- can't rely on Watchman state any more
             if state.walk_on_invalidate:
                 return bail(b'ignore rules changed')
             notefiles = []
             clock = b'c:0:0'
     else:
         # always ignore
         ignore = util.always
         dirignore = util.always
 
     matchfn = match.matchfn
     matchalways = match.always()
     dmap = self._map
-    if util.safehasattr(dmap, b'_map'):
+    if hasattr(dmap, b'_map'):
         # for better performance, directly access the inner dirstate map if the
         # standard dirstate implementation is in use.
         dmap = dmap._map
 
     has_mtime = parsers.DIRSTATE_V2_HAS_MTIME
     mtime_is_ambiguous = parsers.DIRSTATE_V2_MTIME_SECOND_AMBIGUOUS
     mask = has_mtime | mtime_is_ambiguous
 
     # All entries that may not be clean
     nonnormalset = {
         f
         for f, e in self._map.items()
         if not e.maybe_clean
         # same as "not has_time or has_ambiguous_time", but factored to only
         # need a single access to flags for performance.
         # `mask` removes all irrelevant bits, then we flip the `mtime` bit so
         # its `true` value is NOT having a mtime, then check if either bit
         # is set.
         or bool((e.v2_data()[0] & mask) ^ has_mtime)
     }
 
     copymap = self._map.copymap
     getkind = stat.S_IFMT
     dirkind = stat.S_IFDIR
     regkind = stat.S_IFREG
     lnkkind = stat.S_IFLNK
     join = self._join
     normcase = util.normcase
     fresh_instance = False
 
     exact = skipstep3 = False
     if match.isexact():  # match.exact
         exact = True
         dirignore = util.always  # skip step 2
     elif match.prefix():  # match.match, no patterns
         skipstep3 = True
 
     if not exact and self._checkcase:
         # note that even though we could receive directory entries, we're only
         # interested in checking if a file with the same name exists. So only
         # normalize files if possible.
         normalize = self._normalizefile
         skipstep3 = False
     else:
         normalize = None
 
     # step 1: find all explicit files
     results, work, dirsnotfound = self._walkexplicit(match, subrepos)
 
     skipstep3 = skipstep3 and not (work or dirsnotfound)
     work = [d for d in work if not dirignore(d[0])]
 
     if not work and (exact or skipstep3):
         for s in subrepos:
             del results[s]
         del results[b'.hg']
         return results
 
     # step 2: query Watchman
     try:
         # Use the user-configured timeout for the query.
         # Add a little slack over the top of the user query to allow for
         # overheads while transferring the data
         self._watchmanclient.settimeout(state.timeout + 0.1)
         result = self._watchmanclient.command(
             b'query',
             {
                 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
                 b'since': clock,
                 b'expression': [
                     b'not',
                     [
                         b'anyof',
                         [b'dirname', b'.hg'],
                         [b'name', b'.hg', b'wholename'],
                     ],
                 ],
                 b'sync_timeout': int(state.timeout * 1000),
                 b'empty_on_fresh_instance': state.walk_on_invalidate,
             },
         )
     except Exception as ex:
         _handleunavailable(self._ui, state, ex)
         self._watchmanclient.clearconnection()
         return bail(b'exception during run')
     else:
         # We need to propagate the last observed clock up so that we
         # can use it for our next query
         state.setlastclock(pycompat.sysbytes(result[b'clock']))
         if result[b'is_fresh_instance']:
             if state.walk_on_invalidate:
                 state.invalidate()
                 return bail(b'fresh instance')
             fresh_instance = True
             # Ignore any prior noteable files from the state info
             notefiles = []
 
     # for file paths which require normalization and we encounter a case
     # collision, we store our own foldmap
     if normalize:
         foldmap = {normcase(k): k for k in results}
 
     switch_slashes = pycompat.ossep == b'\\'
     # The order of the results is, strictly speaking, undefined.
     # For case changes on a case insensitive filesystem we may receive
     # two entries, one with exists=True and another with exists=False.
     # The exists=True entries in the same response should be interpreted
     # as being happens-after the exists=False entries due to the way that
     # Watchman tracks files.  We use this property to reconcile deletes
     # for name case changes.
     for entry in result[b'files']:
         fname = entry[b'name']
 
         # Watchman always give us a str. Normalize to bytes on Python 3
         # using Watchman's encoding, if needed.
         if not isinstance(fname, bytes):
             fname = fname.encode(_watchmanencoding)
 
         if _fixencoding:
             fname = _watchmantofsencoding(fname)
 
         if switch_slashes:
             fname = fname.replace(b'\\', b'/')
         if normalize:
             normed = normcase(fname)
             fname = normalize(fname, True, True)
             foldmap[normed] = fname
         fmode = entry[b'mode']
         fexists = entry[b'exists']
         kind = getkind(fmode)
 
         if b'/.hg/' in fname or fname.endswith(b'/.hg'):
             return bail(b'nested-repo-detected')
 
         if not fexists:
             # if marked as deleted and we don't already have a change
             # record, mark it as deleted.  If we already have an entry
             # for fname then it was either part of walkexplicit or was
             # an earlier result that was a case change
             if (
                 fname not in results
                 and fname in dmap
                 and (matchalways or matchfn(fname))
             ):
                 results[fname] = None
         elif kind == dirkind:
             if fname in dmap and (matchalways or matchfn(fname)):
                 results[fname] = None
         elif kind == regkind or kind == lnkkind:
             if fname in dmap:
                 if matchalways or matchfn(fname):
                     results[fname] = entry
             elif (matchalways or matchfn(fname)) and not ignore(fname):
                 results[fname] = entry
         elif fname in dmap and (matchalways or matchfn(fname)):
             results[fname] = None
 
     # step 3: query notable files we don't already know about
     # XXX try not to iterate over the entire dmap
     if normalize:
         # any notable files that have changed case will already be handled
         # above, so just check membership in the foldmap
         notefiles = {
             normalize(f, True, True)
             for f in notefiles
             if normcase(f) not in foldmap
         }
     visit = {
         f
         for f in notefiles
         if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
     }
 
     if not fresh_instance:
         if matchalways:
             visit.update(f for f in nonnormalset if f not in results)
             visit.update(f for f in copymap if f not in results)
         else:
             visit.update(
                 f for f in nonnormalset if f not in results and matchfn(f)
             )
             visit.update(f for f in copymap if f not in results and matchfn(f))
     else:
         if matchalways:
             visit.update(f for f, st in dmap.items() if f not in results)
             visit.update(f for f in copymap if f not in results)
         else:
             visit.update(
                 f for f, st in dmap.items() if f not in results and matchfn(f)
             )
             visit.update(f for f in copymap if f not in results and matchfn(f))
 
     audit = pathutil.pathauditor(self._root, cached=True).check
     auditpass = [f for f in visit if audit(f)]
     auditpass.sort()
     auditfail = visit.difference(auditpass)
     for f in auditfail:
         results[f] = None
 
     nf = iter(auditpass)
     for st in util.statfiles([join(f) for f in auditpass]):
         f = next(nf)
         if st or f in dmap:
             results[f] = st
 
     for s in subrepos:
         del results[s]
     del results[b'.hg']
     return results
 
 
 def overridestatus(
     orig,
     self,
     node1=b'.',
     node2=None,
     match=None,
     ignored=False,
     clean=False,
     unknown=False,
     listsubrepos=False,
 ):
     listignored = ignored
     listclean = clean
     listunknown = unknown
 
     def _cmpsets(l1, l2):
         try:
             if b'FSMONITOR_LOG_FILE' in encoding.environ:
                 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
                 f = open(fn, b'wb')
             else:
                 fn = b'fsmonitorfail.log'
                 f = self.vfs.open(fn, b'wb')
         except (IOError, OSError):
             self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
             return
 
         try:
             for i, (s1, s2) in enumerate(zip(l1, l2)):
                 if set(s1) != set(s2):
                     f.write(b'sets at position %d are unequal\n' % i)
                     f.write(b'watchman returned: %r\n' % s1)
                     f.write(b'stat returned: %r\n' % s2)
         finally:
             f.close()
 
     if isinstance(node1, context.changectx):
         ctx1 = node1
     else:
         ctx1 = self[node1]
     if isinstance(node2, context.changectx):
         ctx2 = node2
     else:
         ctx2 = self[node2]
 
     working = ctx2.rev() is None
     parentworking = working and ctx1 == self[b'.']
     match = match or matchmod.always()
 
     # Maybe we can use this opportunity to update Watchman's state.
     # Mercurial uses workingcommitctx and/or memctx to represent the part of
     # the workingctx that is to be committed. So don't update the state in
     # that case.
     # HG_PENDING is set in the environment when the dirstate is being updated
     # in the middle of a transaction; we must not update our state in that
     # case, or we risk forgetting about changes in the working copy.
     updatestate = (
         parentworking
         and match.always()
         and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
         and b'HG_PENDING' not in encoding.environ
     )
 
     try:
         if self._fsmonitorstate.walk_on_invalidate:
             # Use a short timeout to query the current clock.  If that
             # takes too long then we assume that the service will be slow
             # to answer our query.
             # walk_on_invalidate indicates that we prefer to walk the
             # tree ourselves because we can ignore portions that Watchman
             # cannot and we tend to be faster in the warmer buffer cache
             # cases.
             self._watchmanclient.settimeout(0.1)
         else:
             # Give Watchman more time to potentially complete its walk
             # and return the initial clock.  In this mode we assume that
             # the filesystem will be slower than parsing a potentially
             # very large Watchman result set.
             self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
         startclock = self._watchmanclient.getcurrentclock()
     except Exception as ex:
         self._watchmanclient.clearconnection()
         _handleunavailable(self.ui, self._fsmonitorstate, ex)
         # boo, Watchman failed. bail
         return orig(
             node1,
             node2,
             match,
             listignored,
             listclean,
             listunknown,
             listsubrepos,
         )
 
     if updatestate:
         # We need info about unknown files. This may make things slower the
         # first time, but whatever.
         stateunknown = True
     else:
         stateunknown = listunknown
 
     if updatestate:
         ps = poststatus(startclock)
         self.addpostdsstatus(ps)
 
     r = orig(
         node1, node2, match, listignored, listclean, stateunknown, listsubrepos
     )
     modified, added, removed, deleted, unknown, ignored, clean = r
 
     if not listunknown:
         unknown = []
 
     # don't do paranoid checks if we're not going to query Watchman anyway
     full = listclean or match.traversedir is not None
     if self._fsmonitorstate.mode == b'paranoid' and not full:
         # run status again and fall back to the old walk this time
         self.dirstate._fsmonitordisable = True
 
         # shut the UI up
         quiet = self.ui.quiet
         self.ui.quiet = True
         fout, ferr = self.ui.fout, self.ui.ferr
         self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
 
         try:
             rv2 = orig(
                 node1,
                 node2,
                 match,
                 listignored,
                 listclean,
                 listunknown,
                 listsubrepos,
             )
         finally:
             self.dirstate._fsmonitordisable = False
             self.ui.quiet = quiet
             self.ui.fout, self.ui.ferr = fout, ferr
 
         # clean isn't tested since it's set to True above
         with self.wlock():
             _cmpsets(
                 [modified, added, removed, deleted, unknown, ignored, clean],
                 rv2,
             )
         modified, added, removed, deleted, unknown, ignored, clean = rv2
 
     return scmutil.status(
         modified, added, removed, deleted, unknown, ignored, clean
     )
 
 
 class poststatus:
     def __init__(self, startclock):
         self._startclock = pycompat.sysbytes(startclock)
 
     def __call__(self, wctx, status):
         clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
         hashignore = _hashignore(wctx.repo().dirstate._ignore)
         notefiles = (
             status.modified
             + status.added
             + status.removed
             + status.deleted
             + status.unknown
         )
         wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
 
 
 def makedirstate(repo, dirstate):
     class fsmonitordirstate(dirstate.__class__):
         def _fsmonitorinit(self, repo):
             # _fsmonitordisable is used in paranoid mode
             self._fsmonitordisable = False
             self._fsmonitorstate = repo._fsmonitorstate
             self._watchmanclient = repo._watchmanclient
             self._repo = weakref.proxy(repo)
 
         def walk(self, *args, **kwargs):
             orig = super(fsmonitordirstate, self).walk
             if self._fsmonitordisable:
                 return orig(*args, **kwargs)
             return overridewalk(orig, self, *args, **kwargs)
 
         def rebuild(self, *args, **kwargs):
             self._fsmonitorstate.invalidate()
             return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
 
         def invalidate(self, *args, **kwargs):
             self._fsmonitorstate.invalidate()
             return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
 
     dirstate.__class__ = fsmonitordirstate
     dirstate._fsmonitorinit(repo)
 
 
 def wrapdirstate(orig, self):
     ds = orig(self)
     # only override the dirstate when Watchman is available for the repo
-    if util.safehasattr(self, b'_fsmonitorstate'):
+    if hasattr(self, b'_fsmonitorstate'):
         makedirstate(self, ds)
     return ds
 
 
 def extsetup(ui):
     extensions.wrapfilecache(
         localrepo.localrepository, b'dirstate', wrapdirstate
     )
     if pycompat.isdarwin:
         # An assist for avoiding the dangling-symlink fsevents bug
         extensions.wrapfunction(os, 'symlink', wrapsymlink)
 
     extensions.wrapfunction(merge, '_update', wrapupdate)
 
 
 def wrapsymlink(orig, source, link_name):
     """if we create a dangling symlink, also touch the parent dir
     to encourage fsevents notifications to work more correctly"""
     try:
         return orig(source, link_name)
     finally:
         try:
             os.utime(os.path.dirname(link_name), None)
         except OSError:
             pass
 
 
 class state_update:
     """This context manager is responsible for dispatching the state-enter
     and state-leave signals to the watchman service. The enter and leave
     methods can be invoked manually (for scenarios where context manager
     semantics are not possible). If parameters oldnode and newnode are None,
     they will be populated based on current working copy in enter and
     leave, respectively. Similarly, if the distance is none, it will be
     calculated based on the oldnode and newnode in the leave method."""
 
     def __init__(
         self,
         repo,
         name,
         oldnode=None,
         newnode=None,
         distance=None,
         partial=False,
     ):
         self.repo = repo.unfiltered()
         self.name = name
         self.oldnode = oldnode
         self.newnode = newnode
         self.distance = distance
         self.partial = partial
         self._lock = None
         self.need_leave = False
 
     def __enter__(self):
         self.enter()
 
     def enter(self):
         # Make sure we have a wlock prior to sending notifications to watchman.
         # We don't want to race with other actors. In the update case,
         # merge.update is going to take the wlock almost immediately. We are
         # effectively extending the lock around several short sanity checks.
         if self.oldnode is None:
             self.oldnode = self.repo[b'.'].node()
 
         if self.repo.currentwlock() is None:
-            if util.safehasattr(self.repo, b'wlocknostateupdate'):
+            if hasattr(self.repo, b'wlocknostateupdate'):
                 self._lock = self.repo.wlocknostateupdate()
             else:
                 self._lock = self.repo.wlock()
         self.need_leave = self._state(b'state-enter', hex(self.oldnode))
         return self
 
     def __exit__(self, type_, value, tb):
         abort = True if type_ else False
         self.exit(abort=abort)
 
     def exit(self, abort=False):
         try:
             if self.need_leave:
                 status = b'failed' if abort else b'ok'
                 if self.newnode is None:
                     self.newnode = self.repo[b'.'].node()
                 if self.distance is None:
                     self.distance = calcdistance(
                         self.repo, self.oldnode, self.newnode
                     )
                 self._state(b'state-leave', hex(self.newnode), status=status)
         finally:
             self.need_leave = False
             if self._lock:
                 self._lock.release()
 
     def _state(self, cmd, commithash, status=b'ok'):
-        if not util.safehasattr(self.repo, b'_watchmanclient'):
+        if not hasattr(self.repo, b'_watchmanclient'):
             return False
         try:
             self.repo._watchmanclient.command(
                 cmd,
                 {
                     b'name': self.name,
                     b'metadata': {
                         # the target revision
                         b'rev': commithash,
                         # approximate number of commits between current and target
                         b'distance': self.distance if self.distance else 0,
                         # success/failure (only really meaningful for state-leave)
                         b'status': status,
                         # whether the working copy parent is changing
                         b'partial': self.partial,
                     },
                 },
             )
             return True
         except Exception as e:
             # Swallow any errors; fire and forget
             self.repo.ui.log(
                 b'watchman', b'Exception %s while running %s\n', e, cmd
             )
             return False
 
 
 # Estimate the distance between two nodes
 def calcdistance(repo, oldnode, newnode):
     anc = repo.changelog.ancestor(oldnode, newnode)
     ancrev = repo[anc].rev()
     distance = abs(repo[oldnode].rev() - ancrev) + abs(
         repo[newnode].rev() - ancrev
     )
     return distance
 
 
 # Bracket working copy updates with calls to the watchman state-enter
 # and state-leave commands.  This allows clients to perform more intelligent
 # settling during bulk file change scenarios
 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
 def wrapupdate(
     orig,
     repo,
     node,
     branchmerge,
     force,
     ancestor=None,
     mergeancestor=False,
     labels=None,
     matcher=None,
     **kwargs
 ):
 
     distance = 0
     partial = True
     oldnode = repo[b'.'].node()
     newnode = repo[node].node()
     if matcher is None or matcher.always():
         partial = False
         distance = calcdistance(repo.unfiltered(), oldnode, newnode)
 
     with state_update(
         repo,
         name=b"hg.update",
         oldnode=oldnode,
         newnode=newnode,
         distance=distance,
         partial=partial,
     ):
         return orig(
             repo,
             node,
             branchmerge,
             force,
             ancestor,
             mergeancestor,
             labels,
             matcher,
             **kwargs
         )
 
 
 def repo_has_depth_one_nested_repo(repo):
     for f in repo.wvfs.listdir():
         if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
             msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
             repo.ui.debug(msg % f)
             return True
     return False
 
 
 def reposetup(ui, repo):
     # We don't work with largefiles or inotify
     exts = extensions.enabled()
     for ext in _blacklist:
         if ext in exts:
             ui.warn(
                 _(
                     b'The fsmonitor extension is incompatible with the %s '
                     b'extension and has been disabled.\n'
                 )
                 % ext
             )
             return
 
     if repo.local():
         # We don't work with subrepos either.
         #
         # if repo[None].substate can cause a dirstate parse, which is too
         # slow. Instead, look for a file called hgsubstate,
         if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
             return
 
         if repo_has_depth_one_nested_repo(repo):
             return
 
         fsmonitorstate = state.state(repo)
         if fsmonitorstate.mode == b'off':
             return
 
         try:
             client = watchmanclient.client(repo.ui, repo.root)
         except Exception as ex:
             _handleunavailable(ui, fsmonitorstate, ex)
             return
 
         repo._fsmonitorstate = fsmonitorstate
         repo._watchmanclient = client
 
         dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
         if cached:
             # at this point since fsmonitorstate wasn't present,
             # repo.dirstate is not a fsmonitordirstate
             makedirstate(repo, dirstate)
 
         class fsmonitorrepo(repo.__class__):
             def status(self, *args, **kwargs):
                 orig = super(fsmonitorrepo, self).status
                 return overridestatus(orig, self, *args, **kwargs)
 
             def wlocknostateupdate(self, *args, **kwargs):
                 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
 
             def wlock(self, *args, **kwargs):
                 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
                 if not ui.configbool(
                     b"experimental", b"fsmonitor.transaction_notify"
                 ):
                     return l
                 if l.held != 1:
                     return l
                 origrelease = l.releasefn
 
                 def staterelease():
                     if origrelease:
                         origrelease()
                     if l.stateupdate:
                         l.stateupdate.exit()
                         l.stateupdate = None
 
                 try:
                     l.stateupdate = None
                     l.stateupdate = state_update(self, name=b"hg.transaction")
                     l.stateupdate.enter()
                     l.releasefn = staterelease
                 except Exception as e:
                     # Swallow any errors; fire and forget
                     self.ui.log(
                         b'watchman', b'Exception in state update %s\n', e
                     )
                 return l
 
         repo.__class__ = fsmonitorrepo

hgext/fsmonitor/watchmanclient.py

@@ -1,128 +1,128 b''
 # watchmanclient.py - Watchman client for the fsmonitor extension
 #
 # Copyright 2013-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import getpass
 
 from mercurial import (
     encoding,
     util,
 )
 from mercurial.utils import (
     procutil,
     stringutil,
 )
 
 from . import pywatchman
 
 
 class Unavailable(Exception):
     def __init__(self, msg, warn=True, invalidate=False):
         self.msg = msg
         self.warn = warn
         if self.msg == b'timed out waiting for response':
             self.warn = False
         self.invalidate = invalidate
 
     def __bytes__(self):
         if self.warn:
             return b'warning: Watchman unavailable: %s' % self.msg
         else:
             return b'Watchman unavailable: %s' % self.msg
 
     __str__ = encoding.strmethod(__bytes__)
 
 
 class WatchmanNoRoot(Unavailable):
     def __init__(self, root, msg):
         self.root = root
         super(WatchmanNoRoot, self).__init__(msg)
 
 
 class client:
     def __init__(self, ui, root, timeout=1.0):
         err = None
         if not self._user:
             err = b"couldn't get user"
             warn = True
         if self._user in ui.configlist(b'fsmonitor', b'blacklistusers'):
             err = b'user %s in blacklist' % self._user
             warn = False
 
         if err:
             raise Unavailable(err, warn)
 
         self._timeout = timeout
         self._watchmanclient = None
         self._root = root
         self._ui = ui
         self._firsttime = True
 
     def settimeout(self, timeout):
         self._timeout = timeout
         if self._watchmanclient is not None:
             self._watchmanclient.setTimeout(timeout)
 
     def getcurrentclock(self):
         result = self.command(b'clock')
-        if not util.safehasattr(result, 'clock'):
+        if not hasattr(result, 'clock'):
             raise Unavailable(
                 b'clock result is missing clock value', invalidate=True
             )
         return result.clock
 
     def clearconnection(self):
         self._watchmanclient = None
 
     def available(self):
         return self._watchmanclient is not None or self._firsttime
 
     @util.propertycache
     def _user(self):
         try:
             return getpass.getuser()
         except KeyError:
             # couldn't figure out our user
             return None
 
     def _command(self, *args):
         watchmanargs = (args[0], self._root) + args[1:]
         try:
             if self._watchmanclient is None:
                 self._firsttime = False
                 watchman_exe = self._ui.configpath(
                     b'fsmonitor', b'watchman_exe'
                 )
                 self._watchmanclient = pywatchman.client(
                     timeout=self._timeout,
                     useImmutableBser=True,
                     binpath=procutil.tonativestr(watchman_exe),
                 )
             return self._watchmanclient.query(*watchmanargs)
         except pywatchman.CommandError as ex:
             if 'unable to resolve root' in ex.msg:
                 raise WatchmanNoRoot(
                     self._root, stringutil.forcebytestr(ex.msg)
                 )
             raise Unavailable(stringutil.forcebytestr(ex.msg))
         except pywatchman.WatchmanError as ex:
             raise Unavailable(stringutil.forcebytestr(ex))
 
     def command(self, *args):
         try:
             try:
                 return self._command(*args)
             except WatchmanNoRoot:
                 # this 'watch' command can also raise a WatchmanNoRoot if
                 # watchman refuses to accept this root
                 self._command(b'watch')
                 return self._command(*args)
         except Unavailable:
             # this is in an outer scope to catch Unavailable form any of the
             # above _command calls
             self._watchmanclient = None
             raise

hgext/journal.py

@@ -1,610 +1,606 b''
 # journal.py
 #
 # Copyright 2014-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """track previous positions of bookmarks (EXPERIMENTAL)
 
 This extension adds a new command: `hg journal`, which shows you where
 bookmarks were previously located.
 
 """
 
 
 import collections
 import os
 import weakref
 
 from mercurial.i18n import _
 from mercurial.node import (
     bin,
     hex,
 )
 
 from mercurial import (
     bookmarks,
     cmdutil,
     dispatch,
     encoding,
     error,
     extensions,
     hg,
     localrepo,
     lock,
     logcmdutil,
     pycompat,
     registrar,
     util,
 )
 from mercurial.utils import (
     dateutil,
     procutil,
     stringutil,
 )
 
 cmdtable = {}
 command = registrar.command(cmdtable)
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 # storage format version; increment when the format changes
 storageversion = 0
 
 # namespaces
 bookmarktype = b'bookmark'
 wdirparenttype = b'wdirparent'
 # In a shared repository, what shared feature name is used
 # to indicate this namespace is shared with the source?
 sharednamespaces = {
     bookmarktype: hg.sharedbookmarks,
 }
 
 # Journal recording, register hooks and storage object
 def extsetup(ui):
     extensions.wrapfunction(dispatch, 'runcommand', runcommand)
     extensions.wrapfunction(bookmarks.bmstore, '_write', recordbookmarks)
     extensions.wrapfilecache(
         localrepo.localrepository, b'dirstate', wrapdirstate
     )
     extensions.wrapfunction(hg, 'postshare', wrappostshare)
     extensions.wrapfunction(hg, 'copystore', unsharejournal)
 
 
 def reposetup(ui, repo):
     if repo.local():
         repo.journal = journalstorage(repo)
         repo._wlockfreeprefix.add(b'namejournal')
 
         dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
         if cached:
             # already instantiated dirstate isn't yet marked as
             # "journal"-ing, even though repo.dirstate() was already
             # wrapped by own wrapdirstate()
             _setupdirstate(repo, dirstate)
 
 
 def runcommand(orig, lui, repo, cmd, fullargs, *args):
     """Track the command line options for recording in the journal"""
     journalstorage.recordcommand(*fullargs)
     return orig(lui, repo, cmd, fullargs, *args)
 
 
 def _setupdirstate(repo, dirstate):
     dirstate.journalstorage = repo.journal
     dirstate.addparentchangecallback(b'journal', recorddirstateparents)
 
 
 # hooks to record dirstate changes
 def wrapdirstate(orig, repo):
     """Make journal storage available to the dirstate object"""
     dirstate = orig(repo)
-    if util.safehasattr(repo, 'journal'):
+    if hasattr(repo, 'journal'):
         _setupdirstate(repo, dirstate)
     return dirstate
 
 
 def recorddirstateparents(dirstate, old, new):
     """Records all dirstate parent changes in the journal."""
     old = list(old)
     new = list(new)
-    if util.safehasattr(dirstate, 'journalstorage'):
+    if hasattr(dirstate, 'journalstorage'):
         # only record two hashes if there was a merge
         oldhashes = old[:1] if old[1] == dirstate._nodeconstants.nullid else old
         newhashes = new[:1] if new[1] == dirstate._nodeconstants.nullid else new
         dirstate.journalstorage.record(
             wdirparenttype, b'.', oldhashes, newhashes
         )
 
 
 # hooks to record bookmark changes (both local and remote)
 def recordbookmarks(orig, store, fp):
     """Records all bookmark changes in the journal."""
     repo = store._repo
-    if util.safehasattr(repo, 'journal'):
+    if hasattr(repo, 'journal'):
         oldmarks = bookmarks.bmstore(repo)
         all_marks = set(b for b, n in oldmarks.items())
         all_marks.update(b for b, n in store.items())
         for mark in sorted(all_marks):
             value = store.get(mark, repo.nullid)
             oldvalue = oldmarks.get(mark, repo.nullid)
             if value != oldvalue:
                 repo.journal.record(bookmarktype, mark, oldvalue, value)
     return orig(store, fp)
 
 
 # shared repository support
 def _readsharedfeatures(repo):
     """A set of shared features for this repository"""
     try:
         return set(repo.vfs.read(b'shared').splitlines())
     except FileNotFoundError:
         return set()
 
 
 def _mergeentriesiter(*iterables, **kwargs):
     """Given a set of sorted iterables, yield the next entry in merged order
 
     Note that by default entries go from most recent to oldest.
     """
     order = kwargs.pop('order', max)
     iterables = [iter(it) for it in iterables]
     # this tracks still active iterables; iterables are deleted as they are
     # exhausted, which is why this is a dictionary and why each entry also
     # stores the key. Entries are mutable so we can store the next value each
     # time.
     iterable_map = {}
     for key, it in enumerate(iterables):
         try:
             iterable_map[key] = [next(it), key, it]
         except StopIteration:
             # empty entry, can be ignored
             pass
 
     while iterable_map:
         value, key, it = order(iterable_map.values())
         yield value
         try:
             iterable_map[key][0] = next(it)
         except StopIteration:
             # this iterable is empty, remove it from consideration
             del iterable_map[key]
 
 
 def wrappostshare(orig, sourcerepo, destrepo, **kwargs):
     """Mark this shared working copy as sharing journal information"""
     with destrepo.wlock():
         orig(sourcerepo, destrepo, **kwargs)
         with destrepo.vfs(b'shared', b'a') as fp:
             fp.write(b'journal\n')
 
 
 def unsharejournal(orig, ui, repo, repopath):
     """Copy shared journal entries into this repo when unsharing"""
-    if (
-        repo.path == repopath
-        and repo.shared()
-        and util.safehasattr(repo, 'journal')
-    ):
+    if repo.path == repopath and repo.shared() and hasattr(repo, 'journal'):
         sharedrepo = hg.sharedreposource(repo)
         sharedfeatures = _readsharedfeatures(repo)
         if sharedrepo and sharedfeatures > {b'journal'}:
             # there is a shared repository and there are shared journal entries
             # to copy. move shared date over from source to destination but
             # move the local file first
             if repo.vfs.exists(b'namejournal'):
                 journalpath = repo.vfs.join(b'namejournal')
                 util.rename(journalpath, journalpath + b'.bak')
             storage = repo.journal
             local = storage._open(
                 repo.vfs, filename=b'namejournal.bak', _newestfirst=False
             )
             shared = (
                 e
                 for e in storage._open(sharedrepo.vfs, _newestfirst=False)
                 if sharednamespaces.get(e.namespace) in sharedfeatures
             )
             for entry in _mergeentriesiter(local, shared, order=min):
                 storage._write(repo.vfs, entry)
 
     return orig(ui, repo, repopath)
 
 
 class journalentry(
     collections.namedtuple(
         'journalentry',
         'timestamp user command namespace name oldhashes newhashes',
     )
 ):
     """Individual journal entry
 
     * timestamp: a mercurial (time, timezone) tuple
     * user: the username that ran the command
     * namespace: the entry namespace, an opaque string
     * name: the name of the changed item, opaque string with meaning in the
       namespace
     * command: the hg command that triggered this record
     * oldhashes: a tuple of one or more binary hashes for the old location
     * newhashes: a tuple of one or more binary hashes for the new location
 
     Handles serialisation from and to the storage format. Fields are
     separated by newlines, hashes are written out in hex separated by commas,
     timestamp and timezone are separated by a space.
 
     """
 
     @classmethod
     def fromstorage(cls, line):
         (
             time,
             user,
             command,
             namespace,
             name,
             oldhashes,
             newhashes,
         ) = line.split(b'\n')
         timestamp, tz = time.split()
         timestamp, tz = float(timestamp), int(tz)
         oldhashes = tuple(bin(hash) for hash in oldhashes.split(b','))
         newhashes = tuple(bin(hash) for hash in newhashes.split(b','))
         return cls(
             (timestamp, tz),
             user,
             command,
             namespace,
             name,
             oldhashes,
             newhashes,
         )
 
     def __bytes__(self):
         """bytes representation for storage"""
         time = b' '.join(map(pycompat.bytestr, self.timestamp))
         oldhashes = b','.join([hex(hash) for hash in self.oldhashes])
         newhashes = b','.join([hex(hash) for hash in self.newhashes])
         return b'\n'.join(
             (
                 time,
                 self.user,
                 self.command,
                 self.namespace,
                 self.name,
                 oldhashes,
                 newhashes,
             )
         )
 
     __str__ = encoding.strmethod(__bytes__)
 
 
 class journalstorage:
     """Storage for journal entries
 
     Entries are divided over two files; one with entries that pertain to the
     local working copy *only*, and one with entries that are shared across
     multiple working copies when shared using the share extension.
 
     Entries are stored with NUL bytes as separators. See the journalentry
     class for the per-entry structure.
 
     The file format starts with an integer version, delimited by a NUL.
 
     This storage uses a dedicated lock; this makes it easier to avoid issues
     with adding entries that added when the regular wlock is unlocked (e.g.
     the dirstate).
 
     """
 
     _currentcommand = ()
     _lockref = None
 
     def __init__(self, repo):
         self.user = procutil.getuser()
         self.ui = repo.ui
         self.vfs = repo.vfs
 
         # is this working copy using a shared storage?
         self.sharedfeatures = self.sharedvfs = None
         if repo.shared():
             features = _readsharedfeatures(repo)
             sharedrepo = hg.sharedreposource(repo)
             if sharedrepo is not None and b'journal' in features:
                 self.sharedvfs = sharedrepo.vfs
                 self.sharedfeatures = features
 
     # track the current command for recording in journal entries
     @property
     def command(self):
         commandstr = b' '.join(
             map(procutil.shellquote, journalstorage._currentcommand)
         )
         if b'\n' in commandstr:
             # truncate multi-line commands
             commandstr = commandstr.partition(b'\n')[0] + b' ...'
         return commandstr
 
     @classmethod
     def recordcommand(cls, *fullargs):
         """Set the current hg arguments, stored with recorded entries"""
         # Set the current command on the class because we may have started
         # with a non-local repo (cloning for example).
         cls._currentcommand = fullargs
 
     def _currentlock(self, lockref):
         """Returns the lock if it's held, or None if it's not.
 
         (This is copied from the localrepo class)
         """
         if lockref is None:
             return None
         l = lockref()
         if l is None or not l.held:
             return None
         return l
 
     def jlock(self, vfs):
         """Create a lock for the journal file"""
         if self._currentlock(self._lockref) is not None:
             raise error.Abort(_(b'journal lock does not support nesting'))
         desc = _(b'journal of %s') % vfs.base
         try:
             l = lock.lock(vfs, b'namejournal.lock', 0, desc=desc)
         except error.LockHeld as inst:
             self.ui.warn(
                 _(b"waiting for lock on %s held by %r\n") % (desc, inst.locker)
             )
             # default to 600 seconds timeout
             l = lock.lock(
                 vfs,
                 b'namejournal.lock',
                 self.ui.configint(b"ui", b"timeout"),
                 desc=desc,
             )
             self.ui.warn(_(b"got lock after %s seconds\n") % l.delay)
         self._lockref = weakref.ref(l)
         return l
 
     def record(self, namespace, name, oldhashes, newhashes):
         """Record a new journal entry
 
         * namespace: an opaque string; this can be used to filter on the type
           of recorded entries.
         * name: the name defining this entry; for bookmarks, this is the
           bookmark name. Can be filtered on when retrieving entries.
         * oldhashes and newhashes: each a single binary hash, or a list of
           binary hashes. These represent the old and new position of the named
           item.
 
         """
         if not isinstance(oldhashes, list):
             oldhashes = [oldhashes]
         if not isinstance(newhashes, list):
             newhashes = [newhashes]
 
         entry = journalentry(
             dateutil.makedate(),
             self.user,
             self.command,
             namespace,
             name,
             oldhashes,
             newhashes,
         )
 
         vfs = self.vfs
         if self.sharedvfs is not None:
             # write to the shared repository if this feature is being
             # shared between working copies.
             if sharednamespaces.get(namespace) in self.sharedfeatures:
                 vfs = self.sharedvfs
 
         self._write(vfs, entry)
 
     def _write(self, vfs, entry):
         with self.jlock(vfs):
             # open file in amend mode to ensure it is created if missing
             with vfs(b'namejournal', mode=b'a+b') as f:
                 f.seek(0, os.SEEK_SET)
                 # Read just enough bytes to get a version number (up to 2
                 # digits plus separator)
                 version = f.read(3).partition(b'\0')[0]
                 if version and version != b"%d" % storageversion:
                     # different version of the storage. Exit early (and not
                     # write anything) if this is not a version we can handle or
                     # the file is corrupt. In future, perhaps rotate the file
                     # instead?
                     self.ui.warn(
                         _(b"unsupported journal file version '%s'\n") % version
                     )
                     return
                 if not version:
                     # empty file, write version first
                     f.write((b"%d" % storageversion) + b'\0')
                 f.seek(0, os.SEEK_END)
                 f.write(bytes(entry) + b'\0')
 
     def filtered(self, namespace=None, name=None):
         """Yield all journal entries with the given namespace or name
 
         Both the namespace and the name are optional; if neither is given all
         entries in the journal are produced.
 
         Matching supports regular expressions by using the `re:` prefix
         (use `literal:` to match names or namespaces that start with `re:`)
 
         """
         if namespace is not None:
             namespace = stringutil.stringmatcher(namespace)[-1]
         if name is not None:
             name = stringutil.stringmatcher(name)[-1]
         for entry in self:
             if namespace is not None and not namespace(entry.namespace):
                 continue
             if name is not None and not name(entry.name):
                 continue
             yield entry
 
     def __iter__(self):
         """Iterate over the storage
 
         Yields journalentry instances for each contained journal record.
 
         """
         local = self._open(self.vfs)
 
         if self.sharedvfs is None:
             return local
 
         # iterate over both local and shared entries, but only those
         # shared entries that are among the currently shared features
         shared = (
             e
             for e in self._open(self.sharedvfs)
             if sharednamespaces.get(e.namespace) in self.sharedfeatures
         )
         return _mergeentriesiter(local, shared)
 
     def _open(self, vfs, filename=b'namejournal', _newestfirst=True):
         if not vfs.exists(filename):
             return
 
         with vfs(filename) as f:
             raw = f.read()
 
         lines = raw.split(b'\0')
         version = lines and lines[0]
         if version != b"%d" % storageversion:
             version = version or _(b'not available')
             raise error.Abort(_(b"unknown journal file version '%s'") % version)
 
         # Skip the first line, it's a version number. Normally we iterate over
         # these in reverse order to list newest first; only when copying across
         # a shared storage do we forgo reversing.
         lines = lines[1:]
         if _newestfirst:
             lines = reversed(lines)
         for line in lines:
             if not line:
                 continue
             yield journalentry.fromstorage(line)
 
 
 # journal reading
 # log options that don't make sense for journal
 _ignoreopts = (b'no-merges', b'graph')
 
 
 @command(
     b'journal',
     [
         (b'', b'all', None, b'show history for all names'),
         (b'c', b'commits', None, b'show commit metadata'),
     ]
     + [opt for opt in cmdutil.logopts if opt[1] not in _ignoreopts],
     b'[OPTION]... [BOOKMARKNAME]',
     helpcategory=command.CATEGORY_CHANGE_ORGANIZATION,
 )
 def journal(ui, repo, *args, **opts):
     """show the previous position of bookmarks and the working copy
 
     The journal is used to see the previous commits that bookmarks and the
     working copy pointed to. By default the previous locations for the working
     copy.  Passing a bookmark name will show all the previous positions of
     that bookmark. Use the --all switch to show previous locations for all
     bookmarks and the working copy; each line will then include the bookmark
     name, or '.' for the working copy, as well.
 
     If `name` starts with `re:`, the remainder of the name is treated as
     a regular expression. To match a name that actually starts with `re:`,
     use the prefix `literal:`.
 
     By default hg journal only shows the commit hash and the command that was
     running at that time. -v/--verbose will show the prior hash, the user, and
     the time at which it happened.
 
     Use -c/--commits to output log information on each commit hash; at this
     point you can use the usual `--patch`, `--git`, `--stat` and `--template`
     switches to alter the log output for these.
 
     `hg journal -T json` can be used to produce machine readable output.
 
     """
     opts = pycompat.byteskwargs(opts)
     name = b'.'
     if opts.get(b'all'):
         if args:
             raise error.Abort(
                 _(b"You can't combine --all and filtering on a name")
             )
         name = None
     if args:
         name = args[0]
 
     fm = ui.formatter(b'journal', opts)
 
     def formatnodes(nodes):
         return fm.formatlist(map(fm.hexfunc, nodes), name=b'node', sep=b',')
 
     if opts.get(b"template") != b"json":
         if name is None:
             displayname = _(b'the working copy and bookmarks')
         else:
             displayname = b"'%s'" % name
         ui.status(_(b"previous locations of %s:\n") % displayname)
 
     limit = logcmdutil.getlimit(opts)
     entry = None
     ui.pager(b'journal')
     for count, entry in enumerate(repo.journal.filtered(name=name)):
         if count == limit:
             break
 
         fm.startitem()
         fm.condwrite(
             ui.verbose, b'oldnodes', b'%s -> ', formatnodes(entry.oldhashes)
         )
         fm.write(b'newnodes', b'%s', formatnodes(entry.newhashes))
         fm.condwrite(ui.verbose, b'user', b' %-8s', entry.user)
 
         # ``name`` is bytes, or None only if 'all' was an option.
         fm.condwrite(
             # pytype: disable=attribute-error
             opts.get(b'all') or name.startswith(b're:'),
             # pytype: enable=attribute-error
             b'name',
             b'  %-8s',
             entry.name,
         )
 
         fm.condwrite(
             ui.verbose,
             b'date',
             b' %s',
             fm.formatdate(entry.timestamp, b'%Y-%m-%d %H:%M %1%2'),
         )
         fm.write(b'command', b'  %s\n', entry.command)
 
         if opts.get(b"commits"):
             if fm.isplain():
                 displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
             else:
                 displayer = logcmdutil.changesetformatter(
                     ui, repo, fm.nested(b'changesets'), diffopts=opts
                 )
             for hash in entry.newhashes:
                 try:
                     ctx = repo[hash]
                     displayer.show(ctx)
                 except error.RepoLookupError as e:
                     fm.plain(b"%s\n\n" % pycompat.bytestr(e))
             displayer.close()
 
     fm.end()
 
     if entry is None:
         ui.status(_(b"no recorded locations\n"))

hgext/largefiles/lfutil.py

@@ -1,823 +1,823 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''largefiles utility code: must not import other modules in this package.'''
 
 import contextlib
 import copy
 import os
 import stat
 
 from mercurial.i18n import _
 from mercurial.node import hex
 from mercurial.pycompat import open
 
 from mercurial import (
     dirstate,
     encoding,
     error,
     httpconnection,
     match as matchmod,
     pycompat,
     requirements,
     scmutil,
     sparse,
     util,
     vfs as vfsmod,
 )
 from mercurial.utils import hashutil
 from mercurial.dirstateutils import timestamp
 
 shortname = b'.hglf'
 shortnameslash = shortname + b'/'
 longname = b'largefiles'
 
 # -- Private worker functions ------------------------------------------
 
 
 @contextlib.contextmanager
 def lfstatus(repo, value=True):
     oldvalue = getattr(repo, 'lfstatus', False)
     repo.lfstatus = value
     try:
         yield
     finally:
         repo.lfstatus = oldvalue
 
 
 def getminsize(ui, assumelfiles, opt, default=10):
     lfsize = opt
     if not lfsize and assumelfiles:
         lfsize = ui.config(longname, b'minsize', default=default)
     if lfsize:
         try:
             lfsize = float(lfsize)
         except ValueError:
             raise error.Abort(
                 _(b'largefiles: size must be number (not %s)\n') % lfsize
             )
     if lfsize is None:
         raise error.Abort(_(b'minimum size for largefiles must be specified'))
     return lfsize
 
 
 def link(src, dest):
     """Try to create hardlink - if that fails, efficiently make a copy."""
     util.makedirs(os.path.dirname(dest))
     try:
         util.oslink(src, dest)
     except OSError:
         # if hardlinks fail, fallback on atomic copy
         with open(src, b'rb') as srcf, util.atomictempfile(dest) as dstf:
             for chunk in util.filechunkiter(srcf):
                 dstf.write(chunk)
         os.chmod(dest, os.stat(src).st_mode)
 
 
 def usercachepath(ui, hash):
     """Return the correct location in the "global" largefiles cache for a file
     with the given hash.
     This cache is used for sharing of largefiles across repositories - both
     to preserve download bandwidth and storage space."""
     return os.path.join(_usercachedir(ui), hash)
 
 
 def _usercachedir(ui, name=longname):
     '''Return the location of the "global" largefiles cache.'''
     path = ui.configpath(name, b'usercache')
     if path:
         return path
 
     hint = None
 
     if pycompat.iswindows:
         appdata = encoding.environ.get(
             b'LOCALAPPDATA', encoding.environ.get(b'APPDATA')
         )
         if appdata:
             return os.path.join(appdata, name)
 
         hint = _(b"define %s or %s in the environment, or set %s.usercache") % (
             b"LOCALAPPDATA",
             b"APPDATA",
             name,
         )
     elif pycompat.isdarwin:
         home = encoding.environ.get(b'HOME')
         if home:
             return os.path.join(home, b'Library', b'Caches', name)
 
         hint = _(b"define %s in the environment, or set %s.usercache") % (
             b"HOME",
             name,
         )
     elif pycompat.isposix:
         path = encoding.environ.get(b'XDG_CACHE_HOME')
         if path:
             return os.path.join(path, name)
         home = encoding.environ.get(b'HOME')
         if home:
             return os.path.join(home, b'.cache', name)
 
         hint = _(b"define %s or %s in the environment, or set %s.usercache") % (
             b"XDG_CACHE_HOME",
             b"HOME",
             name,
         )
     else:
         raise error.Abort(
             _(b'unknown operating system: %s\n') % pycompat.osname
         )
 
     raise error.Abort(_(b'unknown %s usercache location') % name, hint=hint)
 
 
 def inusercache(ui, hash):
     path = usercachepath(ui, hash)
     return os.path.exists(path)
 
 
 def findfile(repo, hash):
     """Return store path of the largefile with the specified hash.
     As a side effect, the file might be linked from user cache.
     Return None if the file can't be found locally."""
     path, exists = findstorepath(repo, hash)
     if exists:
         repo.ui.note(_(b'found %s in store\n') % hash)
         return path
     elif inusercache(repo.ui, hash):
         repo.ui.note(_(b'found %s in system cache\n') % hash)
         path = storepath(repo, hash)
         link(usercachepath(repo.ui, hash), path)
         return path
     return None
 
 
 class largefilesdirstate(dirstate.dirstate):
     _large_file_dirstate = True
     _tr_key_suffix = b'-large-files'
 
     def __getitem__(self, key):
         return super(largefilesdirstate, self).__getitem__(unixpath(key))
 
     def set_tracked(self, f):
         return super(largefilesdirstate, self).set_tracked(unixpath(f))
 
     def set_untracked(self, f):
         return super(largefilesdirstate, self).set_untracked(unixpath(f))
 
     def normal(self, f, parentfiledata=None):
         # not sure if we should pass the `parentfiledata` down or throw it
         # away. So throwing it away to stay on the safe side.
         return super(largefilesdirstate, self).normal(unixpath(f))
 
     def remove(self, f):
         return super(largefilesdirstate, self).remove(unixpath(f))
 
     def add(self, f):
         return super(largefilesdirstate, self).add(unixpath(f))
 
     def drop(self, f):
         return super(largefilesdirstate, self).drop(unixpath(f))
 
     def forget(self, f):
         return super(largefilesdirstate, self).forget(unixpath(f))
 
     def normallookup(self, f):
         return super(largefilesdirstate, self).normallookup(unixpath(f))
 
     def _ignore(self, f):
         return False
 
     def write(self, tr):
         # (1) disable PENDING mode always
         #     (lfdirstate isn't yet managed as a part of the transaction)
         # (2) avoid develwarn 'use dirstate.write with ....'
         if tr:
             tr.addbackup(b'largefiles/dirstate', location=b'plain')
         super(largefilesdirstate, self).write(None)
 
 
 def openlfdirstate(ui, repo, create=True):
     """
     Return a dirstate object that tracks largefiles: i.e. its root is
     the repo root, but it is saved in .hg/largefiles/dirstate.
 
     If a dirstate object already exists and is being used for a 'changing_*'
     context, it will be returned.
     """
     sub_dirstate = getattr(repo.dirstate, '_sub_dirstate', None)
     if sub_dirstate is not None:
         return sub_dirstate
     vfs = repo.vfs
     lfstoredir = longname
     opener = vfsmod.vfs(vfs.join(lfstoredir))
     use_dirstate_v2 = requirements.DIRSTATE_V2_REQUIREMENT in repo.requirements
     lfdirstate = largefilesdirstate(
         opener,
         ui,
         repo.root,
         repo.dirstate._validate,
         lambda: sparse.matcher(repo),
         repo.nodeconstants,
         use_dirstate_v2,
     )
 
     # If the largefiles dirstate does not exist, populate and create
     # it. This ensures that we create it on the first meaningful
     # largefiles operation in a new clone.
     if create and not vfs.exists(vfs.join(lfstoredir, b'dirstate')):
         try:
             with repo.wlock(wait=False), lfdirstate.changing_files(repo):
                 matcher = getstandinmatcher(repo)
                 standins = repo.dirstate.walk(
                     matcher, subrepos=[], unknown=False, ignored=False
                 )
 
                 if len(standins) > 0:
                     vfs.makedirs(lfstoredir)
 
                 for standin in standins:
                     lfile = splitstandin(standin)
                     lfdirstate.hacky_extension_update_file(
                         lfile,
                         p1_tracked=True,
                         wc_tracked=True,
                         possibly_dirty=True,
                     )
         except error.LockError:
             # Assume that whatever was holding the lock was important.
             # If we were doing something important, we would already have
             # either the lock or a largefile dirstate.
             pass
     return lfdirstate
 
 
 def lfdirstatestatus(lfdirstate, repo):
     pctx = repo[b'.']
     match = matchmod.always()
     unsure, s, mtime_boundary = lfdirstate.status(
         match, subrepos=[], ignored=False, clean=False, unknown=False
     )
     modified, clean = s.modified, s.clean
     wctx = repo[None]
     for lfile in unsure:
         try:
             fctx = pctx[standin(lfile)]
         except LookupError:
             fctx = None
         if not fctx or readasstandin(fctx) != hashfile(repo.wjoin(lfile)):
             modified.append(lfile)
         else:
             clean.append(lfile)
             st = wctx[lfile].lstat()
             mode = st.st_mode
             size = st.st_size
             mtime = timestamp.reliable_mtime_of(st, mtime_boundary)
             if mtime is not None:
                 cache_data = (mode, size, mtime)
                 lfdirstate.set_clean(lfile, cache_data)
     return s
 
 
 def listlfiles(repo, rev=None, matcher=None):
     """return a list of largefiles in the working copy or the
     specified changeset"""
 
     if matcher is None:
         matcher = getstandinmatcher(repo)
 
     # ignore unknown files in working directory
     return [
         splitstandin(f)
         for f in repo[rev].walk(matcher)
         if rev is not None or repo.dirstate.get_entry(f).any_tracked
     ]
 
 
 def instore(repo, hash, forcelocal=False):
     '''Return true if a largefile with the given hash exists in the store'''
     return os.path.exists(storepath(repo, hash, forcelocal))
 
 
 def storepath(repo, hash, forcelocal=False):
     """Return the correct location in the repository largefiles store for a
     file with the given hash."""
     if not forcelocal and repo.shared():
         return repo.vfs.reljoin(repo.sharedpath, longname, hash)
     return repo.vfs.join(longname, hash)
 
 
 def findstorepath(repo, hash):
     """Search through the local store path(s) to find the file for the given
     hash.  If the file is not found, its path in the primary store is returned.
     The return value is a tuple of (path, exists(path)).
     """
     # For shared repos, the primary store is in the share source.  But for
     # backward compatibility, force a lookup in the local store if it wasn't
     # found in the share source.
     path = storepath(repo, hash, False)
 
     if instore(repo, hash):
         return (path, True)
     elif repo.shared() and instore(repo, hash, True):
         return storepath(repo, hash, True), True
 
     return (path, False)
 
 
 def copyfromcache(repo, hash, filename):
     """Copy the specified largefile from the repo or system cache to
     filename in the repository. Return true on success or false if the
     file was not found in either cache (which should not happened:
     this is meant to be called only after ensuring that the needed
     largefile exists in the cache)."""
     wvfs = repo.wvfs
     path = findfile(repo, hash)
     if path is None:
         return False
     wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
     # The write may fail before the file is fully written, but we
     # don't use atomic writes in the working copy.
     with open(path, b'rb') as srcfd, wvfs(filename, b'wb') as destfd:
         gothash = copyandhash(util.filechunkiter(srcfd), destfd)
     if gothash != hash:
         repo.ui.warn(
             _(b'%s: data corruption in %s with hash %s\n')
             % (filename, path, gothash)
         )
         wvfs.unlink(filename)
         return False
     return True
 
 
 def copytostore(repo, ctx, file, fstandin):
     wvfs = repo.wvfs
     hash = readasstandin(ctx[fstandin])
     if instore(repo, hash):
         return
     if wvfs.exists(file):
         copytostoreabsolute(repo, wvfs.join(file), hash)
     else:
         repo.ui.warn(
             _(b"%s: largefile %s not available from local store\n")
             % (file, hash)
         )
 
 
 def copyalltostore(repo, node):
     '''Copy all largefiles in a given revision to the store'''
 
     ctx = repo[node]
     for filename in ctx.files():
         realfile = splitstandin(filename)
         if realfile is not None and filename in ctx.manifest():
             copytostore(repo, ctx, realfile, filename)
 
 
 def copytostoreabsolute(repo, file, hash):
     if inusercache(repo.ui, hash):
         link(usercachepath(repo.ui, hash), storepath(repo, hash))
     else:
         util.makedirs(os.path.dirname(storepath(repo, hash)))
         with open(file, b'rb') as srcf:
             with util.atomictempfile(
                 storepath(repo, hash), createmode=repo.store.createmode
             ) as dstf:
                 for chunk in util.filechunkiter(srcf):
                     dstf.write(chunk)
         linktousercache(repo, hash)
 
 
 def linktousercache(repo, hash):
     """Link / copy the largefile with the specified hash from the store
     to the cache."""
     path = usercachepath(repo.ui, hash)
     link(storepath(repo, hash), path)
 
 
 def getstandinmatcher(repo, rmatcher=None):
     '''Return a match object that applies rmatcher to the standin directory'''
     wvfs = repo.wvfs
     standindir = shortname
 
     # no warnings about missing files or directories
     badfn = lambda f, msg: None
 
     if rmatcher and not rmatcher.always():
         pats = [wvfs.join(standindir, pat) for pat in rmatcher.files()]
         if not pats:
             pats = [wvfs.join(standindir)]
         match = scmutil.match(repo[None], pats, badfn=badfn)
     else:
         # no patterns: relative to repo root
         match = scmutil.match(repo[None], [wvfs.join(standindir)], badfn=badfn)
     return match
 
 
 def composestandinmatcher(repo, rmatcher):
     """Return a matcher that accepts standins corresponding to the
     files accepted by rmatcher. Pass the list of files in the matcher
     as the paths specified by the user."""
     smatcher = getstandinmatcher(repo, rmatcher)
     isstandin = smatcher.matchfn
 
     def composedmatchfn(f):
         return isstandin(f) and rmatcher.matchfn(splitstandin(f))
 
     smatcher.matchfn = composedmatchfn
 
     return smatcher
 
 
 def standin(filename):
     """Return the repo-relative path to the standin for the specified big
     file."""
     # Notes:
     # 1) Some callers want an absolute path, but for instance addlargefiles
     #    needs it repo-relative so it can be passed to repo[None].add().  So
     #    leave it up to the caller to use repo.wjoin() to get an absolute path.
     # 2) Join with '/' because that's what dirstate always uses, even on
     #    Windows. Change existing separator to '/' first in case we are
     #    passed filenames from an external source (like the command line).
     return shortnameslash + util.pconvert(filename)
 
 
 def isstandin(filename):
     """Return true if filename is a big file standin. filename must be
     in Mercurial's internal form (slash-separated)."""
     return filename.startswith(shortnameslash)
 
 
 def splitstandin(filename):
     # Split on / because that's what dirstate always uses, even on Windows.
     # Change local separator to / first just in case we are passed filenames
     # from an external source (like the command line).
     bits = util.pconvert(filename).split(b'/', 1)
     if len(bits) == 2 and bits[0] == shortname:
         return bits[1]
     else:
         return None
 
 
 def updatestandin(repo, lfile, standin):
     """Re-calculate hash value of lfile and write it into standin
 
     This assumes that "lfutil.standin(lfile) == standin", for efficiency.
     """
     file = repo.wjoin(lfile)
     if repo.wvfs.exists(lfile):
         hash = hashfile(file)
         executable = getexecutable(file)
         writestandin(repo, standin, hash, executable)
     else:
         raise error.Abort(_(b'%s: file not found!') % lfile)
 
 
 def readasstandin(fctx):
     """read hex hash from given filectx of standin file
 
     This encapsulates how "standin" data is stored into storage layer."""
     return fctx.data().strip()
 
 
 def writestandin(repo, standin, hash, executable):
     '''write hash to <repo.root>/<standin>'''
     repo.wwrite(standin, hash + b'\n', executable and b'x' or b'')
 
 
 def copyandhash(instream, outfile):
     """Read bytes from instream (iterable) and write them to outfile,
     computing the SHA-1 hash of the data along the way. Return the hash."""
     hasher = hashutil.sha1(b'')
     for data in instream:
         hasher.update(data)
         outfile.write(data)
     return hex(hasher.digest())
 
 
 def hashfile(file):
     if not os.path.exists(file):
         return b''
     with open(file, b'rb') as fd:
         return hexsha1(fd)
 
 
 def getexecutable(filename):
     mode = os.stat(filename).st_mode
     return (
         (mode & stat.S_IXUSR)
         and (mode & stat.S_IXGRP)
         and (mode & stat.S_IXOTH)
     )
 
 
 def urljoin(first, second, *arg):
     def join(left, right):
         if not left.endswith(b'/'):
             left += b'/'
         if right.startswith(b'/'):
             right = right[1:]
         return left + right
 
     url = join(first, second)
     for a in arg:
         url = join(url, a)
     return url
 
 
 def hexsha1(fileobj):
     """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
     object data"""
     h = hashutil.sha1()
     for chunk in util.filechunkiter(fileobj):
         h.update(chunk)
     return hex(h.digest())
 
 
 def httpsendfile(ui, filename):
     return httpconnection.httpsendfile(ui, filename, b'rb')
 
 
 def unixpath(path):
     '''Return a version of path normalized for use with the lfdirstate.'''
     return util.pconvert(os.path.normpath(path))
 
 
 def islfilesrepo(repo):
     '''Return true if the repo is a largefile repo.'''
     if b'largefiles' in repo.requirements:
         for entry in repo.store.data_entries():
             if entry.is_revlog and shortnameslash in entry.target_id:
                 return True
 
     return any(openlfdirstate(repo.ui, repo, False))
 
 
 class storeprotonotcapable(Exception):
     def __init__(self, storetypes):
         self.storetypes = storetypes
 
 
 def getstandinsstate(repo):
     standins = []
     matcher = getstandinmatcher(repo)
     wctx = repo[None]
     for standin in repo.dirstate.walk(
         matcher, subrepos=[], unknown=False, ignored=False
     ):
         lfile = splitstandin(standin)
         try:
             hash = readasstandin(wctx[standin])
         except IOError:
             hash = None
         standins.append((lfile, hash))
     return standins
 
 
 def synclfdirstate(repo, lfdirstate, lfile, normallookup):
     lfstandin = standin(lfile)
     if lfstandin not in repo.dirstate:
         lfdirstate.hacky_extension_update_file(
             lfile,
             p1_tracked=False,
             wc_tracked=False,
         )
     else:
         entry = repo.dirstate.get_entry(lfstandin)
         lfdirstate.hacky_extension_update_file(
             lfile,
             wc_tracked=entry.tracked,
             p1_tracked=entry.p1_tracked,
             p2_info=entry.p2_info,
             possibly_dirty=True,
         )
 
 
 def markcommitted(orig, ctx, node):
     repo = ctx.repo()
 
     with repo.dirstate.changing_parents(repo):
         orig(node)
 
         # ATTENTION: "ctx.files()" may differ from "repo[node].files()"
         # because files coming from the 2nd parent are omitted in the latter.
         #
         # The former should be used to get targets of "synclfdirstate",
         # because such files:
         # - are marked as "a" by "patch.patch()" (e.g. via transplant), and
         # - have to be marked as "n" after commit, but
         # - aren't listed in "repo[node].files()"
 
         lfdirstate = openlfdirstate(repo.ui, repo)
         for f in ctx.files():
             lfile = splitstandin(f)
             if lfile is not None:
                 synclfdirstate(repo, lfdirstate, lfile, False)
 
     # As part of committing, copy all of the largefiles into the cache.
     #
     # Using "node" instead of "ctx" implies additional "repo[node]"
     # lookup while copyalltostore(), but can omit redundant check for
     # files comming from the 2nd parent, which should exist in store
     # at merging.
     copyalltostore(repo, node)
 
 
 def getlfilestoupdate(oldstandins, newstandins):
     changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
     filelist = []
     for f in changedstandins:
         if f[0] not in filelist:
             filelist.append(f[0])
     return filelist
 
 
 def getlfilestoupload(repo, missing, addfunc):
     makeprogress = repo.ui.makeprogress
     with makeprogress(
         _(b'finding outgoing largefiles'),
         unit=_(b'revisions'),
         total=len(missing),
     ) as progress:
         for i, n in enumerate(missing):
             progress.update(i)
             parents = [p for p in repo[n].parents() if p != repo.nullid]
 
             with lfstatus(repo, value=False):
                 ctx = repo[n]
 
             files = set(ctx.files())
             if len(parents) == 2:
                 mc = ctx.manifest()
                 mp1 = ctx.p1().manifest()
                 mp2 = ctx.p2().manifest()
                 for f in mp1:
                     if f not in mc:
                         files.add(f)
                 for f in mp2:
                     if f not in mc:
                         files.add(f)
                 for f in mc:
                     if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
                         files.add(f)
             for fn in files:
                 if isstandin(fn) and fn in ctx:
                     addfunc(fn, readasstandin(ctx[fn]))
 
 
 def updatestandinsbymatch(repo, match):
     """Update standins in the working directory according to specified match
 
     This returns (possibly modified) ``match`` object to be used for
     subsequent commit process.
     """
 
     ui = repo.ui
 
     # Case 1: user calls commit with no specific files or
     # include/exclude patterns: refresh and commit all files that
     # are "dirty".
     if match is None or match.always():
         # Spend a bit of time here to get a list of files we know
         # are modified so we can compare only against those.
         # It can cost a lot of time (several seconds)
         # otherwise to update all standins if the largefiles are
         # large.
         dirtymatch = matchmod.always()
         with repo.dirstate.running_status(repo):
             lfdirstate = openlfdirstate(ui, repo)
             unsure, s, mtime_boundary = lfdirstate.status(
                 dirtymatch,
                 subrepos=[],
                 ignored=False,
                 clean=False,
                 unknown=False,
             )
         modifiedfiles = unsure + s.modified + s.added + s.removed
         lfiles = listlfiles(repo)
         # this only loops through largefiles that exist (not
         # removed/renamed)
         for lfile in lfiles:
             if lfile in modifiedfiles:
                 fstandin = standin(lfile)
                 if repo.wvfs.exists(fstandin):
                     # this handles the case where a rebase is being
                     # performed and the working copy is not updated
                     # yet.
                     if repo.wvfs.exists(lfile):
                         updatestandin(repo, lfile, fstandin)
 
         return match
 
     lfiles = listlfiles(repo)
     match._files = repo._subdirlfs(match.files(), lfiles)
 
     # Case 2: user calls commit with specified patterns: refresh
     # any matching big files.
     smatcher = composestandinmatcher(repo, match)
     standins = repo.dirstate.walk(
         smatcher, subrepos=[], unknown=False, ignored=False
     )
 
     # No matching big files: get out of the way and pass control to
     # the usual commit() method.
     if not standins:
         return match
 
     # Refresh all matching big files.  It's possible that the
     # commit will end up failing, in which case the big files will
     # stay refreshed.  No harm done: the user modified them and
     # asked to commit them, so sooner or later we're going to
     # refresh the standins.  Might as well leave them refreshed.
     lfdirstate = openlfdirstate(ui, repo)
     for fstandin in standins:
         lfile = splitstandin(fstandin)
         if lfdirstate.get_entry(lfile).tracked:
             updatestandin(repo, lfile, fstandin)
 
     # Cook up a new matcher that only matches regular files or
     # standins corresponding to the big files requested by the
     # user.  Have to modify _files to prevent commit() from
     # complaining "not tracked" for big files.
     match = copy.copy(match)
     origmatchfn = match.matchfn
 
     # Check both the list of largefiles and the list of
     # standins because if a largefile was removed, it
     # won't be in the list of largefiles at this point
     match._files += sorted(standins)
 
     actualfiles = []
     for f in match._files:
         fstandin = standin(f)
 
         # For largefiles, only one of the normal and standin should be
         # committed (except if one of them is a remove).  In the case of a
         # standin removal, drop the normal file if it is unknown to dirstate.
         # Thus, skip plain largefile names but keep the standin.
         if f in lfiles or fstandin in standins:
             if not repo.dirstate.get_entry(fstandin).removed:
                 if not repo.dirstate.get_entry(f).removed:
                     continue
             elif not repo.dirstate.get_entry(f).any_tracked:
                 continue
 
         actualfiles.append(f)
     match._files = actualfiles
 
     def matchfn(f):
         if origmatchfn(f):
             return f not in lfiles
         else:
             return f in standins
 
     match.matchfn = matchfn
 
     return match
 
 
 class automatedcommithook:
     """Stateful hook to update standins at the 1st commit of resuming
 
     For efficiency, updating standins in the working directory should
     be avoided while automated committing (like rebase, transplant and
     so on), because they should be updated before committing.
 
     But the 1st commit of resuming automated committing (e.g. ``rebase
     --continue``) should update them, because largefiles may be
     modified manually.
     """
 
     def __init__(self, resuming):
         self.resuming = resuming
 
     def __call__(self, repo, match):
         if self.resuming:
             self.resuming = False  # avoids updating at subsequent commits
             return updatestandinsbymatch(repo, match)
         else:
             return match
 
 
 def getstatuswriter(ui, repo, forcibly=None):
     """Return the function to write largefiles specific status out
 
     If ``forcibly`` is ``None``, this returns the last element of
     ``repo._lfstatuswriters`` as "default" writer function.
 
     Otherwise, this returns the function to always write out (or
     ignore if ``not forcibly``) status.
     """
-    if forcibly is None and util.safehasattr(repo, '_largefilesenabled'):
+    if forcibly is None and hasattr(repo, '_largefilesenabled'):
         return repo._lfstatuswriters[-1]
     else:
         if forcibly:
             return ui.status  # forcibly WRITE OUT
         else:
             return lambda *msg, **opts: None  # forcibly IGNORE

hgext/largefiles/overrides.py

@@ -1,1924 +1,1924 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''Overridden Mercurial commands and functions for the largefiles extension'''
 
 import contextlib
 import copy
 import os
 
 from mercurial.i18n import _
 
 from mercurial.pycompat import open
 
 from mercurial.hgweb import webcommands
 
 from mercurial import (
     archival,
     cmdutil,
     copies as copiesmod,
     dirstate,
     error,
     exchange,
     extensions,
     exthelper,
     filemerge,
     hg,
     logcmdutil,
     match as matchmod,
     merge,
     mergestate as mergestatemod,
     pathutil,
     pycompat,
     scmutil,
     smartset,
     subrepo,
     url as urlmod,
     util,
 )
 
 from mercurial.upgrade_utils import (
     actions as upgrade_actions,
 )
 
 from . import (
     lfcommands,
     lfutil,
     storefactory,
 )
 
 ACTION_ADD = mergestatemod.ACTION_ADD
 ACTION_DELETED_CHANGED = mergestatemod.ACTION_DELETED_CHANGED
 ACTION_GET = mergestatemod.ACTION_GET
 ACTION_KEEP = mergestatemod.ACTION_KEEP
 ACTION_REMOVE = mergestatemod.ACTION_REMOVE
 
 eh = exthelper.exthelper()
 
 lfstatus = lfutil.lfstatus
 
 MERGE_ACTION_LARGEFILE_MARK_REMOVED = mergestatemod.MergeAction('lfmr')
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
 
 def composelargefilematcher(match, manifest):
     """create a matcher that matches only the largefiles in the original
     matcher"""
     m = copy.copy(match)
     lfile = lambda f: lfutil.standin(f) in manifest
     m._files = [lf for lf in m._files if lfile(lf)]
     m._fileset = set(m._files)
     m.always = lambda: False
     origmatchfn = m.matchfn
     m.matchfn = lambda f: lfile(f) and origmatchfn(f)
     return m
 
 
 def composenormalfilematcher(match, manifest, exclude=None):
     excluded = set()
     if exclude is not None:
         excluded.update(exclude)
 
     m = copy.copy(match)
     notlfile = lambda f: not (
         lfutil.isstandin(f) or lfutil.standin(f) in manifest or f in excluded
     )
     m._files = [lf for lf in m._files if notlfile(lf)]
     m._fileset = set(m._files)
     m.always = lambda: False
     origmatchfn = m.matchfn
     m.matchfn = lambda f: notlfile(f) and origmatchfn(f)
     return m
 
 
 def addlargefiles(ui, repo, isaddremove, matcher, uipathfn, **opts):
     large = opts.get('large')
     lfsize = lfutil.getminsize(
         ui, lfutil.islfilesrepo(repo), opts.get('lfsize')
     )
 
     lfmatcher = None
     if lfutil.islfilesrepo(repo):
         lfpats = ui.configlist(lfutil.longname, b'patterns')
         if lfpats:
             lfmatcher = matchmod.match(repo.root, b'', list(lfpats))
 
     lfnames = []
     m = matcher
 
     wctx = repo[None]
     for f in wctx.walk(matchmod.badmatch(m, lambda x, y: None)):
         exact = m.exact(f)
         lfile = lfutil.standin(f) in wctx
         nfile = f in wctx
         exists = lfile or nfile
 
         # Don't warn the user when they attempt to add a normal tracked file.
         # The normal add code will do that for us.
         if exact and exists:
             if lfile:
                 ui.warn(_(b'%s already a largefile\n') % uipathfn(f))
             continue
 
         if (exact or not exists) and not lfutil.isstandin(f):
             # In case the file was removed previously, but not committed
             # (issue3507)
             if not repo.wvfs.exists(f):
                 continue
 
             abovemin = (
                 lfsize and repo.wvfs.lstat(f).st_size >= lfsize * 1024 * 1024
             )
             if large or abovemin or (lfmatcher and lfmatcher(f)):
                 lfnames.append(f)
                 if ui.verbose or not exact:
                     ui.status(_(b'adding %s as a largefile\n') % uipathfn(f))
 
     bad = []
 
     # Need to lock, otherwise there could be a race condition between
     # when standins are created and added to the repo.
     with repo.wlock():
         if not opts.get('dry_run'):
             standins = []
             lfdirstate = lfutil.openlfdirstate(ui, repo)
             for f in lfnames:
                 standinname = lfutil.standin(f)
                 lfutil.writestandin(
                     repo,
                     standinname,
                     hash=b'',
                     executable=lfutil.getexecutable(repo.wjoin(f)),
                 )
                 standins.append(standinname)
                 lfdirstate.set_tracked(f)
             lfdirstate.write(repo.currenttransaction())
             bad += [
                 lfutil.splitstandin(f)
                 for f in repo[None].add(standins)
                 if f in m.files()
             ]
 
         added = [f for f in lfnames if f not in bad]
     return added, bad
 
 
 def removelargefiles(ui, repo, isaddremove, matcher, uipathfn, dryrun, **opts):
     after = opts.get('after')
     m = composelargefilematcher(matcher, repo[None].manifest())
     with lfstatus(repo):
         s = repo.status(match=m, clean=not isaddremove)
     manifest = repo[None].manifest()
     modified, added, deleted, clean = [
         [f for f in list if lfutil.standin(f) in manifest]
         for list in (s.modified, s.added, s.deleted, s.clean)
     ]
 
     def warn(files, msg):
         for f in files:
             ui.warn(msg % uipathfn(f))
         return int(len(files) > 0)
 
     if after:
         remove = deleted
         result = warn(
             modified + added + clean, _(b'not removing %s: file still exists\n')
         )
     else:
         remove = deleted + clean
         result = warn(
             modified,
             _(
                 b'not removing %s: file is modified (use -f'
                 b' to force removal)\n'
             ),
         )
         result = (
             warn(
                 added,
                 _(
                     b'not removing %s: file has been marked for add'
                     b' (use forget to undo)\n'
                 ),
             )
             or result
         )
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in sorted(remove):
             if ui.verbose or not m.exact(f):
                 ui.status(_(b'removing %s\n') % uipathfn(f))
 
             if not dryrun:
                 if not after:
                     repo.wvfs.unlinkpath(f, ignoremissing=True)
 
         if dryrun:
             return result
 
         remove = [lfutil.standin(f) for f in remove]
         # If this is being called by addremove, let the original addremove
         # function handle this.
         if not isaddremove:
             for f in remove:
                 repo.wvfs.unlinkpath(f, ignoremissing=True)
         repo[None].forget(remove)
 
         for f in remove:
             lfdirstate.set_untracked(lfutil.splitstandin(f))
 
         lfdirstate.write(repo.currenttransaction())
 
     return result
 
 
 # For overriding mercurial.hgweb.webcommands so that largefiles will
 # appear at their right place in the manifests.
 @eh.wrapfunction(webcommands, 'decodepath')
 def decodepath(orig, path):
     return lfutil.splitstandin(path) or path
 
 
 # -- Wrappers: modify existing commands --------------------------------
 
 
 @eh.wrapcommand(
     b'add',
     opts=[
         (b'', b'large', None, _(b'add as largefile')),
         (b'', b'normal', None, _(b'add as normal file')),
         (
             b'',
             b'lfsize',
             b'',
             _(
                 b'add all files above this size (in megabytes) '
                 b'as largefiles (default: 10)'
             ),
         ),
     ],
 )
 def overrideadd(orig, ui, repo, *pats, **opts):
     if opts.get('normal') and opts.get('large'):
         raise error.Abort(_(b'--normal cannot be used with --large'))
     return orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapfunction(cmdutil, 'add')
 def cmdutiladd(orig, ui, repo, matcher, prefix, uipathfn, explicitonly, **opts):
     # The --normal flag short circuits this override
     if opts.get('normal'):
         return orig(ui, repo, matcher, prefix, uipathfn, explicitonly, **opts)
 
     ladded, lbad = addlargefiles(ui, repo, False, matcher, uipathfn, **opts)
     normalmatcher = composenormalfilematcher(
         matcher, repo[None].manifest(), ladded
     )
     bad = orig(ui, repo, normalmatcher, prefix, uipathfn, explicitonly, **opts)
 
     bad.extend(f for f in lbad)
     return bad
 
 
 @eh.wrapfunction(cmdutil, 'remove')
 def cmdutilremove(
     orig, ui, repo, matcher, prefix, uipathfn, after, force, subrepos, dryrun
 ):
     normalmatcher = composenormalfilematcher(matcher, repo[None].manifest())
     result = orig(
         ui,
         repo,
         normalmatcher,
         prefix,
         uipathfn,
         after,
         force,
         subrepos,
         dryrun,
     )
     return (
         removelargefiles(
             ui, repo, False, matcher, uipathfn, dryrun, after=after, force=force
         )
         or result
     )
 
 
 @eh.wrapfunction(dirstate.dirstate, '_changing')
 @contextlib.contextmanager
 def _changing(orig, self, repo, change_type):
     pre = sub_dirstate = getattr(self, '_sub_dirstate', None)
     try:
         lfd = getattr(self, '_large_file_dirstate', False)
         if sub_dirstate is None and not lfd:
             sub_dirstate = lfutil.openlfdirstate(repo.ui, repo)
             self._sub_dirstate = sub_dirstate
         if not lfd:
             assert self._sub_dirstate is not None
         with orig(self, repo, change_type):
             if sub_dirstate is None:
                 yield
             else:
                 with sub_dirstate._changing(repo, change_type):
                     yield
     finally:
         self._sub_dirstate = pre
 
 
 @eh.wrapfunction(dirstate.dirstate, 'running_status')
 @contextlib.contextmanager
 def running_status(orig, self, repo):
     pre = sub_dirstate = getattr(self, '_sub_dirstate', None)
     try:
         lfd = getattr(self, '_large_file_dirstate', False)
         if sub_dirstate is None and not lfd:
             sub_dirstate = lfutil.openlfdirstate(repo.ui, repo)
             self._sub_dirstate = sub_dirstate
         if not lfd:
             assert self._sub_dirstate is not None
         with orig(self, repo):
             if sub_dirstate is None:
                 yield
             else:
                 with sub_dirstate.running_status(repo):
                     yield
     finally:
         self._sub_dirstate = pre
 
 
 @eh.wrapfunction(subrepo.hgsubrepo, 'status')
 def overridestatusfn(orig, repo, rev2, **opts):
     with lfstatus(repo._repo):
         return orig(repo, rev2, **opts)
 
 
 @eh.wrapcommand(b'status')
 def overridestatus(orig, ui, repo, *pats, **opts):
     with lfstatus(repo):
         return orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapfunction(subrepo.hgsubrepo, 'dirty')
 def overridedirty(orig, repo, ignoreupdate=False, missing=False):
     with lfstatus(repo._repo):
         return orig(repo, ignoreupdate=ignoreupdate, missing=missing)
 
 
 @eh.wrapcommand(b'log')
 def overridelog(orig, ui, repo, *pats, **opts):
     def overridematchandpats(
         orig,
         ctx,
         pats=(),
         opts=None,
         globbed=False,
         default=b'relpath',
         badfn=None,
     ):
         """Matcher that merges root directory with .hglf, suitable for log.
         It is still possible to match .hglf directly.
         For any listed files run log on the standin too.
         matchfn tries both the given filename and with .hglf stripped.
         """
         if opts is None:
             opts = {}
         matchandpats = orig(ctx, pats, opts, globbed, default, badfn=badfn)
         m, p = copy.copy(matchandpats)
 
         if m.always():
             # We want to match everything anyway, so there's no benefit trying
             # to add standins.
             return matchandpats
 
         pats = set(p)
 
         def fixpats(pat, tostandin=lfutil.standin):
             if pat.startswith(b'set:'):
                 return pat
 
             kindpat = matchmod._patsplit(pat, None)
 
             if kindpat[0] is not None:
                 return kindpat[0] + b':' + tostandin(kindpat[1])
             return tostandin(kindpat[1])
 
         cwd = repo.getcwd()
         if cwd:
             hglf = lfutil.shortname
             back = util.pconvert(repo.pathto(hglf)[: -len(hglf)])
 
             def tostandin(f):
                 # The file may already be a standin, so truncate the back
                 # prefix and test before mangling it.  This avoids turning
                 # 'glob:../.hglf/foo*' into 'glob:../.hglf/../.hglf/foo*'.
                 if f.startswith(back) and lfutil.splitstandin(f[len(back) :]):
                     return f
 
                 # An absolute path is from outside the repo, so truncate the
                 # path to the root before building the standin.  Otherwise cwd
                 # is somewhere in the repo, relative to root, and needs to be
                 # prepended before building the standin.
                 if os.path.isabs(cwd):
                     f = f[len(back) :]
                 else:
                     f = cwd + b'/' + f
                 return back + lfutil.standin(f)
 
         else:
 
             def tostandin(f):
                 if lfutil.isstandin(f):
                     return f
                 return lfutil.standin(f)
 
         pats.update(fixpats(f, tostandin) for f in p)
 
         for i in range(0, len(m._files)):
             # Don't add '.hglf' to m.files, since that is already covered by '.'
             if m._files[i] == b'.':
                 continue
             standin = lfutil.standin(m._files[i])
             # If the "standin" is a directory, append instead of replace to
             # support naming a directory on the command line with only
             # largefiles.  The original directory is kept to support normal
             # files.
             if standin in ctx:
                 m._files[i] = standin
             elif m._files[i] not in ctx and repo.wvfs.isdir(standin):
                 m._files.append(standin)
 
         m._fileset = set(m._files)
         m.always = lambda: False
         origmatchfn = m.matchfn
 
         def lfmatchfn(f):
             lf = lfutil.splitstandin(f)
             if lf is not None and origmatchfn(lf):
                 return True
             r = origmatchfn(f)
             return r
 
         m.matchfn = lfmatchfn
 
         ui.debug(b'updated patterns: %s\n' % b', '.join(sorted(pats)))
         return m, pats
 
     # For hg log --patch, the match object is used in two different senses:
     # (1) to determine what revisions should be printed out, and
     # (2) to determine what files to print out diffs for.
     # The magic matchandpats override should be used for case (1) but not for
     # case (2).
     oldmatchandpats = scmutil.matchandpats
 
     def overridemakefilematcher(orig, repo, pats, opts, badfn=None):
         wctx = repo[None]
         match, pats = oldmatchandpats(wctx, pats, opts, badfn=badfn)
         return lambda ctx: match
 
     wrappedmatchandpats = extensions.wrappedfunction(
         scmutil, 'matchandpats', overridematchandpats
     )
     wrappedmakefilematcher = extensions.wrappedfunction(
         logcmdutil, '_makenofollowfilematcher', overridemakefilematcher
     )
     with wrappedmatchandpats, wrappedmakefilematcher:
         return orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapcommand(
     b'verify',
     opts=[
         (
             b'',
             b'large',
             None,
             _(b'verify that all largefiles in current revision exists'),
         ),
         (
             b'',
             b'lfa',
             None,
             _(b'verify largefiles in all revisions, not just current'),
         ),
         (
             b'',
             b'lfc',
             None,
             _(b'verify local largefile contents, not just existence'),
         ),
     ],
 )
 def overrideverify(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     all = opts.pop('lfa', False)
     contents = opts.pop('lfc', False)
 
     result = orig(ui, repo, *pats, **opts)
     if large or all or contents:
         result = result or lfcommands.verifylfiles(ui, repo, all, contents)
     return result
 
 
 @eh.wrapcommand(
     b'debugstate',
     opts=[(b'', b'large', None, _(b'display largefiles dirstate'))],
 )
 def overridedebugstate(orig, ui, repo, *pats, **opts):
     large = opts.pop('large', False)
     if large:
 
         class fakerepo:
             dirstate = lfutil.openlfdirstate(ui, repo)
 
         orig(ui, fakerepo, *pats, **opts)
     else:
         orig(ui, repo, *pats, **opts)
 
 
 # Before starting the manifest merge, merge.updates will call
 # _checkunknownfile to check if there are any files in the merged-in
 # changeset that collide with unknown files in the working copy.
 #
 # The largefiles are seen as unknown, so this prevents us from merging
 # in a file 'foo' if we already have a largefile with the same name.
 #
 # The overridden function filters the unknown files by removing any
 # largefiles. This makes the merge proceed and we can then handle this
 # case further in the overridden calculateupdates function below.
 @eh.wrapfunction(merge, '_checkunknownfile')
 def overridecheckunknownfile(
     origfn, dirstate, wvfs, dircache, wctx, mctx, f, f2=None
 ):
     if lfutil.standin(dirstate.normalize(f)) in wctx:
         return False
     return origfn(dirstate, wvfs, dircache, wctx, mctx, f, f2)
 
 
 # The manifest merge handles conflicts on the manifest level. We want
 # to handle changes in largefile-ness of files at this level too.
 #
 # The strategy is to run the original calculateupdates and then process
 # the action list it outputs. There are two cases we need to deal with:
 #
 # 1. Normal file in p1, largefile in p2. Here the largefile is
 #    detected via its standin file, which will enter the working copy
 #    with a "get" action. It is not "merge" since the standin is all
 #    Mercurial is concerned with at this level -- the link to the
 #    existing normal file is not relevant here.
 #
 # 2. Largefile in p1, normal file in p2. Here we get a "merge" action
 #    since the largefile will be present in the working copy and
 #    different from the normal file in p2. Mercurial therefore
 #    triggers a merge action.
 #
 # In both cases, we prompt the user and emit new actions to either
 # remove the standin (if the normal file was kept) or to remove the
 # normal file and get the standin (if the largefile was kept). The
 # default prompt answer is to use the largefile version since it was
 # presumably changed on purpose.
 #
 # Finally, the merge.applyupdates function will then take care of
 # writing the files into the working copy and lfcommands.updatelfiles
 # will update the largefiles.
 @eh.wrapfunction(merge, 'calculateupdates')
 def overridecalculateupdates(
     origfn, repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs
 ):
     overwrite = force and not branchmerge
     mresult = origfn(
         repo, p1, p2, pas, branchmerge, force, acceptremote, *args, **kwargs
     )
 
     if overwrite:
         return mresult
 
     # Convert to dictionary with filename as key and action as value.
     lfiles = set()
     for f in mresult.files():
         splitstandin = lfutil.splitstandin(f)
         if splitstandin is not None and splitstandin in p1:
             lfiles.add(splitstandin)
         elif lfutil.standin(f) in p1:
             lfiles.add(f)
 
     for lfile in sorted(lfiles):
         standin = lfutil.standin(lfile)
         (lm, largs, lmsg) = mresult.getfile(lfile, (None, None, None))
         (sm, sargs, smsg) = mresult.getfile(standin, (None, None, None))
 
         if sm in (ACTION_GET, ACTION_DELETED_CHANGED) and lm != ACTION_REMOVE:
             if sm == ACTION_DELETED_CHANGED:
                 f1, f2, fa, move, anc = sargs
                 sargs = (p2[f2].flags(), False)
             # Case 1: normal file in the working copy, largefile in
             # the second parent
             usermsg = (
                 _(
                     b'remote turned local normal file %s into a largefile\n'
                     b'use (l)argefile or keep (n)ormal file?'
                     b'$$ &Largefile $$ &Normal file'
                 )
                 % lfile
             )
             if repo.ui.promptchoice(usermsg, 0) == 0:  # pick remote largefile
                 mresult.addfile(
                     lfile, ACTION_REMOVE, None, b'replaced by standin'
                 )
                 mresult.addfile(standin, ACTION_GET, sargs, b'replaces standin')
             else:  # keep local normal file
                 mresult.addfile(lfile, ACTION_KEEP, None, b'replaces standin')
                 if branchmerge:
                     mresult.addfile(
                         standin,
                         ACTION_KEEP,
                         None,
                         b'replaced by non-standin',
                     )
                 else:
                     mresult.addfile(
                         standin,
                         ACTION_REMOVE,
                         None,
                         b'replaced by non-standin',
                     )
         if lm in (ACTION_GET, ACTION_DELETED_CHANGED) and sm != ACTION_REMOVE:
             if lm == ACTION_DELETED_CHANGED:
                 f1, f2, fa, move, anc = largs
                 largs = (p2[f2].flags(), False)
             # Case 2: largefile in the working copy, normal file in
             # the second parent
             usermsg = (
                 _(
                     b'remote turned local largefile %s into a normal file\n'
                     b'keep (l)argefile or use (n)ormal file?'
                     b'$$ &Largefile $$ &Normal file'
                 )
                 % lfile
             )
             if repo.ui.promptchoice(usermsg, 0) == 0:  # keep local largefile
                 if branchmerge:
                     # largefile can be restored from standin safely
                     mresult.addfile(
                         lfile,
                         ACTION_KEEP,
                         None,
                         b'replaced by standin',
                     )
                     mresult.addfile(
                         standin, ACTION_KEEP, None, b'replaces standin'
                     )
                 else:
                     # "lfile" should be marked as "removed" without
                     # removal of itself
                     mresult.addfile(
                         lfile,
                         MERGE_ACTION_LARGEFILE_MARK_REMOVED,
                         None,
                         b'forget non-standin largefile',
                     )
 
                     # linear-merge should treat this largefile as 're-added'
                     mresult.addfile(standin, ACTION_ADD, None, b'keep standin')
             else:  # pick remote normal file
                 mresult.addfile(lfile, ACTION_GET, largs, b'replaces standin')
                 mresult.addfile(
                     standin,
                     ACTION_REMOVE,
                     None,
                     b'replaced by non-standin',
                 )
 
     return mresult
 
 
 @eh.wrapfunction(mergestatemod, 'recordupdates')
 def mergerecordupdates(orig, repo, actions, branchmerge, getfiledata):
     if MERGE_ACTION_LARGEFILE_MARK_REMOVED in actions:
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
         for lfile, args, msg in actions[MERGE_ACTION_LARGEFILE_MARK_REMOVED]:
             # this should be executed before 'orig', to execute 'remove'
             # before all other actions
             repo.dirstate.update_file(lfile, p1_tracked=True, wc_tracked=False)
             # make sure lfile doesn't get synclfdirstate'd as normal
             lfdirstate.update_file(lfile, p1_tracked=False, wc_tracked=True)
 
     return orig(repo, actions, branchmerge, getfiledata)
 
 
 # Override filemerge to prompt the user about how they wish to merge
 # largefiles. This will handle identical edits without prompting the user.
 @eh.wrapfunction(filemerge, 'filemerge')
 def overridefilemerge(
     origfn, repo, wctx, mynode, orig, fcd, fco, fca, labels=None
 ):
     if not lfutil.isstandin(orig) or fcd.isabsent() or fco.isabsent():
         return origfn(repo, wctx, mynode, orig, fcd, fco, fca, labels=labels)
 
     ahash = lfutil.readasstandin(fca).lower()
     dhash = lfutil.readasstandin(fcd).lower()
     ohash = lfutil.readasstandin(fco).lower()
     if (
         ohash != ahash
         and ohash != dhash
         and (
             dhash == ahash
             or repo.ui.promptchoice(
                 _(
                     b'largefile %s has a merge conflict\nancestor was %s\n'
                     b'you can keep (l)ocal %s or take (o)ther %s.\n'
                     b'what do you want to do?'
                     b'$$ &Local $$ &Other'
                 )
                 % (lfutil.splitstandin(orig), ahash, dhash, ohash),
                 0,
             )
             == 1
         )
     ):
         repo.wwrite(fcd.path(), fco.data(), fco.flags())
     return 0, False
 
 
 @eh.wrapfunction(copiesmod, 'pathcopies')
 def copiespathcopies(orig, ctx1, ctx2, match=None):
     copies = orig(ctx1, ctx2, match=match)
     updated = {}
 
     for k, v in copies.items():
         updated[lfutil.splitstandin(k) or k] = lfutil.splitstandin(v) or v
 
     return updated
 
 
 # Copy first changes the matchers to match standins instead of
 # largefiles.  Then it overrides util.copyfile in that function it
 # checks if the destination largefile already exists. It also keeps a
 # list of copied files so that the largefiles can be copied and the
 # dirstate updated.
 @eh.wrapfunction(cmdutil, 'copy')
 def overridecopy(orig, ui, repo, pats, opts, rename=False):
     # doesn't remove largefile on rename
     if len(pats) < 2:
         # this isn't legal, let the original function deal with it
         return orig(ui, repo, pats, opts, rename)
 
     # This could copy both lfiles and normal files in one command,
     # but we don't want to do that. First replace their matcher to
     # only match normal files and run it, then replace it to just
     # match largefiles and run it again.
     nonormalfiles = False
     nolfiles = False
     manifest = repo[None].manifest()
 
     def normalfilesmatchfn(
         orig,
         ctx,
         pats=(),
         opts=None,
         globbed=False,
         default=b'relpath',
         badfn=None,
     ):
         if opts is None:
             opts = {}
         match = orig(ctx, pats, opts, globbed, default, badfn=badfn)
         return composenormalfilematcher(match, manifest)
 
     with extensions.wrappedfunction(scmutil, 'match', normalfilesmatchfn):
         try:
             result = orig(ui, repo, pats, opts, rename)
         except error.Abort as e:
             if e.message != _(b'no files to copy'):
                 raise e
             else:
                 nonormalfiles = True
             result = 0
 
     # The first rename can cause our current working directory to be removed.
     # In that case there is nothing left to copy/rename so just quit.
     try:
         repo.getcwd()
     except OSError:
         return result
 
     def makestandin(relpath):
         path = pathutil.canonpath(repo.root, repo.getcwd(), relpath)
         return repo.wvfs.join(lfutil.standin(path))
 
     fullpats = scmutil.expandpats(pats)
     dest = fullpats[-1]
 
     if os.path.isdir(dest):
         if not os.path.isdir(makestandin(dest)):
             os.makedirs(makestandin(dest))
 
     try:
         # When we call orig below it creates the standins but we don't add
         # them to the dir state until later so lock during that time.
         wlock = repo.wlock()
 
         manifest = repo[None].manifest()
 
         def overridematch(
             orig,
             ctx,
             pats=(),
             opts=None,
             globbed=False,
             default=b'relpath',
             badfn=None,
         ):
             if opts is None:
                 opts = {}
             newpats = []
             # The patterns were previously mangled to add the standin
             # directory; we need to remove that now
             for pat in pats:
                 if matchmod.patkind(pat) is None and lfutil.shortname in pat:
                     newpats.append(pat.replace(lfutil.shortname, b''))
                 else:
                     newpats.append(pat)
             match = orig(ctx, newpats, opts, globbed, default, badfn=badfn)
             m = copy.copy(match)
             lfile = lambda f: lfutil.standin(f) in manifest
             m._files = [lfutil.standin(f) for f in m._files if lfile(f)]
             m._fileset = set(m._files)
             origmatchfn = m.matchfn
 
             def matchfn(f):
                 lfile = lfutil.splitstandin(f)
                 return (
                     lfile is not None
                     and (f in manifest)
                     and origmatchfn(lfile)
                     or None
                 )
 
             m.matchfn = matchfn
             return m
 
         listpats = []
         for pat in pats:
             if matchmod.patkind(pat) is not None:
                 listpats.append(pat)
             else:
                 listpats.append(makestandin(pat))
 
         copiedfiles = []
 
         def overridecopyfile(orig, src, dest, *args, **kwargs):
             if lfutil.shortname in src and dest.startswith(
                 repo.wjoin(lfutil.shortname)
             ):
                 destlfile = dest.replace(lfutil.shortname, b'')
                 if not opts[b'force'] and os.path.exists(destlfile):
                     raise IOError(
                         b'', _(b'destination largefile already exists')
                     )
             copiedfiles.append((src, dest))
             orig(src, dest, *args, **kwargs)
 
         with extensions.wrappedfunction(util, 'copyfile', overridecopyfile):
             with extensions.wrappedfunction(scmutil, 'match', overridematch):
                 result += orig(ui, repo, listpats, opts, rename)
 
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for (src, dest) in copiedfiles:
             if lfutil.shortname in src and dest.startswith(
                 repo.wjoin(lfutil.shortname)
             ):
                 srclfile = src.replace(repo.wjoin(lfutil.standin(b'')), b'')
                 destlfile = dest.replace(repo.wjoin(lfutil.standin(b'')), b'')
                 destlfiledir = repo.wvfs.dirname(repo.wjoin(destlfile)) or b'.'
                 if not os.path.isdir(destlfiledir):
                     os.makedirs(destlfiledir)
                 if rename:
                     os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile))
 
                     # The file is gone, but this deletes any empty parent
                     # directories as a side-effect.
                     repo.wvfs.unlinkpath(srclfile, ignoremissing=True)
                     lfdirstate.set_untracked(srclfile)
                 else:
                     util.copyfile(repo.wjoin(srclfile), repo.wjoin(destlfile))
 
                 lfdirstate.set_tracked(destlfile)
         lfdirstate.write(repo.currenttransaction())
     except error.Abort as e:
         if e.message != _(b'no files to copy'):
             raise e
         else:
             nolfiles = True
     finally:
         wlock.release()
 
     if nolfiles and nonormalfiles:
         raise error.Abort(_(b'no files to copy'))
 
     return result
 
 
 # When the user calls revert, we have to be careful to not revert any
 # changes to other largefiles accidentally. This means we have to keep
 # track of the largefiles that are being reverted so we only pull down
 # the necessary largefiles.
 #
 # Standins are only updated (to match the hash of largefiles) before
 # commits. Update the standins then run the original revert, changing
 # the matcher to hit standins instead of largefiles. Based on the
 # resulting standins update the largefiles.
 @eh.wrapfunction(cmdutil, 'revert')
 def overriderevert(orig, ui, repo, ctx, *pats, **opts):
     # Because we put the standins in a bad state (by updating them)
     # and then return them to a correct state we need to lock to
     # prevent others from changing them in their incorrect state.
     with repo.wlock(), repo.dirstate.running_status(repo):
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         s = lfutil.lfdirstatestatus(lfdirstate, repo)
         lfdirstate.write(repo.currenttransaction())
         for lfile in s.modified:
             lfutil.updatestandin(repo, lfile, lfutil.standin(lfile))
         for lfile in s.deleted:
             fstandin = lfutil.standin(lfile)
             if repo.wvfs.exists(fstandin):
                 repo.wvfs.unlink(fstandin)
 
         oldstandins = lfutil.getstandinsstate(repo)
 
         def overridematch(
             orig,
             mctx,
             pats=(),
             opts=None,
             globbed=False,
             default=b'relpath',
             badfn=None,
         ):
             if opts is None:
                 opts = {}
             match = orig(mctx, pats, opts, globbed, default, badfn=badfn)
             m = copy.copy(match)
 
             # revert supports recursing into subrepos, and though largefiles
             # currently doesn't work correctly in that case, this match is
             # called, so the lfdirstate above may not be the correct one for
             # this invocation of match.
             lfdirstate = lfutil.openlfdirstate(
                 mctx.repo().ui, mctx.repo(), False
             )
 
             wctx = repo[None]
             matchfiles = []
             for f in m._files:
                 standin = lfutil.standin(f)
                 if standin in ctx or standin in mctx:
                     matchfiles.append(standin)
                 elif standin in wctx or lfdirstate.get_entry(f).removed:
                     continue
                 else:
                     matchfiles.append(f)
             m._files = matchfiles
             m._fileset = set(m._files)
             origmatchfn = m.matchfn
 
             def matchfn(f):
                 lfile = lfutil.splitstandin(f)
                 if lfile is not None:
                     return origmatchfn(lfile) and (f in ctx or f in mctx)
                 return origmatchfn(f)
 
             m.matchfn = matchfn
             return m
 
         with extensions.wrappedfunction(scmutil, 'match', overridematch):
             orig(ui, repo, ctx, *pats, **opts)
 
         newstandins = lfutil.getstandinsstate(repo)
         filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
         # lfdirstate should be 'normallookup'-ed for updated files,
         # because reverting doesn't touch dirstate for 'normal' files
         # when target revision is explicitly specified: in such case,
         # 'n' and valid timestamp in dirstate doesn't ensure 'clean'
         # of target (standin) file.
         lfcommands.updatelfiles(
             ui, repo, filelist, printmessage=False, normallookup=True
         )
 
 
 # after pulling changesets, we need to take some extra care to get
 # largefiles updated remotely
 @eh.wrapcommand(
     b'pull',
     opts=[
         (
             b'',
             b'all-largefiles',
             None,
             _(b'download all pulled versions of largefiles (DEPRECATED)'),
         ),
         (
             b'',
             b'lfrev',
             [],
             _(b'download largefiles for these revisions'),
             _(b'REV'),
         ),
     ],
 )
 def overridepull(orig, ui, repo, source=None, **opts):
     revsprepull = len(repo)
     if not source:
         source = b'default'
     repo.lfpullsource = source
     result = orig(ui, repo, source, **opts)
     revspostpull = len(repo)
     lfrevs = opts.get('lfrev', [])
     if opts.get('all_largefiles'):
         lfrevs.append(b'pulled()')
     if lfrevs and revspostpull > revsprepull:
         numcached = 0
         repo.firstpulled = revsprepull  # for pulled() revset expression
         try:
             for rev in logcmdutil.revrange(repo, lfrevs):
                 ui.note(_(b'pulling largefiles for revision %d\n') % rev)
                 (cached, missing) = lfcommands.cachelfiles(ui, repo, rev)
                 numcached += len(cached)
         finally:
             del repo.firstpulled
         ui.status(_(b"%d largefiles cached\n") % numcached)
     return result
 
 
 @eh.wrapcommand(
     b'push',
     opts=[
         (
             b'',
             b'lfrev',
             [],
             _(b'upload largefiles for these revisions'),
             _(b'REV'),
         )
     ],
 )
 def overridepush(orig, ui, repo, *args, **kwargs):
     """Override push command and store --lfrev parameters in opargs"""
     lfrevs = kwargs.pop('lfrev', None)
     if lfrevs:
         opargs = kwargs.setdefault('opargs', {})
         opargs[b'lfrevs'] = logcmdutil.revrange(repo, lfrevs)
     return orig(ui, repo, *args, **kwargs)
 
 
 @eh.wrapfunction(exchange, 'pushoperation')
 def exchangepushoperation(orig, *args, **kwargs):
     """Override pushoperation constructor and store lfrevs parameter"""
     lfrevs = kwargs.pop('lfrevs', None)
     pushop = orig(*args, **kwargs)
     pushop.lfrevs = lfrevs
     return pushop
 
 
 @eh.revsetpredicate(b'pulled()')
 def pulledrevsetsymbol(repo, subset, x):
     """Changesets that just has been pulled.
 
     Only available with largefiles from pull --lfrev expressions.
 
     .. container:: verbose
 
       Some examples:
 
       - pull largefiles for all new changesets::
 
           hg pull -lfrev "pulled()"
 
       - pull largefiles for all new branch heads::
 
           hg pull -lfrev "head(pulled()) and not closed()"
 
     """
 
     try:
         firstpulled = repo.firstpulled
     except AttributeError:
         raise error.Abort(_(b"pulled() only available in --lfrev"))
     return smartset.baseset([r for r in subset if r >= firstpulled])
 
 
 @eh.wrapcommand(
     b'clone',
     opts=[
         (
             b'',
             b'all-largefiles',
             None,
             _(b'download all versions of all largefiles'),
         )
     ],
 )
 def overrideclone(orig, ui, source, dest=None, **opts):
     d = dest
     if d is None:
         d = hg.defaultdest(source)
     if opts.get('all_largefiles') and not hg.islocal(d):
         raise error.Abort(
             _(b'--all-largefiles is incompatible with non-local destination %s')
             % d
         )
 
     return orig(ui, source, dest, **opts)
 
 
 @eh.wrapfunction(hg, 'clone')
 def hgclone(orig, ui, opts, *args, **kwargs):
     result = orig(ui, opts, *args, **kwargs)
 
     if result is not None:
         sourcerepo, destrepo = result
         repo = destrepo.local()
 
         # When cloning to a remote repo (like through SSH), no repo is available
         # from the peer.   Therefore the largefiles can't be downloaded and the
         # hgrc can't be updated.
         if not repo:
             return result
 
         # Caching is implicitly limited to 'rev' option, since the dest repo was
         # truncated at that point.  The user may expect a download count with
         # this option, so attempt whether or not this is a largefile repo.
         if opts.get(b'all_largefiles'):
             success, missing = lfcommands.downloadlfiles(ui, repo)
 
             if missing != 0:
                 return None
 
     return result
 
 
 @eh.wrapcommand(b'rebase', extension=b'rebase')
 def overriderebasecmd(orig, ui, repo, **opts):
-    if not util.safehasattr(repo, '_largefilesenabled'):
+    if not hasattr(repo, '_largefilesenabled'):
         return orig(ui, repo, **opts)
 
     resuming = opts.get('continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
     repo._lfstatuswriters.append(lambda *msg, **opts: None)
     try:
         with ui.configoverride(
             {(b'rebase', b'experimental.inmemory'): False}, b"largefiles"
         ):
             return orig(ui, repo, **opts)
     finally:
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
 
 
 @eh.extsetup
 def overriderebase(ui):
     try:
         rebase = extensions.find(b'rebase')
     except KeyError:
         pass
     else:
 
         def _dorebase(orig, *args, **kwargs):
             kwargs['inmemory'] = False
             return orig(*args, **kwargs)
 
         extensions.wrapfunction(rebase, '_dorebase', _dorebase)
 
 
 @eh.wrapcommand(b'archive')
 def overridearchivecmd(orig, ui, repo, dest, **opts):
     with lfstatus(repo.unfiltered()):
         return orig(ui, repo.unfiltered(), dest, **opts)
 
 
 @eh.wrapfunction(webcommands, 'archive')
 def hgwebarchive(orig, web):
     with lfstatus(web.repo):
         return orig(web)
 
 
 @eh.wrapfunction(archival, 'archive')
 def overridearchive(
     orig,
     repo,
     dest,
     node,
     kind,
     decode=True,
     match=None,
     prefix=b'',
     mtime=None,
     subrepos=None,
 ):
     # For some reason setting repo.lfstatus in hgwebarchive only changes the
     # unfiltered repo's attr, so check that as well.
     if not repo.lfstatus and not repo.unfiltered().lfstatus:
         return orig(
             repo, dest, node, kind, decode, match, prefix, mtime, subrepos
         )
 
     # No need to lock because we are only reading history and
     # largefile caches, neither of which are modified.
     if node is not None:
         lfcommands.cachelfiles(repo.ui, repo, node)
 
     if kind not in archival.archivers:
         raise error.Abort(_(b"unknown archive type '%s'") % kind)
 
     ctx = repo[node]
 
     if kind == b'files':
         if prefix:
             raise error.Abort(_(b'cannot give prefix when archiving to files'))
     else:
         prefix = archival.tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         if match and not match(name):
             return
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     archiver = archival.archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool(b"ui", b"archivemeta"):
         write(
             b'.hg_archival.txt',
             0o644,
             False,
             lambda: archival.buildmetadata(ctx),
         )
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         lfile = lfutil.splitstandin(f)
         if lfile is not None:
             if node is not None:
                 path = lfutil.findfile(repo, getdata().strip())
 
                 if path is None:
                     raise error.Abort(
                         _(
                             b'largefile %s not found in repo store or system cache'
                         )
                         % lfile
                     )
             else:
                 path = lfile
 
             f = lfile
 
             getdata = lambda: util.readfile(path)
         write(f, b'x' in ff and 0o755 or 0o644, b'l' in ff, getdata)
 
     if subrepos:
         for subpath in sorted(ctx.substate):
             sub = ctx.workingsub(subpath)
             submatch = matchmod.subdirmatcher(subpath, match)
             subprefix = prefix + subpath + b'/'
 
             # TODO: Only hgsubrepo instances have `_repo`, so figure out how to
             # infer and possibly set lfstatus in hgsubrepoarchive.  That would
             # allow only hgsubrepos to set this, instead of the current scheme
             # where the parent sets this for the child.
             with (
-                util.safehasattr(sub, '_repo')
+                hasattr(sub, '_repo')
                 and lfstatus(sub._repo)
                 or util.nullcontextmanager()
             ):
                 sub.archive(archiver, subprefix, submatch)
 
     archiver.done()
 
 
 @eh.wrapfunction(subrepo.hgsubrepo, 'archive')
 def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
-    lfenabled = util.safehasattr(repo._repo, '_largefilesenabled')
+    lfenabled = hasattr(repo._repo, '_largefilesenabled')
     if not lfenabled or not repo._repo.lfstatus:
         return orig(repo, archiver, prefix, match, decode)
 
     repo._get(repo._state + (b'hg',))
     rev = repo._state[1]
     ctx = repo._repo[rev]
 
     if ctx.node() is not None:
         lfcommands.cachelfiles(repo.ui, repo._repo, ctx.node())
 
     def write(name, mode, islink, getdata):
         # At this point, the standin has been replaced with the largefile name,
         # so the normal matcher works here without the lfutil variants.
         if match and not match(f):
             return
         data = getdata()
         if decode:
             data = repo._repo.wwritedata(name, data)
 
         archiver.addfile(prefix + name, mode, islink, data)
 
     for f in ctx:
         ff = ctx.flags(f)
         getdata = ctx[f].data
         lfile = lfutil.splitstandin(f)
         if lfile is not None:
             if ctx.node() is not None:
                 path = lfutil.findfile(repo._repo, getdata().strip())
 
                 if path is None:
                     raise error.Abort(
                         _(
                             b'largefile %s not found in repo store or system cache'
                         )
                         % lfile
                     )
             else:
                 path = lfile
 
             f = lfile
 
             getdata = lambda: util.readfile(os.path.join(prefix, path))
 
         write(f, b'x' in ff and 0o755 or 0o644, b'l' in ff, getdata)
 
     for subpath in sorted(ctx.substate):
         sub = ctx.workingsub(subpath)
         submatch = matchmod.subdirmatcher(subpath, match)
         subprefix = prefix + subpath + b'/'
         # TODO: Only hgsubrepo instances have `_repo`, so figure out how to
         # infer and possibly set lfstatus at the top of this function.  That
         # would allow only hgsubrepos to set this, instead of the current scheme
         # where the parent sets this for the child.
         with (
-            util.safehasattr(sub, '_repo')
+            hasattr(sub, '_repo')
             and lfstatus(sub._repo)
             or util.nullcontextmanager()
         ):
             sub.archive(archiver, subprefix, submatch, decode)
 
 
 # If a largefile is modified, the change is not reflected in its
 # standin until a commit. cmdutil.bailifchanged() raises an exception
 # if the repo has uncommitted changes. Wrap it to also check if
 # largefiles were changed. This is used by bisect, backout and fetch.
 @eh.wrapfunction(cmdutil, 'bailifchanged')
 def overridebailifchanged(orig, repo, *args, **kwargs):
     orig(repo, *args, **kwargs)
     with lfstatus(repo):
         s = repo.status()
     if s.modified or s.added or s.removed or s.deleted:
         raise error.Abort(_(b'uncommitted changes'))
 
 
 @eh.wrapfunction(cmdutil, 'postcommitstatus')
 def postcommitstatus(orig, repo, *args, **kwargs):
     with lfstatus(repo):
         return orig(repo, *args, **kwargs)
 
 
 @eh.wrapfunction(cmdutil, 'forget')
 def cmdutilforget(
     orig, ui, repo, match, prefix, uipathfn, explicitonly, dryrun, interactive
 ):
     normalmatcher = composenormalfilematcher(match, repo[None].manifest())
     bad, forgot = orig(
         ui,
         repo,
         normalmatcher,
         prefix,
         uipathfn,
         explicitonly,
         dryrun,
         interactive,
     )
     m = composelargefilematcher(match, repo[None].manifest())
 
     with lfstatus(repo):
         s = repo.status(match=m, clean=True)
     manifest = repo[None].manifest()
     forget = sorted(s.modified + s.added + s.deleted + s.clean)
     forget = [f for f in forget if lfutil.standin(f) in manifest]
 
     for f in forget:
         fstandin = lfutil.standin(f)
         if fstandin not in repo.dirstate and not repo.wvfs.isdir(fstandin):
             ui.warn(
                 _(b'not removing %s: file is already untracked\n') % uipathfn(f)
             )
             bad.append(f)
 
     for f in forget:
         if ui.verbose or not m.exact(f):
             ui.status(_(b'removing %s\n') % uipathfn(f))
 
     # Need to lock because standin files are deleted then removed from the
     # repository and we could race in-between.
     with repo.wlock():
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         for f in forget:
             lfdirstate.set_untracked(f)
         lfdirstate.write(repo.currenttransaction())
         standins = [lfutil.standin(f) for f in forget]
         for f in standins:
             repo.wvfs.unlinkpath(f, ignoremissing=True)
         rejected = repo[None].forget(standins)
 
     bad.extend(f for f in rejected if f in m.files())
     forgot.extend(f for f in forget if f not in rejected)
     return bad, forgot
 
 
 def _getoutgoings(repo, other, missing, addfunc):
     """get pairs of filename and largefile hash in outgoing revisions
     in 'missing'.
 
     largefiles already existing on 'other' repository are ignored.
 
     'addfunc' is invoked with each unique pairs of filename and
     largefile hash value.
     """
     knowns = set()
     lfhashes = set()
 
     def dedup(fn, lfhash):
         k = (fn, lfhash)
         if k not in knowns:
             knowns.add(k)
             lfhashes.add(lfhash)
 
     lfutil.getlfilestoupload(repo, missing, dedup)
     if lfhashes:
         lfexists = storefactory.openstore(repo, other).exists(lfhashes)
         for fn, lfhash in knowns:
             if not lfexists[lfhash]:  # lfhash doesn't exist on "other"
                 addfunc(fn, lfhash)
 
 
 def outgoinghook(ui, repo, other, opts, missing):
     if opts.pop(b'large', None):
         lfhashes = set()
         if ui.debugflag:
             toupload = {}
 
             def addfunc(fn, lfhash):
                 if fn not in toupload:
                     toupload[fn] = []  # pytype: disable=unsupported-operands
                 toupload[fn].append(lfhash)
                 lfhashes.add(lfhash)
 
             def showhashes(fn):
                 for lfhash in sorted(toupload[fn]):
                     ui.debug(b'    %s\n' % lfhash)
 
         else:
             toupload = set()
 
             def addfunc(fn, lfhash):
                 toupload.add(fn)
                 lfhashes.add(lfhash)
 
             def showhashes(fn):
                 pass
 
         _getoutgoings(repo, other, missing, addfunc)
 
         if not toupload:
             ui.status(_(b'largefiles: no files to upload\n'))
         else:
             ui.status(
                 _(b'largefiles to upload (%d entities):\n') % (len(lfhashes))
             )
             for file in sorted(toupload):
                 ui.status(lfutil.splitstandin(file) + b'\n')
                 showhashes(file)
             ui.status(b'\n')
 
 
 @eh.wrapcommand(
     b'outgoing', opts=[(b'', b'large', None, _(b'display outgoing largefiles'))]
 )
 def _outgoingcmd(orig, *args, **kwargs):
     # Nothing to do here other than add the extra help option- the hook above
     # processes it.
     return orig(*args, **kwargs)
 
 
 def summaryremotehook(ui, repo, opts, changes):
     largeopt = opts.get(b'large', False)
     if changes is None:
         if largeopt:
             return (False, True)  # only outgoing check is needed
         else:
             return (False, False)
     elif largeopt:
         url, branch, peer, outgoing = changes[1]
         if peer is None:
             # i18n: column positioning for "hg summary"
             ui.status(_(b'largefiles: (no remote repo)\n'))
             return
 
         toupload = set()
         lfhashes = set()
 
         def addfunc(fn, lfhash):
             toupload.add(fn)
             lfhashes.add(lfhash)
 
         _getoutgoings(repo, peer, outgoing.missing, addfunc)
 
         if not toupload:
             # i18n: column positioning for "hg summary"
             ui.status(_(b'largefiles: (no files to upload)\n'))
         else:
             # i18n: column positioning for "hg summary"
             ui.status(
                 _(b'largefiles: %d entities for %d files to upload\n')
                 % (len(lfhashes), len(toupload))
             )
 
 
 @eh.wrapcommand(
     b'summary', opts=[(b'', b'large', None, _(b'display outgoing largefiles'))]
 )
 def overridesummary(orig, ui, repo, *pats, **opts):
     with lfstatus(repo):
         orig(ui, repo, *pats, **opts)
 
 
 @eh.wrapfunction(scmutil, 'addremove')
 def scmutiladdremove(
     orig,
     repo,
     matcher,
     prefix,
     uipathfn,
     opts=None,
     open_tr=None,
 ):
     if opts is None:
         opts = {}
     if not lfutil.islfilesrepo(repo):
         return orig(repo, matcher, prefix, uipathfn, opts, open_tr=open_tr)
 
     # open the transaction and changing_files context
     if open_tr is not None:
         open_tr()
 
     # Get the list of missing largefiles so we can remove them
     with repo.dirstate.running_status(repo):
         lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
         unsure, s, mtime_boundary = lfdirstate.status(
             matchmod.always(),
             subrepos=[],
             ignored=False,
             clean=False,
             unknown=False,
         )
 
     # Call into the normal remove code, but the removing of the standin, we want
     # to have handled by original addremove.  Monkey patching here makes sure
     # we don't remove the standin in the largefiles code, preventing a very
     # confused state later.
     if s.deleted:
         m = copy.copy(matcher)
 
         # The m._files and m._map attributes are not changed to the deleted list
         # because that affects the m.exact() test, which in turn governs whether
         # or not the file name is printed, and how.  Simply limit the original
         # matches to those in the deleted status list.
         matchfn = m.matchfn
         m.matchfn = lambda f: f in s.deleted and matchfn(f)
 
         removelargefiles(
             repo.ui,
             repo,
             True,
             m,
             uipathfn,
             opts.get(b'dry_run'),
             **pycompat.strkwargs(opts)
         )
     # Call into the normal add code, and any files that *should* be added as
     # largefiles will be
     added, bad = addlargefiles(
         repo.ui, repo, True, matcher, uipathfn, **pycompat.strkwargs(opts)
     )
     # Now that we've handled largefiles, hand off to the original addremove
     # function to take care of the rest.  Make sure it doesn't do anything with
     # largefiles by passing a matcher that will ignore them.
     matcher = composenormalfilematcher(matcher, repo[None].manifest(), added)
 
     return orig(repo, matcher, prefix, uipathfn, opts, open_tr=open_tr)
 
 
 # Calling purge with --all will cause the largefiles to be deleted.
 # Override repo.status to prevent this from happening.
 @eh.wrapcommand(b'purge')
 def overridepurge(orig, ui, repo, *dirs, **opts):
     # XXX Monkey patching a repoview will not work. The assigned attribute will
     # be set on the unfiltered repo, but we will only lookup attributes in the
     # unfiltered repo if the lookup in the repoview object itself fails. As the
     # monkey patched method exists on the repoview class the lookup will not
     # fail. As a result, the original version will shadow the monkey patched
     # one, defeating the monkey patch.
     #
     # As a work around we use an unfiltered repo here. We should do something
     # cleaner instead.
     repo = repo.unfiltered()
     oldstatus = repo.status
 
     def overridestatus(
         node1=b'.',
         node2=None,
         match=None,
         ignored=False,
         clean=False,
         unknown=False,
         listsubrepos=False,
     ):
         r = oldstatus(
             node1, node2, match, ignored, clean, unknown, listsubrepos
         )
         lfdirstate = lfutil.openlfdirstate(ui, repo)
         unknown = [
             f for f in r.unknown if not lfdirstate.get_entry(f).any_tracked
         ]
         ignored = [
             f for f in r.ignored if not lfdirstate.get_entry(f).any_tracked
         ]
         return scmutil.status(
             r.modified, r.added, r.removed, r.deleted, unknown, ignored, r.clean
         )
 
     repo.status = overridestatus
     orig(ui, repo, *dirs, **opts)
     repo.status = oldstatus
 
 
 @eh.wrapcommand(b'rollback')
 def overriderollback(orig, ui, repo, **opts):
     with repo.wlock():
         before = repo.dirstate.parents()
         orphans = {
             f
             for f in repo.dirstate
             if lfutil.isstandin(f) and not repo.dirstate.get_entry(f).removed
         }
         result = orig(ui, repo, **opts)
         after = repo.dirstate.parents()
         if before == after:
             return result  # no need to restore standins
 
         pctx = repo[b'.']
         for f in repo.dirstate:
             if lfutil.isstandin(f):
                 orphans.discard(f)
                 if repo.dirstate.get_entry(f).removed:
                     repo.wvfs.unlinkpath(f, ignoremissing=True)
                 elif f in pctx:
                     fctx = pctx[f]
                     repo.wwrite(f, fctx.data(), fctx.flags())
                 else:
                     # content of standin is not so important in 'a',
                     # 'm' or 'n' (coming from the 2nd parent) cases
                     lfutil.writestandin(repo, f, b'', False)
         for standin in orphans:
             repo.wvfs.unlinkpath(standin, ignoremissing=True)
 
     return result
 
 
 @eh.wrapcommand(b'transplant', extension=b'transplant')
 def overridetransplant(orig, ui, repo, *revs, **opts):
     resuming = opts.get('continue')
     repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
     repo._lfstatuswriters.append(lambda *msg, **opts: None)
     try:
         result = orig(ui, repo, *revs, **opts)
     finally:
         repo._lfstatuswriters.pop()
         repo._lfcommithooks.pop()
     return result
 
 
 @eh.wrapcommand(b'cat')
 def overridecat(orig, ui, repo, file1, *pats, **opts):
     ctx = logcmdutil.revsingle(repo, opts.get('rev'))
     err = 1
     notbad = set()
     m = scmutil.match(ctx, (file1,) + pats, pycompat.byteskwargs(opts))
     origmatchfn = m.matchfn
 
     def lfmatchfn(f):
         if origmatchfn(f):
             return True
         lf = lfutil.splitstandin(f)
         if lf is None:
             return False
         notbad.add(lf)
         return origmatchfn(lf)
 
     m.matchfn = lfmatchfn
     origbadfn = m.bad
 
     def lfbadfn(f, msg):
         if not f in notbad:
             origbadfn(f, msg)
 
     m.bad = lfbadfn
 
     origvisitdirfn = m.visitdir
 
     def lfvisitdirfn(dir):
         if dir == lfutil.shortname:
             return True
         ret = origvisitdirfn(dir)
         if ret:
             return ret
         lf = lfutil.splitstandin(dir)
         if lf is None:
             return False
         return origvisitdirfn(lf)
 
     m.visitdir = lfvisitdirfn
 
     for f in ctx.walk(m):
         with cmdutil.makefileobj(ctx, opts.get('output'), pathname=f) as fp:
             lf = lfutil.splitstandin(f)
             if lf is None or origmatchfn(f):
                 # duplicating unreachable code from commands.cat
                 data = ctx[f].data()
                 if opts.get('decode'):
                     data = repo.wwritedata(f, data)
                 fp.write(data)
             else:
                 hash = lfutil.readasstandin(ctx[f])
                 if not lfutil.inusercache(repo.ui, hash):
                     store = storefactory.openstore(repo)
                     success, missing = store.get([(lf, hash)])
                     if len(success) != 1:
                         raise error.Abort(
                             _(
                                 b'largefile %s is not in cache and could not be '
                                 b'downloaded'
                             )
                             % lf
                         )
                 path = lfutil.usercachepath(repo.ui, hash)
                 with open(path, b"rb") as fpin:
                     for chunk in util.filechunkiter(fpin):
                         fp.write(chunk)
         err = 0
     return err
 
 
 @eh.wrapfunction(merge, '_update')
 def mergeupdate(orig, repo, node, branchmerge, force, *args, **kwargs):
     matcher = kwargs.get('matcher', None)
     # note if this is a partial update
     partial = matcher and not matcher.always()
     with repo.wlock(), repo.dirstate.changing_parents(repo):
         # branch |       |         |
         #  merge | force | partial | action
         # -------+-------+---------+--------------
         #    x   |   x   |    x    | linear-merge
         #    o   |   x   |    x    | branch-merge
         #    x   |   o   |    x    | overwrite (as clean update)
         #    o   |   o   |    x    | force-branch-merge (*1)
         #    x   |   x   |    o    |   (*)
         #    o   |   x   |    o    |   (*)
         #    x   |   o   |    o    | overwrite (as revert)
         #    o   |   o   |    o    |   (*)
         #
         # (*) don't care
         # (*1) deprecated, but used internally (e.g: "rebase --collapse")
         with repo.dirstate.running_status(repo):
             lfdirstate = lfutil.openlfdirstate(repo.ui, repo)
             unsure, s, mtime_boundary = lfdirstate.status(
                 matchmod.always(),
                 subrepos=[],
                 ignored=False,
                 clean=True,
                 unknown=False,
             )
         oldclean = set(s.clean)
         pctx = repo[b'.']
         dctx = repo[node]
         for lfile in unsure + s.modified:
             lfileabs = repo.wvfs.join(lfile)
             if not repo.wvfs.exists(lfileabs):
                 continue
             lfhash = lfutil.hashfile(lfileabs)
             standin = lfutil.standin(lfile)
             lfutil.writestandin(
                 repo, standin, lfhash, lfutil.getexecutable(lfileabs)
             )
             if standin in pctx and lfhash == lfutil.readasstandin(
                 pctx[standin]
             ):
                 oldclean.add(lfile)
         for lfile in s.added:
             fstandin = lfutil.standin(lfile)
             if fstandin not in dctx:
                 # in this case, content of standin file is meaningless
                 # (in dctx, lfile is unknown, or normal file)
                 continue
             lfutil.updatestandin(repo, lfile, fstandin)
         # mark all clean largefiles as dirty, just in case the update gets
         # interrupted before largefiles and lfdirstate are synchronized
         for lfile in oldclean:
             entry = lfdirstate.get_entry(lfile)
             lfdirstate.hacky_extension_update_file(
                 lfile,
                 wc_tracked=entry.tracked,
                 p1_tracked=entry.p1_tracked,
                 p2_info=entry.p2_info,
                 possibly_dirty=True,
             )
         lfdirstate.write(repo.currenttransaction())
 
         oldstandins = lfutil.getstandinsstate(repo)
         wc = kwargs.get('wc')
         if wc and wc.isinmemory():
             # largefiles is not a good candidate for in-memory merge (large
             # files, custom dirstate, matcher usage).
             raise error.ProgrammingError(
                 b'largefiles is not compatible with in-memory merge'
             )
         result = orig(repo, node, branchmerge, force, *args, **kwargs)
 
         newstandins = lfutil.getstandinsstate(repo)
         filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
 
         # to avoid leaving all largefiles as dirty and thus rehash them, mark
         # all the ones that didn't change as clean
         for lfile in oldclean.difference(filelist):
             lfdirstate.update_file(lfile, p1_tracked=True, wc_tracked=True)
 
         if branchmerge or force or partial:
             filelist.extend(s.deleted + s.removed)
 
         lfcommands.updatelfiles(
             repo.ui, repo, filelist=filelist, normallookup=partial
         )
 
         return result
 
 
 @eh.wrapfunction(scmutil, 'marktouched')
 def scmutilmarktouched(orig, repo, files, *args, **kwargs):
     result = orig(repo, files, *args, **kwargs)
 
     filelist = []
     for f in files:
         lf = lfutil.splitstandin(f)
         if lf is not None:
             filelist.append(lf)
     if filelist:
         lfcommands.updatelfiles(
             repo.ui,
             repo,
             filelist=filelist,
             printmessage=False,
             normallookup=True,
         )
 
     return result
 
 
 @eh.wrapfunction(upgrade_actions, 'preservedrequirements')
 @eh.wrapfunction(upgrade_actions, 'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if b'largefiles' in repo.requirements:
         reqs.add(b'largefiles')
     return reqs
 
 
 _lfscheme = b'largefile://'
 
 
 @eh.wrapfunction(urlmod, 'open')
 def openlargefile(orig, ui, url_, data=None, **kwargs):
     if url_.startswith(_lfscheme):
         if data:
             msg = b"cannot use data on a 'largefile://' url"
             raise error.ProgrammingError(msg)
         lfid = url_[len(_lfscheme) :]
         return storefactory.getlfile(ui, lfid)
     else:
         return orig(ui, url_, data=data, **kwargs)

hgext/largefiles/storefactory.py

@@ -1,96 +1,96 b''
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import getattr
 from mercurial import (
     error,
     hg,
     util,
 )
 from mercurial.utils import (
     urlutil,
 )
 
 from . import (
     lfutil,
     localstore,
     wirestore,
 )
 
 
 # During clone this function is passed the src's ui object
 # but it needs the dest's ui object so it can read out of
 # the config file. Use repo.ui instead.
 def openstore(repo=None, remote=None, put=False, ui=None):
     if ui is None:
         ui = repo.ui
 
     if not remote:
         lfpullsource = getattr(repo, 'lfpullsource', None)
         if put:
             path = urlutil.get_unique_push_path(
                 b'lfpullsource', repo, ui, lfpullsource
             )
         else:
             path = urlutil.get_unique_pull_path_obj(
                 b'lfpullsource', ui, lfpullsource
             )
 
         # XXX we should not explicitly pass b'default', as this will result in
         # b'default' being returned if no `paths.default` was defined. We
         # should explicitely handle the lack of value instead.
         if repo is None:
             path = urlutil.get_unique_pull_path_obj(
                 b'lfs',
                 ui,
                 b'default',
             )
             remote = hg.peer(repo or ui, {}, path)
         elif path.loc == b'default-push' or path.loc == b'default':
             remote = repo
         else:
             remote = hg.peer(repo or ui, {}, path)
 
     # The path could be a scheme so use Mercurial's normal functionality
     # to resolve the scheme to a repository and use its path
-    path = util.safehasattr(remote, 'url') and remote.url() or remote.path
+    path = hasattr(remote, 'url') and remote.url() or remote.path
 
     match = _scheme_re.match(path)
     if not match:  # regular filesystem path
         scheme = b'file'
     else:
         scheme = match.group(1)
 
     try:
         storeproviders = _storeprovider[scheme]
     except KeyError:
         raise error.Abort(_(b'unsupported URL scheme %r') % scheme)
 
     for classobj in storeproviders:
         try:
             return classobj(ui, repo, remote)
         except lfutil.storeprotonotcapable:
             pass
 
     raise error.Abort(
         _(b'%s does not appear to be a largefile store')
         % urlutil.hidepassword(path)
     )
 
 
 _storeprovider = {
     b'file': [localstore.localstore],
     b'http': [wirestore.wirestore],
     b'https': [wirestore.wirestore],
     b'ssh': [wirestore.wirestore],
 }
 
 _scheme_re = re.compile(br'^([a-zA-Z0-9+-.]+)://')
 
 
 def getlfile(ui, hash):
     return util.chunkbuffer(openstore(ui=ui)._get(hash))

hgext/lfs/blobstore.py

@@ -1,789 +1,789 b''
 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import contextlib
 import errno
 import hashlib
 import json
 import os
 import re
 import socket
 
 from mercurial.i18n import _
 from mercurial.pycompat import getattr
 from mercurial.node import hex
 
 from mercurial import (
     encoding,
     error,
     httpconnection as httpconnectionmod,
     pathutil,
     pycompat,
     url as urlmod,
     util,
     vfs as vfsmod,
     worker,
 )
 
 from mercurial.utils import (
     stringutil,
     urlutil,
 )
 
 from ..largefiles import lfutil
 
 # 64 bytes for SHA256
 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z')
 
 
 class lfsvfs(vfsmod.vfs):
     def join(self, path):
         """split the path at first two characters, like: XX/XXXXX..."""
         if not _lfsre.match(path):
             raise error.ProgrammingError(b'unexpected lfs path: %s' % path)
         return super(lfsvfs, self).join(path[0:2], path[2:])
 
     def walk(self, path=None, onerror=None):
         """Yield (dirpath, [], oids) tuple for blobs under path
 
         Oids only exist in the root of this vfs, so dirpath is always ''.
         """
         root = os.path.normpath(self.base)
         # when dirpath == root, dirpath[prefixlen:] becomes empty
         # because len(dirpath) < prefixlen.
         prefixlen = len(pathutil.normasprefix(root))
         oids = []
 
         for dirpath, dirs, files in os.walk(
             self.reljoin(self.base, path or b''), onerror=onerror
         ):
             dirpath = dirpath[prefixlen:]
 
             # Silently skip unexpected files and directories
             if len(dirpath) == 2:
                 oids.extend(
                     [dirpath + f for f in files if _lfsre.match(dirpath + f)]
                 )
 
         yield (b'', [], oids)
 
 
 class nullvfs(lfsvfs):
     def __init__(self):
         pass
 
     def exists(self, oid):
         return False
 
     def read(self, oid):
         # store.read() calls into here if the blob doesn't exist in its
         # self.vfs.  Raise the same error as a normal vfs when asked to read a
         # file that doesn't exist.  The only difference is the full file path
         # isn't available in the error.
         raise IOError(
             errno.ENOENT,
             pycompat.sysstr(b'%s: No such file or directory' % oid),
         )
 
     def walk(self, path=None, onerror=None):
         return (b'', [], [])
 
     def write(self, oid, data):
         pass
 
 
 class lfsuploadfile(httpconnectionmod.httpsendfile):
     """a file-like object that supports keepalive."""
 
     def __init__(self, ui, filename):
         super(lfsuploadfile, self).__init__(ui, filename, b'rb')
         self.read = self._data.read
 
     def _makeprogress(self):
         return None  # progress is handled by the worker client
 
 
 class local:
     """Local blobstore for large file contents.
 
     This blobstore is used both as a cache and as a staging area for large blobs
     to be uploaded to the remote blobstore.
     """
 
     def __init__(self, repo):
         fullpath = repo.svfs.join(b'lfs/objects')
         self.vfs = lfsvfs(fullpath)
 
         if repo.ui.configbool(b'experimental', b'lfs.disableusercache'):
             self.cachevfs = nullvfs()
         else:
             usercache = lfutil._usercachedir(repo.ui, b'lfs')
             self.cachevfs = lfsvfs(usercache)
         self.ui = repo.ui
 
     def open(self, oid):
         """Open a read-only file descriptor to the named blob, in either the
         usercache or the local store."""
         return open(self.path(oid), 'rb')
 
     def path(self, oid):
         """Build the path for the given blob ``oid``.
 
         If the blob exists locally, the path may point to either the usercache
         or the local store.  If it doesn't, it will point to the local store.
         This is meant for situations where existing code that isn't LFS aware
         needs to open a blob.  Generally, prefer the ``open`` method on this
         class.
         """
         # The usercache is the most likely place to hold the file.  Commit will
         # write to both it and the local store, as will anything that downloads
         # the blobs.  However, things like clone without an update won't
         # populate the local store.  For an init + push of a local clone,
         # the usercache is the only place it _could_ be.  If not present, the
         # missing file msg here will indicate the local repo, not the usercache.
         if self.cachevfs.exists(oid):
             return self.cachevfs.join(oid)
 
         return self.vfs.join(oid)
 
     def download(self, oid, src, content_length):
         """Read the blob from the remote source in chunks, verify the content,
         and write to this local blobstore."""
         sha256 = hashlib.sha256()
         size = 0
 
         with self.vfs(oid, b'wb', atomictemp=True) as fp:
             for chunk in util.filechunkiter(src, size=1048576):
                 fp.write(chunk)
                 sha256.update(chunk)
                 size += len(chunk)
 
             # If the server advertised a length longer than what we actually
             # received, then we should expect that the server crashed while
             # producing the response (but the server has no way of telling us
             # that), and we really don't need to try to write the response to
             # the localstore, because it's not going to match the expected.
             # The server also uses this method to store data uploaded by the
             # client, so if this happens on the server side, it's possible
             # that the client crashed or an antivirus interfered with the
             # upload.
             if content_length is not None and int(content_length) != size:
                 msg = (
                     b"Response length (%d) does not match Content-Length "
                     b"header (%d) for %s"
                 )
                 raise LfsRemoteError(_(msg) % (size, int(content_length), oid))
 
             realoid = hex(sha256.digest())
             if realoid != oid:
                 raise LfsCorruptionError(
                     _(b'corrupt remote lfs object: %s') % oid
                 )
 
         self._linktousercache(oid)
 
     def write(self, oid, data):
         """Write blob to local blobstore.
 
         This should only be called from the filelog during a commit or similar.
         As such, there is no need to verify the data.  Imports from a remote
         store must use ``download()`` instead."""
         with self.vfs(oid, b'wb', atomictemp=True) as fp:
             fp.write(data)
 
         self._linktousercache(oid)
 
     def linkfromusercache(self, oid):
         """Link blobs found in the user cache into this store.
 
         The server module needs to do this when it lets the client know not to
         upload the blob, to ensure it is always available in this store.
         Normally this is done implicitly when the client reads or writes the
         blob, but that doesn't happen when the server tells the client that it
         already has the blob.
         """
         if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid):
             self.ui.note(_(b'lfs: found %s in the usercache\n') % oid)
             lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
 
     def _linktousercache(self, oid):
         # XXX: should we verify the content of the cache, and hardlink back to
         # the local store on success, but truncate, write and link on failure?
         if not self.cachevfs.exists(oid) and not isinstance(
             self.cachevfs, nullvfs
         ):
             self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid)
             lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
 
     def read(self, oid, verify=True):
         """Read blob from local blobstore."""
         if not self.vfs.exists(oid):
             blob = self._read(self.cachevfs, oid, verify)
 
             # Even if revlog will verify the content, it needs to be verified
             # now before making the hardlink to avoid propagating corrupt blobs.
             # Don't abort if corruption is detected, because `hg verify` will
             # give more useful info about the corruption- simply don't add the
             # hardlink.
             if verify or hex(hashlib.sha256(blob).digest()) == oid:
                 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid)
                 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
         else:
             self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid)
             blob = self._read(self.vfs, oid, verify)
         return blob
 
     def _read(self, vfs, oid, verify):
         """Read blob (after verifying) from the given store"""
         blob = vfs.read(oid)
         if verify:
             _verify(oid, blob)
         return blob
 
     def verify(self, oid):
         """Indicate whether or not the hash of the underlying file matches its
         name."""
         sha256 = hashlib.sha256()
 
         with self.open(oid) as fp:
             for chunk in util.filechunkiter(fp, size=1048576):
                 sha256.update(chunk)
 
         return oid == hex(sha256.digest())
 
     def has(self, oid):
         """Returns True if the local blobstore contains the requested blob,
         False otherwise."""
         return self.cachevfs.exists(oid) or self.vfs.exists(oid)
 
 
 def _urlerrorreason(urlerror):
     """Create a friendly message for the given URLError to be used in an
     LfsRemoteError message.
     """
     inst = urlerror
 
     if isinstance(urlerror.reason, Exception):
         inst = urlerror.reason
 
-    if util.safehasattr(inst, 'reason'):
+    if hasattr(inst, 'reason'):
         try:  # usually it is in the form (errno, strerror)
             reason = inst.reason.args[1]
         except (AttributeError, IndexError):
             # it might be anything, for example a string
             reason = inst.reason
         if isinstance(reason, str):
             # SSLError of Python 2.7.9 contains a unicode
             reason = encoding.unitolocal(reason)
         return reason
     elif getattr(inst, "strerror", None):
         return encoding.strtolocal(inst.strerror)
     else:
         return stringutil.forcebytestr(urlerror)
 
 
 class lfsauthhandler(util.urlreq.basehandler):
     handler_order = 480  # Before HTTPDigestAuthHandler (== 490)
 
     def http_error_401(self, req, fp, code, msg, headers):
         """Enforces that any authentication performed is HTTP Basic
         Authentication.  No authentication is also acceptable.
         """
         authreq = headers.get('www-authenticate', None)
         if authreq:
             scheme = authreq.split()[0]
 
             if scheme.lower() != 'basic':
                 msg = _(b'the server must support Basic Authentication')
                 raise util.urlerr.httperror(
                     req.get_full_url(),
                     code,
                     encoding.strfromlocal(msg),
                     headers,
                     fp,
                 )
         return None
 
 
 class _gitlfsremote:
     def __init__(self, repo, url):
         ui = repo.ui
         self.ui = ui
         baseurl, authinfo = url.authinfo()
         self.baseurl = baseurl.rstrip(b'/')
         useragent = repo.ui.config(b'experimental', b'lfs.user-agent')
         if not useragent:
             useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version()
         self.urlopener = urlmod.opener(ui, authinfo, useragent)
         self.urlopener.add_handler(lfsauthhandler())
         self.retry = ui.configint(b'lfs', b'retry')
 
     def writebatch(self, pointers, fromstore):
         """Batch upload from local to remote blobstore."""
         self._batch(_deduplicate(pointers), fromstore, b'upload')
 
     def readbatch(self, pointers, tostore):
         """Batch download from remote to local blostore."""
         self._batch(_deduplicate(pointers), tostore, b'download')
 
     def _batchrequest(self, pointers, action):
         """Get metadata about objects pointed by pointers for given action
 
         Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
         See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
         """
         objects = [
             {'oid': pycompat.strurl(p.oid()), 'size': p.size()}
             for p in pointers
         ]
         requestdata = pycompat.bytesurl(
             json.dumps(
                 {
                     'objects': objects,
                     'operation': pycompat.strurl(action),
                 }
             )
         )
         url = b'%s/objects/batch' % self.baseurl
         batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata)
         batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
         batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
         try:
             with contextlib.closing(self.urlopener.open(batchreq)) as rsp:
                 rawjson = rsp.read()
         except util.urlerr.httperror as ex:
             hints = {
                 400: _(
                     b'check that lfs serving is enabled on %s and "%s" is '
                     b'supported'
                 )
                 % (self.baseurl, action),
                 404: _(b'the "lfs.url" config may be used to override %s')
                 % self.baseurl,
             }
             hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action))
             raise LfsRemoteError(
                 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex),
                 hint=hint,
             )
         except util.urlerr.urlerror as ex:
             hint = (
                 _(b'the "lfs.url" config may be used to override %s')
                 % self.baseurl
             )
             raise LfsRemoteError(
                 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
             )
         try:
             response = pycompat.json_loads(rawjson)
         except ValueError:
             raise LfsRemoteError(
                 _(b'LFS server returns invalid JSON: %s')
                 % rawjson.encode("utf-8")
             )
 
         if self.ui.debugflag:
             self.ui.debug(b'Status: %d\n' % rsp.status)
             # lfs-test-server and hg serve return headers in different order
             headers = pycompat.bytestr(rsp.info()).strip()
             self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
 
             if 'objects' in response:
                 response['objects'] = sorted(
                     response['objects'], key=lambda p: p['oid']
                 )
             self.ui.debug(
                 b'%s\n'
                 % pycompat.bytesurl(
                     json.dumps(
                         response,
                         indent=2,
                         separators=('', ': '),
                         sort_keys=True,
                     )
                 )
             )
 
         def encodestr(x):
             if isinstance(x, str):
                 return x.encode('utf-8')
             return x
 
         return pycompat.rapply(encodestr, response)
 
     def _checkforservererror(self, pointers, responses, action):
         """Scans errors from objects
 
         Raises LfsRemoteError if any objects have an error"""
         for response in responses:
             # The server should return 404 when objects cannot be found. Some
             # server implementation (ex. lfs-test-server)  does not set "error"
             # but just removes "download" from "actions". Treat that case
             # as the same as 404 error.
             if b'error' not in response:
                 if action == b'download' and action not in response.get(
                     b'actions', []
                 ):
                     code = 404
                 else:
                     continue
             else:
                 # An error dict without a code doesn't make much sense, so
                 # treat as a server error.
                 code = response.get(b'error').get(b'code', 500)
 
             ptrmap = {p.oid(): p for p in pointers}
             p = ptrmap.get(response[b'oid'], None)
             if p:
                 filename = getattr(p, 'filename', b'unknown')
                 errors = {
                     404: b'The object does not exist',
                     410: b'The object was removed by the owner',
                     422: b'Validation error',
                     500: b'Internal server error',
                 }
                 msg = errors.get(code, b'status code %d' % code)
                 raise LfsRemoteError(
                     _(b'LFS server error for "%s": %s') % (filename, msg)
                 )
             else:
                 raise LfsRemoteError(
                     _(b'LFS server error. Unsolicited response for oid %s')
                     % response[b'oid']
                 )
 
     def _extractobjects(self, response, pointers, action):
         """extract objects from response of the batch API
 
         response: parsed JSON object returned by batch API
         return response['objects'] filtered by action
         raise if any object has an error
         """
         # Scan errors from objects - fail early
         objects = response.get(b'objects', [])
         self._checkforservererror(pointers, objects, action)
 
         # Filter objects with given action. Practically, this skips uploading
         # objects which exist in the server.
         filteredobjects = [
             o for o in objects if action in o.get(b'actions', [])
         ]
 
         return filteredobjects
 
     def _basictransfer(self, obj, action, localstore):
         """Download or upload a single object using basic transfer protocol
 
         obj: dict, an object description returned by batch API
         action: string, one of ['upload', 'download']
         localstore: blobstore.local
 
         See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
         basic-transfers.md
         """
         oid = obj[b'oid']
         href = obj[b'actions'][action].get(b'href')
         headers = obj[b'actions'][action].get(b'header', {}).items()
 
         request = util.urlreq.request(pycompat.strurl(href))
         if action == b'upload':
             # If uploading blobs, read data from local blobstore.
             if not localstore.verify(oid):
                 raise error.Abort(
                     _(b'detected corrupt lfs object: %s') % oid,
                     hint=_(b'run hg verify'),
                 )
 
         for k, v in headers:
             request.add_header(pycompat.strurl(k), pycompat.strurl(v))
 
         try:
             if action == b'upload':
                 request.data = lfsuploadfile(self.ui, localstore.path(oid))
                 request.get_method = lambda: 'PUT'
                 request.add_header('Content-Type', 'application/octet-stream')
                 request.add_header('Content-Length', request.data.length)
 
             with contextlib.closing(self.urlopener.open(request)) as res:
                 contentlength = res.info().get(b"content-length")
                 ui = self.ui  # Shorten debug lines
                 if self.ui.debugflag:
                     ui.debug(b'Status: %d\n' % res.status)
                     # lfs-test-server and hg serve return headers in different
                     # order
                     headers = pycompat.bytestr(res.info()).strip()
                     ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
 
                 if action == b'download':
                     # If downloading blobs, store downloaded data to local
                     # blobstore
                     localstore.download(oid, res, contentlength)
                 else:
                     blocks = []
                     while True:
                         data = res.read(1048576)
                         if not data:
                             break
                         blocks.append(data)
 
                     response = b"".join(blocks)
                     if response:
                         ui.debug(b'lfs %s response: %s' % (action, response))
         except util.urlerr.httperror as ex:
             if self.ui.debugflag:
                 self.ui.debug(
                     b'%s: %s\n' % (oid, ex.read())
                 )  # XXX: also bytes?
             raise LfsRemoteError(
                 _(b'LFS HTTP error: %s (oid=%s, action=%s)')
                 % (stringutil.forcebytestr(ex), oid, action)
             )
         except util.urlerr.urlerror as ex:
             hint = _(b'attempted connection to %s') % pycompat.bytesurl(
                 util.urllibcompat.getfullurl(request)
             )
             raise LfsRemoteError(
                 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
             )
         finally:
             if request.data:
                 request.data.close()
 
     def _batch(self, pointers, localstore, action):
         if action not in [b'upload', b'download']:
             raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action)
 
         response = self._batchrequest(pointers, action)
         objects = self._extractobjects(response, pointers, action)
         total = sum(x.get(b'size', 0) for x in objects)
         sizes = {}
         for obj in objects:
             sizes[obj.get(b'oid')] = obj.get(b'size', 0)
         topic = {
             b'upload': _(b'lfs uploading'),
             b'download': _(b'lfs downloading'),
         }[action]
         if len(objects) > 1:
             self.ui.note(
                 _(b'lfs: need to transfer %d objects (%s)\n')
                 % (len(objects), util.bytecount(total))
             )
 
         def transfer(chunk):
             for obj in chunk:
                 objsize = obj.get(b'size', 0)
                 if self.ui.verbose:
                     if action == b'download':
                         msg = _(b'lfs: downloading %s (%s)\n')
                     elif action == b'upload':
                         msg = _(b'lfs: uploading %s (%s)\n')
                     self.ui.note(
                         msg % (obj.get(b'oid'), util.bytecount(objsize))
                     )
                 retry = self.retry
                 while True:
                     try:
                         self._basictransfer(obj, action, localstore)
                         yield 1, obj.get(b'oid')
                         break
                     except socket.error as ex:
                         if retry > 0:
                             self.ui.note(
                                 _(b'lfs: failed: %r (remaining retry %d)\n')
                                 % (stringutil.forcebytestr(ex), retry)
                             )
                             retry -= 1
                             continue
                         raise
 
         # Until https multiplexing gets sorted out.  It's not clear if
         # ConnectionManager.set_ready() is externally synchronized for thread
         # safety with Windows workers.
         if self.ui.configbool(b'experimental', b'lfs.worker-enable'):
             # The POSIX workers are forks of this process, so before spinning
             # them up, close all pooled connections.  Otherwise, there's no way
             # to coordinate between them about who is using what, and the
             # transfers will get corrupted.
             #
             # TODO: add a function to keepalive.ConnectionManager to mark all
             #  ready connections as in use, and roll that back after the fork?
             #  That would allow the existing pool of connections in this process
             #  to be preserved.
             def prefork():
                 for h in self.urlopener.handlers:
                     getattr(h, "close_all", lambda: None)()
 
             oids = worker.worker(
                 self.ui,
                 0.1,
                 transfer,
                 (),
                 sorted(objects, key=lambda o: o.get(b'oid')),
                 prefork=prefork,
             )
         else:
             oids = transfer(sorted(objects, key=lambda o: o.get(b'oid')))
 
         with self.ui.makeprogress(
             topic, unit=_(b"bytes"), total=total
         ) as progress:
             progress.update(0)
             processed = 0
             blobs = 0
             for _one, oid in oids:
                 processed += sizes[oid]
                 blobs += 1
                 progress.update(processed)
                 self.ui.note(_(b'lfs: processed: %s\n') % oid)
 
         if blobs > 0:
             if action == b'upload':
                 self.ui.status(
                     _(b'lfs: uploaded %d files (%s)\n')
                     % (blobs, util.bytecount(processed))
                 )
             elif action == b'download':
                 self.ui.status(
                     _(b'lfs: downloaded %d files (%s)\n')
                     % (blobs, util.bytecount(processed))
                 )
 
     def __del__(self):
         # copied from mercurial/httppeer.py
         urlopener = getattr(self, 'urlopener', None)
         if urlopener:
             for h in urlopener.handlers:
                 h.close()
                 getattr(h, "close_all", lambda: None)()
 
 
 class _dummyremote:
     """Dummy store storing blobs to temp directory."""
 
     def __init__(self, repo, url):
         fullpath = repo.vfs.join(b'lfs', url.path)
         self.vfs = lfsvfs(fullpath)
 
     def writebatch(self, pointers, fromstore):
         for p in _deduplicate(pointers):
             content = fromstore.read(p.oid(), verify=True)
             with self.vfs(p.oid(), b'wb', atomictemp=True) as fp:
                 fp.write(content)
 
     def readbatch(self, pointers, tostore):
         for p in _deduplicate(pointers):
             with self.vfs(p.oid(), b'rb') as fp:
                 tostore.download(p.oid(), fp, None)
 
 
 class _nullremote:
     """Null store storing blobs to /dev/null."""
 
     def __init__(self, repo, url):
         pass
 
     def writebatch(self, pointers, fromstore):
         pass
 
     def readbatch(self, pointers, tostore):
         pass
 
 
 class _promptremote:
     """Prompt user to set lfs.url when accessed."""
 
     def __init__(self, repo, url):
         pass
 
     def writebatch(self, pointers, fromstore, ui=None):
         self._prompt()
 
     def readbatch(self, pointers, tostore, ui=None):
         self._prompt()
 
     def _prompt(self):
         raise error.Abort(_(b'lfs.url needs to be configured'))
 
 
 _storemap = {
     b'https': _gitlfsremote,
     b'http': _gitlfsremote,
     b'file': _dummyremote,
     b'null': _nullremote,
     None: _promptremote,
 }
 
 
 def _deduplicate(pointers):
     """Remove any duplicate oids that exist in the list"""
     reduced = util.sortdict()
     for p in pointers:
         reduced[p.oid()] = p
     return reduced.values()
 
 
 def _verify(oid, content):
     realoid = hex(hashlib.sha256(content).digest())
     if realoid != oid:
         raise LfsCorruptionError(
             _(b'detected corrupt lfs object: %s') % oid,
             hint=_(b'run hg verify'),
         )
 
 
 def remote(repo, remote=None):
     """remotestore factory. return a store in _storemap depending on config
 
     If ``lfs.url`` is specified, use that remote endpoint.  Otherwise, try to
     infer the endpoint, based on the remote repository using the same path
     adjustments as git.  As an extension, 'http' is supported as well so that
     ``hg serve`` works out of the box.
 
     https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md
     """
     lfsurl = repo.ui.config(b'lfs', b'url')
     url = urlutil.url(lfsurl or b'')
     if lfsurl is None:
         if remote:
             path = remote
-        elif util.safehasattr(repo, '_subtoppath'):
+        elif hasattr(repo, '_subtoppath'):
             # The pull command sets this during the optional update phase, which
             # tells exactly where the pull originated, whether 'paths.default'
             # or explicit.
             path = repo._subtoppath
         else:
             # TODO: investigate 'paths.remote:lfsurl' style path customization,
             # and fall back to inferring from 'paths.remote' if unspecified.
             path = repo.ui.config(b'paths', b'default') or b''
 
         defaulturl = urlutil.url(path)
 
         # TODO: support local paths as well.
         # TODO: consider the ssh -> https transformation that git applies
         if defaulturl.scheme in (b'http', b'https'):
             if defaulturl.path and defaulturl.path[:-1] != b'/':
                 defaulturl.path += b'/'
             defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs'
 
             url = urlutil.url(bytes(defaulturl))
             repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url)
 
     scheme = url.scheme
     if scheme not in _storemap:
         raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme)
     return _storemap[scheme](repo, url)
 
 
 class LfsRemoteError(error.StorageError):
     pass
 
 
 class LfsCorruptionError(error.Abort):
     """Raised when a corrupt blob is detected, aborting an operation
 
     It exists to allow specialized handling on the server side."""

hgext/lfs/wireprotolfsserver.py

@@ -1,369 +1,368 b''
 # wireprotolfsserver.py - lfs protocol server side implementation
 #
 # Copyright 2018 Matt Harbison <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import datetime
 import errno
 import json
 import traceback
 
 from mercurial.hgweb import common as hgwebcommon
 
 from mercurial import (
     exthelper,
     pycompat,
-    util,
     wireprotoserver,
 )
 
 from . import blobstore
 
 HTTP_OK = hgwebcommon.HTTP_OK
 HTTP_CREATED = hgwebcommon.HTTP_CREATED
 HTTP_BAD_REQUEST = hgwebcommon.HTTP_BAD_REQUEST
 HTTP_NOT_FOUND = hgwebcommon.HTTP_NOT_FOUND
 HTTP_METHOD_NOT_ALLOWED = hgwebcommon.HTTP_METHOD_NOT_ALLOWED
 HTTP_NOT_ACCEPTABLE = hgwebcommon.HTTP_NOT_ACCEPTABLE
 HTTP_UNSUPPORTED_MEDIA_TYPE = hgwebcommon.HTTP_UNSUPPORTED_MEDIA_TYPE
 
 eh = exthelper.exthelper()
 
 
 @eh.wrapfunction(wireprotoserver, 'handlewsgirequest')
 def handlewsgirequest(orig, rctx, req, res, checkperm):
     """Wrap wireprotoserver.handlewsgirequest() to possibly process an LFS
     request if it is left unprocessed by the wrapped method.
     """
     if orig(rctx, req, res, checkperm):
         return True
 
     if not rctx.repo.ui.configbool(b'experimental', b'lfs.serve'):
         return False
 
-    if not util.safehasattr(rctx.repo.svfs, 'lfslocalblobstore'):
+    if not hasattr(rctx.repo.svfs, 'lfslocalblobstore'):
         return False
 
     if not req.dispatchpath:
         return False
 
     try:
         if req.dispatchpath == b'.git/info/lfs/objects/batch':
             checkperm(rctx, req, b'pull')
             return _processbatchrequest(rctx.repo, req, res)
         # TODO: reserve and use a path in the proposed http wireprotocol /api/
         #       namespace?
         elif req.dispatchpath.startswith(b'.hg/lfs/objects'):
             return _processbasictransfer(
                 rctx.repo, req, res, lambda perm: checkperm(rctx, req, perm)
             )
         return False
     except hgwebcommon.ErrorResponse as e:
         # XXX: copied from the handler surrounding wireprotoserver._callhttp()
         #      in the wrapped function.  Should this be moved back to hgweb to
         #      be a common handler?
         for k, v in e.headers:
             res.headers[k] = v
         res.status = hgwebcommon.statusmessage(e.code, pycompat.bytestr(e))
         res.setbodybytes(b'0\n%s\n' % pycompat.bytestr(e))
         return True
 
 
 def _sethttperror(res, code, message=None):
     res.status = hgwebcommon.statusmessage(code, message=message)
     res.headers[b'Content-Type'] = b'text/plain; charset=utf-8'
     res.setbodybytes(b'')
 
 
 def _logexception(req):
     """Write information about the current exception to wsgi.errors."""
     tb = pycompat.sysbytes(traceback.format_exc())
     errorlog = req.rawenv[b'wsgi.errors']
 
     uri = b''
     if req.apppath:
         uri += req.apppath
     uri += b'/' + req.dispatchpath
 
     errorlog.write(
         b"Exception happened while processing request '%s':\n%s" % (uri, tb)
     )
 
 
 def _processbatchrequest(repo, req, res):
     """Handle a request for the Batch API, which is the gateway to granting file
     access.
 
     https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
     """
 
     # Mercurial client request:
     #
     #   HOST: localhost:$HGPORT
     #   ACCEPT: application/vnd.git-lfs+json
     #   ACCEPT-ENCODING: identity
     #   USER-AGENT: git-lfs/2.3.4 (Mercurial 4.5.2+1114-f48b9754f04c+20180316)
     #   Content-Length: 125
     #   Content-Type: application/vnd.git-lfs+json
     #
     #   {
     #     "objects": [
     #       {
     #         "oid": "31cf...8e5b"
     #         "size": 12
     #       }
     #     ]
     #     "operation": "upload"
     #  }
 
     if req.method != b'POST':
         _sethttperror(res, HTTP_METHOD_NOT_ALLOWED)
         return True
 
     if req.headers[b'Content-Type'] != b'application/vnd.git-lfs+json':
         _sethttperror(res, HTTP_UNSUPPORTED_MEDIA_TYPE)
         return True
 
     if req.headers[b'Accept'] != b'application/vnd.git-lfs+json':
         _sethttperror(res, HTTP_NOT_ACCEPTABLE)
         return True
 
     # XXX: specify an encoding?
     lfsreq = pycompat.json_loads(req.bodyfh.read())
 
     # If no transfer handlers are explicitly requested, 'basic' is assumed.
     if 'basic' not in lfsreq.get('transfers', ['basic']):
         _sethttperror(
             res,
             HTTP_BAD_REQUEST,
             b'Only the basic LFS transfer handler is supported',
         )
         return True
 
     operation = lfsreq.get('operation')
     operation = pycompat.bytestr(operation)
 
     if operation not in (b'upload', b'download'):
         _sethttperror(
             res,
             HTTP_BAD_REQUEST,
             b'Unsupported LFS transfer operation: %s' % operation,
         )
         return True
 
     localstore = repo.svfs.lfslocalblobstore
 
     objects = [
         p
         for p in _batchresponseobjects(
             req, lfsreq.get('objects', []), operation, localstore
         )
     ]
 
     rsp = {
         'transfer': 'basic',
         'objects': objects,
     }
 
     res.status = hgwebcommon.statusmessage(HTTP_OK)
     res.headers[b'Content-Type'] = b'application/vnd.git-lfs+json'
     res.setbodybytes(pycompat.bytestr(json.dumps(rsp)))
 
     return True
 
 
 def _batchresponseobjects(req, objects, action, store):
     """Yield one dictionary of attributes for the Batch API response for each
     object in the list.
 
     req: The parsedrequest for the Batch API request
     objects: The list of objects in the Batch API object request list
     action: 'upload' or 'download'
     store: The local blob store for servicing requests"""
 
     # Successful lfs-test-server response to solict an upload:
     # {
     #    u'objects': [{
     #       u'size': 12,
     #       u'oid': u'31cf...8e5b',
     #       u'actions': {
     #           u'upload': {
     #               u'href': u'http://localhost:$HGPORT/objects/31cf...8e5b',
     #               u'expires_at': u'0001-01-01T00:00:00Z',
     #               u'header': {
     #                   u'Accept': u'application/vnd.git-lfs'
     #               }
     #           }
     #       }
     #    }]
     # }
 
     # TODO: Sort out the expires_at/expires_in/authenticated keys.
 
     for obj in objects:
         # Convert unicode to ASCII to create a filesystem path
         soid = obj.get('oid')
         oid = soid.encode('ascii')
         rsp = {
             'oid': soid,
             'size': obj.get('size'),  # XXX: should this check the local size?
             # 'authenticated': True,
         }
 
         exists = True
         verifies = False
 
         # Verify an existing file on the upload request, so that the client is
         # solicited to re-upload if it corrupt locally.  Download requests are
         # also verified, so the error can be flagged in the Batch API response.
         # (Maybe we can use this to short circuit the download for `hg verify`,
         # IFF the client can assert that the remote end is an hg server.)
         # Otherwise, it's potentially overkill on download, since it is also
         # verified as the file is streamed to the caller.
         try:
             verifies = store.verify(oid)
             if verifies and action == b'upload':
                 # The client will skip this upload, but make sure it remains
                 # available locally.
                 store.linkfromusercache(oid)
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 _logexception(req)
 
                 rsp['error'] = {
                     'code': 500,
                     'message': inst.strerror or 'Internal Server Server',
                 }
                 yield rsp
                 continue
 
             exists = False
 
         # Items are always listed for downloads.  They are dropped for uploads
         # IFF they already exist locally.
         if action == b'download':
             if not exists:
                 rsp['error'] = {
                     'code': 404,
                     'message': "The object does not exist",
                 }
                 yield rsp
                 continue
 
             elif not verifies:
                 rsp['error'] = {
                     'code': 422,  # XXX: is this the right code?
                     'message': "The object is corrupt",
                 }
                 yield rsp
                 continue
 
         elif verifies:
             yield rsp  # Skip 'actions': already uploaded
             continue
 
         expiresat = datetime.datetime.now() + datetime.timedelta(minutes=10)
 
         def _buildheader():
             # The spec doesn't mention the Accept header here, but avoid
             # a gratuitous deviation from lfs-test-server in the test
             # output.
             hdr = {'Accept': 'application/vnd.git-lfs'}
 
             auth = req.headers.get(b'Authorization', b'')
             if auth.startswith(b'Basic '):
                 hdr['Authorization'] = pycompat.strurl(auth)
 
             return hdr
 
         rsp['actions'] = {
             '%s'
             % pycompat.strurl(action): {
                 'href': pycompat.strurl(
                     b'%s%s/.hg/lfs/objects/%s' % (req.baseurl, req.apppath, oid)
                 ),
                 # datetime.isoformat() doesn't include the 'Z' suffix
                 "expires_at": expiresat.strftime('%Y-%m-%dT%H:%M:%SZ'),
                 'header': _buildheader(),
             }
         }
 
         yield rsp
 
 
 def _processbasictransfer(repo, req, res, checkperm):
     """Handle a single file upload (PUT) or download (GET) action for the Basic
     Transfer Adapter.
 
     After determining if the request is for an upload or download, the access
     must be checked by calling ``checkperm()`` with either 'pull' or 'upload'
     before accessing the files.
 
     https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md
     """
 
     method = req.method
     oid = req.dispatchparts[-1]
     localstore = repo.svfs.lfslocalblobstore
 
     if len(req.dispatchparts) != 4:
         _sethttperror(res, HTTP_NOT_FOUND)
         return True
 
     if method == b'PUT':
         checkperm(b'upload')
 
         # TODO: verify Content-Type?
 
         existed = localstore.has(oid)
 
         # TODO: how to handle timeouts?  The body proxy handles limiting to
         #       Content-Length, but what happens if a client sends less than it
         #       says it will?
 
         statusmessage = hgwebcommon.statusmessage
         try:
             localstore.download(oid, req.bodyfh, req.headers[b'Content-Length'])
             res.status = statusmessage(HTTP_OK if existed else HTTP_CREATED)
         except blobstore.LfsCorruptionError:
             _logexception(req)
 
             # XXX: Is this the right code?
             res.status = statusmessage(422, b'corrupt blob')
 
         # There's no payload here, but this is the header that lfs-test-server
         # sends back.  This eliminates some gratuitous test output conditionals.
         res.headers[b'Content-Type'] = b'text/plain; charset=utf-8'
         res.setbodybytes(b'')
 
         return True
     elif method == b'GET':
         checkperm(b'pull')
 
         res.status = hgwebcommon.statusmessage(HTTP_OK)
         res.headers[b'Content-Type'] = b'application/octet-stream'
 
         try:
             # TODO: figure out how to send back the file in chunks, instead of
             #       reading the whole thing.  (Also figure out how to send back
             #       an error status if an IOError occurs after a partial write
             #       in that case.  Here, everything is read before starting.)
             res.setbodybytes(localstore.read(oid))
         except blobstore.LfsCorruptionError:
             _logexception(req)
 
             # XXX: Is this the right code?
             res.status = hgwebcommon.statusmessage(422, b'corrupt blob')
             res.setbodybytes(b'')
 
         return True
     else:
         _sethttperror(
             res,
             HTTP_METHOD_NOT_ALLOWED,
             message=b'Unsupported LFS transfer method: %s' % method,
         )
         return True

hgext/lfs/wrapper.py

@@ -1,545 +1,544 b''
 # wrapper.py - methods wrapping core mercurial logic
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import hashlib
 
 from mercurial.i18n import _
 from mercurial.node import bin, hex, short
 from mercurial.pycompat import (
     getattr,
     setattr,
 )
 
 from mercurial import (
     bundle2,
     changegroup,
     cmdutil,
     context,
     error,
     exchange,
     exthelper,
     localrepo,
     revlog,
     scmutil,
-    util,
     vfs as vfsmod,
     wireprotov1server,
 )
 
 from mercurial.upgrade_utils import (
     actions as upgrade_actions,
     engine as upgrade_engine,
 )
 
 from mercurial.interfaces import repository
 
 from mercurial.utils import (
     storageutil,
     stringutil,
 )
 
 from ..largefiles import lfutil
 
 from . import (
     blobstore,
     pointer,
 )
 
 eh = exthelper.exthelper()
 
 
 @eh.wrapfunction(localrepo, 'makefilestorage')
 def localrepomakefilestorage(orig, requirements, features, **kwargs):
     if b'lfs' in requirements:
         features.add(repository.REPO_FEATURE_LFS)
 
     return orig(requirements=requirements, features=features, **kwargs)
 
 
 @eh.wrapfunction(changegroup, 'allsupportedversions')
 def allsupportedversions(orig, ui):
     versions = orig(ui)
     versions.add(b'03')
     return versions
 
 
 @eh.wrapfunction(wireprotov1server, '_capabilities')
 def _capabilities(orig, repo, proto):
     '''Wrap server command to announce lfs server capability'''
     caps = orig(repo, proto)
-    if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
+    if hasattr(repo.svfs, 'lfslocalblobstore'):
         # Advertise a slightly different capability when lfs is *required*, so
         # that the client knows it MUST load the extension.  If lfs is not
         # required on the server, there's no reason to autoload the extension
         # on the client.
         if b'lfs' in repo.requirements:
             caps.append(b'lfs-serve')
 
         caps.append(b'lfs')
     return caps
 
 
 def bypasscheckhash(self, text):
     return False
 
 
 def readfromstore(self, text):
     """Read filelog content from local blobstore transform for flagprocessor.
 
     Default tranform for flagprocessor, returning contents from blobstore.
     Returns a 2-typle (text, validatehash) where validatehash is True as the
     contents of the blobstore should be checked using checkhash.
     """
     p = pointer.deserialize(text)
     oid = p.oid()
     store = self.opener.lfslocalblobstore
     if not store.has(oid):
         p.filename = self.filename
         self.opener.lfsremoteblobstore.readbatch([p], store)
 
     # The caller will validate the content
     text = store.read(oid, verify=False)
 
     # pack hg filelog metadata
     hgmeta = {}
     for k in p.keys():
         if k.startswith(b'x-hg-'):
             name = k[len(b'x-hg-') :]
             hgmeta[name] = p[k]
     if hgmeta or text.startswith(b'\1\n'):
         text = storageutil.packmeta(hgmeta, text)
 
     return (text, True)
 
 
 def writetostore(self, text):
     # hg filelog metadata (includes rename, etc)
     hgmeta, offset = storageutil.parsemeta(text)
     if offset and offset > 0:
         # lfs blob does not contain hg filelog metadata
         text = text[offset:]
 
     # git-lfs only supports sha256
     oid = hex(hashlib.sha256(text).digest())
     self.opener.lfslocalblobstore.write(oid, text)
 
     # replace contents with metadata
     longoid = b'sha256:%s' % oid
     metadata = pointer.gitlfspointer(oid=longoid, size=b'%d' % len(text))
 
     # by default, we expect the content to be binary. however, LFS could also
     # be used for non-binary content. add a special entry for non-binary data.
     # this will be used by filectx.isbinary().
     if not stringutil.binary(text):
         # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
         metadata[b'x-is-binary'] = b'0'
 
     # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
     if hgmeta is not None:
         for k, v in hgmeta.items():
             metadata[b'x-hg-%s' % k] = v
 
     rawtext = metadata.serialize()
     return (rawtext, False)
 
 
 def _islfs(rlog, node=None, rev=None):
     if rev is None:
         if node is None:
             # both None - likely working copy content where node is not ready
             return False
         rev = rlog.rev(node)
     else:
         node = rlog.node(rev)
     if node == rlog.nullid:
         return False
     flags = rlog.flags(rev)
     return bool(flags & revlog.REVIDX_EXTSTORED)
 
 
 # Wrapping may also be applied by remotefilelog
 def filelogaddrevision(
     orig,
     self,
     text,
     transaction,
     link,
     p1,
     p2,
     cachedelta=None,
     node=None,
     flags=revlog.REVIDX_DEFAULT_FLAGS,
     **kwds
 ):
     # The matcher isn't available if reposetup() wasn't called.
     lfstrack = self._revlog.opener.options.get(b'lfstrack')
 
     if lfstrack:
         textlen = len(text)
         # exclude hg rename meta from file size
         meta, offset = storageutil.parsemeta(text)
         if offset:
             textlen -= offset
 
         if lfstrack(self._revlog.filename, textlen):
             flags |= revlog.REVIDX_EXTSTORED
 
     return orig(
         self,
         text,
         transaction,
         link,
         p1,
         p2,
         cachedelta=cachedelta,
         node=node,
         flags=flags,
         **kwds
     )
 
 
 # Wrapping may also be applied by remotefilelog
 def filelogrenamed(orig, self, node):
     if _islfs(self._revlog, node):
         rawtext = self._revlog.rawdata(node)
         if not rawtext:
             return False
         metadata = pointer.deserialize(rawtext)
         if b'x-hg-copy' in metadata and b'x-hg-copyrev' in metadata:
             return metadata[b'x-hg-copy'], bin(metadata[b'x-hg-copyrev'])
         else:
             return False
     return orig(self, node)
 
 
 # Wrapping may also be applied by remotefilelog
 def filelogsize(orig, self, rev):
     if _islfs(self._revlog, rev=rev):
         # fast path: use lfs metadata to answer size
         rawtext = self._revlog.rawdata(rev)
         metadata = pointer.deserialize(rawtext)
         return int(metadata[b'size'])
     return orig(self, rev)
 
 
 @eh.wrapfunction(revlog, '_verify_revision')
 def _verify_revision(orig, rl, skipflags, state, node):
     if _islfs(rl, node=node):
         rawtext = rl.rawdata(node)
         metadata = pointer.deserialize(rawtext)
 
         # Don't skip blobs that are stored locally, as local verification is
         # relatively cheap and there's no other way to verify the raw data in
         # the revlog.
         if rl.opener.lfslocalblobstore.has(metadata.oid()):
             skipflags &= ~revlog.REVIDX_EXTSTORED
         elif skipflags & revlog.REVIDX_EXTSTORED:
             # The wrapped method will set `skipread`, but there's enough local
             # info to check renames.
             state[b'safe_renamed'].add(node)
 
     orig(rl, skipflags, state, node)
 
 
 @eh.wrapfunction(context.basefilectx, 'cmp')
 def filectxcmp(orig, self, fctx):
     """returns True if text is different than fctx"""
     # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
     if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
         # fast path: check LFS oid
         p1 = pointer.deserialize(self.rawdata())
         p2 = pointer.deserialize(fctx.rawdata())
         return p1.oid() != p2.oid()
     return orig(self, fctx)
 
 
 @eh.wrapfunction(context.basefilectx, 'isbinary')
 def filectxisbinary(orig, self):
     if self.islfs():
         # fast path: use lfs metadata to answer isbinary
         metadata = pointer.deserialize(self.rawdata())
         # if lfs metadata says nothing, assume it's binary by default
         return bool(int(metadata.get(b'x-is-binary', 1)))
     return orig(self)
 
 
 def filectxislfs(self):
     return _islfs(self.filelog()._revlog, self.filenode())
 
 
 @eh.wrapfunction(cmdutil, '_updatecatformatter')
 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
     orig(fm, ctx, matcher, path, decode)
     fm.data(rawdata=ctx[path].rawdata())
 
 
 @eh.wrapfunction(scmutil, 'wrapconvertsink')
 def convertsink(orig, sink):
     sink = orig(sink)
     if sink.repotype == b'hg':
 
         class lfssink(sink.__class__):
             def putcommit(
                 self,
                 files,
                 copies,
                 parents,
                 commit,
                 source,
                 revmap,
                 full,
                 cleanp2,
             ):
                 pc = super(lfssink, self).putcommit
                 node = pc(
                     files,
                     copies,
                     parents,
                     commit,
                     source,
                     revmap,
                     full,
                     cleanp2,
                 )
 
                 if b'lfs' not in self.repo.requirements:
                     ctx = self.repo[node]
 
                     # The file list may contain removed files, so check for
                     # membership before assuming it is in the context.
                     if any(f in ctx and ctx[f].islfs() for f, n in files):
                         self.repo.requirements.add(b'lfs')
                         scmutil.writereporequirements(self.repo)
 
                 return node
 
         sink.__class__ = lfssink
 
     return sink
 
 
 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
 # options and blob stores are passed from othervfs to the new readonlyvfs.
 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
 def vfsinit(orig, self, othervfs):
     orig(self, othervfs)
     # copy lfs related options
     for k, v in othervfs.options.items():
         if k.startswith(b'lfs'):
             self.options[k] = v
     # also copy lfs blobstores. note: this can run before reposetup, so lfs
     # blobstore attributes are not always ready at this time.
     for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
-        if util.safehasattr(othervfs, name):
+        if hasattr(othervfs, name):
             setattr(self, name, getattr(othervfs, name))
 
 
 def _prefetchfiles(repo, revmatches):
     """Ensure that required LFS blobs are present, fetching them as a group if
     needed."""
-    if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
+    if not hasattr(repo.svfs, 'lfslocalblobstore'):
         return
 
     pointers = []
     oids = set()
     localstore = repo.svfs.lfslocalblobstore
 
     for rev, match in revmatches:
         ctx = repo[rev]
         for f in ctx.walk(match):
             p = pointerfromctx(ctx, f)
             if p and p.oid() not in oids and not localstore.has(p.oid()):
                 p.filename = f
                 pointers.append(p)
                 oids.add(p.oid())
 
     if pointers:
         # Recalculating the repo store here allows 'paths.default' that is set
         # on the repo by a clone command to be used for the update.
         blobstore.remote(repo).readbatch(pointers, localstore)
 
 
 def _canskipupload(repo):
     # Skip if this hasn't been passed to reposetup()
-    if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
+    if not hasattr(repo.svfs, 'lfsremoteblobstore'):
         return True
 
     # if remotestore is a null store, upload is a no-op and can be skipped
     return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 
 def candownload(repo):
     # Skip if this hasn't been passed to reposetup()
-    if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
+    if not hasattr(repo.svfs, 'lfsremoteblobstore'):
         return False
 
     # if remotestore is a null store, downloads will lead to nothing
     return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 
 def uploadblobsfromrevs(repo, revs):
     """upload lfs blobs introduced by revs"""
     if _canskipupload(repo):
         return
     pointers = extractpointers(repo, revs)
     uploadblobs(repo, pointers)
 
 
 def prepush(pushop):
     """Prepush hook.
 
     Read through the revisions to push, looking for filelog entries that can be
     deserialized into metadata so that we can block the push on their upload to
     the remote blobstore.
     """
     return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
 
 
 @eh.wrapfunction(exchange, 'push')
 def push(orig, repo, remote, *args, **kwargs):
     """bail on push if the extension isn't enabled on remote when needed, and
     update the remote store based on the destination path."""
     if b'lfs' in repo.requirements:
         # If the remote peer is for a local repo, the requirement tests in the
         # base class method enforce lfs support.  Otherwise, some revisions in
         # this repo use lfs, and the remote repo needs the extension loaded.
         if not remote.local() and not remote.capable(b'lfs'):
             # This is a copy of the message in exchange.push() when requirements
             # are missing between local repos.
             m = _(b"required features are not supported in the destination: %s")
             raise error.Abort(
                 m % b'lfs', hint=_(b'enable the lfs extension on the server')
             )
 
         # Repositories where this extension is disabled won't have the field.
         # But if there's a requirement, then the extension must be loaded AND
         # there may be blobs to push.
         remotestore = repo.svfs.lfsremoteblobstore
         try:
             repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
             return orig(repo, remote, *args, **kwargs)
         finally:
             repo.svfs.lfsremoteblobstore = remotestore
     else:
         return orig(repo, remote, *args, **kwargs)
 
 
 # when writing a bundle via "hg bundle" command, upload related LFS blobs
 @eh.wrapfunction(bundle2, 'writenewbundle')
 def writenewbundle(
     orig, ui, repo, source, filename, bundletype, outgoing, *args, **kwargs
 ):
     """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
     uploadblobsfromrevs(repo, outgoing.missing)
     return orig(
         ui, repo, source, filename, bundletype, outgoing, *args, **kwargs
     )
 
 
 def extractpointers(repo, revs):
     """return a list of lfs pointers added by given revs"""
     repo.ui.debug(b'lfs: computing set of blobs to upload\n')
     pointers = {}
 
     makeprogress = repo.ui.makeprogress
     with makeprogress(
         _(b'lfs search'), _(b'changesets'), len(revs)
     ) as progress:
         for r in revs:
             ctx = repo[r]
             for p in pointersfromctx(ctx).values():
                 pointers[p.oid()] = p
             progress.increment()
         return sorted(pointers.values(), key=lambda p: p.oid())
 
 
 def pointerfromctx(ctx, f, removed=False):
     """return a pointer for the named file from the given changectx, or None if
     the file isn't LFS.
 
     Optionally, the pointer for a file deleted from the context can be returned.
     Since no such pointer is actually stored, and to distinguish from a non LFS
     file, this pointer is represented by an empty dict.
     """
     _ctx = ctx
     if f not in ctx:
         if not removed:
             return None
         if f in ctx.p1():
             _ctx = ctx.p1()
         elif f in ctx.p2():
             _ctx = ctx.p2()
         else:
             return None
     fctx = _ctx[f]
     if not _islfs(fctx.filelog()._revlog, fctx.filenode()):
         return None
     try:
         p = pointer.deserialize(fctx.rawdata())
         if ctx == _ctx:
             return p
         return {}
     except pointer.InvalidPointer as ex:
         raise error.Abort(
             _(b'lfs: corrupted pointer (%s@%s): %s\n')
             % (f, short(_ctx.node()), ex)
         )
 
 
 def pointersfromctx(ctx, removed=False):
     """return a dict {path: pointer} for given single changectx.
 
     If ``removed`` == True and the LFS file was removed from ``ctx``, the value
     stored for the path is an empty dict.
     """
     result = {}
     m = ctx.repo().narrowmatch()
 
     # TODO: consider manifest.fastread() instead
     for f in ctx.files():
         if not m(f):
             continue
         p = pointerfromctx(ctx, f, removed=removed)
         if p is not None:
             result[f] = p
     return result
 
 
 def uploadblobs(repo, pointers):
     """upload given pointers from local blobstore"""
     if not pointers:
         return
 
     remoteblob = repo.svfs.lfsremoteblobstore
     remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
 
 
 @eh.wrapfunction(upgrade_engine, 'finishdatamigration')
 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
     orig(ui, srcrepo, dstrepo, requirements)
 
     # Skip if this hasn't been passed to reposetup()
-    if util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and util.safehasattr(
+    if hasattr(srcrepo.svfs, 'lfslocalblobstore') and hasattr(
         dstrepo.svfs, 'lfslocalblobstore'
     ):
         srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
         dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
 
         for dirpath, dirs, files in srclfsvfs.walk():
             for oid in files:
                 ui.write(_(b'copying lfs blob %s\n') % oid)
                 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
 
 
 @eh.wrapfunction(upgrade_actions, 'preservedrequirements')
 @eh.wrapfunction(upgrade_actions, 'supporteddestrequirements')
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if b'lfs' in repo.requirements:
         reqs.add(b'lfs')
     return reqs