##// END OF EJS Templates
revlog: optionally cache the full text when adding revisions...
revlog: optionally cache the full text when adding revisions revlog instances can cache the full text of a single revision. Typically the most recently read revision is cached. When adding a delta group via addgroup() and _addrevision(), the full text isn't always computed: sometimes only the passed in delta is sufficient for adding a new revision to the revlog. When writing the changelog from a delta group, the just-added full text revision is always read immediately after it is written because the changegroup code needs to extract the set of files from the entry. In other words, revision() is *always* being called and caching the full text of the just-added revision is guaranteed to result in a cache hit, making the cache worthwhile. This patch adds support to _addrevision() for always building and caching the full text. This option is currently only active when processing changelog entries from a changegroup. While the total number of revision() calls is the same, the location matters: buildtext() calls into revision() on the base revision when building the full text of the just-added revision. Since the previous revision's _addrevision() built the full text and the the previous revision is likely the base revision, this means that the base revision's full text is likely cached and can be used to compute the current full text from just a delta. No extra I/O required. The end result is the changelog isn't opened and read after adding every revision from a changegroup. On my 2013 MacBook Pro running OS X 10.10.5 from an SSD and Python 2.7, this patch impacted the time taken to apply ~262,000 changesets from a mozilla-central gzip bundle: before: ~43s after: ~32s ~25% reduction in changelog processing times. Not bad.

File last commit:

r26230:6b16a353 default
r26243:83629142 default
Show More
unionrepo.py
259 lines | 8.9 KiB | text/x-python | PythonLexer
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # unionrepo.py - repository class for viewing union of repository changesets
#
# Derived from bundlerepo.py
# Copyright 2006, 2007 Benoit Boissinot <bboissin@gmail.com>
# Copyright 2013 Unity Technologies, Mads Kiilerich <madski@unity3d.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""Repository class for "in-memory pull" of one local repository to another,
allowing operations like diff and log with revsets.
"""
Gregory Szorc
unionrepo: use absolute_import
r25988 from __future__ import absolute_import
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 import os
Gregory Szorc
unionrepo: use absolute_import
r25988
from .i18n import _
from .node import nullid
from . import (
changelog,
cmdutil,
filelog,
localrepo,
manifest,
mdiff,
pathutil,
revlog,
scmutil,
util,
)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
class unionrevlog(revlog.revlog):
def __init__(self, opener, indexfile, revlog2, linkmapper):
# How it works:
# To retrieve a revision, we just need to know the node id so we can
# look it up in revlog2.
#
# To differentiate a rev in the second revlog from a rev in the revlog,
# we check revision against repotiprev.
opener = scmutil.readonlyvfs(opener)
revlog.revlog.__init__(self, opener, indexfile)
self.revlog2 = revlog2
n = len(self)
self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression
for rev2 in self.revlog2:
rev = self.revlog2.index[rev2]
# rev numbers - in revlog2, very different from self.rev
Pierre-Yves David
unionrepo: take delta base in account with building unified revlog...
r26230 _start, _csize, _rsize, base, linkrev, p1rev, p2rev, node = rev
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
if linkmapper is None: # link is to same revlog
assert linkrev == rev2 # we never link back
link = n
else: # rev must be mapped from repo2 cl to unified cl by linkmapper
link = linkmapper(linkrev)
Pierre-Yves David
unionrepo: take delta base in account with building unified revlog...
r26230 if linkmapper is not None: # link is to same revlog
base = linkmapper(base)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if node in self.nodemap:
# this happens for the common revlog revisions
self.bundlerevs.add(self.nodemap[node])
continue
p1node = self.revlog2.node(p1rev)
p2node = self.revlog2.node(p2rev)
Pierre-Yves David
unionrepo: take delta base in account with building unified revlog...
r26230 e = (None, None, None, base,
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 link, self.rev(p1node), self.rev(p2node), node)
self.index.insert(-1, e)
self.nodemap[node] = n
self.bundlerevs.add(n)
n += 1
def _chunk(self, rev):
if rev <= self.repotiprev:
return revlog.revlog._chunk(self, rev)
return self.revlog2._chunk(self.node(rev))
def revdiff(self, rev1, rev2):
"""return or calculate a delta between two revisions"""
if rev1 > self.repotiprev and rev2 > self.repotiprev:
return self.revlog2.revdiff(
self.revlog2.rev(self.node(rev1)),
self.revlog2.rev(self.node(rev2)))
elif rev1 <= self.repotiprev and rev2 <= self.repotiprev:
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 return self.baserevdiff(rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
return mdiff.textdiff(self.revision(self.node(rev1)),
self.revision(self.node(rev2)))
def revision(self, nodeorrev):
"""return an uncompressed revision of a given node or revision
number.
"""
if isinstance(nodeorrev, int):
rev = nodeorrev
node = self.node(rev)
else:
node = nodeorrev
rev = self.rev(node)
if node == nullid:
return ""
if rev > self.repotiprev:
text = self.revlog2.revision(node)
self._cache = (node, rev, text)
else:
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 text = self.baserevision(rev)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # already cached
return text
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
# Revlog subclasses may override 'revision' method to modify format of
# content retrieved from revlog. To use unionrevlog with such class one
# needs to override 'baserevision' and make more specific call here.
return revlog.revlog.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
# Exists for the same purpose as baserevision.
return revlog.revlog.revdiff(self, rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def addrevision(self, text, transaction, link, p1=None, p2=None, d=None):
raise NotImplementedError
def addgroup(self, revs, linkmapper, transaction):
raise NotImplementedError
def strip(self, rev, minlink):
raise NotImplementedError
def checksize(self):
raise NotImplementedError
class unionchangelog(unionrevlog, changelog.changelog):
def __init__(self, opener, opener2):
changelog.changelog.__init__(self, opener)
linkmapper = None
changelog2 = changelog.changelog(opener2)
unionrevlog.__init__(self, opener, self.indexfile, changelog2,
linkmapper)
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
# Although changelog doesn't override 'revision' method, some extensions
# may replace this class with another that does. Same story with
# manifest and filelog classes.
return changelog.changelog.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
return changelog.changelog.revdiff(self, rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionmanifest(unionrevlog, manifest.manifest):
def __init__(self, opener, opener2, linkmapper):
manifest.manifest.__init__(self, opener)
manifest2 = manifest.manifest(opener2)
unionrevlog.__init__(self, opener, self.indexfile, manifest2,
linkmapper)
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
return manifest.manifest.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
return manifest.manifest.revdiff(self, rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionfilelog(unionrevlog, filelog.filelog):
def __init__(self, opener, path, opener2, linkmapper, repo):
filelog.filelog.__init__(self, opener, path)
filelog2 = filelog.filelog(opener2, path)
unionrevlog.__init__(self, opener, self.indexfile, filelog2,
linkmapper)
self._repo = repo
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
return filelog.filelog.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
return filelog.filelog.revdiff(self, rev1, rev2)
Mike Edgar
revlog: add "iscensored()" to revlog public API...
r24118 def iscensored(self, rev):
"""Check if a revision is censored."""
if rev <= self.repotiprev:
return filelog.filelog.iscensored(self, rev)
return self.revlog2.iscensored(rev)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionpeer(localrepo.localpeer):
def canpush(self):
return False
class unionrepository(localrepo.localrepository):
def __init__(self, ui, path, path2):
localrepo.localrepository.__init__(self, ui, path)
Mads Kiilerich
config: set a 'source' in most cases where config don't come from file but code...
r20790 self.ui.setconfig('phases', 'publish', False, 'unionrepo')
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
self._url = 'union:%s+%s' % (util.expandpath(path),
util.expandpath(path2))
self.repo2 = localrepo.localrepository(ui, path2)
@localrepo.unfilteredpropertycache
def changelog(self):
Angel Ezquerra
localrepo: remove all external users of localrepo.sopener...
r23878 return unionchangelog(self.svfs, self.repo2.svfs)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
def _clrev(self, rev2):
"""map from repo2 changelog rev to temporary rev in self.changelog"""
node = self.repo2.changelog.node(rev2)
return self.changelog.rev(node)
@localrepo.unfilteredpropertycache
def manifest(self):
Angel Ezquerra
localrepo: remove all external users of localrepo.sopener...
r23878 return unionmanifest(self.svfs, self.repo2.svfs,
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self._clrev)
def url(self):
return self._url
def file(self, f):
Angel Ezquerra
localrepo: remove all external users of localrepo.sopener...
r23878 return unionfilelog(self.svfs, f, self.repo2.svfs,
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self._clrev, self)
def close(self):
self.repo2.close()
def cancopy(self):
return False
def peer(self):
return unionpeer(self)
def getcwd(self):
return os.getcwd() # always outside the repo
def instance(ui, path, create):
if create:
raise util.Abort(_('cannot create new union repository'))
parentpath = ui.config("bundle", "mainreporoot", "")
if not parentpath:
# try to find the correct path to the working directory repo
parentpath = cmdutil.findrepo(os.getcwd())
if parentpath is None:
parentpath = ''
if parentpath:
# Try to make the full path relative so we get a nice, short URL.
# In particular, we don't want temp dir names in test outputs.
cwd = os.getcwd()
if parentpath == cwd:
parentpath = ''
else:
FUJIWARA Katsunori
unionrepo: use pathutil.normasprefix to ensure os.sep at the end of cwd...
r24835 cwd = pathutil.normasprefix(cwd)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath.startswith(cwd):
parentpath = parentpath[len(cwd):]
if path.startswith('union:'):
s = path.split(":", 1)[1].split("+", 1)
if len(s) == 1:
repopath, repopath2 = parentpath, s[0]
else:
repopath, repopath2 = s
else:
repopath, repopath2 = parentpath, path
return unionrepository(ui, repopath, repopath2)