# bundlerepo.py - repository class for viewing uncompressed bundles # # Copyright 2006, 2007 Benoit Boissinot # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. """Repository class for viewing uncompressed bundles. This provides a read-only repository interface to bundles as if they were part of the actual repository. """ from __future__ import absolute_import import os import shutil import tempfile from .i18n import _ from .node import nullid from . import ( bundle2, changegroup, changelog, cmdutil, discovery, error, exchange, filelog, localrepo, manifest, mdiff, node as nodemod, pathutil, phases, pycompat, revlog, util, vfs as vfsmod, ) class bundlerevlog(revlog.revlog): def __init__(self, opener, indexfile, bundle, linkmapper): # How it works: # To retrieve a revision, we need to know the offset of the revision in # the bundle (an unbundle object). We store this offset in the index # (start). The base of the delta is stored in the base field. # # To differentiate a rev in the bundle from a rev in the revlog, we # check revision against repotiprev. opener = vfsmod.readonlyvfs(opener) revlog.revlog.__init__(self, opener, indexfile) self.bundle = bundle n = len(self) self.repotiprev = n - 1 self.bundlerevs = set() # used by 'bundle()' revset expression for deltadata in bundle.deltaiter(): node, p1, p2, cs, deltabase, delta, flags = deltadata size = len(delta) start = bundle.tell() - size link = linkmapper(cs) if node in self.nodemap: # this can happen if two branches make the same change self.bundlerevs.add(self.nodemap[node]) continue for p in (p1, p2): if p not in self.nodemap: raise error.LookupError(p, self.indexfile, _("unknown parent")) if deltabase not in self.nodemap: raise LookupError(deltabase, self.indexfile, _('unknown delta base')) baserev = self.rev(deltabase) # start, size, full unc. size, base (unused), link, p1, p2, node e = (revlog.offset_type(start, flags), size, -1, baserev, link, self.rev(p1), self.rev(p2), node) self.index.insert(-1, e) self.nodemap[node] = n self.bundlerevs.add(n) n += 1 def _chunk(self, rev, df=None): # Warning: in case of bundle, the diff is against what we stored as # delta base, not against rev - 1 # XXX: could use some caching if rev <= self.repotiprev: return revlog.revlog._chunk(self, rev) self.bundle.seek(self.start(rev)) return self.bundle.read(self.length(rev)) def revdiff(self, rev1, rev2): """return or calculate a delta between two revisions""" if rev1 > self.repotiprev and rev2 > self.repotiprev: # hot path for bundle revb = self.index[rev2][3] if revb == rev1: return self._chunk(rev2) elif rev1 <= self.repotiprev and rev2 <= self.repotiprev: return revlog.revlog.revdiff(self, rev1, rev2) return mdiff.textdiff(self.revision(rev1, raw=True), self.revision(rev2, raw=True)) def revision(self, nodeorrev, _df=None, raw=False): """return an uncompressed revision of a given node or revision number. """ if isinstance(nodeorrev, int): rev = nodeorrev node = self.node(rev) else: node = nodeorrev rev = self.rev(node) if node == nullid: return "" rawtext = None chain = [] iterrev = rev # reconstruct the revision if it is from a changegroup while iterrev > self.repotiprev: if self._cache and self._cache[1] == iterrev: rawtext = self._cache[2] break chain.append(iterrev) iterrev = self.index[iterrev][3] if rawtext is None: rawtext = self.baserevision(iterrev) while chain: delta = self._chunk(chain.pop()) rawtext = mdiff.patches(rawtext, [delta]) text, validatehash = self._processflags(rawtext, self.flags(rev), 'read', raw=raw) if validatehash: self.checkhash(text, node, rev=rev) self._cache = (node, rev, rawtext) return text def baserevision(self, nodeorrev): # Revlog subclasses may override 'revision' method to modify format of # content retrieved from revlog. To use bundlerevlog with such class one # needs to override 'baserevision' and make more specific call here. return revlog.revlog.revision(self, nodeorrev, raw=True) def addrevision(self, *args, **kwargs): raise NotImplementedError def addgroup(self, *args, **kwargs): raise NotImplementedError def strip(self, *args, **kwargs): raise NotImplementedError def checksize(self): raise NotImplementedError class bundlechangelog(bundlerevlog, changelog.changelog): def __init__(self, opener, bundle): changelog.changelog.__init__(self, opener) linkmapper = lambda x: x bundlerevlog.__init__(self, opener, self.indexfile, bundle, linkmapper) def baserevision(self, nodeorrev): # Although changelog doesn't override 'revision' method, some extensions # may replace this class with another that does. Same story with # manifest and filelog classes. # This bypasses filtering on changelog.node() and rev() because we need # revision text of the bundle base even if it is hidden. oldfilter = self.filteredrevs try: self.filteredrevs = () return changelog.changelog.revision(self, nodeorrev, raw=True) finally: self.filteredrevs = oldfilter class bundlemanifest(bundlerevlog, manifest.manifestrevlog): def __init__(self, opener, bundle, linkmapper, dirlogstarts=None, dir=''): manifest.manifestrevlog.__init__(self, opener, dir=dir) bundlerevlog.__init__(self, opener, self.indexfile, bundle, linkmapper) if dirlogstarts is None: dirlogstarts = {} if self.bundle.version == "03": dirlogstarts = _getfilestarts(self.bundle) self._dirlogstarts = dirlogstarts self._linkmapper = linkmapper def baserevision(self, nodeorrev): node = nodeorrev if isinstance(node, int): node = self.node(node) if node in self.fulltextcache: result = '%s' % self.fulltextcache[node] else: result = manifest.manifestrevlog.revision(self, nodeorrev, raw=True) return result def dirlog(self, d): if d in self._dirlogstarts: self.bundle.seek(self._dirlogstarts[d]) return bundlemanifest( self.opener, self.bundle, self._linkmapper, self._dirlogstarts, dir=d) return super(bundlemanifest, self).dirlog(d) class bundlefilelog(bundlerevlog, filelog.filelog): def __init__(self, opener, path, bundle, linkmapper): filelog.filelog.__init__(self, opener, path) bundlerevlog.__init__(self, opener, self.indexfile, bundle, linkmapper) def baserevision(self, nodeorrev): return filelog.filelog.revision(self, nodeorrev, raw=True) class bundlepeer(localrepo.localpeer): def canpush(self): return False class bundlephasecache(phases.phasecache): def __init__(self, *args, **kwargs): super(bundlephasecache, self).__init__(*args, **kwargs) if util.safehasattr(self, 'opener'): self.opener = vfsmod.readonlyvfs(self.opener) def write(self): raise NotImplementedError def _write(self, fp): raise NotImplementedError def _updateroots(self, phase, newroots, tr): self.phaseroots[phase] = newroots self.invalidate() self.dirty = True def _getfilestarts(bundle): bundlefilespos = {} for chunkdata in iter(bundle.filelogheader, {}): fname = chunkdata['filename'] bundlefilespos[fname] = bundle.tell() for chunk in iter(lambda: bundle.deltachunk(None), {}): pass return bundlefilespos class bundlerepository(localrepo.localrepository): """A repository instance that is a union of a local repo and a bundle. Instances represent a read-only repository composed of a local repository with the contents of a bundle file applied. The repository instance is conceptually similar to the state of a repository after an ``hg unbundle`` operation. However, the contents of the bundle are never applied to the actual base repository. """ def __init__(self, ui, repopath, bundlepath): self._tempparent = None try: localrepo.localrepository.__init__(self, ui, repopath) except error.RepoError: self._tempparent = tempfile.mkdtemp() localrepo.instance(ui, self._tempparent, 1) localrepo.localrepository.__init__(self, ui, self._tempparent) self.ui.setconfig('phases', 'publish', False, 'bundlerepo') if repopath: self._url = 'bundle:' + util.expandpath(repopath) + '+' + bundlepath else: self._url = 'bundle:' + bundlepath self.tempfile = None f = util.posixfile(bundlepath, "rb") bundle = exchange.readbundle(ui, f, bundlepath) if isinstance(bundle, bundle2.unbundle20): self._bundlefile = bundle self._cgunpacker = None hadchangegroup = False for part in bundle.iterparts(): if part.type == 'changegroup': if hadchangegroup: raise NotImplementedError("can't process " "multiple changegroups") hadchangegroup = True self._handlebundle2part(bundle, part) if not hadchangegroup: raise error.Abort(_("No changegroups found")) elif isinstance(bundle, changegroup.cg1unpacker): if bundle.compressed(): f = self._writetempbundle(bundle.read, '.hg10un', header='HG10UN') bundle = exchange.readbundle(ui, f, bundlepath, self.vfs) self._bundlefile = bundle self._cgunpacker = bundle else: raise error.Abort(_('bundle type %s cannot be read') % type(bundle)) # dict with the mapping 'filename' -> position in the bundle self.bundlefilespos = {} self.firstnewrev = self.changelog.repotiprev + 1 phases.retractboundary(self, None, phases.draft, [ctx.node() for ctx in self[self.firstnewrev:]]) def _handlebundle2part(self, bundle, part): if part.type == 'changegroup': cgstream = part version = part.params.get('version', '01') legalcgvers = changegroup.supportedincomingversions(self) if version not in legalcgvers: msg = _('Unsupported changegroup version: %s') raise error.Abort(msg % version) if bundle.compressed(): cgstream = self._writetempbundle(part.read, ".cg%sun" % version) self._cgunpacker = changegroup.getunbundler(version, cgstream, 'UN') def _writetempbundle(self, readfn, suffix, header=''): """Write a temporary file to disk """ fdtemp, temp = self.vfs.mkstemp(prefix="hg-bundle-", suffix=suffix) self.tempfile = temp with os.fdopen(fdtemp, pycompat.sysstr('wb')) as fptemp: fptemp.write(header) while True: chunk = readfn(2**18) if not chunk: break fptemp.write(chunk) return self.vfs.open(self.tempfile, mode="rb") @localrepo.unfilteredpropertycache def _phasecache(self): return bundlephasecache(self, self._phasedefaults) @localrepo.unfilteredpropertycache def changelog(self): # consume the header if it exists self._cgunpacker.changelogheader() c = bundlechangelog(self.svfs, self._cgunpacker) self.manstart = self._cgunpacker.tell() return c def _constructmanifest(self): self._cgunpacker.seek(self.manstart) # consume the header if it exists self._cgunpacker.manifestheader() linkmapper = self.unfiltered().changelog.rev m = bundlemanifest(self.svfs, self._cgunpacker, linkmapper) self.filestart = self._cgunpacker.tell() return m def _consumemanifest(self): """Consumes the manifest portion of the bundle, setting filestart so the file portion can be read.""" self._cgunpacker.seek(self.manstart) self._cgunpacker.manifestheader() for delta in self._cgunpacker.deltaiter(): pass self.filestart = self._cgunpacker.tell() @localrepo.unfilteredpropertycache def manstart(self): self.changelog return self.manstart @localrepo.unfilteredpropertycache def filestart(self): self.manifestlog # If filestart was not set by self.manifestlog, that means the # manifestlog implementation did not consume the manifests from the # changegroup (ex: it might be consuming trees from a separate bundle2 # part instead). So we need to manually consume it. if 'filestart' not in self.__dict__: self._consumemanifest() return self.filestart def url(self): return self._url def file(self, f): if not self.bundlefilespos: self._cgunpacker.seek(self.filestart) self.bundlefilespos = _getfilestarts(self._cgunpacker) if f in self.bundlefilespos: self._cgunpacker.seek(self.bundlefilespos[f]) linkmapper = self.unfiltered().changelog.rev return bundlefilelog(self.svfs, f, self._cgunpacker, linkmapper) else: return filelog.filelog(self.svfs, f) def close(self): """Close assigned bundle file immediately.""" self._bundlefile.close() if self.tempfile is not None: self.vfs.unlink(self.tempfile) if self._tempparent: shutil.rmtree(self._tempparent, True) def cancopy(self): return False def peer(self): return bundlepeer(self) def getcwd(self): return pycompat.getcwd() # always outside the repo # Check if parents exist in localrepo before setting def setparents(self, p1, p2=nullid): p1rev = self.changelog.rev(p1) p2rev = self.changelog.rev(p2) msg = _("setting parent to node %s that only exists in the bundle\n") if self.changelog.repotiprev < p1rev: self.ui.warn(msg % nodemod.hex(p1)) if self.changelog.repotiprev < p2rev: self.ui.warn(msg % nodemod.hex(p2)) return super(bundlerepository, self).setparents(p1, p2) def instance(ui, path, create): if create: raise error.Abort(_('cannot create new bundle repository')) # internal config: bundle.mainreporoot parentpath = ui.config("bundle", "mainreporoot") if not parentpath: # try to find the correct path to the working directory repo parentpath = cmdutil.findrepo(pycompat.getcwd()) if parentpath is None: parentpath = '' if parentpath: # Try to make the full path relative so we get a nice, short URL. # In particular, we don't want temp dir names in test outputs. cwd = pycompat.getcwd() if parentpath == cwd: parentpath = '' else: cwd = pathutil.normasprefix(cwd) if parentpath.startswith(cwd): parentpath = parentpath[len(cwd):] u = util.url(path) path = u.localpath() if u.scheme == 'bundle': s = path.split("+", 1) if len(s) == 1: repopath, bundlename = parentpath, s[0] else: repopath, bundlename = s else: repopath, bundlename = parentpath, path return bundlerepository(ui, repopath, bundlename) class bundletransactionmanager(object): def transaction(self): return None def close(self): raise NotImplementedError def release(self): raise NotImplementedError def getremotechanges(ui, repo, other, onlyheads=None, bundlename=None, force=False): '''obtains a bundle of changes incoming from other "onlyheads" restricts the returned changes to those reachable from the specified heads. "bundlename", if given, stores the bundle to this file path permanently; otherwise it's stored to a temp file and gets deleted again when you call the returned "cleanupfn". "force" indicates whether to proceed on unrelated repos. Returns a tuple (local, csets, cleanupfn): "local" is a local repo from which to obtain the actual incoming changesets; it is a bundlerepo for the obtained bundle when the original "other" is remote. "csets" lists the incoming changeset node ids. "cleanupfn" must be called without arguments when you're done processing the changes; it closes both the original "other" and the one returned here. ''' tmp = discovery.findcommonincoming(repo, other, heads=onlyheads, force=force) common, incoming, rheads = tmp if not incoming: try: if bundlename: os.unlink(bundlename) except OSError: pass return repo, [], other.close commonset = set(common) rheads = [x for x in rheads if x not in commonset] bundle = None bundlerepo = None localrepo = other.local() if bundlename or not localrepo: # create a bundle (uncompressed if other repo is not local) # developer config: devel.legacy.exchange legexc = ui.configlist('devel', 'legacy.exchange') forcebundle1 = 'bundle2' not in legexc and 'bundle1' in legexc canbundle2 = (not forcebundle1 and other.capable('getbundle') and other.capable('bundle2')) if canbundle2: kwargs = {} kwargs['common'] = common kwargs['heads'] = rheads kwargs['bundlecaps'] = exchange.caps20to10(repo) kwargs['cg'] = True b2 = other.getbundle('incoming', **kwargs) fname = bundle = changegroup.writechunks(ui, b2._forwardchunks(), bundlename) else: if other.capable('getbundle'): cg = other.getbundle('incoming', common=common, heads=rheads) elif onlyheads is None and not other.capable('changegroupsubset'): # compat with older servers when pulling all remote heads cg = other.changegroup(incoming, "incoming") rheads = None else: cg = other.changegroupsubset(incoming, rheads, 'incoming') if localrepo: bundletype = "HG10BZ" else: bundletype = "HG10UN" fname = bundle = bundle2.writebundle(ui, cg, bundlename, bundletype) # keep written bundle? if bundlename: bundle = None if not localrepo: # use the created uncompressed bundlerepo localrepo = bundlerepo = bundlerepository(repo.baseui, repo.root, fname) # this repo contains local and other now, so filter out local again common = repo.heads() if localrepo: # Part of common may be remotely filtered # So use an unfiltered version # The discovery process probably need cleanup to avoid that localrepo = localrepo.unfiltered() csets = localrepo.changelog.findmissing(common, rheads) if bundlerepo: reponodes = [ctx.node() for ctx in bundlerepo[bundlerepo.firstnewrev:]] remotephases = other.listkeys('phases') pullop = exchange.pulloperation(bundlerepo, other, heads=reponodes) pullop.trmanager = bundletransactionmanager() exchange._pullapplyphases(pullop, remotephases) def cleanup(): if bundlerepo: bundlerepo.close() if bundle: os.unlink(bundle) other.close() return (localrepo, csets, cleanup)