shrink-revlog.py
289 lines
| 9.3 KiB
| text/x-python
|
PythonLexer
/ contrib / shrink-revlog.py
|
r9515 | #!/usr/bin/env python | ||
"""\ | ||||
|
r10236 | reorder a revlog (the manifest by default) to save space | ||
Specifically, this topologically sorts the revisions in the revlog so that | ||||
revisions on the same branch are adjacent as much as possible. This is a | ||||
workaround for the fact that Mercurial computes deltas relative to the | ||||
|
r10216 | previous revision rather than relative to a parent revision. | ||
This is *not* safe to run on a changelog. | ||||
|
r9515 | """ | ||
# Originally written by Benoit Boissinot <benoit.boissinot at ens-lyon.org> | ||||
|
r10216 | # as a patch to rewrite-log. Cleaned up, refactored, documented, and | ||
|
r9515 | # renamed by Greg Ward <greg at gerg.ca>. | ||
# XXX would be nice to have a way to verify the repository after shrinking, | ||||
# e.g. by comparing "before" and "after" states of random changesets | ||||
# (maybe: export before, shrink, export after, diff). | ||||
|
r10542 | import os, tempfile, errno | ||
|
r10509 | from mercurial import revlog, transaction, node, util | ||
|
r10009 | from mercurial import changegroup | ||
|
r10508 | from mercurial.i18n import _ | ||
|
r9515 | |||
|
r10627 | def postorder(start, edges): | ||
result = [] | ||||
visit = list(start) | ||||
finished = set() | ||||
|
r9515 | |||
|
r10627 | while visit: | ||
cur = visit[-1] | ||||
for p in edges[cur]: | ||||
if p not in finished: | ||||
visit.append(p) | ||||
break | ||||
else: | ||||
result.append(cur) | ||||
finished.add(cur) | ||||
visit.pop() | ||||
return result | ||||
|
r9515 | |||
|
r10623 | def toposort_reversepostorder(ui, rl): | ||
# postorder of the reverse directed graph | ||||
# map rev to list of parent revs (p2 first) | ||||
parents = {} | ||||
heads = set() | ||||
|
r10508 | ui.status(_('reading revs\n')) | ||
|
r9515 | try: | ||
|
r10623 | for rev in rl: | ||
ui.progress(_('reading'), rev, total=len(rl)) | ||||
(p1, p2) = rl.parentrevs(rev) | ||||
if p1 == p2 == node.nullrev: | ||||
parents[rev] = () # root node | ||||
elif p1 == p2 or p2 == node.nullrev: | ||||
parents[rev] = (p1,) # normal node | ||||
else: | ||||
parents[rev] = (p2, p1) # merge node | ||||
heads.add(rev) | ||||
for p in parents[rev]: | ||||
heads.discard(p) | ||||
finally: | ||||
|
r10724 | ui.progress(_('reading'), None) | ||
|
r9515 | |||
|
r10627 | heads = list(heads) | ||
heads.sort(reverse=True) | ||||
|
r10623 | |||
|
r10627 | ui.status(_('sorting revs\n')) | ||
return postorder(heads, parents) | ||||
|
r10623 | |||
def toposort_postorderreverse(ui, rl): | ||||
# reverse-postorder of the reverse directed graph | ||||
|
r9515 | |||
|
r10623 | children = {} | ||
roots = set() | ||||
ui.status(_('reading revs\n')) | ||||
try: | ||||
for rev in rl: | ||||
ui.progress(_('reading'), rev, total=len(rl)) | ||||
(p1, p2) = rl.parentrevs(rev) | ||||
if p1 == p2 == node.nullrev: | ||||
roots.add(rev) | ||||
children[rev] = [] | ||||
if p1 != node.nullrev: | ||||
children[p1].append(rev) | ||||
if p2 != node.nullrev: | ||||
children[p2].append(rev) | ||||
finally: | ||||
|
r10724 | ui.progress(_('reading'), None) | ||
|
r10623 | |||
|
r11298 | roots = list(roots) | ||
|
r10627 | roots.sort() | ||
|
r10624 | |||
|
r10508 | ui.status(_('sorting revs\n')) | ||
|
r10627 | result = postorder(roots, children) | ||
|
r10623 | result.reverse() | ||
return result | ||||
|
r9515 | |||
|
r10213 | def writerevs(ui, r1, r2, order, tr): | ||
|
r10009 | |||
|
r10508 | ui.status(_('writing revs\n')) | ||
|
r10440 | |||
|
r10009 | count = [0] | ||
def progress(*args): | ||||
|
r10496 | ui.progress(_('writing'), count[0], total=len(order)) | ||
|
r10009 | count[0] += 1 | ||
order = [r1.node(r) for r in order] | ||||
# this is a bit ugly, but it works | ||||
lookup = lambda x: "%020d" % r1.linkrev(r1.rev(x)) | ||||
unlookup = lambda x: int(x, 10) | ||||
|
r9515 | try: | ||
|
r10009 | group = util.chunkbuffer(r1.group(order, lookup, progress)) | ||
chunkiter = changegroup.chunkiter(group) | ||||
r2.addgroup(chunkiter, unlookup, tr) | ||||
|
r9515 | finally: | ||
|
r10724 | ui.progress(_('writing'), None) | ||
|
r9515 | |||
|
r10542 | def report(ui, r1, r2): | ||
def getsize(r): | ||||
s = 0 | ||||
for fn in (r.indexfile, r.datafile): | ||||
try: | ||||
s += os.stat(fn).st_size | ||||
except OSError, inst: | ||||
if inst.errno != errno.ENOENT: | ||||
raise | ||||
return s | ||||
oldsize = float(getsize(r1)) | ||||
newsize = float(getsize(r2)) | ||||
|
r9515 | |||
|
r9712 | # argh: have to pass an int to %d, because a float >= 2^32 | ||
|
r9515 | # blows up under Python 2.5 or earlier | ||
|
r10508 | ui.write(_('old file size: %12d bytes (%6.1f MiB)\n') | ||
|
r10282 | % (int(oldsize), oldsize / 1024 / 1024)) | ||
|
r10508 | ui.write(_('new file size: %12d bytes (%6.1f MiB)\n') | ||
|
r10282 | % (int(newsize), newsize / 1024 / 1024)) | ||
|
r9515 | |||
shrink_percent = (oldsize - newsize) / oldsize * 100 | ||||
shrink_factor = oldsize / newsize | ||||
|
r10508 | ui.write(_('shrinkage: %.1f%% (%.1fx)\n') | ||
% (shrink_percent, shrink_factor)) | ||||
|
r9515 | |||
|
r10215 | def shrink(ui, repo, **opts): | ||
|
r10622 | """shrink a revlog by reordering revisions | ||
Rewrites all the entries in some revlog of the current repository | ||||
(by default, the manifest log) to save space. | ||||
Different sort algorithms have different performance | ||||
characteristics. Use ``--sort`` to select a sort algorithm so you | ||||
|
r10625 | can determine which works best for your data. | ||
|
r10215 | """ | ||
|
r9515 | |||
|
r10215 | if not repo.local(): | ||
|
r10508 | raise util.Abort(_('not a local repository: %s') % repo.root) | ||
|
r9515 | |||
|
r10215 | fn = opts.get('revlog') | ||
if not fn: | ||||
|
r9515 | indexfn = repo.sjoin('00manifest.i') | ||
else: | ||||
|
r10215 | if not fn.endswith('.i'): | ||
|
r10508 | raise util.Abort(_('--revlog option must specify the revlog index ' | ||
'file (*.i), not %s') % opts.get('revlog')) | ||||
|
r9515 | |||
|
r10215 | indexfn = os.path.realpath(fn) | ||
|
r9515 | store = repo.sjoin('') | ||
if not indexfn.startswith(store): | ||||
|
r10508 | raise util.Abort(_('--revlog option must specify a revlog in %s, ' | ||
'not %s') % (store, indexfn)) | ||||
|
r9515 | |||
|
r10622 | sortname = opts['sort'] | ||
try: | ||||
toposort = globals()['toposort_' + sortname] | ||||
except KeyError: | ||||
raise util.Abort(_('no such toposort algorithm: %s') % sortname) | ||||
|
r9515 | if not os.path.exists(indexfn): | ||
|
r10508 | raise util.Abort(_('no such file: %s') % indexfn) | ||
|
r9515 | if '00changelog' in indexfn: | ||
|
r10508 | raise util.Abort(_('shrinking the changelog ' | ||
'will corrupt your repository')) | ||||
|
r10542 | |||
ui.write(_('shrinking %s\n') % indexfn) | ||||
prefix = os.path.basename(indexfn)[:-1] | ||||
|
r11294 | tmpindexfn = util.mktempcopy(indexfn, emptyok=True) | ||
|
r10542 | |||
r1 = revlog.revlog(util.opener(os.getcwd(), audit=False), indexfn) | ||||
r2 = revlog.revlog(util.opener(os.getcwd(), audit=False), tmpindexfn) | ||||
datafn, tmpdatafn = r1.datafile, r2.datafile | ||||
|
r9515 | |||
oldindexfn = indexfn + '.old' | ||||
olddatafn = datafn + '.old' | ||||
if os.path.exists(oldindexfn) or os.path.exists(olddatafn): | ||||
|
r10508 | raise util.Abort(_('one or both of\n' | ||
' %s\n' | ||||
' %s\n' | ||||
'exists from a previous run; please clean up ' | ||||
'before running again') % (oldindexfn, olddatafn)) | ||||
|
r9515 | |||
# Don't use repo.transaction(), because then things get hairy with | ||||
# paths: some need to be relative to .hg, and some need to be | ||||
|
r10216 | # absolute. Doing it this way keeps things simple: everything is an | ||
|
r9515 | # absolute path. | ||
lock = repo.lock(wait=False) | ||||
|
r10234 | tr = transaction.transaction(ui.warn, | ||
|
r9515 | open, | ||
repo.sjoin('journal')) | ||||
|
r10542 | def ignoremissing(func): | ||
def f(*args, **kw): | ||||
try: | ||||
return func(*args, **kw) | ||||
except OSError, inst: | ||||
if inst.errno != errno.ENOENT: | ||||
raise | ||||
return f | ||||
|
r9515 | try: | ||
try: | ||||
|
r10213 | order = toposort(ui, r1) | ||
|
r10626 | |||
suboptimal = 0 | ||||
for i in xrange(1, len(order)): | ||||
parents = [p for p in r1.parentrevs(order[i]) | ||||
if p != node.nullrev] | ||||
|
r10655 | if parents and order[i - 1] not in parents: | ||
|
r10626 | suboptimal += 1 | ||
ui.note(_('%d suboptimal nodes\n') % suboptimal) | ||||
|
r10213 | writerevs(ui, r1, r2, order, tr) | ||
|
r10542 | report(ui, r1, r2) | ||
|
r9515 | tr.close() | ||
except: | ||||
# Abort transaction first, so we truncate the files before | ||||
# deleting them. | ||||
tr.abort() | ||||
|
r10542 | for fn in (tmpindexfn, tmpdatafn): | ||
ignoremissing(os.unlink)(fn) | ||||
|
r9515 | raise | ||
|
r10241 | if not opts.get('dry_run'): | ||
|
r10542 | # racy, both files cannot be renamed atomically | ||
# copy files | ||||
|
r10241 | util.os_link(indexfn, oldindexfn) | ||
|
r10542 | ignoremissing(util.os_link)(datafn, olddatafn) | ||
|
r11267 | |||
|
r10542 | # rename | ||
|
r10241 | util.rename(tmpindexfn, indexfn) | ||
|
r10542 | try: | ||
|
r11267 | os.chmod(tmpdatafn, os.stat(datafn).st_mode) | ||
|
r10542 | util.rename(tmpdatafn, datafn) | ||
except OSError, inst: | ||||
if inst.errno != errno.ENOENT: | ||||
raise | ||||
ignoremissing(os.unlink)(datafn) | ||||
|
r10241 | else: | ||
|
r10542 | for fn in (tmpindexfn, tmpdatafn): | ||
ignoremissing(os.unlink)(fn) | ||||
|
r9515 | finally: | ||
lock.release() | ||||
|
r10241 | if not opts.get('dry_run'): | ||
|
r10508 | ui.write(_('note: old revlog saved in:\n' | ||
' %s\n' | ||||
' %s\n' | ||||
'(You can delete those files when you are satisfied that your\n' | ||||
'repository is still sane. ' | ||||
'Running \'hg verify\' is strongly recommended.)\n') | ||||
|
r10241 | % (oldindexfn, olddatafn)) | ||
|
r9515 | |||
|
r10215 | cmdtable = { | ||
'shrink': (shrink, | ||||
|
r10508 | [('', 'revlog', '', _('index (.i) file of the revlog to shrink')), | ||
('n', 'dry-run', None, _('do not shrink, simulate only')), | ||||
|
r10625 | ('', 'sort', 'reversepostorder', 'name of sort algorithm to use'), | ||
|
r10241 | ], | ||
|
r10508 | _('hg shrink [--revlog PATH]')) | ||
|
r10215 | } | ||
|
r10236 | |||
if __name__ == "__main__": | ||||
|
r10282 | print "shrink-revlog.py is now an extension (see hg help extensions)" | ||