##// END OF EJS Templates
strip: calculate list of extra nodes to save and pass it to changegroupsubset...
Alexis S. L. Carvalho -
r5909:f45f7390 default
parent child Browse files
Show More
@@ -0,0 +1,61 b''
1 #!/bin/sh
2
3 # test stripping of filelogs where the linkrev doesn't always increase
4
5 echo '[extensions]' >> $HGRCPATH
6 echo 'hgext.mq =' >> $HGRCPATH
7
8 hg init orig
9 cd orig
10
11 hidefilename()
12 {
13 sed -e 's/saving bundle to .*strip-backup/saving bundle to strip-backup/'
14 }
15
16 commit()
17 {
18 hg up -qC null
19 count=1
20 for i in "$@"; do
21 for f in $i; do
22 echo $count > $f
23 done
24 count=`expr $count + 1`
25 done
26 hg commit -qAm "$*"
27 }
28
29 # 2 1 0 2 0 1 2
30 commit '201 210'
31
32 commit '102 120' '210'
33
34 commit '021'
35
36 commit '201' '021 120'
37
38 commit '012 021' '102 201' '120 210'
39
40 commit '102 120' '012 210' '021 201'
41
42 commit '201 210' '021 120' '012 102'
43
44 cd ..
45 hg clone -q -U -r -1 -r -2 -r -3 orig crossed
46
47 for i in crossed/.hg/store/{00manifest.i,data/*.i}; do
48 echo $i
49 hg debugindex $i
50 echo
51 done
52
53 for i in 0 1 2; do
54 hg clone -q -U --pull crossed $i
55 echo "% Trying to strip revision $i"
56 hg --cwd $i strip $i 2>&1 | hidefilename
57 echo "% Verifying"
58 hg --cwd $i verify
59 echo
60 done
61
@@ -0,0 +1,87 b''
1 crossed/.hg/store/00manifest.i
2 rev offset length base linkrev nodeid p1 p2
3 0 0 112 0 0 6f105cbb914d 000000000000 000000000000
4 1 112 123 0 1 8f3d04e263e5 000000000000 000000000000
5 2 235 122 0 2 f0ef8726ac4f 000000000000 000000000000
6
7 crossed/.hg/store/data/012.i
8 rev offset length base linkrev nodeid p1 p2
9 0 0 3 0 0 b8e02f643373 000000000000 000000000000
10 1 3 3 1 1 5d9299349fc0 000000000000 000000000000
11 2 6 3 2 2 2661d26c6496 000000000000 000000000000
12
13 crossed/.hg/store/data/021.i
14 rev offset length base linkrev nodeid p1 p2
15 0 0 3 0 0 b8e02f643373 000000000000 000000000000
16 1 3 3 1 2 5d9299349fc0 000000000000 000000000000
17 2 6 3 2 1 2661d26c6496 000000000000 000000000000
18
19 crossed/.hg/store/data/102.i
20 rev offset length base linkrev nodeid p1 p2
21 0 0 3 0 1 b8e02f643373 000000000000 000000000000
22 1 3 3 1 0 5d9299349fc0 000000000000 000000000000
23 2 6 3 2 2 2661d26c6496 000000000000 000000000000
24
25 crossed/.hg/store/data/120.i
26 rev offset length base linkrev nodeid p1 p2
27 0 0 3 0 1 b8e02f643373 000000000000 000000000000
28 1 3 3 1 2 5d9299349fc0 000000000000 000000000000
29 2 6 3 2 0 2661d26c6496 000000000000 000000000000
30
31 crossed/.hg/store/data/201.i
32 rev offset length base linkrev nodeid p1 p2
33 0 0 3 0 2 b8e02f643373 000000000000 000000000000
34 1 3 3 1 0 5d9299349fc0 000000000000 000000000000
35 2 6 3 2 1 2661d26c6496 000000000000 000000000000
36
37 crossed/.hg/store/data/210.i
38 rev offset length base linkrev nodeid p1 p2
39 0 0 3 0 2 b8e02f643373 000000000000 000000000000
40 1 3 3 1 1 5d9299349fc0 000000000000 000000000000
41 2 6 3 2 0 2661d26c6496 000000000000 000000000000
42
43 % Trying to strip revision 0
44 saving bundle to strip-backup/cbb8c2f0a2e3-backup
45 saving bundle to strip-backup/cbb8c2f0a2e3-temp
46 adding branch
47 adding changesets
48 adding manifests
49 adding file changes
50 added 2 changesets with 12 changes to 6 files (+1 heads)
51 % Verifying
52 checking changesets
53 checking manifests
54 crosschecking files in changesets and manifests
55 checking files
56 6 files, 2 changesets, 12 total revisions
57
58 % Trying to strip revision 1
59 saving bundle to strip-backup/124ecc0cbec9-backup
60 saving bundle to strip-backup/124ecc0cbec9-temp
61 adding branch
62 adding changesets
63 adding manifests
64 adding file changes
65 added 1 changesets with 10 changes to 6 files (+1 heads)
66 % Verifying
67 checking changesets
68 checking manifests
69 crosschecking files in changesets and manifests
70 checking files
71 6 files, 2 changesets, 12 total revisions
72
73 % Trying to strip revision 2
74 saving bundle to strip-backup/f6439b304a1a-backup
75 saving bundle to strip-backup/f6439b304a1a-temp
76 adding branch
77 adding changesets
78 adding manifests
79 adding file changes
80 added 0 changesets with 6 changes to 4 files
81 % Verifying
82 checking changesets
83 checking manifests
84 crosschecking files in changesets and manifests
85 checking files
86 6 files, 2 changesets, 12 total revisions
87
@@ -1,125 +1,165 b''
1 # repair.py - functions for repository repair for mercurial
1 # repair.py - functions for repository repair for mercurial
2 #
2 #
3 # Copyright 2005, 2006 Chris Mason <mason@suse.com>
3 # Copyright 2005, 2006 Chris Mason <mason@suse.com>
4 # Copyright 2007 Matt Mackall
4 # Copyright 2007 Matt Mackall
5 #
5 #
6 # This software may be used and distributed according to the terms
6 # This software may be used and distributed according to the terms
7 # of the GNU General Public License, incorporated herein by reference.
7 # of the GNU General Public License, incorporated herein by reference.
8
8
9 import changegroup, os
9 import changegroup, os
10 from node import *
10 from node import *
11
11
12 def _limitheads(cl, stoprev):
12 def _limitheads(cl, stoprev):
13 """return the list of all revs >= stoprev that have no children"""
13 """return the list of all revs >= stoprev that have no children"""
14 seen = {}
14 seen = {}
15 heads = []
15 heads = []
16
16
17 for r in xrange(cl.count() - 1, stoprev - 1, -1):
17 for r in xrange(cl.count() - 1, stoprev - 1, -1):
18 if r not in seen:
18 if r not in seen:
19 heads.append(r)
19 heads.append(r)
20 for p in cl.parentrevs(r):
20 for p in cl.parentrevs(r):
21 seen[p] = 1
21 seen[p] = 1
22 return heads
22 return heads
23
23
24 def _bundle(repo, bases, heads, node, suffix):
24 def _bundle(repo, bases, heads, node, suffix, extranodes=None):
25 """create a bundle with the specified revisions as a backup"""
25 """create a bundle with the specified revisions as a backup"""
26 cg = repo.changegroupsubset(bases, heads, 'strip')
26 cg = repo.changegroupsubset(bases, heads, 'strip', extranodes)
27 backupdir = repo.join("strip-backup")
27 backupdir = repo.join("strip-backup")
28 if not os.path.isdir(backupdir):
28 if not os.path.isdir(backupdir):
29 os.mkdir(backupdir)
29 os.mkdir(backupdir)
30 name = os.path.join(backupdir, "%s-%s" % (short(node), suffix))
30 name = os.path.join(backupdir, "%s-%s" % (short(node), suffix))
31 repo.ui.warn("saving bundle to %s\n" % name)
31 repo.ui.warn("saving bundle to %s\n" % name)
32 return changegroup.writebundle(cg, name, "HG10BZ")
32 return changegroup.writebundle(cg, name, "HG10BZ")
33
33
34 def _collectfilenodes(repo, striprev):
34 def _collectfilenodes(repo, striprev):
35 """find out the first node that should be stripped from each filelog"""
35 """find out the first node that should be stripped from each filelog"""
36 mm = repo.changectx(striprev).manifest()
36 mm = repo.changectx(striprev).manifest()
37 filenodes = {}
37 filenodes = {}
38
38
39 for x in xrange(striprev, repo.changelog.count()):
39 for x in xrange(striprev, repo.changelog.count()):
40 for name in repo.changectx(x).files():
40 for name in repo.changectx(x).files():
41 if name in filenodes:
41 if name in filenodes:
42 continue
42 continue
43 filenodes[name] = mm.get(name)
43 filenodes[name] = mm.get(name)
44
44
45 return filenodes
45 return filenodes
46
46
47 def _collectextranodes(repo, files, link):
48 """return the nodes that have to be saved before the strip"""
49 def collectone(revlog):
50 extra = []
51 startrev = count = revlog.count()
52 # find the truncation point of the revlog
53 for i in xrange(0, count):
54 node = revlog.node(i)
55 lrev = revlog.linkrev(node)
56 if lrev >= link:
57 startrev = i + 1
58 break
59
60 # see if any revision after that point has a linkrev less than link
61 # (we have to manually save these guys)
62 for i in xrange(startrev, count):
63 node = revlog.node(i)
64 lrev = revlog.linkrev(node)
65 if lrev < link:
66 extra.append((node, cl.node(lrev)))
67
68 return extra
69
70 extranodes = {}
71 cl = repo.changelog
72 extra = collectone(repo.manifest)
73 if extra:
74 extranodes[1] = extra
75 for fname in files:
76 f = repo.file(fname)
77 extra = collectone(f)
78 if extra:
79 extranodes[fname] = extra
80
81 return extranodes
82
47 def _stripall(repo, striprev, filenodes):
83 def _stripall(repo, striprev, filenodes):
48 """strip the requested nodes from the filelogs"""
84 """strip the requested nodes from the filelogs"""
49 # we go in two steps here so the strip loop happens in a
85 # we go in two steps here so the strip loop happens in a
50 # sensible order. When stripping many files, this helps keep
86 # sensible order. When stripping many files, this helps keep
51 # our disk access patterns under control.
87 # our disk access patterns under control.
52
88
53 files = filenodes.keys()
89 files = filenodes.keys()
54 files.sort()
90 files.sort()
55 for name in files:
91 for name in files:
56 f = repo.file(name)
92 f = repo.file(name)
57 fnode = filenodes[name]
93 fnode = filenodes[name]
58 frev = 0
94 frev = 0
59 if fnode is not None and fnode in f.nodemap:
95 if fnode is not None and fnode in f.nodemap:
60 frev = f.rev(fnode)
96 frev = f.rev(fnode)
61 f.strip(frev, striprev)
97 f.strip(frev, striprev)
62
98
63 def strip(ui, repo, node, backup="all"):
99 def strip(ui, repo, node, backup="all"):
64 cl = repo.changelog
100 cl = repo.changelog
65 # TODO delete the undo files, and handle undo of merge sets
101 # TODO delete the undo files, and handle undo of merge sets
66 pp = cl.parents(node)
102 pp = cl.parents(node)
67 striprev = cl.rev(node)
103 striprev = cl.rev(node)
68
104
69 # save is a list of all the branches we are truncating away
105 # save is a list of all the branches we are truncating away
70 # that we actually want to keep. changegroup will be used
106 # that we actually want to keep. changegroup will be used
71 # to preserve them and add them back after the truncate
107 # to preserve them and add them back after the truncate
72 saveheads = []
108 saveheads = []
73 savebases = {}
109 savebases = {}
74
110
75 heads = [cl.node(r) for r in _limitheads(cl, striprev)]
111 heads = [cl.node(r) for r in _limitheads(cl, striprev)]
76 seen = {}
112 seen = {}
77
113
78 # search through all the heads, finding those where the revision
114 # search through all the heads, finding those where the revision
79 # we want to strip away is an ancestor. Also look for merges
115 # we want to strip away is an ancestor. Also look for merges
80 # that might be turned into new heads by the strip.
116 # that might be turned into new heads by the strip.
81 while heads:
117 while heads:
82 h = heads.pop()
118 h = heads.pop()
83 n = h
119 n = h
84 while True:
120 while True:
85 seen[n] = 1
121 seen[n] = 1
86 pp = cl.parents(n)
122 pp = cl.parents(n)
87 if pp[1] != nullid:
123 if pp[1] != nullid:
88 for p in pp:
124 for p in pp:
89 if cl.rev(p) > striprev and p not in seen:
125 if cl.rev(p) > striprev and p not in seen:
90 heads.append(p)
126 heads.append(p)
91 if pp[0] == nullid:
127 if pp[0] == nullid:
92 break
128 break
93 if cl.rev(pp[0]) < striprev:
129 if cl.rev(pp[0]) < striprev:
94 break
130 break
95 n = pp[0]
131 n = pp[0]
96 if n == node:
132 if n == node:
97 break
133 break
98 r = cl.reachable(h, node)
134 r = cl.reachable(h, node)
99 if node not in r:
135 if node not in r:
100 saveheads.append(h)
136 saveheads.append(h)
101 for x in r:
137 for x in r:
102 if cl.rev(x) > striprev:
138 if cl.rev(x) > striprev:
103 savebases[x] = 1
139 savebases[x] = 1
104
140
141 filenodes = _collectfilenodes(repo, striprev)
142
143 extranodes = _collectextranodes(repo, filenodes, striprev)
144
105 # create a changegroup for all the branches we need to keep
145 # create a changegroup for all the branches we need to keep
106 if backup == "all":
146 if backup == "all":
107 _bundle(repo, [node], cl.heads(), node, 'backup')
147 _bundle(repo, [node], cl.heads(), node, 'backup')
108 if saveheads:
148 if saveheads or extranodes:
109 chgrpfile = _bundle(repo, savebases.keys(), saveheads, node, 'temp')
149 chgrpfile = _bundle(repo, savebases.keys(), saveheads, node, 'temp',
150 extranodes)
110
151
111 filenodes = _collectfilenodes(repo, striprev)
112 _stripall(repo, striprev, filenodes)
152 _stripall(repo, striprev, filenodes)
113
153
114 change = cl.read(node)
154 change = cl.read(node)
115 cl.strip(striprev, striprev)
155 cl.strip(striprev, striprev)
116 repo.manifest.strip(repo.manifest.rev(change[0]), striprev)
156 repo.manifest.strip(repo.manifest.rev(change[0]), striprev)
117 if saveheads:
157 if saveheads or extranodes:
118 ui.status("adding branch\n")
158 ui.status("adding branch\n")
119 f = open(chgrpfile, "rb")
159 f = open(chgrpfile, "rb")
120 gen = changegroup.readbundle(f, chgrpfile)
160 gen = changegroup.readbundle(f, chgrpfile)
121 repo.addchangegroup(gen, 'strip', 'bundle:' + chgrpfile)
161 repo.addchangegroup(gen, 'strip', 'bundle:' + chgrpfile, True)
122 f.close()
162 f.close()
123 if backup != "strip":
163 if backup != "strip":
124 os.unlink(chgrpfile)
164 os.unlink(chgrpfile)
125
165
@@ -1,1309 +1,1306 b''
1 """
1 """
2 revlog.py - storage back-end for mercurial
2 revlog.py - storage back-end for mercurial
3
3
4 This provides efficient delta storage with O(1) retrieve and append
4 This provides efficient delta storage with O(1) retrieve and append
5 and O(changes) merge between branches
5 and O(changes) merge between branches
6
6
7 Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
7 Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
8
8
9 This software may be used and distributed according to the terms
9 This software may be used and distributed according to the terms
10 of the GNU General Public License, incorporated herein by reference.
10 of the GNU General Public License, incorporated herein by reference.
11 """
11 """
12
12
13 from node import *
13 from node import *
14 from i18n import _
14 from i18n import _
15 import binascii, changegroup, errno, ancestor, mdiff, os
15 import binascii, changegroup, errno, ancestor, mdiff, os
16 import sha, struct, util, zlib
16 import sha, struct, util, zlib
17
17
18 _pack = struct.pack
18 _pack = struct.pack
19 _unpack = struct.unpack
19 _unpack = struct.unpack
20 _compress = zlib.compress
20 _compress = zlib.compress
21 _decompress = zlib.decompress
21 _decompress = zlib.decompress
22 _sha = sha.new
22 _sha = sha.new
23
23
24 # revlog flags
24 # revlog flags
25 REVLOGV0 = 0
25 REVLOGV0 = 0
26 REVLOGNG = 1
26 REVLOGNG = 1
27 REVLOGNGINLINEDATA = (1 << 16)
27 REVLOGNGINLINEDATA = (1 << 16)
28 REVLOG_DEFAULT_FLAGS = REVLOGNGINLINEDATA
28 REVLOG_DEFAULT_FLAGS = REVLOGNGINLINEDATA
29 REVLOG_DEFAULT_FORMAT = REVLOGNG
29 REVLOG_DEFAULT_FORMAT = REVLOGNG
30 REVLOG_DEFAULT_VERSION = REVLOG_DEFAULT_FORMAT | REVLOG_DEFAULT_FLAGS
30 REVLOG_DEFAULT_VERSION = REVLOG_DEFAULT_FORMAT | REVLOG_DEFAULT_FLAGS
31
31
32 class RevlogError(Exception):
32 class RevlogError(Exception):
33 pass
33 pass
34
34
35 class LookupError(RevlogError):
35 class LookupError(RevlogError):
36 def __init__(self, name, message=None):
36 def __init__(self, name, message=None):
37 if message is None:
37 if message is None:
38 message = _('not found: %s') % name
38 message = _('not found: %s') % name
39 RevlogError.__init__(self, message)
39 RevlogError.__init__(self, message)
40 self.name = name
40 self.name = name
41
41
42 def getoffset(q):
42 def getoffset(q):
43 return int(q >> 16)
43 return int(q >> 16)
44
44
45 def gettype(q):
45 def gettype(q):
46 return int(q & 0xFFFF)
46 return int(q & 0xFFFF)
47
47
48 def offset_type(offset, type):
48 def offset_type(offset, type):
49 return long(long(offset) << 16 | type)
49 return long(long(offset) << 16 | type)
50
50
51 def hash(text, p1, p2):
51 def hash(text, p1, p2):
52 """generate a hash from the given text and its parent hashes
52 """generate a hash from the given text and its parent hashes
53
53
54 This hash combines both the current file contents and its history
54 This hash combines both the current file contents and its history
55 in a manner that makes it easy to distinguish nodes with the same
55 in a manner that makes it easy to distinguish nodes with the same
56 content in the revision graph.
56 content in the revision graph.
57 """
57 """
58 l = [p1, p2]
58 l = [p1, p2]
59 l.sort()
59 l.sort()
60 s = _sha(l[0])
60 s = _sha(l[0])
61 s.update(l[1])
61 s.update(l[1])
62 s.update(text)
62 s.update(text)
63 return s.digest()
63 return s.digest()
64
64
65 def compress(text):
65 def compress(text):
66 """ generate a possibly-compressed representation of text """
66 """ generate a possibly-compressed representation of text """
67 if not text:
67 if not text:
68 return ("", text)
68 return ("", text)
69 l = len(text)
69 l = len(text)
70 bin = None
70 bin = None
71 if l < 44:
71 if l < 44:
72 pass
72 pass
73 elif l > 1000000:
73 elif l > 1000000:
74 # zlib makes an internal copy, thus doubling memory usage for
74 # zlib makes an internal copy, thus doubling memory usage for
75 # large files, so lets do this in pieces
75 # large files, so lets do this in pieces
76 z = zlib.compressobj()
76 z = zlib.compressobj()
77 p = []
77 p = []
78 pos = 0
78 pos = 0
79 while pos < l:
79 while pos < l:
80 pos2 = pos + 2**20
80 pos2 = pos + 2**20
81 p.append(z.compress(text[pos:pos2]))
81 p.append(z.compress(text[pos:pos2]))
82 pos = pos2
82 pos = pos2
83 p.append(z.flush())
83 p.append(z.flush())
84 if sum(map(len, p)) < l:
84 if sum(map(len, p)) < l:
85 bin = "".join(p)
85 bin = "".join(p)
86 else:
86 else:
87 bin = _compress(text)
87 bin = _compress(text)
88 if bin is None or len(bin) > l:
88 if bin is None or len(bin) > l:
89 if text[0] == '\0':
89 if text[0] == '\0':
90 return ("", text)
90 return ("", text)
91 return ('u', text)
91 return ('u', text)
92 return ("", bin)
92 return ("", bin)
93
93
94 def decompress(bin):
94 def decompress(bin):
95 """ decompress the given input """
95 """ decompress the given input """
96 if not bin:
96 if not bin:
97 return bin
97 return bin
98 t = bin[0]
98 t = bin[0]
99 if t == '\0':
99 if t == '\0':
100 return bin
100 return bin
101 if t == 'x':
101 if t == 'x':
102 return _decompress(bin)
102 return _decompress(bin)
103 if t == 'u':
103 if t == 'u':
104 return bin[1:]
104 return bin[1:]
105 raise RevlogError(_("unknown compression type %r") % t)
105 raise RevlogError(_("unknown compression type %r") % t)
106
106
107 class lazyparser(object):
107 class lazyparser(object):
108 """
108 """
109 this class avoids the need to parse the entirety of large indices
109 this class avoids the need to parse the entirety of large indices
110 """
110 """
111
111
112 # lazyparser is not safe to use on windows if win32 extensions not
112 # lazyparser is not safe to use on windows if win32 extensions not
113 # available. it keeps file handle open, which make it not possible
113 # available. it keeps file handle open, which make it not possible
114 # to break hardlinks on local cloned repos.
114 # to break hardlinks on local cloned repos.
115
115
116 def __init__(self, dataf, size):
116 def __init__(self, dataf, size):
117 self.dataf = dataf
117 self.dataf = dataf
118 self.s = struct.calcsize(indexformatng)
118 self.s = struct.calcsize(indexformatng)
119 self.datasize = size
119 self.datasize = size
120 self.l = size/self.s
120 self.l = size/self.s
121 self.index = [None] * self.l
121 self.index = [None] * self.l
122 self.map = {nullid: nullrev}
122 self.map = {nullid: nullrev}
123 self.allmap = 0
123 self.allmap = 0
124 self.all = 0
124 self.all = 0
125 self.mapfind_count = 0
125 self.mapfind_count = 0
126
126
127 def loadmap(self):
127 def loadmap(self):
128 """
128 """
129 during a commit, we need to make sure the rev being added is
129 during a commit, we need to make sure the rev being added is
130 not a duplicate. This requires loading the entire index,
130 not a duplicate. This requires loading the entire index,
131 which is fairly slow. loadmap can load up just the node map,
131 which is fairly slow. loadmap can load up just the node map,
132 which takes much less time.
132 which takes much less time.
133 """
133 """
134 if self.allmap:
134 if self.allmap:
135 return
135 return
136 end = self.datasize
136 end = self.datasize
137 self.allmap = 1
137 self.allmap = 1
138 cur = 0
138 cur = 0
139 count = 0
139 count = 0
140 blocksize = self.s * 256
140 blocksize = self.s * 256
141 self.dataf.seek(0)
141 self.dataf.seek(0)
142 while cur < end:
142 while cur < end:
143 data = self.dataf.read(blocksize)
143 data = self.dataf.read(blocksize)
144 off = 0
144 off = 0
145 for x in xrange(256):
145 for x in xrange(256):
146 n = data[off + ngshaoffset:off + ngshaoffset + 20]
146 n = data[off + ngshaoffset:off + ngshaoffset + 20]
147 self.map[n] = count
147 self.map[n] = count
148 count += 1
148 count += 1
149 if count >= self.l:
149 if count >= self.l:
150 break
150 break
151 off += self.s
151 off += self.s
152 cur += blocksize
152 cur += blocksize
153
153
154 def loadblock(self, blockstart, blocksize, data=None):
154 def loadblock(self, blockstart, blocksize, data=None):
155 if self.all:
155 if self.all:
156 return
156 return
157 if data is None:
157 if data is None:
158 self.dataf.seek(blockstart)
158 self.dataf.seek(blockstart)
159 if blockstart + blocksize > self.datasize:
159 if blockstart + blocksize > self.datasize:
160 # the revlog may have grown since we've started running,
160 # the revlog may have grown since we've started running,
161 # but we don't have space in self.index for more entries.
161 # but we don't have space in self.index for more entries.
162 # limit blocksize so that we don't get too much data.
162 # limit blocksize so that we don't get too much data.
163 blocksize = max(self.datasize - blockstart, 0)
163 blocksize = max(self.datasize - blockstart, 0)
164 data = self.dataf.read(blocksize)
164 data = self.dataf.read(blocksize)
165 lend = len(data) / self.s
165 lend = len(data) / self.s
166 i = blockstart / self.s
166 i = blockstart / self.s
167 off = 0
167 off = 0
168 # lazyindex supports __delitem__
168 # lazyindex supports __delitem__
169 if lend > len(self.index) - i:
169 if lend > len(self.index) - i:
170 lend = len(self.index) - i
170 lend = len(self.index) - i
171 for x in xrange(lend):
171 for x in xrange(lend):
172 if self.index[i + x] == None:
172 if self.index[i + x] == None:
173 b = data[off : off + self.s]
173 b = data[off : off + self.s]
174 self.index[i + x] = b
174 self.index[i + x] = b
175 n = b[ngshaoffset:ngshaoffset + 20]
175 n = b[ngshaoffset:ngshaoffset + 20]
176 self.map[n] = i + x
176 self.map[n] = i + x
177 off += self.s
177 off += self.s
178
178
179 def findnode(self, node):
179 def findnode(self, node):
180 """search backwards through the index file for a specific node"""
180 """search backwards through the index file for a specific node"""
181 if self.allmap:
181 if self.allmap:
182 return None
182 return None
183
183
184 # hg log will cause many many searches for the manifest
184 # hg log will cause many many searches for the manifest
185 # nodes. After we get called a few times, just load the whole
185 # nodes. After we get called a few times, just load the whole
186 # thing.
186 # thing.
187 if self.mapfind_count > 8:
187 if self.mapfind_count > 8:
188 self.loadmap()
188 self.loadmap()
189 if node in self.map:
189 if node in self.map:
190 return node
190 return node
191 return None
191 return None
192 self.mapfind_count += 1
192 self.mapfind_count += 1
193 last = self.l - 1
193 last = self.l - 1
194 while self.index[last] != None:
194 while self.index[last] != None:
195 if last == 0:
195 if last == 0:
196 self.all = 1
196 self.all = 1
197 self.allmap = 1
197 self.allmap = 1
198 return None
198 return None
199 last -= 1
199 last -= 1
200 end = (last + 1) * self.s
200 end = (last + 1) * self.s
201 blocksize = self.s * 256
201 blocksize = self.s * 256
202 while end >= 0:
202 while end >= 0:
203 start = max(end - blocksize, 0)
203 start = max(end - blocksize, 0)
204 self.dataf.seek(start)
204 self.dataf.seek(start)
205 data = self.dataf.read(end - start)
205 data = self.dataf.read(end - start)
206 findend = end - start
206 findend = end - start
207 while True:
207 while True:
208 # we're searching backwards, so we have to make sure
208 # we're searching backwards, so we have to make sure
209 # we don't find a changeset where this node is a parent
209 # we don't find a changeset where this node is a parent
210 off = data.find(node, 0, findend)
210 off = data.find(node, 0, findend)
211 findend = off
211 findend = off
212 if off >= 0:
212 if off >= 0:
213 i = off / self.s
213 i = off / self.s
214 off = i * self.s
214 off = i * self.s
215 n = data[off + ngshaoffset:off + ngshaoffset + 20]
215 n = data[off + ngshaoffset:off + ngshaoffset + 20]
216 if n == node:
216 if n == node:
217 self.map[n] = i + start / self.s
217 self.map[n] = i + start / self.s
218 return node
218 return node
219 else:
219 else:
220 break
220 break
221 end -= blocksize
221 end -= blocksize
222 return None
222 return None
223
223
224 def loadindex(self, i=None, end=None):
224 def loadindex(self, i=None, end=None):
225 if self.all:
225 if self.all:
226 return
226 return
227 all = False
227 all = False
228 if i == None:
228 if i == None:
229 blockstart = 0
229 blockstart = 0
230 blocksize = (65536 / self.s) * self.s
230 blocksize = (65536 / self.s) * self.s
231 end = self.datasize
231 end = self.datasize
232 all = True
232 all = True
233 else:
233 else:
234 if end:
234 if end:
235 blockstart = i * self.s
235 blockstart = i * self.s
236 end = end * self.s
236 end = end * self.s
237 blocksize = end - blockstart
237 blocksize = end - blockstart
238 else:
238 else:
239 blockstart = (i & ~1023) * self.s
239 blockstart = (i & ~1023) * self.s
240 blocksize = self.s * 1024
240 blocksize = self.s * 1024
241 end = blockstart + blocksize
241 end = blockstart + blocksize
242 while blockstart < end:
242 while blockstart < end:
243 self.loadblock(blockstart, blocksize)
243 self.loadblock(blockstart, blocksize)
244 blockstart += blocksize
244 blockstart += blocksize
245 if all:
245 if all:
246 self.all = True
246 self.all = True
247
247
248 class lazyindex(object):
248 class lazyindex(object):
249 """a lazy version of the index array"""
249 """a lazy version of the index array"""
250 def __init__(self, parser):
250 def __init__(self, parser):
251 self.p = parser
251 self.p = parser
252 def __len__(self):
252 def __len__(self):
253 return len(self.p.index)
253 return len(self.p.index)
254 def load(self, pos):
254 def load(self, pos):
255 if pos < 0:
255 if pos < 0:
256 pos += len(self.p.index)
256 pos += len(self.p.index)
257 self.p.loadindex(pos)
257 self.p.loadindex(pos)
258 return self.p.index[pos]
258 return self.p.index[pos]
259 def __getitem__(self, pos):
259 def __getitem__(self, pos):
260 return _unpack(indexformatng, self.p.index[pos] or self.load(pos))
260 return _unpack(indexformatng, self.p.index[pos] or self.load(pos))
261 def __setitem__(self, pos, item):
261 def __setitem__(self, pos, item):
262 self.p.index[pos] = _pack(indexformatng, *item)
262 self.p.index[pos] = _pack(indexformatng, *item)
263 def __delitem__(self, pos):
263 def __delitem__(self, pos):
264 del self.p.index[pos]
264 del self.p.index[pos]
265 def insert(self, pos, e):
265 def insert(self, pos, e):
266 self.p.index.insert(pos, _pack(indexformatng, *e))
266 self.p.index.insert(pos, _pack(indexformatng, *e))
267 def append(self, e):
267 def append(self, e):
268 self.p.index.append(_pack(indexformatng, *e))
268 self.p.index.append(_pack(indexformatng, *e))
269
269
270 class lazymap(object):
270 class lazymap(object):
271 """a lazy version of the node map"""
271 """a lazy version of the node map"""
272 def __init__(self, parser):
272 def __init__(self, parser):
273 self.p = parser
273 self.p = parser
274 def load(self, key):
274 def load(self, key):
275 n = self.p.findnode(key)
275 n = self.p.findnode(key)
276 if n == None:
276 if n == None:
277 raise KeyError(key)
277 raise KeyError(key)
278 def __contains__(self, key):
278 def __contains__(self, key):
279 if key in self.p.map:
279 if key in self.p.map:
280 return True
280 return True
281 self.p.loadmap()
281 self.p.loadmap()
282 return key in self.p.map
282 return key in self.p.map
283 def __iter__(self):
283 def __iter__(self):
284 yield nullid
284 yield nullid
285 for i in xrange(self.p.l):
285 for i in xrange(self.p.l):
286 ret = self.p.index[i]
286 ret = self.p.index[i]
287 if not ret:
287 if not ret:
288 self.p.loadindex(i)
288 self.p.loadindex(i)
289 ret = self.p.index[i]
289 ret = self.p.index[i]
290 if isinstance(ret, str):
290 if isinstance(ret, str):
291 ret = _unpack(indexformatng, ret)
291 ret = _unpack(indexformatng, ret)
292 yield ret[7]
292 yield ret[7]
293 def __getitem__(self, key):
293 def __getitem__(self, key):
294 try:
294 try:
295 return self.p.map[key]
295 return self.p.map[key]
296 except KeyError:
296 except KeyError:
297 try:
297 try:
298 self.load(key)
298 self.load(key)
299 return self.p.map[key]
299 return self.p.map[key]
300 except KeyError:
300 except KeyError:
301 raise KeyError("node " + hex(key))
301 raise KeyError("node " + hex(key))
302 def __setitem__(self, key, val):
302 def __setitem__(self, key, val):
303 self.p.map[key] = val
303 self.p.map[key] = val
304 def __delitem__(self, key):
304 def __delitem__(self, key):
305 del self.p.map[key]
305 del self.p.map[key]
306
306
307 indexformatv0 = ">4l20s20s20s"
307 indexformatv0 = ">4l20s20s20s"
308 v0shaoffset = 56
308 v0shaoffset = 56
309
309
310 class revlogoldio(object):
310 class revlogoldio(object):
311 def __init__(self):
311 def __init__(self):
312 self.size = struct.calcsize(indexformatv0)
312 self.size = struct.calcsize(indexformatv0)
313
313
314 def parseindex(self, fp, inline):
314 def parseindex(self, fp, inline):
315 s = self.size
315 s = self.size
316 index = []
316 index = []
317 nodemap = {nullid: nullrev}
317 nodemap = {nullid: nullrev}
318 n = off = 0
318 n = off = 0
319 data = fp.read()
319 data = fp.read()
320 l = len(data)
320 l = len(data)
321 while off + s <= l:
321 while off + s <= l:
322 cur = data[off:off + s]
322 cur = data[off:off + s]
323 off += s
323 off += s
324 e = _unpack(indexformatv0, cur)
324 e = _unpack(indexformatv0, cur)
325 # transform to revlogv1 format
325 # transform to revlogv1 format
326 e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
326 e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
327 nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
327 nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
328 index.append(e2)
328 index.append(e2)
329 nodemap[e[6]] = n
329 nodemap[e[6]] = n
330 n += 1
330 n += 1
331
331
332 return index, nodemap, None
332 return index, nodemap, None
333
333
334 def packentry(self, entry, node, version, rev):
334 def packentry(self, entry, node, version, rev):
335 e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
335 e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
336 node(entry[5]), node(entry[6]), entry[7])
336 node(entry[5]), node(entry[6]), entry[7])
337 return _pack(indexformatv0, *e2)
337 return _pack(indexformatv0, *e2)
338
338
339 # index ng:
339 # index ng:
340 # 6 bytes offset
340 # 6 bytes offset
341 # 2 bytes flags
341 # 2 bytes flags
342 # 4 bytes compressed length
342 # 4 bytes compressed length
343 # 4 bytes uncompressed length
343 # 4 bytes uncompressed length
344 # 4 bytes: base rev
344 # 4 bytes: base rev
345 # 4 bytes link rev
345 # 4 bytes link rev
346 # 4 bytes parent 1 rev
346 # 4 bytes parent 1 rev
347 # 4 bytes parent 2 rev
347 # 4 bytes parent 2 rev
348 # 32 bytes: nodeid
348 # 32 bytes: nodeid
349 indexformatng = ">Qiiiiii20s12x"
349 indexformatng = ">Qiiiiii20s12x"
350 ngshaoffset = 32
350 ngshaoffset = 32
351 versionformat = ">I"
351 versionformat = ">I"
352
352
353 class revlogio(object):
353 class revlogio(object):
354 def __init__(self):
354 def __init__(self):
355 self.size = struct.calcsize(indexformatng)
355 self.size = struct.calcsize(indexformatng)
356
356
357 def parseindex(self, fp, inline):
357 def parseindex(self, fp, inline):
358 try:
358 try:
359 size = util.fstat(fp).st_size
359 size = util.fstat(fp).st_size
360 except AttributeError:
360 except AttributeError:
361 size = 0
361 size = 0
362
362
363 if util.openhardlinks() and not inline and size > 1000000:
363 if util.openhardlinks() and not inline and size > 1000000:
364 # big index, let's parse it on demand
364 # big index, let's parse it on demand
365 parser = lazyparser(fp, size)
365 parser = lazyparser(fp, size)
366 index = lazyindex(parser)
366 index = lazyindex(parser)
367 nodemap = lazymap(parser)
367 nodemap = lazymap(parser)
368 e = list(index[0])
368 e = list(index[0])
369 type = gettype(e[0])
369 type = gettype(e[0])
370 e[0] = offset_type(0, type)
370 e[0] = offset_type(0, type)
371 index[0] = e
371 index[0] = e
372 return index, nodemap, None
372 return index, nodemap, None
373
373
374 s = self.size
374 s = self.size
375 cache = None
375 cache = None
376 index = []
376 index = []
377 nodemap = {nullid: nullrev}
377 nodemap = {nullid: nullrev}
378 n = off = 0
378 n = off = 0
379 # if we're not using lazymap, always read the whole index
379 # if we're not using lazymap, always read the whole index
380 data = fp.read()
380 data = fp.read()
381 l = len(data) - s
381 l = len(data) - s
382 append = index.append
382 append = index.append
383 if inline:
383 if inline:
384 cache = (0, data)
384 cache = (0, data)
385 while off <= l:
385 while off <= l:
386 e = _unpack(indexformatng, data[off:off + s])
386 e = _unpack(indexformatng, data[off:off + s])
387 nodemap[e[7]] = n
387 nodemap[e[7]] = n
388 append(e)
388 append(e)
389 n += 1
389 n += 1
390 if e[1] < 0:
390 if e[1] < 0:
391 break
391 break
392 off += e[1] + s
392 off += e[1] + s
393 else:
393 else:
394 while off <= l:
394 while off <= l:
395 e = _unpack(indexformatng, data[off:off + s])
395 e = _unpack(indexformatng, data[off:off + s])
396 nodemap[e[7]] = n
396 nodemap[e[7]] = n
397 append(e)
397 append(e)
398 n += 1
398 n += 1
399 off += s
399 off += s
400
400
401 e = list(index[0])
401 e = list(index[0])
402 type = gettype(e[0])
402 type = gettype(e[0])
403 e[0] = offset_type(0, type)
403 e[0] = offset_type(0, type)
404 index[0] = e
404 index[0] = e
405
405
406 return index, nodemap, cache
406 return index, nodemap, cache
407
407
408 def packentry(self, entry, node, version, rev):
408 def packentry(self, entry, node, version, rev):
409 p = _pack(indexformatng, *entry)
409 p = _pack(indexformatng, *entry)
410 if rev == 0:
410 if rev == 0:
411 p = _pack(versionformat, version) + p[4:]
411 p = _pack(versionformat, version) + p[4:]
412 return p
412 return p
413
413
414 class revlog(object):
414 class revlog(object):
415 """
415 """
416 the underlying revision storage object
416 the underlying revision storage object
417
417
418 A revlog consists of two parts, an index and the revision data.
418 A revlog consists of two parts, an index and the revision data.
419
419
420 The index is a file with a fixed record size containing
420 The index is a file with a fixed record size containing
421 information on each revision, includings its nodeid (hash), the
421 information on each revision, includings its nodeid (hash), the
422 nodeids of its parents, the position and offset of its data within
422 nodeids of its parents, the position and offset of its data within
423 the data file, and the revision it's based on. Finally, each entry
423 the data file, and the revision it's based on. Finally, each entry
424 contains a linkrev entry that can serve as a pointer to external
424 contains a linkrev entry that can serve as a pointer to external
425 data.
425 data.
426
426
427 The revision data itself is a linear collection of data chunks.
427 The revision data itself is a linear collection of data chunks.
428 Each chunk represents a revision and is usually represented as a
428 Each chunk represents a revision and is usually represented as a
429 delta against the previous chunk. To bound lookup time, runs of
429 delta against the previous chunk. To bound lookup time, runs of
430 deltas are limited to about 2 times the length of the original
430 deltas are limited to about 2 times the length of the original
431 version data. This makes retrieval of a version proportional to
431 version data. This makes retrieval of a version proportional to
432 its size, or O(1) relative to the number of revisions.
432 its size, or O(1) relative to the number of revisions.
433
433
434 Both pieces of the revlog are written to in an append-only
434 Both pieces of the revlog are written to in an append-only
435 fashion, which means we never need to rewrite a file to insert or
435 fashion, which means we never need to rewrite a file to insert or
436 remove data, and can use some simple techniques to avoid the need
436 remove data, and can use some simple techniques to avoid the need
437 for locking while reading.
437 for locking while reading.
438 """
438 """
439 def __init__(self, opener, indexfile):
439 def __init__(self, opener, indexfile):
440 """
440 """
441 create a revlog object
441 create a revlog object
442
442
443 opener is a function that abstracts the file opening operation
443 opener is a function that abstracts the file opening operation
444 and can be used to implement COW semantics or the like.
444 and can be used to implement COW semantics or the like.
445 """
445 """
446 self.indexfile = indexfile
446 self.indexfile = indexfile
447 self.datafile = indexfile[:-2] + ".d"
447 self.datafile = indexfile[:-2] + ".d"
448 self.opener = opener
448 self.opener = opener
449 self._cache = None
449 self._cache = None
450 self._chunkcache = None
450 self._chunkcache = None
451 self.nodemap = {nullid: nullrev}
451 self.nodemap = {nullid: nullrev}
452 self.index = []
452 self.index = []
453
453
454 v = REVLOG_DEFAULT_VERSION
454 v = REVLOG_DEFAULT_VERSION
455 if hasattr(opener, "defversion"):
455 if hasattr(opener, "defversion"):
456 v = opener.defversion
456 v = opener.defversion
457 if v & REVLOGNG:
457 if v & REVLOGNG:
458 v |= REVLOGNGINLINEDATA
458 v |= REVLOGNGINLINEDATA
459
459
460 i = ""
460 i = ""
461 try:
461 try:
462 f = self.opener(self.indexfile)
462 f = self.opener(self.indexfile)
463 i = f.read(4)
463 i = f.read(4)
464 f.seek(0)
464 f.seek(0)
465 if len(i) > 0:
465 if len(i) > 0:
466 v = struct.unpack(versionformat, i)[0]
466 v = struct.unpack(versionformat, i)[0]
467 except IOError, inst:
467 except IOError, inst:
468 if inst.errno != errno.ENOENT:
468 if inst.errno != errno.ENOENT:
469 raise
469 raise
470
470
471 self.version = v
471 self.version = v
472 self._inline = v & REVLOGNGINLINEDATA
472 self._inline = v & REVLOGNGINLINEDATA
473 flags = v & ~0xFFFF
473 flags = v & ~0xFFFF
474 fmt = v & 0xFFFF
474 fmt = v & 0xFFFF
475 if fmt == REVLOGV0 and flags:
475 if fmt == REVLOGV0 and flags:
476 raise RevlogError(_("index %s unknown flags %#04x for format v0")
476 raise RevlogError(_("index %s unknown flags %#04x for format v0")
477 % (self.indexfile, flags >> 16))
477 % (self.indexfile, flags >> 16))
478 elif fmt == REVLOGNG and flags & ~REVLOGNGINLINEDATA:
478 elif fmt == REVLOGNG and flags & ~REVLOGNGINLINEDATA:
479 raise RevlogError(_("index %s unknown flags %#04x for revlogng")
479 raise RevlogError(_("index %s unknown flags %#04x for revlogng")
480 % (self.indexfile, flags >> 16))
480 % (self.indexfile, flags >> 16))
481 elif fmt > REVLOGNG:
481 elif fmt > REVLOGNG:
482 raise RevlogError(_("index %s unknown format %d")
482 raise RevlogError(_("index %s unknown format %d")
483 % (self.indexfile, fmt))
483 % (self.indexfile, fmt))
484
484
485 self._io = revlogio()
485 self._io = revlogio()
486 if self.version == REVLOGV0:
486 if self.version == REVLOGV0:
487 self._io = revlogoldio()
487 self._io = revlogoldio()
488 if i:
488 if i:
489 d = self._io.parseindex(f, self._inline)
489 d = self._io.parseindex(f, self._inline)
490 self.index, self.nodemap, self._chunkcache = d
490 self.index, self.nodemap, self._chunkcache = d
491
491
492 # add the magic null revision at -1
492 # add the magic null revision at -1
493 self.index.append((0, 0, 0, -1, -1, -1, -1, nullid))
493 self.index.append((0, 0, 0, -1, -1, -1, -1, nullid))
494
494
495 def _loadindex(self, start, end):
495 def _loadindex(self, start, end):
496 """load a block of indexes all at once from the lazy parser"""
496 """load a block of indexes all at once from the lazy parser"""
497 if isinstance(self.index, lazyindex):
497 if isinstance(self.index, lazyindex):
498 self.index.p.loadindex(start, end)
498 self.index.p.loadindex(start, end)
499
499
500 def _loadindexmap(self):
500 def _loadindexmap(self):
501 """loads both the map and the index from the lazy parser"""
501 """loads both the map and the index from the lazy parser"""
502 if isinstance(self.index, lazyindex):
502 if isinstance(self.index, lazyindex):
503 p = self.index.p
503 p = self.index.p
504 p.loadindex()
504 p.loadindex()
505 self.nodemap = p.map
505 self.nodemap = p.map
506
506
507 def _loadmap(self):
507 def _loadmap(self):
508 """loads the map from the lazy parser"""
508 """loads the map from the lazy parser"""
509 if isinstance(self.nodemap, lazymap):
509 if isinstance(self.nodemap, lazymap):
510 self.nodemap.p.loadmap()
510 self.nodemap.p.loadmap()
511 self.nodemap = self.nodemap.p.map
511 self.nodemap = self.nodemap.p.map
512
512
513 def tip(self):
513 def tip(self):
514 return self.node(len(self.index) - 2)
514 return self.node(len(self.index) - 2)
515 def count(self):
515 def count(self):
516 return len(self.index) - 1
516 return len(self.index) - 1
517
517
518 def rev(self, node):
518 def rev(self, node):
519 try:
519 try:
520 return self.nodemap[node]
520 return self.nodemap[node]
521 except KeyError:
521 except KeyError:
522 raise LookupError(hex(node), _('%s: no node %s') % (self.indexfile, hex(node)))
522 raise LookupError(hex(node), _('%s: no node %s') % (self.indexfile, hex(node)))
523 def node(self, rev):
523 def node(self, rev):
524 return self.index[rev][7]
524 return self.index[rev][7]
525 def linkrev(self, node):
525 def linkrev(self, node):
526 return self.index[self.rev(node)][4]
526 return self.index[self.rev(node)][4]
527 def parents(self, node):
527 def parents(self, node):
528 d = self.index[self.rev(node)][5:7]
528 d = self.index[self.rev(node)][5:7]
529 return (self.node(d[0]), self.node(d[1]))
529 return (self.node(d[0]), self.node(d[1]))
530 def parentrevs(self, rev):
530 def parentrevs(self, rev):
531 return self.index[rev][5:7]
531 return self.index[rev][5:7]
532 def start(self, rev):
532 def start(self, rev):
533 return int(self.index[rev][0] >> 16)
533 return int(self.index[rev][0] >> 16)
534 def end(self, rev):
534 def end(self, rev):
535 return self.start(rev) + self.length(rev)
535 return self.start(rev) + self.length(rev)
536 def length(self, rev):
536 def length(self, rev):
537 return self.index[rev][1]
537 return self.index[rev][1]
538 def base(self, rev):
538 def base(self, rev):
539 return self.index[rev][3]
539 return self.index[rev][3]
540
540
541 def size(self, rev):
541 def size(self, rev):
542 """return the length of the uncompressed text for a given revision"""
542 """return the length of the uncompressed text for a given revision"""
543 l = self.index[rev][2]
543 l = self.index[rev][2]
544 if l >= 0:
544 if l >= 0:
545 return l
545 return l
546
546
547 t = self.revision(self.node(rev))
547 t = self.revision(self.node(rev))
548 return len(t)
548 return len(t)
549
549
550 # alternate implementation, The advantage to this code is it
550 # alternate implementation, The advantage to this code is it
551 # will be faster for a single revision. But, the results are not
551 # will be faster for a single revision. But, the results are not
552 # cached, so finding the size of every revision will be slower.
552 # cached, so finding the size of every revision will be slower.
553 """
553 """
554 if self.cache and self.cache[1] == rev:
554 if self.cache and self.cache[1] == rev:
555 return len(self.cache[2])
555 return len(self.cache[2])
556
556
557 base = self.base(rev)
557 base = self.base(rev)
558 if self.cache and self.cache[1] >= base and self.cache[1] < rev:
558 if self.cache and self.cache[1] >= base and self.cache[1] < rev:
559 base = self.cache[1]
559 base = self.cache[1]
560 text = self.cache[2]
560 text = self.cache[2]
561 else:
561 else:
562 text = self.revision(self.node(base))
562 text = self.revision(self.node(base))
563
563
564 l = len(text)
564 l = len(text)
565 for x in xrange(base + 1, rev + 1):
565 for x in xrange(base + 1, rev + 1):
566 l = mdiff.patchedsize(l, self.chunk(x))
566 l = mdiff.patchedsize(l, self.chunk(x))
567 return l
567 return l
568 """
568 """
569
569
570 def reachable(self, node, stop=None):
570 def reachable(self, node, stop=None):
571 """return a hash of all nodes ancestral to a given node, including
571 """return a hash of all nodes ancestral to a given node, including
572 the node itself, stopping when stop is matched"""
572 the node itself, stopping when stop is matched"""
573 reachable = {}
573 reachable = {}
574 visit = [node]
574 visit = [node]
575 reachable[node] = 1
575 reachable[node] = 1
576 if stop:
576 if stop:
577 stopn = self.rev(stop)
577 stopn = self.rev(stop)
578 else:
578 else:
579 stopn = 0
579 stopn = 0
580 while visit:
580 while visit:
581 n = visit.pop(0)
581 n = visit.pop(0)
582 if n == stop:
582 if n == stop:
583 continue
583 continue
584 if n == nullid:
584 if n == nullid:
585 continue
585 continue
586 for p in self.parents(n):
586 for p in self.parents(n):
587 if self.rev(p) < stopn:
587 if self.rev(p) < stopn:
588 continue
588 continue
589 if p not in reachable:
589 if p not in reachable:
590 reachable[p] = 1
590 reachable[p] = 1
591 visit.append(p)
591 visit.append(p)
592 return reachable
592 return reachable
593
593
594 def nodesbetween(self, roots=None, heads=None):
594 def nodesbetween(self, roots=None, heads=None):
595 """Return a tuple containing three elements. Elements 1 and 2 contain
595 """Return a tuple containing three elements. Elements 1 and 2 contain
596 a final list bases and heads after all the unreachable ones have been
596 a final list bases and heads after all the unreachable ones have been
597 pruned. Element 0 contains a topologically sorted list of all
597 pruned. Element 0 contains a topologically sorted list of all
598
598
599 nodes that satisfy these constraints:
599 nodes that satisfy these constraints:
600 1. All nodes must be descended from a node in roots (the nodes on
600 1. All nodes must be descended from a node in roots (the nodes on
601 roots are considered descended from themselves).
601 roots are considered descended from themselves).
602 2. All nodes must also be ancestors of a node in heads (the nodes in
602 2. All nodes must also be ancestors of a node in heads (the nodes in
603 heads are considered to be their own ancestors).
603 heads are considered to be their own ancestors).
604
604
605 If roots is unspecified, nullid is assumed as the only root.
605 If roots is unspecified, nullid is assumed as the only root.
606 If heads is unspecified, it is taken to be the output of the
606 If heads is unspecified, it is taken to be the output of the
607 heads method (i.e. a list of all nodes in the repository that
607 heads method (i.e. a list of all nodes in the repository that
608 have no children)."""
608 have no children)."""
609 nonodes = ([], [], [])
609 nonodes = ([], [], [])
610 if roots is not None:
610 if roots is not None:
611 roots = list(roots)
611 roots = list(roots)
612 if not roots:
612 if not roots:
613 return nonodes
613 return nonodes
614 lowestrev = min([self.rev(n) for n in roots])
614 lowestrev = min([self.rev(n) for n in roots])
615 else:
615 else:
616 roots = [nullid] # Everybody's a descendent of nullid
616 roots = [nullid] # Everybody's a descendent of nullid
617 lowestrev = nullrev
617 lowestrev = nullrev
618 if (lowestrev == nullrev) and (heads is None):
618 if (lowestrev == nullrev) and (heads is None):
619 # We want _all_ the nodes!
619 # We want _all_ the nodes!
620 return ([self.node(r) for r in xrange(0, self.count())],
620 return ([self.node(r) for r in xrange(0, self.count())],
621 [nullid], list(self.heads()))
621 [nullid], list(self.heads()))
622 if heads is None:
622 if heads is None:
623 # All nodes are ancestors, so the latest ancestor is the last
623 # All nodes are ancestors, so the latest ancestor is the last
624 # node.
624 # node.
625 highestrev = self.count() - 1
625 highestrev = self.count() - 1
626 # Set ancestors to None to signal that every node is an ancestor.
626 # Set ancestors to None to signal that every node is an ancestor.
627 ancestors = None
627 ancestors = None
628 # Set heads to an empty dictionary for later discovery of heads
628 # Set heads to an empty dictionary for later discovery of heads
629 heads = {}
629 heads = {}
630 else:
630 else:
631 heads = list(heads)
631 heads = list(heads)
632 if not heads:
632 if not heads:
633 return nonodes
633 return nonodes
634 ancestors = {}
634 ancestors = {}
635 # Turn heads into a dictionary so we can remove 'fake' heads.
635 # Turn heads into a dictionary so we can remove 'fake' heads.
636 # Also, later we will be using it to filter out the heads we can't
636 # Also, later we will be using it to filter out the heads we can't
637 # find from roots.
637 # find from roots.
638 heads = dict.fromkeys(heads, 0)
638 heads = dict.fromkeys(heads, 0)
639 # Start at the top and keep marking parents until we're done.
639 # Start at the top and keep marking parents until we're done.
640 nodestotag = heads.keys()
640 nodestotag = heads.keys()
641 # Remember where the top was so we can use it as a limit later.
641 # Remember where the top was so we can use it as a limit later.
642 highestrev = max([self.rev(n) for n in nodestotag])
642 highestrev = max([self.rev(n) for n in nodestotag])
643 while nodestotag:
643 while nodestotag:
644 # grab a node to tag
644 # grab a node to tag
645 n = nodestotag.pop()
645 n = nodestotag.pop()
646 # Never tag nullid
646 # Never tag nullid
647 if n == nullid:
647 if n == nullid:
648 continue
648 continue
649 # A node's revision number represents its place in a
649 # A node's revision number represents its place in a
650 # topologically sorted list of nodes.
650 # topologically sorted list of nodes.
651 r = self.rev(n)
651 r = self.rev(n)
652 if r >= lowestrev:
652 if r >= lowestrev:
653 if n not in ancestors:
653 if n not in ancestors:
654 # If we are possibly a descendent of one of the roots
654 # If we are possibly a descendent of one of the roots
655 # and we haven't already been marked as an ancestor
655 # and we haven't already been marked as an ancestor
656 ancestors[n] = 1 # Mark as ancestor
656 ancestors[n] = 1 # Mark as ancestor
657 # Add non-nullid parents to list of nodes to tag.
657 # Add non-nullid parents to list of nodes to tag.
658 nodestotag.extend([p for p in self.parents(n) if
658 nodestotag.extend([p for p in self.parents(n) if
659 p != nullid])
659 p != nullid])
660 elif n in heads: # We've seen it before, is it a fake head?
660 elif n in heads: # We've seen it before, is it a fake head?
661 # So it is, real heads should not be the ancestors of
661 # So it is, real heads should not be the ancestors of
662 # any other heads.
662 # any other heads.
663 heads.pop(n)
663 heads.pop(n)
664 if not ancestors:
664 if not ancestors:
665 return nonodes
665 return nonodes
666 # Now that we have our set of ancestors, we want to remove any
666 # Now that we have our set of ancestors, we want to remove any
667 # roots that are not ancestors.
667 # roots that are not ancestors.
668
668
669 # If one of the roots was nullid, everything is included anyway.
669 # If one of the roots was nullid, everything is included anyway.
670 if lowestrev > nullrev:
670 if lowestrev > nullrev:
671 # But, since we weren't, let's recompute the lowest rev to not
671 # But, since we weren't, let's recompute the lowest rev to not
672 # include roots that aren't ancestors.
672 # include roots that aren't ancestors.
673
673
674 # Filter out roots that aren't ancestors of heads
674 # Filter out roots that aren't ancestors of heads
675 roots = [n for n in roots if n in ancestors]
675 roots = [n for n in roots if n in ancestors]
676 # Recompute the lowest revision
676 # Recompute the lowest revision
677 if roots:
677 if roots:
678 lowestrev = min([self.rev(n) for n in roots])
678 lowestrev = min([self.rev(n) for n in roots])
679 else:
679 else:
680 # No more roots? Return empty list
680 # No more roots? Return empty list
681 return nonodes
681 return nonodes
682 else:
682 else:
683 # We are descending from nullid, and don't need to care about
683 # We are descending from nullid, and don't need to care about
684 # any other roots.
684 # any other roots.
685 lowestrev = nullrev
685 lowestrev = nullrev
686 roots = [nullid]
686 roots = [nullid]
687 # Transform our roots list into a 'set' (i.e. a dictionary where the
687 # Transform our roots list into a 'set' (i.e. a dictionary where the
688 # values don't matter.
688 # values don't matter.
689 descendents = dict.fromkeys(roots, 1)
689 descendents = dict.fromkeys(roots, 1)
690 # Also, keep the original roots so we can filter out roots that aren't
690 # Also, keep the original roots so we can filter out roots that aren't
691 # 'real' roots (i.e. are descended from other roots).
691 # 'real' roots (i.e. are descended from other roots).
692 roots = descendents.copy()
692 roots = descendents.copy()
693 # Our topologically sorted list of output nodes.
693 # Our topologically sorted list of output nodes.
694 orderedout = []
694 orderedout = []
695 # Don't start at nullid since we don't want nullid in our output list,
695 # Don't start at nullid since we don't want nullid in our output list,
696 # and if nullid shows up in descedents, empty parents will look like
696 # and if nullid shows up in descedents, empty parents will look like
697 # they're descendents.
697 # they're descendents.
698 for r in xrange(max(lowestrev, 0), highestrev + 1):
698 for r in xrange(max(lowestrev, 0), highestrev + 1):
699 n = self.node(r)
699 n = self.node(r)
700 isdescendent = False
700 isdescendent = False
701 if lowestrev == nullrev: # Everybody is a descendent of nullid
701 if lowestrev == nullrev: # Everybody is a descendent of nullid
702 isdescendent = True
702 isdescendent = True
703 elif n in descendents:
703 elif n in descendents:
704 # n is already a descendent
704 # n is already a descendent
705 isdescendent = True
705 isdescendent = True
706 # This check only needs to be done here because all the roots
706 # This check only needs to be done here because all the roots
707 # will start being marked is descendents before the loop.
707 # will start being marked is descendents before the loop.
708 if n in roots:
708 if n in roots:
709 # If n was a root, check if it's a 'real' root.
709 # If n was a root, check if it's a 'real' root.
710 p = tuple(self.parents(n))
710 p = tuple(self.parents(n))
711 # If any of its parents are descendents, it's not a root.
711 # If any of its parents are descendents, it's not a root.
712 if (p[0] in descendents) or (p[1] in descendents):
712 if (p[0] in descendents) or (p[1] in descendents):
713 roots.pop(n)
713 roots.pop(n)
714 else:
714 else:
715 p = tuple(self.parents(n))
715 p = tuple(self.parents(n))
716 # A node is a descendent if either of its parents are
716 # A node is a descendent if either of its parents are
717 # descendents. (We seeded the dependents list with the roots
717 # descendents. (We seeded the dependents list with the roots
718 # up there, remember?)
718 # up there, remember?)
719 if (p[0] in descendents) or (p[1] in descendents):
719 if (p[0] in descendents) or (p[1] in descendents):
720 descendents[n] = 1
720 descendents[n] = 1
721 isdescendent = True
721 isdescendent = True
722 if isdescendent and ((ancestors is None) or (n in ancestors)):
722 if isdescendent and ((ancestors is None) or (n in ancestors)):
723 # Only include nodes that are both descendents and ancestors.
723 # Only include nodes that are both descendents and ancestors.
724 orderedout.append(n)
724 orderedout.append(n)
725 if (ancestors is not None) and (n in heads):
725 if (ancestors is not None) and (n in heads):
726 # We're trying to figure out which heads are reachable
726 # We're trying to figure out which heads are reachable
727 # from roots.
727 # from roots.
728 # Mark this head as having been reached
728 # Mark this head as having been reached
729 heads[n] = 1
729 heads[n] = 1
730 elif ancestors is None:
730 elif ancestors is None:
731 # Otherwise, we're trying to discover the heads.
731 # Otherwise, we're trying to discover the heads.
732 # Assume this is a head because if it isn't, the next step
732 # Assume this is a head because if it isn't, the next step
733 # will eventually remove it.
733 # will eventually remove it.
734 heads[n] = 1
734 heads[n] = 1
735 # But, obviously its parents aren't.
735 # But, obviously its parents aren't.
736 for p in self.parents(n):
736 for p in self.parents(n):
737 heads.pop(p, None)
737 heads.pop(p, None)
738 heads = [n for n in heads.iterkeys() if heads[n] != 0]
738 heads = [n for n in heads.iterkeys() if heads[n] != 0]
739 roots = roots.keys()
739 roots = roots.keys()
740 assert orderedout
740 assert orderedout
741 assert roots
741 assert roots
742 assert heads
742 assert heads
743 return (orderedout, roots, heads)
743 return (orderedout, roots, heads)
744
744
745 def heads(self, start=None, stop=None):
745 def heads(self, start=None, stop=None):
746 """return the list of all nodes that have no children
746 """return the list of all nodes that have no children
747
747
748 if start is specified, only heads that are descendants of
748 if start is specified, only heads that are descendants of
749 start will be returned
749 start will be returned
750 if stop is specified, it will consider all the revs from stop
750 if stop is specified, it will consider all the revs from stop
751 as if they had no children
751 as if they had no children
752 """
752 """
753 if start is None and stop is None:
753 if start is None and stop is None:
754 count = self.count()
754 count = self.count()
755 if not count:
755 if not count:
756 return [nullid]
756 return [nullid]
757 ishead = [1] * (count + 1)
757 ishead = [1] * (count + 1)
758 index = self.index
758 index = self.index
759 for r in xrange(count):
759 for r in xrange(count):
760 e = index[r]
760 e = index[r]
761 ishead[e[5]] = ishead[e[6]] = 0
761 ishead[e[5]] = ishead[e[6]] = 0
762 return [self.node(r) for r in xrange(count) if ishead[r]]
762 return [self.node(r) for r in xrange(count) if ishead[r]]
763
763
764 if start is None:
764 if start is None:
765 start = nullid
765 start = nullid
766 if stop is None:
766 if stop is None:
767 stop = []
767 stop = []
768 stoprevs = dict.fromkeys([self.rev(n) for n in stop])
768 stoprevs = dict.fromkeys([self.rev(n) for n in stop])
769 startrev = self.rev(start)
769 startrev = self.rev(start)
770 reachable = {startrev: 1}
770 reachable = {startrev: 1}
771 heads = {startrev: 1}
771 heads = {startrev: 1}
772
772
773 parentrevs = self.parentrevs
773 parentrevs = self.parentrevs
774 for r in xrange(startrev + 1, self.count()):
774 for r in xrange(startrev + 1, self.count()):
775 for p in parentrevs(r):
775 for p in parentrevs(r):
776 if p in reachable:
776 if p in reachable:
777 if r not in stoprevs:
777 if r not in stoprevs:
778 reachable[r] = 1
778 reachable[r] = 1
779 heads[r] = 1
779 heads[r] = 1
780 if p in heads and p not in stoprevs:
780 if p in heads and p not in stoprevs:
781 del heads[p]
781 del heads[p]
782
782
783 return [self.node(r) for r in heads]
783 return [self.node(r) for r in heads]
784
784
785 def children(self, node):
785 def children(self, node):
786 """find the children of a given node"""
786 """find the children of a given node"""
787 c = []
787 c = []
788 p = self.rev(node)
788 p = self.rev(node)
789 for r in range(p + 1, self.count()):
789 for r in range(p + 1, self.count()):
790 prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
790 prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
791 if prevs:
791 if prevs:
792 for pr in prevs:
792 for pr in prevs:
793 if pr == p:
793 if pr == p:
794 c.append(self.node(r))
794 c.append(self.node(r))
795 elif p == nullrev:
795 elif p == nullrev:
796 c.append(self.node(r))
796 c.append(self.node(r))
797 return c
797 return c
798
798
799 def _match(self, id):
799 def _match(self, id):
800 if isinstance(id, (long, int)):
800 if isinstance(id, (long, int)):
801 # rev
801 # rev
802 return self.node(id)
802 return self.node(id)
803 if len(id) == 20:
803 if len(id) == 20:
804 # possibly a binary node
804 # possibly a binary node
805 # odds of a binary node being all hex in ASCII are 1 in 10**25
805 # odds of a binary node being all hex in ASCII are 1 in 10**25
806 try:
806 try:
807 node = id
807 node = id
808 r = self.rev(node) # quick search the index
808 r = self.rev(node) # quick search the index
809 return node
809 return node
810 except LookupError:
810 except LookupError:
811 pass # may be partial hex id
811 pass # may be partial hex id
812 try:
812 try:
813 # str(rev)
813 # str(rev)
814 rev = int(id)
814 rev = int(id)
815 if str(rev) != id:
815 if str(rev) != id:
816 raise ValueError
816 raise ValueError
817 if rev < 0:
817 if rev < 0:
818 rev = self.count() + rev
818 rev = self.count() + rev
819 if rev < 0 or rev >= self.count():
819 if rev < 0 or rev >= self.count():
820 raise ValueError
820 raise ValueError
821 return self.node(rev)
821 return self.node(rev)
822 except (ValueError, OverflowError):
822 except (ValueError, OverflowError):
823 pass
823 pass
824 if len(id) == 40:
824 if len(id) == 40:
825 try:
825 try:
826 # a full hex nodeid?
826 # a full hex nodeid?
827 node = bin(id)
827 node = bin(id)
828 r = self.rev(node)
828 r = self.rev(node)
829 return node
829 return node
830 except TypeError:
830 except TypeError:
831 pass
831 pass
832
832
833 def _partialmatch(self, id):
833 def _partialmatch(self, id):
834 if len(id) < 40:
834 if len(id) < 40:
835 try:
835 try:
836 # hex(node)[:...]
836 # hex(node)[:...]
837 bin_id = bin(id[:len(id) & ~1]) # grab an even number of digits
837 bin_id = bin(id[:len(id) & ~1]) # grab an even number of digits
838 node = None
838 node = None
839 for n in self.nodemap:
839 for n in self.nodemap:
840 if n.startswith(bin_id) and hex(n).startswith(id):
840 if n.startswith(bin_id) and hex(n).startswith(id):
841 if node is not None:
841 if node is not None:
842 raise LookupError(hex(node),
842 raise LookupError(hex(node),
843 _("Ambiguous identifier"))
843 _("Ambiguous identifier"))
844 node = n
844 node = n
845 if node is not None:
845 if node is not None:
846 return node
846 return node
847 except TypeError:
847 except TypeError:
848 pass
848 pass
849
849
850 def lookup(self, id):
850 def lookup(self, id):
851 """locate a node based on:
851 """locate a node based on:
852 - revision number or str(revision number)
852 - revision number or str(revision number)
853 - nodeid or subset of hex nodeid
853 - nodeid or subset of hex nodeid
854 """
854 """
855 n = self._match(id)
855 n = self._match(id)
856 if n is not None:
856 if n is not None:
857 return n
857 return n
858 n = self._partialmatch(id)
858 n = self._partialmatch(id)
859 if n:
859 if n:
860 return n
860 return n
861
861
862 raise LookupError(id, _("No match found"))
862 raise LookupError(id, _("No match found"))
863
863
864 def cmp(self, node, text):
864 def cmp(self, node, text):
865 """compare text with a given file revision"""
865 """compare text with a given file revision"""
866 p1, p2 = self.parents(node)
866 p1, p2 = self.parents(node)
867 return hash(text, p1, p2) != node
867 return hash(text, p1, p2) != node
868
868
869 def chunk(self, rev, df=None):
869 def chunk(self, rev, df=None):
870 def loadcache(df):
870 def loadcache(df):
871 if not df:
871 if not df:
872 if self._inline:
872 if self._inline:
873 df = self.opener(self.indexfile)
873 df = self.opener(self.indexfile)
874 else:
874 else:
875 df = self.opener(self.datafile)
875 df = self.opener(self.datafile)
876 df.seek(start)
876 df.seek(start)
877 self._chunkcache = (start, df.read(cache_length))
877 self._chunkcache = (start, df.read(cache_length))
878
878
879 start, length = self.start(rev), self.length(rev)
879 start, length = self.start(rev), self.length(rev)
880 if self._inline:
880 if self._inline:
881 start += (rev + 1) * self._io.size
881 start += (rev + 1) * self._io.size
882 end = start + length
882 end = start + length
883
883
884 offset = 0
884 offset = 0
885 if not self._chunkcache:
885 if not self._chunkcache:
886 cache_length = max(65536, length)
886 cache_length = max(65536, length)
887 loadcache(df)
887 loadcache(df)
888 else:
888 else:
889 cache_start = self._chunkcache[0]
889 cache_start = self._chunkcache[0]
890 cache_length = len(self._chunkcache[1])
890 cache_length = len(self._chunkcache[1])
891 cache_end = cache_start + cache_length
891 cache_end = cache_start + cache_length
892 if start >= cache_start and end <= cache_end:
892 if start >= cache_start and end <= cache_end:
893 # it is cached
893 # it is cached
894 offset = start - cache_start
894 offset = start - cache_start
895 else:
895 else:
896 cache_length = max(65536, length)
896 cache_length = max(65536, length)
897 loadcache(df)
897 loadcache(df)
898
898
899 # avoid copying large chunks
899 # avoid copying large chunks
900 c = self._chunkcache[1]
900 c = self._chunkcache[1]
901 if cache_length != length:
901 if cache_length != length:
902 c = c[offset:offset + length]
902 c = c[offset:offset + length]
903
903
904 return decompress(c)
904 return decompress(c)
905
905
906 def delta(self, node):
906 def delta(self, node):
907 """return or calculate a delta between a node and its predecessor"""
907 """return or calculate a delta between a node and its predecessor"""
908 r = self.rev(node)
908 r = self.rev(node)
909 return self.revdiff(r - 1, r)
909 return self.revdiff(r - 1, r)
910
910
911 def revdiff(self, rev1, rev2):
911 def revdiff(self, rev1, rev2):
912 """return or calculate a delta between two revisions"""
912 """return or calculate a delta between two revisions"""
913 if rev1 + 1 == rev2 and self.base(rev1) == self.base(rev2):
913 if rev1 + 1 == rev2 and self.base(rev1) == self.base(rev2):
914 return self.chunk(rev2)
914 return self.chunk(rev2)
915
915
916 return mdiff.textdiff(self.revision(self.node(rev1)),
916 return mdiff.textdiff(self.revision(self.node(rev1)),
917 self.revision(self.node(rev2)))
917 self.revision(self.node(rev2)))
918
918
919 def revision(self, node):
919 def revision(self, node):
920 """return an uncompressed revision of a given"""
920 """return an uncompressed revision of a given"""
921 if node == nullid:
921 if node == nullid:
922 return ""
922 return ""
923 if self._cache and self._cache[0] == node:
923 if self._cache and self._cache[0] == node:
924 return str(self._cache[2])
924 return str(self._cache[2])
925
925
926 # look up what we need to read
926 # look up what we need to read
927 text = None
927 text = None
928 rev = self.rev(node)
928 rev = self.rev(node)
929 base = self.base(rev)
929 base = self.base(rev)
930
930
931 # check rev flags
931 # check rev flags
932 if self.index[rev][0] & 0xFFFF:
932 if self.index[rev][0] & 0xFFFF:
933 raise RevlogError(_('incompatible revision flag %x') %
933 raise RevlogError(_('incompatible revision flag %x') %
934 (self.index[rev][0] & 0xFFFF))
934 (self.index[rev][0] & 0xFFFF))
935
935
936 if self._inline:
936 if self._inline:
937 # we probably have the whole chunk cached
937 # we probably have the whole chunk cached
938 df = None
938 df = None
939 else:
939 else:
940 df = self.opener(self.datafile)
940 df = self.opener(self.datafile)
941
941
942 # do we have useful data cached?
942 # do we have useful data cached?
943 if self._cache and self._cache[1] >= base and self._cache[1] < rev:
943 if self._cache and self._cache[1] >= base and self._cache[1] < rev:
944 base = self._cache[1]
944 base = self._cache[1]
945 text = str(self._cache[2])
945 text = str(self._cache[2])
946 self._loadindex(base, rev + 1)
946 self._loadindex(base, rev + 1)
947 else:
947 else:
948 self._loadindex(base, rev + 1)
948 self._loadindex(base, rev + 1)
949 text = self.chunk(base, df=df)
949 text = self.chunk(base, df=df)
950
950
951 bins = [self.chunk(r, df) for r in xrange(base + 1, rev + 1)]
951 bins = [self.chunk(r, df) for r in xrange(base + 1, rev + 1)]
952 text = mdiff.patches(text, bins)
952 text = mdiff.patches(text, bins)
953 p1, p2 = self.parents(node)
953 p1, p2 = self.parents(node)
954 if node != hash(text, p1, p2):
954 if node != hash(text, p1, p2):
955 raise RevlogError(_("integrity check failed on %s:%d")
955 raise RevlogError(_("integrity check failed on %s:%d")
956 % (self.datafile, rev))
956 % (self.datafile, rev))
957
957
958 self._cache = (node, rev, text)
958 self._cache = (node, rev, text)
959 return text
959 return text
960
960
961 def checkinlinesize(self, tr, fp=None):
961 def checkinlinesize(self, tr, fp=None):
962 if not self._inline:
962 if not self._inline:
963 return
963 return
964 if not fp:
964 if not fp:
965 fp = self.opener(self.indexfile, 'r')
965 fp = self.opener(self.indexfile, 'r')
966 fp.seek(0, 2)
966 fp.seek(0, 2)
967 size = fp.tell()
967 size = fp.tell()
968 if size < 131072:
968 if size < 131072:
969 return
969 return
970 trinfo = tr.find(self.indexfile)
970 trinfo = tr.find(self.indexfile)
971 if trinfo == None:
971 if trinfo == None:
972 raise RevlogError(_("%s not found in the transaction")
972 raise RevlogError(_("%s not found in the transaction")
973 % self.indexfile)
973 % self.indexfile)
974
974
975 trindex = trinfo[2]
975 trindex = trinfo[2]
976 dataoff = self.start(trindex)
976 dataoff = self.start(trindex)
977
977
978 tr.add(self.datafile, dataoff)
978 tr.add(self.datafile, dataoff)
979 df = self.opener(self.datafile, 'w')
979 df = self.opener(self.datafile, 'w')
980 calc = self._io.size
980 calc = self._io.size
981 for r in xrange(self.count()):
981 for r in xrange(self.count()):
982 start = self.start(r) + (r + 1) * calc
982 start = self.start(r) + (r + 1) * calc
983 length = self.length(r)
983 length = self.length(r)
984 fp.seek(start)
984 fp.seek(start)
985 d = fp.read(length)
985 d = fp.read(length)
986 df.write(d)
986 df.write(d)
987 fp.close()
987 fp.close()
988 df.close()
988 df.close()
989 fp = self.opener(self.indexfile, 'w', atomictemp=True)
989 fp = self.opener(self.indexfile, 'w', atomictemp=True)
990 self.version &= ~(REVLOGNGINLINEDATA)
990 self.version &= ~(REVLOGNGINLINEDATA)
991 self._inline = False
991 self._inline = False
992 for i in xrange(self.count()):
992 for i in xrange(self.count()):
993 e = self._io.packentry(self.index[i], self.node, self.version, i)
993 e = self._io.packentry(self.index[i], self.node, self.version, i)
994 fp.write(e)
994 fp.write(e)
995
995
996 # if we don't call rename, the temp file will never replace the
996 # if we don't call rename, the temp file will never replace the
997 # real index
997 # real index
998 fp.rename()
998 fp.rename()
999
999
1000 tr.replace(self.indexfile, trindex * calc)
1000 tr.replace(self.indexfile, trindex * calc)
1001 self._chunkcache = None
1001 self._chunkcache = None
1002
1002
1003 def addrevision(self, text, transaction, link, p1, p2, d=None):
1003 def addrevision(self, text, transaction, link, p1, p2, d=None):
1004 """add a revision to the log
1004 """add a revision to the log
1005
1005
1006 text - the revision data to add
1006 text - the revision data to add
1007 transaction - the transaction object used for rollback
1007 transaction - the transaction object used for rollback
1008 link - the linkrev data to add
1008 link - the linkrev data to add
1009 p1, p2 - the parent nodeids of the revision
1009 p1, p2 - the parent nodeids of the revision
1010 d - an optional precomputed delta
1010 d - an optional precomputed delta
1011 """
1011 """
1012 dfh = None
1012 dfh = None
1013 if not self._inline:
1013 if not self._inline:
1014 dfh = self.opener(self.datafile, "a")
1014 dfh = self.opener(self.datafile, "a")
1015 ifh = self.opener(self.indexfile, "a+")
1015 ifh = self.opener(self.indexfile, "a+")
1016 return self._addrevision(text, transaction, link, p1, p2, d, ifh, dfh)
1016 return self._addrevision(text, transaction, link, p1, p2, d, ifh, dfh)
1017
1017
1018 def _addrevision(self, text, transaction, link, p1, p2, d, ifh, dfh):
1018 def _addrevision(self, text, transaction, link, p1, p2, d, ifh, dfh):
1019 node = hash(text, p1, p2)
1019 node = hash(text, p1, p2)
1020 if node in self.nodemap:
1020 if node in self.nodemap:
1021 return node
1021 return node
1022
1022
1023 curr = self.count()
1023 curr = self.count()
1024 prev = curr - 1
1024 prev = curr - 1
1025 base = self.base(prev)
1025 base = self.base(prev)
1026 offset = self.end(prev)
1026 offset = self.end(prev)
1027
1027
1028 if curr:
1028 if curr:
1029 if not d:
1029 if not d:
1030 ptext = self.revision(self.node(prev))
1030 ptext = self.revision(self.node(prev))
1031 d = mdiff.textdiff(ptext, text)
1031 d = mdiff.textdiff(ptext, text)
1032 data = compress(d)
1032 data = compress(d)
1033 l = len(data[1]) + len(data[0])
1033 l = len(data[1]) + len(data[0])
1034 dist = l + offset - self.start(base)
1034 dist = l + offset - self.start(base)
1035
1035
1036 # full versions are inserted when the needed deltas
1036 # full versions are inserted when the needed deltas
1037 # become comparable to the uncompressed text
1037 # become comparable to the uncompressed text
1038 if not curr or dist > len(text) * 2:
1038 if not curr or dist > len(text) * 2:
1039 data = compress(text)
1039 data = compress(text)
1040 l = len(data[1]) + len(data[0])
1040 l = len(data[1]) + len(data[0])
1041 base = curr
1041 base = curr
1042
1042
1043 e = (offset_type(offset, 0), l, len(text),
1043 e = (offset_type(offset, 0), l, len(text),
1044 base, link, self.rev(p1), self.rev(p2), node)
1044 base, link, self.rev(p1), self.rev(p2), node)
1045 self.index.insert(-1, e)
1045 self.index.insert(-1, e)
1046 self.nodemap[node] = curr
1046 self.nodemap[node] = curr
1047
1047
1048 entry = self._io.packentry(e, self.node, self.version, curr)
1048 entry = self._io.packentry(e, self.node, self.version, curr)
1049 if not self._inline:
1049 if not self._inline:
1050 transaction.add(self.datafile, offset)
1050 transaction.add(self.datafile, offset)
1051 transaction.add(self.indexfile, curr * len(entry))
1051 transaction.add(self.indexfile, curr * len(entry))
1052 if data[0]:
1052 if data[0]:
1053 dfh.write(data[0])
1053 dfh.write(data[0])
1054 dfh.write(data[1])
1054 dfh.write(data[1])
1055 dfh.flush()
1055 dfh.flush()
1056 ifh.write(entry)
1056 ifh.write(entry)
1057 else:
1057 else:
1058 offset += curr * self._io.size
1058 offset += curr * self._io.size
1059 transaction.add(self.indexfile, offset, curr)
1059 transaction.add(self.indexfile, offset, curr)
1060 ifh.write(entry)
1060 ifh.write(entry)
1061 ifh.write(data[0])
1061 ifh.write(data[0])
1062 ifh.write(data[1])
1062 ifh.write(data[1])
1063 self.checkinlinesize(transaction, ifh)
1063 self.checkinlinesize(transaction, ifh)
1064
1064
1065 self._cache = (node, curr, text)
1065 self._cache = (node, curr, text)
1066 return node
1066 return node
1067
1067
1068 def ancestor(self, a, b):
1068 def ancestor(self, a, b):
1069 """calculate the least common ancestor of nodes a and b"""
1069 """calculate the least common ancestor of nodes a and b"""
1070
1070
1071 def parents(rev):
1071 def parents(rev):
1072 return [p for p in self.parentrevs(rev) if p != nullrev]
1072 return [p for p in self.parentrevs(rev) if p != nullrev]
1073
1073
1074 c = ancestor.ancestor(self.rev(a), self.rev(b), parents)
1074 c = ancestor.ancestor(self.rev(a), self.rev(b), parents)
1075 if c is None:
1075 if c is None:
1076 return nullid
1076 return nullid
1077
1077
1078 return self.node(c)
1078 return self.node(c)
1079
1079
1080 def group(self, nodelist, lookup, infocollect=None):
1080 def group(self, nodelist, lookup, infocollect=None):
1081 """calculate a delta group
1081 """calculate a delta group
1082
1082
1083 Given a list of changeset revs, return a set of deltas and
1083 Given a list of changeset revs, return a set of deltas and
1084 metadata corresponding to nodes. the first delta is
1084 metadata corresponding to nodes. the first delta is
1085 parent(nodes[0]) -> nodes[0] the receiver is guaranteed to
1085 parent(nodes[0]) -> nodes[0] the receiver is guaranteed to
1086 have this parent as it has all history before these
1086 have this parent as it has all history before these
1087 changesets. parent is parent[0]
1087 changesets. parent is parent[0]
1088 """
1088 """
1089 revs = [self.rev(n) for n in nodelist]
1089 revs = [self.rev(n) for n in nodelist]
1090
1090
1091 # if we don't have any revisions touched by these changesets, bail
1091 # if we don't have any revisions touched by these changesets, bail
1092 if not revs:
1092 if not revs:
1093 yield changegroup.closechunk()
1093 yield changegroup.closechunk()
1094 return
1094 return
1095
1095
1096 # add the parent of the first rev
1096 # add the parent of the first rev
1097 p = self.parents(self.node(revs[0]))[0]
1097 p = self.parents(self.node(revs[0]))[0]
1098 revs.insert(0, self.rev(p))
1098 revs.insert(0, self.rev(p))
1099
1099
1100 # build deltas
1100 # build deltas
1101 for d in xrange(0, len(revs) - 1):
1101 for d in xrange(0, len(revs) - 1):
1102 a, b = revs[d], revs[d + 1]
1102 a, b = revs[d], revs[d + 1]
1103 nb = self.node(b)
1103 nb = self.node(b)
1104
1104
1105 if infocollect is not None:
1105 if infocollect is not None:
1106 infocollect(nb)
1106 infocollect(nb)
1107
1107
1108 p = self.parents(nb)
1108 p = self.parents(nb)
1109 meta = nb + p[0] + p[1] + lookup(nb)
1109 meta = nb + p[0] + p[1] + lookup(nb)
1110 if a == -1:
1110 if a == -1:
1111 d = self.revision(nb)
1111 d = self.revision(nb)
1112 meta += mdiff.trivialdiffheader(len(d))
1112 meta += mdiff.trivialdiffheader(len(d))
1113 else:
1113 else:
1114 d = self.revdiff(a, b)
1114 d = self.revdiff(a, b)
1115 yield changegroup.chunkheader(len(meta) + len(d))
1115 yield changegroup.chunkheader(len(meta) + len(d))
1116 yield meta
1116 yield meta
1117 if len(d) > 2**20:
1117 if len(d) > 2**20:
1118 pos = 0
1118 pos = 0
1119 while pos < len(d):
1119 while pos < len(d):
1120 pos2 = pos + 2 ** 18
1120 pos2 = pos + 2 ** 18
1121 yield d[pos:pos2]
1121 yield d[pos:pos2]
1122 pos = pos2
1122 pos = pos2
1123 else:
1123 else:
1124 yield d
1124 yield d
1125
1125
1126 yield changegroup.closechunk()
1126 yield changegroup.closechunk()
1127
1127
1128 def addgroup(self, revs, linkmapper, transaction, unique=0):
1128 def addgroup(self, revs, linkmapper, transaction, unique=0):
1129 """
1129 """
1130 add a delta group
1130 add a delta group
1131
1131
1132 given a set of deltas, add them to the revision log. the
1132 given a set of deltas, add them to the revision log. the
1133 first delta is against its parent, which should be in our
1133 first delta is against its parent, which should be in our
1134 log, the rest are against the previous delta.
1134 log, the rest are against the previous delta.
1135 """
1135 """
1136
1136
1137 #track the base of the current delta log
1137 #track the base of the current delta log
1138 r = self.count()
1138 r = self.count()
1139 t = r - 1
1139 t = r - 1
1140 node = None
1140 node = None
1141
1141
1142 base = prev = nullrev
1142 base = prev = nullrev
1143 start = end = textlen = 0
1143 start = end = textlen = 0
1144 if r:
1144 if r:
1145 end = self.end(t)
1145 end = self.end(t)
1146
1146
1147 ifh = self.opener(self.indexfile, "a+")
1147 ifh = self.opener(self.indexfile, "a+")
1148 isize = r * self._io.size
1148 isize = r * self._io.size
1149 if self._inline:
1149 if self._inline:
1150 transaction.add(self.indexfile, end + isize, r)
1150 transaction.add(self.indexfile, end + isize, r)
1151 dfh = None
1151 dfh = None
1152 else:
1152 else:
1153 transaction.add(self.indexfile, isize, r)
1153 transaction.add(self.indexfile, isize, r)
1154 transaction.add(self.datafile, end)
1154 transaction.add(self.datafile, end)
1155 dfh = self.opener(self.datafile, "a")
1155 dfh = self.opener(self.datafile, "a")
1156
1156
1157 # loop through our set of deltas
1157 # loop through our set of deltas
1158 chain = None
1158 chain = None
1159 for chunk in revs:
1159 for chunk in revs:
1160 node, p1, p2, cs = struct.unpack("20s20s20s20s", chunk[:80])
1160 node, p1, p2, cs = struct.unpack("20s20s20s20s", chunk[:80])
1161 link = linkmapper(cs)
1161 link = linkmapper(cs)
1162 if node in self.nodemap:
1162 if node in self.nodemap:
1163 # this can happen if two branches make the same change
1163 # this can happen if two branches make the same change
1164 # if unique:
1164 # if unique:
1165 # raise RevlogError(_("already have %s") % hex(node[:4]))
1165 # raise RevlogError(_("already have %s") % hex(node[:4]))
1166 chain = node
1166 chain = node
1167 continue
1167 continue
1168 delta = buffer(chunk, 80)
1168 delta = buffer(chunk, 80)
1169 del chunk
1169 del chunk
1170
1170
1171 for p in (p1, p2):
1171 for p in (p1, p2):
1172 if not p in self.nodemap:
1172 if not p in self.nodemap:
1173 raise LookupError(hex(p), _("unknown parent %s") % short(p))
1173 raise LookupError(hex(p), _("unknown parent %s") % short(p))
1174
1174
1175 if not chain:
1175 if not chain:
1176 # retrieve the parent revision of the delta chain
1176 # retrieve the parent revision of the delta chain
1177 chain = p1
1177 chain = p1
1178 if not chain in self.nodemap:
1178 if not chain in self.nodemap:
1179 raise LookupError(hex(chain), _("unknown base %s") % short(chain[:4]))
1179 raise LookupError(hex(chain), _("unknown base %s") % short(chain[:4]))
1180
1180
1181 # full versions are inserted when the needed deltas become
1181 # full versions are inserted when the needed deltas become
1182 # comparable to the uncompressed text or when the previous
1182 # comparable to the uncompressed text or when the previous
1183 # version is not the one we have a delta against. We use
1183 # version is not the one we have a delta against. We use
1184 # the size of the previous full rev as a proxy for the
1184 # the size of the previous full rev as a proxy for the
1185 # current size.
1185 # current size.
1186
1186
1187 if chain == prev:
1187 if chain == prev:
1188 cdelta = compress(delta)
1188 cdelta = compress(delta)
1189 cdeltalen = len(cdelta[0]) + len(cdelta[1])
1189 cdeltalen = len(cdelta[0]) + len(cdelta[1])
1190 textlen = mdiff.patchedsize(textlen, delta)
1190 textlen = mdiff.patchedsize(textlen, delta)
1191
1191
1192 if chain != prev or (end - start + cdeltalen) > textlen * 2:
1192 if chain != prev or (end - start + cdeltalen) > textlen * 2:
1193 # flush our writes here so we can read it in revision
1193 # flush our writes here so we can read it in revision
1194 if dfh:
1194 if dfh:
1195 dfh.flush()
1195 dfh.flush()
1196 ifh.flush()
1196 ifh.flush()
1197 text = self.revision(chain)
1197 text = self.revision(chain)
1198 if len(text) == 0:
1198 if len(text) == 0:
1199 # skip over trivial delta header
1199 # skip over trivial delta header
1200 text = buffer(delta, 12)
1200 text = buffer(delta, 12)
1201 else:
1201 else:
1202 text = mdiff.patches(text, [delta])
1202 text = mdiff.patches(text, [delta])
1203 del delta
1203 del delta
1204 chk = self._addrevision(text, transaction, link, p1, p2, None,
1204 chk = self._addrevision(text, transaction, link, p1, p2, None,
1205 ifh, dfh)
1205 ifh, dfh)
1206 if not dfh and not self._inline:
1206 if not dfh and not self._inline:
1207 # addrevision switched from inline to conventional
1207 # addrevision switched from inline to conventional
1208 # reopen the index
1208 # reopen the index
1209 dfh = self.opener(self.datafile, "a")
1209 dfh = self.opener(self.datafile, "a")
1210 ifh = self.opener(self.indexfile, "a")
1210 ifh = self.opener(self.indexfile, "a")
1211 if chk != node:
1211 if chk != node:
1212 raise RevlogError(_("consistency error adding group"))
1212 raise RevlogError(_("consistency error adding group"))
1213 textlen = len(text)
1213 textlen = len(text)
1214 else:
1214 else:
1215 e = (offset_type(end, 0), cdeltalen, textlen, base,
1215 e = (offset_type(end, 0), cdeltalen, textlen, base,
1216 link, self.rev(p1), self.rev(p2), node)
1216 link, self.rev(p1), self.rev(p2), node)
1217 self.index.insert(-1, e)
1217 self.index.insert(-1, e)
1218 self.nodemap[node] = r
1218 self.nodemap[node] = r
1219 entry = self._io.packentry(e, self.node, self.version, r)
1219 entry = self._io.packentry(e, self.node, self.version, r)
1220 if self._inline:
1220 if self._inline:
1221 ifh.write(entry)
1221 ifh.write(entry)
1222 ifh.write(cdelta[0])
1222 ifh.write(cdelta[0])
1223 ifh.write(cdelta[1])
1223 ifh.write(cdelta[1])
1224 self.checkinlinesize(transaction, ifh)
1224 self.checkinlinesize(transaction, ifh)
1225 if not self._inline:
1225 if not self._inline:
1226 dfh = self.opener(self.datafile, "a")
1226 dfh = self.opener(self.datafile, "a")
1227 ifh = self.opener(self.indexfile, "a")
1227 ifh = self.opener(self.indexfile, "a")
1228 else:
1228 else:
1229 dfh.write(cdelta[0])
1229 dfh.write(cdelta[0])
1230 dfh.write(cdelta[1])
1230 dfh.write(cdelta[1])
1231 ifh.write(entry)
1231 ifh.write(entry)
1232
1232
1233 t, r, chain, prev = r, r + 1, node, node
1233 t, r, chain, prev = r, r + 1, node, node
1234 base = self.base(t)
1234 base = self.base(t)
1235 start = self.start(base)
1235 start = self.start(base)
1236 end = self.end(t)
1236 end = self.end(t)
1237
1237
1238 return node
1238 return node
1239
1239
1240 def strip(self, rev, minlink):
1240 def strip(self, rev, minlink):
1241 if self.count() == 0 or rev >= self.count():
1241 if self.count() == 0:
1242 return
1242 return
1243
1243
1244 if isinstance(self.index, lazyindex):
1244 if isinstance(self.index, lazyindex):
1245 self._loadindexmap()
1245 self._loadindexmap()
1246
1246
1247 # When stripping away a revision, we need to make sure it
1247 for rev in xrange(0, self.count()):
1248 # does not actually belong to an older changeset.
1248 if self.index[rev][4] >= minlink:
1249 # The minlink parameter defines the oldest revision
1249 break
1250 # we're allowed to strip away.
1250 else:
1251 while minlink > self.index[rev][4]:
1251 return
1252 rev += 1
1253 if rev >= self.count():
1254 return
1255
1252
1256 # first truncate the files on disk
1253 # first truncate the files on disk
1257 end = self.start(rev)
1254 end = self.start(rev)
1258 if not self._inline:
1255 if not self._inline:
1259 df = self.opener(self.datafile, "a")
1256 df = self.opener(self.datafile, "a")
1260 df.truncate(end)
1257 df.truncate(end)
1261 end = rev * self._io.size
1258 end = rev * self._io.size
1262 else:
1259 else:
1263 end += rev * self._io.size
1260 end += rev * self._io.size
1264
1261
1265 indexf = self.opener(self.indexfile, "a")
1262 indexf = self.opener(self.indexfile, "a")
1266 indexf.truncate(end)
1263 indexf.truncate(end)
1267
1264
1268 # then reset internal state in memory to forget those revisions
1265 # then reset internal state in memory to forget those revisions
1269 self._cache = None
1266 self._cache = None
1270 self._chunkcache = None
1267 self._chunkcache = None
1271 for x in xrange(rev, self.count()):
1268 for x in xrange(rev, self.count()):
1272 del self.nodemap[self.node(x)]
1269 del self.nodemap[self.node(x)]
1273
1270
1274 del self.index[rev:-1]
1271 del self.index[rev:-1]
1275
1272
1276 def checksize(self):
1273 def checksize(self):
1277 expected = 0
1274 expected = 0
1278 if self.count():
1275 if self.count():
1279 expected = max(0, self.end(self.count() - 1))
1276 expected = max(0, self.end(self.count() - 1))
1280
1277
1281 try:
1278 try:
1282 f = self.opener(self.datafile)
1279 f = self.opener(self.datafile)
1283 f.seek(0, 2)
1280 f.seek(0, 2)
1284 actual = f.tell()
1281 actual = f.tell()
1285 dd = actual - expected
1282 dd = actual - expected
1286 except IOError, inst:
1283 except IOError, inst:
1287 if inst.errno != errno.ENOENT:
1284 if inst.errno != errno.ENOENT:
1288 raise
1285 raise
1289 dd = 0
1286 dd = 0
1290
1287
1291 try:
1288 try:
1292 f = self.opener(self.indexfile)
1289 f = self.opener(self.indexfile)
1293 f.seek(0, 2)
1290 f.seek(0, 2)
1294 actual = f.tell()
1291 actual = f.tell()
1295 s = self._io.size
1292 s = self._io.size
1296 i = max(0, actual / s)
1293 i = max(0, actual / s)
1297 di = actual - (i * s)
1294 di = actual - (i * s)
1298 if self._inline:
1295 if self._inline:
1299 databytes = 0
1296 databytes = 0
1300 for r in xrange(self.count()):
1297 for r in xrange(self.count()):
1301 databytes += max(0, self.length(r))
1298 databytes += max(0, self.length(r))
1302 dd = 0
1299 dd = 0
1303 di = actual - self.count() * s - databytes
1300 di = actual - self.count() * s - databytes
1304 except IOError, inst:
1301 except IOError, inst:
1305 if inst.errno != errno.ENOENT:
1302 if inst.errno != errno.ENOENT:
1306 raise
1303 raise
1307 di = 0
1304 di = 0
1308
1305
1309 return (dd, di)
1306 return (dd, di)
General Comments 0
You need to be logged in to leave comments. Login now