##// END OF EJS Templates
lazyparser speed ups...
mpm@selenic.com -
r323:c6f0673a default
parent child Browse files
Show More
@@ -1,513 +1,522 b''
1 # revlog.py - storage back-end for mercurial
1 # revlog.py - storage back-end for mercurial
2 #
2 #
3 # This provides efficient delta storage with O(1) retrieve and append
3 # This provides efficient delta storage with O(1) retrieve and append
4 # and O(changes) merge between branches
4 # and O(changes) merge between branches
5 #
5 #
6 # Copyright 2005 Matt Mackall <mpm@selenic.com>
6 # Copyright 2005 Matt Mackall <mpm@selenic.com>
7 #
7 #
8 # This software may be used and distributed according to the terms
8 # This software may be used and distributed according to the terms
9 # of the GNU General Public License, incorporated herein by reference.
9 # of the GNU General Public License, incorporated herein by reference.
10
10
11 import zlib, struct, sha, binascii, heapq
11 import zlib, struct, sha, binascii, heapq
12 from mercurial import mdiff
12 from mercurial import mdiff
13
13
14 def hex(node): return binascii.hexlify(node)
14 def hex(node): return binascii.hexlify(node)
15 def bin(node): return binascii.unhexlify(node)
15 def bin(node): return binascii.unhexlify(node)
16 def short(node): return hex(node[:4])
16 def short(node): return hex(node[:4])
17
17
18 def compress(text):
18 def compress(text):
19 if not text: return text
19 if not text: return text
20 if len(text) < 44:
20 if len(text) < 44:
21 if text[0] == '\0': return text
21 if text[0] == '\0': return text
22 return 'u' + text
22 return 'u' + text
23 bin = zlib.compress(text)
23 bin = zlib.compress(text)
24 if len(bin) > len(text):
24 if len(bin) > len(text):
25 if text[0] == '\0': return text
25 if text[0] == '\0': return text
26 return 'u' + text
26 return 'u' + text
27 return bin
27 return bin
28
28
29 def decompress(bin):
29 def decompress(bin):
30 if not bin: return bin
30 if not bin: return bin
31 t = bin[0]
31 t = bin[0]
32 if t == '\0': return bin
32 if t == '\0': return bin
33 if t == 'x': return zlib.decompress(bin)
33 if t == 'x': return zlib.decompress(bin)
34 if t == 'u': return bin[1:]
34 if t == 'u': return bin[1:]
35 raise "unknown compression type %s" % t
35 raise "unknown compression type %s" % t
36
36
37 def hash(text, p1, p2):
37 def hash(text, p1, p2):
38 l = [p1, p2]
38 l = [p1, p2]
39 l.sort()
39 l.sort()
40 return sha.sha(l[0] + l[1] + text).digest()
40 return sha.sha(l[0] + l[1] + text).digest()
41
41
42 nullid = "\0" * 20
42 nullid = "\0" * 20
43 indexformat = ">4l20s20s20s"
43 indexformat = ">4l20s20s20s"
44
44
45 class lazyparser:
45 class lazyparser:
46 def __init__(self, data):
46 def __init__(self, data, revlog):
47 self.data = data
47 self.data = data
48 self.s = struct.calcsize(indexformat)
48 self.s = struct.calcsize(indexformat)
49 self.l = len(data)/self.s
49 self.l = len(data)/self.s
50 self.index = [None] * self.l
50 self.index = [None] * self.l
51 self.map = {nullid: -1}
51 self.map = {nullid: -1}
52 self.all = 0
53 self.revlog = revlog
52
54
53 def load(self, pos):
55 def load(self, pos=None):
56 if self.all: return
57 if pos is not None:
54 block = pos / 1000
58 block = pos / 1000
55 i = block * 1000
59 i = block * 1000
56 end = min(self.l, i + 1000)
60 end = min(self.l, i + 1000)
61 else:
62 self.all = 1
63 i = 0
64 end = self.l
65 self.revlog.index = self.index
66 self.revlog.nodemap = self.map
67
57 while i < end:
68 while i < end:
58 d = self.data[i * self.s: (i + 1) * self.s]
69 d = self.data[i * self.s: (i + 1) * self.s]
59 e = struct.unpack(indexformat, d)
70 e = struct.unpack(indexformat, d)
60 self.index[i] = e
71 self.index[i] = e
61 self.map[e[6]] = i
72 self.map[e[6]] = i
62 i += 1
73 i += 1
63
74
64 class lazyindex:
75 class lazyindex:
65 def __init__(self, parser):
76 def __init__(self, parser):
66 self.p = parser
77 self.p = parser
67 def __len__(self):
78 def __len__(self):
68 return len(self.p.index)
79 return len(self.p.index)
69 def load(self, pos):
80 def load(self, pos):
70 self.p.load(pos)
81 self.p.load(pos)
71 return self.p.index[pos]
82 return self.p.index[pos]
72 def __getitem__(self, pos):
83 def __getitem__(self, pos):
73 return self.p.index[pos] or self.load(pos)
84 return self.p.index[pos] or self.load(pos)
74 def append(self, e):
85 def append(self, e):
75 self.p.index.append(e)
86 self.p.index.append(e)
76
87
77 class lazymap:
88 class lazymap:
78 def __init__(self, parser):
89 def __init__(self, parser):
79 self.p = parser
90 self.p = parser
80 def load(self, key):
91 def load(self, key):
92 if self.p.all: return
81 n = self.p.data.find(key)
93 n = self.p.data.find(key)
82 if n < 0: raise KeyError("node " + hex(key))
94 if n < 0: raise KeyError("node " + hex(key))
83 pos = n / self.p.s
95 pos = n / self.p.s
84 self.p.load(pos)
96 self.p.load(pos)
85 def __contains__(self, key):
97 def __contains__(self, key):
86 try:
98 self.p.load()
87 self[key]
99 return key in self.p.map
88 return True
89 except KeyError:
90 return False
91 def __iter__(self):
100 def __iter__(self):
92 for i in xrange(self.p.l):
101 for i in xrange(self.p.l):
93 try:
102 try:
94 yield self.p.index[i][6]
103 yield self.p.index[i][6]
95 except:
104 except:
96 self.p.load(i)
105 self.p.load(i)
97 yield self.p.index[i][6]
106 yield self.p.index[i][6]
98 def __getitem__(self, key):
107 def __getitem__(self, key):
99 try:
108 try:
100 return self.p.map[key]
109 return self.p.map[key]
101 except KeyError:
110 except KeyError:
102 try:
111 try:
103 self.load(key)
112 self.load(key)
104 return self.p.map[key]
113 return self.p.map[key]
105 except KeyError:
114 except KeyError:
106 raise KeyError("node " + hex(key))
115 raise KeyError("node " + hex(key))
107 def __setitem__(self, key, val):
116 def __setitem__(self, key, val):
108 self.p.map[key] = val
117 self.p.map[key] = val
109
118
110 class revlog:
119 class revlog:
111 def __init__(self, opener, indexfile, datafile):
120 def __init__(self, opener, indexfile, datafile):
112 self.indexfile = indexfile
121 self.indexfile = indexfile
113 self.datafile = datafile
122 self.datafile = datafile
114 self.opener = opener
123 self.opener = opener
115 self.cache = None
124 self.cache = None
116
125
117 try:
126 try:
118 i = self.opener(self.indexfile).read()
127 i = self.opener(self.indexfile).read()
119 except IOError:
128 except IOError:
120 i = ""
129 i = ""
121
130
122 if len(i) > 10000:
131 if len(i) > 10000:
123 # big index, let's parse it on demand
132 # big index, let's parse it on demand
124 parser = lazyparser(i)
133 parser = lazyparser(i, self)
125 self.index = lazyindex(parser)
134 self.index = lazyindex(parser)
126 self.nodemap = lazymap(parser)
135 self.nodemap = lazymap(parser)
127 else:
136 else:
128 s = struct.calcsize(indexformat)
137 s = struct.calcsize(indexformat)
129 l = len(i) / s
138 l = len(i) / s
130 self.index = [None] * l
139 self.index = [None] * l
131 m = [None] * l
140 m = [None] * l
132
141
133 n = 0
142 n = 0
134 for f in xrange(0, len(i), s):
143 for f in xrange(0, len(i), s):
135 # offset, size, base, linkrev, p1, p2, nodeid
144 # offset, size, base, linkrev, p1, p2, nodeid
136 e = struct.unpack(indexformat, i[f:f + s])
145 e = struct.unpack(indexformat, i[f:f + s])
137 m[n] = (e[6], n)
146 m[n] = (e[6], n)
138 self.index[n] = e
147 self.index[n] = e
139 n += 1
148 n += 1
140
149
141 self.nodemap = dict(m)
150 self.nodemap = dict(m)
142 self.nodemap[nullid] = -1
151 self.nodemap[nullid] = -1
143
152
144
153
145 def tip(self): return self.node(len(self.index) - 1)
154 def tip(self): return self.node(len(self.index) - 1)
146 def count(self): return len(self.index)
155 def count(self): return len(self.index)
147 def node(self, rev): return (rev < 0) and nullid or self.index[rev][6]
156 def node(self, rev): return (rev < 0) and nullid or self.index[rev][6]
148 def rev(self, node): return self.nodemap[node]
157 def rev(self, node): return self.nodemap[node]
149 def linkrev(self, node): return self.index[self.nodemap[node]][3]
158 def linkrev(self, node): return self.index[self.nodemap[node]][3]
150 def parents(self, node):
159 def parents(self, node):
151 if node == nullid: return (nullid, nullid)
160 if node == nullid: return (nullid, nullid)
152 return self.index[self.nodemap[node]][4:6]
161 return self.index[self.nodemap[node]][4:6]
153
162
154 def start(self, rev): return self.index[rev][0]
163 def start(self, rev): return self.index[rev][0]
155 def length(self, rev): return self.index[rev][1]
164 def length(self, rev): return self.index[rev][1]
156 def end(self, rev): return self.start(rev) + self.length(rev)
165 def end(self, rev): return self.start(rev) + self.length(rev)
157 def base(self, rev): return self.index[rev][2]
166 def base(self, rev): return self.index[rev][2]
158
167
159 def heads(self):
168 def heads(self):
160 p = {}
169 p = {}
161 h = []
170 h = []
162 for r in range(self.count() - 1, -1, -1):
171 for r in range(self.count() - 1, -1, -1):
163 n = self.node(r)
172 n = self.node(r)
164 if n not in p:
173 if n not in p:
165 h.append(n)
174 h.append(n)
166 for pn in self.parents(n):
175 for pn in self.parents(n):
167 p[pn] = 1
176 p[pn] = 1
168 return h
177 return h
169
178
170 def lookup(self, id):
179 def lookup(self, id):
171 try:
180 try:
172 rev = int(id)
181 rev = int(id)
173 return self.node(rev)
182 return self.node(rev)
174 except ValueError:
183 except ValueError:
175 c = []
184 c = []
176 for n in self.nodemap:
185 for n in self.nodemap:
177 if id in hex(n):
186 if id in hex(n):
178 c.append(n)
187 c.append(n)
179 if len(c) > 1: raise KeyError("Ambiguous identifier")
188 if len(c) > 1: raise KeyError("Ambiguous identifier")
180 if len(c) < 1: raise KeyError("No match found")
189 if len(c) < 1: raise KeyError("No match found")
181 return c[0]
190 return c[0]
182
191
183 return None
192 return None
184
193
185 def diff(self, a, b):
194 def diff(self, a, b):
186 return mdiff.textdiff(a, b)
195 return mdiff.textdiff(a, b)
187
196
188 def patches(self, t, pl):
197 def patches(self, t, pl):
189 return mdiff.patches(t, pl)
198 return mdiff.patches(t, pl)
190
199
191 def delta(self, node):
200 def delta(self, node):
192 r = self.rev(node)
201 r = self.rev(node)
193 b = self.base(r)
202 b = self.base(r)
194 if r == b:
203 if r == b:
195 return self.diff(self.revision(self.node(r - 1)),
204 return self.diff(self.revision(self.node(r - 1)),
196 self.revision(node))
205 self.revision(node))
197 else:
206 else:
198 f = self.opener(self.datafile)
207 f = self.opener(self.datafile)
199 f.seek(self.start(r))
208 f.seek(self.start(r))
200 data = f.read(self.length(r))
209 data = f.read(self.length(r))
201 return decompress(data)
210 return decompress(data)
202
211
203 def revision(self, node):
212 def revision(self, node):
204 if node == nullid: return ""
213 if node == nullid: return ""
205 if self.cache and self.cache[0] == node: return self.cache[2]
214 if self.cache and self.cache[0] == node: return self.cache[2]
206
215
207 text = None
216 text = None
208 rev = self.rev(node)
217 rev = self.rev(node)
209 start, length, base, link, p1, p2, node = self.index[rev]
218 start, length, base, link, p1, p2, node = self.index[rev]
210 end = start + length
219 end = start + length
211 if base != rev: start = self.start(base)
220 if base != rev: start = self.start(base)
212
221
213 if self.cache and self.cache[1] >= base and self.cache[1] < rev:
222 if self.cache and self.cache[1] >= base and self.cache[1] < rev:
214 base = self.cache[1]
223 base = self.cache[1]
215 start = self.start(base + 1)
224 start = self.start(base + 1)
216 text = self.cache[2]
225 text = self.cache[2]
217 last = 0
226 last = 0
218
227
219 f = self.opener(self.datafile)
228 f = self.opener(self.datafile)
220 f.seek(start)
229 f.seek(start)
221 data = f.read(end - start)
230 data = f.read(end - start)
222
231
223 if not text:
232 if not text:
224 last = self.length(base)
233 last = self.length(base)
225 text = decompress(data[:last])
234 text = decompress(data[:last])
226
235
227 bins = []
236 bins = []
228 for r in xrange(base + 1, rev + 1):
237 for r in xrange(base + 1, rev + 1):
229 s = self.length(r)
238 s = self.length(r)
230 bins.append(decompress(data[last:last + s]))
239 bins.append(decompress(data[last:last + s]))
231 last = last + s
240 last = last + s
232
241
233 text = mdiff.patches(text, bins)
242 text = mdiff.patches(text, bins)
234
243
235 if node != hash(text, p1, p2):
244 if node != hash(text, p1, p2):
236 raise IOError("integrity check failed on %s:%d"
245 raise IOError("integrity check failed on %s:%d"
237 % (self.datafile, rev))
246 % (self.datafile, rev))
238
247
239 self.cache = (node, rev, text)
248 self.cache = (node, rev, text)
240 return text
249 return text
241
250
242 def addrevision(self, text, transaction, link, p1=None, p2=None):
251 def addrevision(self, text, transaction, link, p1=None, p2=None):
243 if text is None: text = ""
252 if text is None: text = ""
244 if p1 is None: p1 = self.tip()
253 if p1 is None: p1 = self.tip()
245 if p2 is None: p2 = nullid
254 if p2 is None: p2 = nullid
246
255
247 node = hash(text, p1, p2)
256 node = hash(text, p1, p2)
248
257
249 if node in self.nodemap:
258 if node in self.nodemap:
250 return node
259 return node
251
260
252 n = self.count()
261 n = self.count()
253 t = n - 1
262 t = n - 1
254
263
255 if n:
264 if n:
256 base = self.base(t)
265 base = self.base(t)
257 start = self.start(base)
266 start = self.start(base)
258 end = self.end(t)
267 end = self.end(t)
259 prev = self.revision(self.tip())
268 prev = self.revision(self.tip())
260 d = self.diff(prev, text)
269 d = self.diff(prev, text)
261 data = compress(d)
270 data = compress(d)
262 dist = end - start + len(data)
271 dist = end - start + len(data)
263
272
264 # full versions are inserted when the needed deltas
273 # full versions are inserted when the needed deltas
265 # become comparable to the uncompressed text
274 # become comparable to the uncompressed text
266 if not n or dist > len(text) * 2:
275 if not n or dist > len(text) * 2:
267 data = compress(text)
276 data = compress(text)
268 base = n
277 base = n
269 else:
278 else:
270 base = self.base(t)
279 base = self.base(t)
271
280
272 offset = 0
281 offset = 0
273 if t >= 0:
282 if t >= 0:
274 offset = self.end(t)
283 offset = self.end(t)
275
284
276 e = (offset, len(data), base, link, p1, p2, node)
285 e = (offset, len(data), base, link, p1, p2, node)
277
286
278 self.index.append(e)
287 self.index.append(e)
279 self.nodemap[node] = n
288 self.nodemap[node] = n
280 entry = struct.pack(indexformat, *e)
289 entry = struct.pack(indexformat, *e)
281
290
282 transaction.add(self.datafile, e[0])
291 transaction.add(self.datafile, e[0])
283 self.opener(self.datafile, "a").write(data)
292 self.opener(self.datafile, "a").write(data)
284 transaction.add(self.indexfile, n * len(entry))
293 transaction.add(self.indexfile, n * len(entry))
285 self.opener(self.indexfile, "a").write(entry)
294 self.opener(self.indexfile, "a").write(entry)
286
295
287 self.cache = (node, n, text)
296 self.cache = (node, n, text)
288 return node
297 return node
289
298
290 def ancestor(self, a, b):
299 def ancestor(self, a, b):
291 # calculate the distance of every node from root
300 # calculate the distance of every node from root
292 dist = {nullid: 0}
301 dist = {nullid: 0}
293 for i in xrange(self.count()):
302 for i in xrange(self.count()):
294 n = self.node(i)
303 n = self.node(i)
295 p1, p2 = self.parents(n)
304 p1, p2 = self.parents(n)
296 dist[n] = max(dist[p1], dist[p2]) + 1
305 dist[n] = max(dist[p1], dist[p2]) + 1
297
306
298 # traverse ancestors in order of decreasing distance from root
307 # traverse ancestors in order of decreasing distance from root
299 def ancestors(node):
308 def ancestors(node):
300 # we store negative distances because heap returns smallest member
309 # we store negative distances because heap returns smallest member
301 h = [(-dist[node], node)]
310 h = [(-dist[node], node)]
302 seen = {}
311 seen = {}
303 earliest = self.count()
312 earliest = self.count()
304 while h:
313 while h:
305 d, n = heapq.heappop(h)
314 d, n = heapq.heappop(h)
306 r = self.rev(n)
315 r = self.rev(n)
307 if n not in seen:
316 if n not in seen:
308 seen[n] = 1
317 seen[n] = 1
309 yield (-d, n)
318 yield (-d, n)
310 for p in self.parents(n):
319 for p in self.parents(n):
311 heapq.heappush(h, (-dist[p], p))
320 heapq.heappush(h, (-dist[p], p))
312
321
313 x = ancestors(a)
322 x = ancestors(a)
314 y = ancestors(b)
323 y = ancestors(b)
315 lx = x.next()
324 lx = x.next()
316 ly = y.next()
325 ly = y.next()
317
326
318 # increment each ancestor list until it is closer to root than
327 # increment each ancestor list until it is closer to root than
319 # the other, or they match
328 # the other, or they match
320 while 1:
329 while 1:
321 if lx == ly:
330 if lx == ly:
322 return lx[1]
331 return lx[1]
323 elif lx < ly:
332 elif lx < ly:
324 ly = y.next()
333 ly = y.next()
325 elif lx > ly:
334 elif lx > ly:
326 lx = x.next()
335 lx = x.next()
327
336
328 def group(self, linkmap):
337 def group(self, linkmap):
329 # given a list of changeset revs, return a set of deltas and
338 # given a list of changeset revs, return a set of deltas and
330 # metadata corresponding to nodes. the first delta is
339 # metadata corresponding to nodes. the first delta is
331 # parent(nodes[0]) -> nodes[0] the receiver is guaranteed to
340 # parent(nodes[0]) -> nodes[0] the receiver is guaranteed to
332 # have this parent as it has all history before these
341 # have this parent as it has all history before these
333 # changesets. parent is parent[0]
342 # changesets. parent is parent[0]
334
343
335 revs = []
344 revs = []
336 needed = {}
345 needed = {}
337
346
338 # find file nodes/revs that match changeset revs
347 # find file nodes/revs that match changeset revs
339 for i in xrange(0, self.count()):
348 for i in xrange(0, self.count()):
340 if self.index[i][3] in linkmap:
349 if self.index[i][3] in linkmap:
341 revs.append(i)
350 revs.append(i)
342 needed[i] = 1
351 needed[i] = 1
343
352
344 # if we don't have any revisions touched by these changesets, bail
353 # if we don't have any revisions touched by these changesets, bail
345 if not revs:
354 if not revs:
346 yield struct.pack(">l", 0)
355 yield struct.pack(">l", 0)
347 return
356 return
348
357
349 # add the parent of the first rev
358 # add the parent of the first rev
350 p = self.parents(self.node(revs[0]))[0]
359 p = self.parents(self.node(revs[0]))[0]
351 revs.insert(0, self.rev(p))
360 revs.insert(0, self.rev(p))
352
361
353 # for each delta that isn't contiguous in the log, we need to
362 # for each delta that isn't contiguous in the log, we need to
354 # reconstruct the base, reconstruct the result, and then
363 # reconstruct the base, reconstruct the result, and then
355 # calculate the delta. We also need to do this where we've
364 # calculate the delta. We also need to do this where we've
356 # stored a full version and not a delta
365 # stored a full version and not a delta
357 for i in xrange(0, len(revs) - 1):
366 for i in xrange(0, len(revs) - 1):
358 a, b = revs[i], revs[i + 1]
367 a, b = revs[i], revs[i + 1]
359 if a + 1 != b or self.base(b) == b:
368 if a + 1 != b or self.base(b) == b:
360 for j in xrange(self.base(a), a + 1):
369 for j in xrange(self.base(a), a + 1):
361 needed[j] = 1
370 needed[j] = 1
362 for j in xrange(self.base(b), b + 1):
371 for j in xrange(self.base(b), b + 1):
363 needed[j] = 1
372 needed[j] = 1
364
373
365 # calculate spans to retrieve from datafile
374 # calculate spans to retrieve from datafile
366 needed = needed.keys()
375 needed = needed.keys()
367 needed.sort()
376 needed.sort()
368 spans = []
377 spans = []
369 oo = -1
378 oo = -1
370 ol = 0
379 ol = 0
371 for n in needed:
380 for n in needed:
372 if n < 0: continue
381 if n < 0: continue
373 o = self.start(n)
382 o = self.start(n)
374 l = self.length(n)
383 l = self.length(n)
375 if oo + ol == o: # can we merge with the previous?
384 if oo + ol == o: # can we merge with the previous?
376 nl = spans[-1][2]
385 nl = spans[-1][2]
377 nl.append((n, l))
386 nl.append((n, l))
378 ol += l
387 ol += l
379 spans[-1] = (oo, ol, nl)
388 spans[-1] = (oo, ol, nl)
380 else:
389 else:
381 oo = o
390 oo = o
382 ol = l
391 ol = l
383 spans.append((oo, ol, [(n, l)]))
392 spans.append((oo, ol, [(n, l)]))
384
393
385 # read spans in, divide up chunks
394 # read spans in, divide up chunks
386 chunks = {}
395 chunks = {}
387 for span in spans:
396 for span in spans:
388 # we reopen the file for each span to make http happy for now
397 # we reopen the file for each span to make http happy for now
389 f = self.opener(self.datafile)
398 f = self.opener(self.datafile)
390 f.seek(span[0])
399 f.seek(span[0])
391 data = f.read(span[1])
400 data = f.read(span[1])
392
401
393 # divide up the span
402 # divide up the span
394 pos = 0
403 pos = 0
395 for r, l in span[2]:
404 for r, l in span[2]:
396 chunks[r] = decompress(data[pos: pos + l])
405 chunks[r] = decompress(data[pos: pos + l])
397 pos += l
406 pos += l
398
407
399 # helper to reconstruct intermediate versions
408 # helper to reconstruct intermediate versions
400 def construct(text, base, rev):
409 def construct(text, base, rev):
401 bins = [chunks[r] for r in xrange(base + 1, rev + 1)]
410 bins = [chunks[r] for r in xrange(base + 1, rev + 1)]
402 return mdiff.patches(text, bins)
411 return mdiff.patches(text, bins)
403
412
404 # build deltas
413 # build deltas
405 deltas = []
414 deltas = []
406 for d in xrange(0, len(revs) - 1):
415 for d in xrange(0, len(revs) - 1):
407 a, b = revs[d], revs[d + 1]
416 a, b = revs[d], revs[d + 1]
408 n = self.node(b)
417 n = self.node(b)
409
418
410 # do we need to construct a new delta?
419 # do we need to construct a new delta?
411 if a + 1 != b or self.base(b) == b:
420 if a + 1 != b or self.base(b) == b:
412 if a >= 0:
421 if a >= 0:
413 base = self.base(a)
422 base = self.base(a)
414 ta = chunks[self.base(a)]
423 ta = chunks[self.base(a)]
415 ta = construct(ta, base, a)
424 ta = construct(ta, base, a)
416 else:
425 else:
417 ta = ""
426 ta = ""
418
427
419 base = self.base(b)
428 base = self.base(b)
420 if a > base:
429 if a > base:
421 base = a
430 base = a
422 tb = ta
431 tb = ta
423 else:
432 else:
424 tb = chunks[self.base(b)]
433 tb = chunks[self.base(b)]
425 tb = construct(tb, base, b)
434 tb = construct(tb, base, b)
426 d = self.diff(ta, tb)
435 d = self.diff(ta, tb)
427 else:
436 else:
428 d = chunks[b]
437 d = chunks[b]
429
438
430 p = self.parents(n)
439 p = self.parents(n)
431 meta = n + p[0] + p[1] + linkmap[self.linkrev(n)]
440 meta = n + p[0] + p[1] + linkmap[self.linkrev(n)]
432 l = struct.pack(">l", len(meta) + len(d) + 4)
441 l = struct.pack(">l", len(meta) + len(d) + 4)
433 yield l
442 yield l
434 yield meta
443 yield meta
435 yield d
444 yield d
436
445
437 yield struct.pack(">l", 0)
446 yield struct.pack(">l", 0)
438
447
439 def addgroup(self, revs, linkmapper, transaction, unique = 0):
448 def addgroup(self, revs, linkmapper, transaction, unique = 0):
440 # given a set of deltas, add them to the revision log. the
449 # given a set of deltas, add them to the revision log. the
441 # first delta is against its parent, which should be in our
450 # first delta is against its parent, which should be in our
442 # log, the rest are against the previous delta.
451 # log, the rest are against the previous delta.
443
452
444 # track the base of the current delta log
453 # track the base of the current delta log
445 r = self.count()
454 r = self.count()
446 t = r - 1
455 t = r - 1
447 node = nullid
456 node = nullid
448
457
449 base = prev = -1
458 base = prev = -1
450 start = end = 0
459 start = end = 0
451 if r:
460 if r:
452 start = self.start(self.base(t))
461 start = self.start(self.base(t))
453 end = self.end(t)
462 end = self.end(t)
454 measure = self.length(self.base(t))
463 measure = self.length(self.base(t))
455 base = self.base(t)
464 base = self.base(t)
456 prev = self.tip()
465 prev = self.tip()
457
466
458 transaction.add(self.datafile, end)
467 transaction.add(self.datafile, end)
459 transaction.add(self.indexfile, r * struct.calcsize(indexformat))
468 transaction.add(self.indexfile, r * struct.calcsize(indexformat))
460 dfh = self.opener(self.datafile, "a")
469 dfh = self.opener(self.datafile, "a")
461 ifh = self.opener(self.indexfile, "a")
470 ifh = self.opener(self.indexfile, "a")
462
471
463 # loop through our set of deltas
472 # loop through our set of deltas
464 chain = None
473 chain = None
465 for chunk in revs:
474 for chunk in revs:
466 node, p1, p2, cs = struct.unpack("20s20s20s20s", chunk[:80])
475 node, p1, p2, cs = struct.unpack("20s20s20s20s", chunk[:80])
467 link = linkmapper(cs)
476 link = linkmapper(cs)
468 if node in self.nodemap:
477 if node in self.nodemap:
469 # this can happen if two branches make the same change
478 # this can happen if two branches make the same change
470 if unique:
479 if unique:
471 raise "already have %s" % hex(node[:4])
480 raise "already have %s" % hex(node[:4])
472 continue
481 continue
473 delta = chunk[80:]
482 delta = chunk[80:]
474
483
475 if not chain:
484 if not chain:
476 # retrieve the parent revision of the delta chain
485 # retrieve the parent revision of the delta chain
477 chain = p1
486 chain = p1
478 if not chain in self.nodemap:
487 if not chain in self.nodemap:
479 raise "unknown base %s" % short(chain[:4])
488 raise "unknown base %s" % short(chain[:4])
480
489
481 # full versions are inserted when the needed deltas become
490 # full versions are inserted when the needed deltas become
482 # comparable to the uncompressed text or when the previous
491 # comparable to the uncompressed text or when the previous
483 # version is not the one we have a delta against. We use
492 # version is not the one we have a delta against. We use
484 # the size of the previous full rev as a proxy for the
493 # the size of the previous full rev as a proxy for the
485 # current size.
494 # current size.
486
495
487 if chain == prev:
496 if chain == prev:
488 cdelta = compress(delta)
497 cdelta = compress(delta)
489
498
490 if chain != prev or (end - start + len(cdelta)) > measure * 2:
499 if chain != prev or (end - start + len(cdelta)) > measure * 2:
491 # flush our writes here so we can read it in revision
500 # flush our writes here so we can read it in revision
492 dfh.flush()
501 dfh.flush()
493 ifh.flush()
502 ifh.flush()
494 text = self.revision(chain)
503 text = self.revision(chain)
495 text = self.patches(text, [delta])
504 text = self.patches(text, [delta])
496 chk = self.addrevision(text, transaction, link, p1, p2)
505 chk = self.addrevision(text, transaction, link, p1, p2)
497 if chk != node:
506 if chk != node:
498 raise "consistency error adding group"
507 raise "consistency error adding group"
499 measure = len(text)
508 measure = len(text)
500 else:
509 else:
501 e = (end, len(cdelta), self.base(t), link, p1, p2, node)
510 e = (end, len(cdelta), self.base(t), link, p1, p2, node)
502 self.index.append(e)
511 self.index.append(e)
503 self.nodemap[node] = r
512 self.nodemap[node] = r
504 dfh.write(cdelta)
513 dfh.write(cdelta)
505 ifh.write(struct.pack(indexformat, *e))
514 ifh.write(struct.pack(indexformat, *e))
506
515
507 t, r, chain, prev = r, r + 1, node, node
516 t, r, chain, prev = r, r + 1, node, node
508 start = self.start(self.base(t))
517 start = self.start(self.base(t))
509 end = self.end(t)
518 end = self.end(t)
510
519
511 dfh.close()
520 dfh.close()
512 ifh.close()
521 ifh.close()
513 return node
522 return node
General Comments 0
You need to be logged in to leave comments. Login now