##// END OF EJS Templates
Implement data inlined with the index file...
mason@suse.com -
r2073:1e6745f7 default
parent child Browse files
Show More
@@ -43,12 +43,23 b' class localrepository(object):'
43
43
44 v = self.ui.revlogopts
44 v = self.ui.revlogopts
45 self.revlogversion = int(v.get('format', 0))
45 self.revlogversion = int(v.get('format', 0))
46 flags = 0
46 for x in v.get('flags', "").split():
47 for x in v.get('flags', "").split():
47 self.revlogversion |= revlog.flagstr(x)
48 flags |= revlog.flagstr(x)
49
50 v = self.revlogversion | flags
51 self.manifest = manifest.manifest(self.opener, v)
52 self.changelog = changelog.changelog(self.opener, v)
48
53
49 self.manifest = manifest.manifest(self.opener, self.revlogversion)
54 # the changelog might not have the inline index flag
50 self.changelog = changelog.changelog(self.opener, self.revlogversion)
55 # on. If the format of the changelog is the same as found in
51 self.revlogversion = self.changelog.version
56 # .hgrc, apply any flags found in the .hgrc as well.
57 # Otherwise, just version from the changelog
58 v = self.changelog.version
59 if v == self.revlogversion:
60 v |= flags
61 self.revlogversion = v
62
52 self.tagscache = None
63 self.tagscache = None
53 self.nodetagscache = None
64 self.nodetagscache = None
54 self.encodepats = None
65 self.encodepats = None
@@ -20,6 +20,14 b' demandload(globals(), "sha struct zlib")'
20 REVLOGV0 = 0
20 REVLOGV0 = 0
21 REVLOGNG = 1
21 REVLOGNG = 1
22
22
23 # revlog flags
24 REVLOGNGINLINEDATA = (1 << 16)
25
26 def flagstr(flag):
27 if flag == "inline":
28 return REVLOGNGINLINEDATA
29 raise RevlogError(_("unknown revlog flag %s" % flag))
30
23 def hash(text, p1, p2):
31 def hash(text, p1, p2):
24 """generate a hash from the given text and its parent hashes
32 """generate a hash from the given text and its parent hashes
25
33
@@ -234,13 +242,19 b' class revlog(object):'
234 self.indexstat = st
242 self.indexstat = st
235 if len(i) > 0:
243 if len(i) > 0:
236 v = struct.unpack(versionformat, i[:4])[0]
244 v = struct.unpack(versionformat, i[:4])[0]
237 if v != 0:
238 flags = v & ~0xFFFF
245 flags = v & ~0xFFFF
239 fmt = v & 0xFFFF
246 fmt = v & 0xFFFF
240 if fmt != REVLOGNG or (flags & ~(REVLOGNGINLINEDATA)):
247 if fmt == 0:
241 raise RevlogError(
248 if flags:
242 _("unknown version format %d or flags %x on %s") %
249 raise RevlogError(_("index %s invalid flags %x for format v0" %
243 (v, flags, self.indexfile))
250 (self.indexfile, flags)))
251 elif fmt == REVLOGNG:
252 if flags & ~REVLOGNGINLINEDATA:
253 raise RevlogError(_("index %s invalid flags %x for revlogng" %
254 (self.indexfile, flags)))
255 else:
256 raise RevlogError(_("index %s invalid format %d" %
257 (self.indexfile, fmt)))
244 self.version = v
258 self.version = v
245 if v == 0:
259 if v == 0:
246 self.indexformat = indexformatv0
260 self.indexformat = indexformatv0
@@ -248,13 +262,17 b' class revlog(object):'
248 self.indexformat = indexformatng
262 self.indexformat = indexformatng
249
263
250 if i:
264 if i:
251 if st and st.st_size > 10000:
265 if not self.inlinedata() and st and st.st_size > 10000:
252 # big index, let's parse it on demand
266 # big index, let's parse it on demand
253 parser = lazyparser(i, self, self.indexformat)
267 parser = lazyparser(i, self, self.indexformat)
254 self.index = lazyindex(parser)
268 self.index = lazyindex(parser)
255 self.nodemap = lazymap(parser)
269 self.nodemap = lazymap(parser)
256 else:
270 else:
257 self.parseindex(i)
271 self.parseindex(i)
272 if self.inlinedata():
273 # we've already got the entire data file read in, save it
274 # in the chunk data
275 self.chunkcache = (0, i)
258 if self.version != 0:
276 if self.version != 0:
259 e = list(self.index[0])
277 e = list(self.index[0])
260 type = self.ngtype(e[0])
278 type = self.ngtype(e[0])
@@ -270,6 +288,7 b' class revlog(object):'
270 l = len(data)
288 l = len(data)
271 self.index = []
289 self.index = []
272 self.nodemap = {nullid: -1}
290 self.nodemap = {nullid: -1}
291 inline = self.inlinedata()
273 off = 0
292 off = 0
274 n = 0
293 n = 0
275 while off < l:
294 while off < l:
@@ -278,6 +297,8 b' class revlog(object):'
278 self.nodemap[e[-1]] = n
297 self.nodemap[e[-1]] = n
279 n += 1
298 n += 1
280 off += s
299 off += s
300 if inline:
301 off += e[1]
281
302
282 def ngoffset(self, q):
303 def ngoffset(self, q):
283 if q & 0xFFFF:
304 if q & 0xFFFF:
@@ -297,6 +318,7 b' class revlog(object):'
297 p = self.index.p
318 p = self.index.p
298 p.load()
319 p.load()
299
320
321 def inlinedata(self): return self.version & REVLOGNGINLINEDATA
300 def tip(self): return self.node(len(self.index) - 1)
322 def tip(self): return self.node(len(self.index) - 1)
301 def count(self): return len(self.index)
323 def count(self): return len(self.index)
302 def node(self, rev):
324 def node(self, rev):
@@ -568,10 +590,16 b' class revlog(object):'
568
590
569 def chunk(self, rev, df=None, cachelen=4096):
591 def chunk(self, rev, df=None, cachelen=4096):
570 start, length = self.start(rev), self.length(rev)
592 start, length = self.start(rev), self.length(rev)
593 inline = self.inlinedata()
594 if inline:
595 start += (rev + 1) * struct.calcsize(self.indexformat)
571 end = start + length
596 end = start + length
572 def loadcache(df):
597 def loadcache(df):
573 cache_length = max(cachelen, length) # 4k
598 cache_length = max(cachelen, length) # 4k
574 if not df:
599 if not df:
600 if inline:
601 df = self.opener(self.indexfile)
602 else:
575 df = self.opener(self.datafile)
603 df = self.opener(self.datafile)
576 df.seek(start)
604 df.seek(start)
577 self.chunkcache = (start, df.read(cache_length))
605 self.chunkcache = (start, df.read(cache_length))
@@ -620,6 +648,10 b' class revlog(object):'
620 rev = self.rev(node)
648 rev = self.rev(node)
621 base = self.base(rev)
649 base = self.base(rev)
622
650
651 if self.inlinedata():
652 # we probably have the whole chunk cached
653 df = None
654 else:
623 df = self.opener(self.datafile)
655 df = self.opener(self.datafile)
624
656
625 # do we have useful data cached?
657 # do we have useful data cached?
@@ -643,6 +675,40 b' class revlog(object):'
643 self.cache = (node, rev, text)
675 self.cache = (node, rev, text)
644 return text
676 return text
645
677
678 def checkinlinesize(self, fp, tr):
679 if not self.inlinedata():
680 return
681 size = fp.tell()
682 if size < 131072:
683 return
684 tr.add(self.datafile, 0)
685 df = self.opener(self.datafile, 'w')
686 calc = struct.calcsize(self.indexformat)
687 for r in xrange(self.count()):
688 start = self.start(r) + (r + 1) * calc
689 length = self.length(r)
690 fp.seek(start)
691 d = fp.read(length)
692 df.write(d)
693 fp.close()
694 df.close()
695 fp = self.opener(self.indexfile, 'w', atomic=True)
696 self.version &= ~(REVLOGNGINLINEDATA)
697 if self.count():
698 x = self.index[0]
699 e = struct.pack(self.indexformat, *x)[4:]
700 l = struct.pack(versionformat, self.version)
701 fp.write(l)
702 fp.write(e)
703
704 for i in xrange(1, self.count()):
705 x = self.index[i]
706 e = struct.pack(self.indexformat, *x)
707 fp.write(e)
708
709 fp.close()
710 self.chunkcache = None
711
646 def addrevision(self, text, transaction, link, p1=None, p2=None, d=None):
712 def addrevision(self, text, transaction, link, p1=None, p2=None, d=None):
647 """add a revision to the log
713 """add a revision to the log
648
714
@@ -698,6 +764,7 b' class revlog(object):'
698 self.nodemap[node] = n
764 self.nodemap[node] = n
699 entry = struct.pack(self.indexformat, *e)
765 entry = struct.pack(self.indexformat, *e)
700
766
767 if not self.inlinedata():
701 transaction.add(self.datafile, offset)
768 transaction.add(self.datafile, offset)
702 transaction.add(self.indexfile, n * len(entry))
769 transaction.add(self.indexfile, n * len(entry))
703 f = self.opener(self.datafile, "a")
770 f = self.opener(self.datafile, "a")
@@ -705,6 +772,9 b' class revlog(object):'
705 f.write(data[0])
772 f.write(data[0])
706 f.write(data[1])
773 f.write(data[1])
707 f = self.opener(self.indexfile, "a")
774 f = self.opener(self.indexfile, "a")
775 else:
776 f = self.opener(self.indexfile, "a+")
777 transaction.add(self.indexfile, f.tell())
708
778
709 if len(self.index) == 1 and self.version != 0:
779 if len(self.index) == 1 and self.version != 0:
710 l = struct.pack(versionformat, self.version)
780 l = struct.pack(versionformat, self.version)
@@ -713,6 +783,11 b' class revlog(object):'
713
783
714 f.write(entry)
784 f.write(entry)
715
785
786 if self.inlinedata():
787 f.write(data[0])
788 f.write(data[1])
789 self.checkinlinesize(f, transaction)
790
716 self.cache = (node, n, text)
791 self.cache = (node, n, text)
717 return node
792 return node
718
793
@@ -830,6 +905,9 b' class revlog(object):'
830
905
831 ifh = self.opener(self.indexfile, "a+")
906 ifh = self.opener(self.indexfile, "a+")
832 transaction.add(self.indexfile, ifh.tell())
907 transaction.add(self.indexfile, ifh.tell())
908 if self.inlinedata():
909 dfh = None
910 else:
833 transaction.add(self.datafile, end)
911 transaction.add(self.datafile, end)
834 dfh = self.opener(self.datafile, "a")
912 dfh = self.opener(self.datafile, "a")
835
913
@@ -885,6 +963,19 b' class revlog(object):'
885 link, self.rev(p1), self.rev(p2), node)
963 link, self.rev(p1), self.rev(p2), node)
886 self.index.append(e)
964 self.index.append(e)
887 self.nodemap[node] = r
965 self.nodemap[node] = r
966 if self.inlinedata():
967 ifh.write(struct.pack(self.indexformat, *e))
968 ifh.write(cdelta)
969 self.checkinlinesize(ifh, transaction)
970 if not self.inlinedata():
971 dfh = self.opener(self.datafile, "a")
972 ifh = self.opener(self.indexfile, "a")
973 else:
974 if not dfh:
975 # addrevision switched from inline to conventional
976 # reopen the index
977 dfh = self.opener(self.datafile, "a")
978 ifh = self.opener(self.indexfile, "a")
888 dfh.write(cdelta)
979 dfh.write(cdelta)
889 ifh.write(struct.pack(self.indexformat, *e))
980 ifh.write(struct.pack(self.indexformat, *e))
890
981
@@ -915,9 +1006,12 b' class revlog(object):'
915
1006
916 # first truncate the files on disk
1007 # first truncate the files on disk
917 end = self.start(rev)
1008 end = self.start(rev)
1009 if not self.inlinedata():
918 df = self.opener(self.datafile, "a")
1010 df = self.opener(self.datafile, "a")
919 df.truncate(end)
1011 df.truncate(end)
920 end = rev * struct.calcsize(self.indexformat)
1012 end = rev * struct.calcsize(self.indexformat)
1013 else:
1014 end += rev * struct.calcsize(self.indexformat)
921
1015
922 indexf = self.opener(self.indexfile, "a")
1016 indexf = self.opener(self.indexfile, "a")
923 indexf.truncate(end)
1017 indexf.truncate(end)
@@ -952,6 +1046,12 b' class revlog(object):'
952 s = struct.calcsize(self.indexformat)
1046 s = struct.calcsize(self.indexformat)
953 i = actual / s
1047 i = actual / s
954 di = actual - (i * s)
1048 di = actual - (i * s)
1049 if self.inlinedata():
1050 databytes = 0
1051 for r in xrange(self.count()):
1052 databytes += self.length(r)
1053 dd = 0
1054 di = actual - self.count() * s - databytes
955 except IOError, inst:
1055 except IOError, inst:
956 if inst.errno != errno.ENOENT:
1056 if inst.errno != errno.ENOENT:
957 raise
1057 raise
General Comments 0
You need to be logged in to leave comments. Login now