##// END OF EJS Templates
appendfile: handle only changelog.i file
Matt Mackall -
r4260:bdbfc219 default
parent child Browse files
Show More
@@ -1,154 +1,152
1 # appendfile.py - special classes to make repo updates atomic
1 # appendfile.py - special classes to make repo updates atomic
2 #
2 #
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms
5 # This software may be used and distributed according to the terms
6 # of the GNU General Public License, incorporated herein by reference.
6 # of the GNU General Public License, incorporated herein by reference.
7
7
8 import cStringIO, changelog, errno, manifest, os, tempfile, util
8 import cStringIO, changelog, errno, manifest, os, tempfile, util
9
9
10 # writes to metadata files are ordered. reads: changelog, manifest,
10 # writes to metadata files are ordered. reads: changelog, manifest,
11 # normal files. writes: normal files, manifest, changelog.
11 # normal files. writes: normal files, manifest, changelog.
12
12
13 # manifest contains pointers to offsets in normal files. changelog
13 # manifest contains pointers to offsets in normal files. changelog
14 # contains pointers to offsets in manifest. if reader reads old
14 # contains pointers to offsets in manifest. if reader reads old
15 # changelog while manifest or normal files are written, it has no
15 # changelog while manifest or normal files are written, it has no
16 # pointers into new parts of those files that are maybe not consistent
16 # pointers into new parts of those files that are maybe not consistent
17 # yet, so will not read them.
17 # yet, so will not read them.
18
18
19 # localrepo.addchangegroup thinks it writes changelog first, then
19 # localrepo.addchangegroup thinks it writes changelog first, then
20 # manifest, then normal files (this is order they are available, and
20 # manifest, then normal files (this is order they are available, and
21 # needed for computing linkrev fields), but uses appendfile to hide
21 # needed for computing linkrev fields), but uses appendfile to hide
22 # updates from readers. data not written to manifest or changelog
22 # updates from readers. data not written to manifest or changelog
23 # until all normal files updated. write manifest first, then
23 # until all normal files updated. write manifest first, then
24 # changelog.
24 # changelog.
25
25
26 # with this write ordering, readers cannot see inconsistent view of
26 # with this write ordering, readers cannot see inconsistent view of
27 # repo during update.
27 # repo during update.
28
28
29 class appendfile(object):
29 class appendfile(object):
30 '''implement enough of file protocol to append to revlog file.
30 '''implement enough of file protocol to append to revlog file.
31 appended data is written to temp file. reads and seeks span real
31 appended data is written to temp file. reads and seeks span real
32 file and temp file. readers cannot see appended data until
32 file and temp file. readers cannot see appended data until
33 writedata called.'''
33 writedata called.'''
34
34
35 def __init__(self, fp, tmpname):
35 def __init__(self, fp, tmpname):
36 if tmpname:
36 if tmpname:
37 self.tmpname = tmpname
37 self.tmpname = tmpname
38 self.tmpfp = util.posixfile(self.tmpname, 'ab+')
38 self.tmpfp = util.posixfile(self.tmpname, 'ab+')
39 else:
39 else:
40 fd, self.tmpname = tempfile.mkstemp(prefix="hg-appendfile-")
40 fd, self.tmpname = tempfile.mkstemp(prefix="hg-appendfile-")
41 os.close(fd)
41 os.close(fd)
42 self.tmpfp = util.posixfile(self.tmpname, 'ab+')
42 self.tmpfp = util.posixfile(self.tmpname, 'ab+')
43 self.realfp = fp
43 self.realfp = fp
44 self.offset = fp.tell()
44 self.offset = fp.tell()
45 # real file is not written by anyone else. cache its size so
45 # real file is not written by anyone else. cache its size so
46 # seek and read can be fast.
46 # seek and read can be fast.
47 self.realsize = util.fstat(fp).st_size
47 self.realsize = util.fstat(fp).st_size
48 self.name = fp.name
48 self.name = fp.name
49
49
50 def end(self):
50 def end(self):
51 self.tmpfp.flush() # make sure the stat is correct
51 self.tmpfp.flush() # make sure the stat is correct
52 return self.realsize + util.fstat(self.tmpfp).st_size
52 return self.realsize + util.fstat(self.tmpfp).st_size
53
53
54 def tell(self):
54 def tell(self):
55 return self.offset
55 return self.offset
56
56
57 def flush(self):
57 def flush(self):
58 self.tmpfp.flush()
58 self.tmpfp.flush()
59
59
60 def close(self):
60 def close(self):
61 self.realfp.close()
61 self.realfp.close()
62 self.tmpfp.close()
62 self.tmpfp.close()
63
63
64 def seek(self, offset, whence=0):
64 def seek(self, offset, whence=0):
65 '''virtual file offset spans real file and temp file.'''
65 '''virtual file offset spans real file and temp file.'''
66 if whence == 0:
66 if whence == 0:
67 self.offset = offset
67 self.offset = offset
68 elif whence == 1:
68 elif whence == 1:
69 self.offset += offset
69 self.offset += offset
70 elif whence == 2:
70 elif whence == 2:
71 self.offset = self.end() + offset
71 self.offset = self.end() + offset
72
72
73 if self.offset < self.realsize:
73 if self.offset < self.realsize:
74 self.realfp.seek(self.offset)
74 self.realfp.seek(self.offset)
75 else:
75 else:
76 self.tmpfp.seek(self.offset - self.realsize)
76 self.tmpfp.seek(self.offset - self.realsize)
77
77
78 def read(self, count=-1):
78 def read(self, count=-1):
79 '''only trick here is reads that span real file and temp file.'''
79 '''only trick here is reads that span real file and temp file.'''
80 fp = cStringIO.StringIO()
80 fp = cStringIO.StringIO()
81 old_offset = self.offset
81 old_offset = self.offset
82 if self.offset < self.realsize:
82 if self.offset < self.realsize:
83 s = self.realfp.read(count)
83 s = self.realfp.read(count)
84 fp.write(s)
84 fp.write(s)
85 self.offset += len(s)
85 self.offset += len(s)
86 if count > 0:
86 if count > 0:
87 count -= len(s)
87 count -= len(s)
88 if count != 0:
88 if count != 0:
89 if old_offset != self.offset:
89 if old_offset != self.offset:
90 self.tmpfp.seek(self.offset - self.realsize)
90 self.tmpfp.seek(self.offset - self.realsize)
91 s = self.tmpfp.read(count)
91 s = self.tmpfp.read(count)
92 fp.write(s)
92 fp.write(s)
93 self.offset += len(s)
93 self.offset += len(s)
94 return fp.getvalue()
94 return fp.getvalue()
95
95
96 def write(self, s):
96 def write(self, s):
97 '''append to temp file.'''
97 '''append to temp file.'''
98 self.tmpfp.seek(0, 2)
98 self.tmpfp.seek(0, 2)
99 self.tmpfp.write(s)
99 self.tmpfp.write(s)
100 # all writes are appends, so offset must go to end of file.
100 # all writes are appends, so offset must go to end of file.
101 self.offset = self.realsize + self.tmpfp.tell()
101 self.offset = self.realsize + self.tmpfp.tell()
102
102
103 class appendopener(object):
103 class appendopener(object):
104 '''special opener for files that only read or append.'''
104 '''special opener for files that only read or append.'''
105
105
106 def __init__(self, opener):
106 def __init__(self, opener):
107 self.realopener = opener
107 self.realopener = opener
108 # key: file name, value: appendfile name
108 self.tmpname = None
109 self.tmpnames = {}
110
109
111 def __call__(self, name, mode='r'):
110 def __call__(self, name, mode='r'):
112 '''open file.'''
111 '''open file.'''
113
112 # only handle .i file
113 if not name.endswith("."):
114 return self.realopener(name, mode)
114 assert mode in 'ra+'
115 assert mode in 'ra+'
115 try:
116 try:
116 realfp = self.realopener(name, 'r')
117 realfp = self.realopener(name, 'r')
117 except IOError, err:
118 except IOError, err:
118 if err.errno != errno.ENOENT: raise
119 if err.errno != errno.ENOENT: raise
119 realfp = self.realopener(name, 'w+')
120 self.realfp = self.realopener(name, 'w+')
120 tmpname = self.tmpnames.get(name)
121 fp = appendfile(realfp, self.tmpname)
121 fp = appendfile(realfp, tmpname)
122 if tmpname is None:
122 if tmpname is None:
123 self.tmpnames[name] = fp.tmpname
123 self.tmpname = fp.tmpname
124 self.name = name
124 return fp
125 return fp
125
126
126 def writedata(self):
127 def writedata(self):
127 '''copy data from temp files to real files.'''
128 '''copy data from temp files to real files.'''
128 # write .d file before .i file.
129 if not self.tmpname:
129 tmpnames = self.tmpnames.items()
130 return
130 tmpnames.sort()
131 ifp = open(self.tmpname, 'rb')
131 for name, tmpname in tmpnames:
132 ofp = self.realopener(self.name, 'a')
132 ifp = open(tmpname, 'rb')
133 for chunk in util.filechunkiter(ifp):
133 ofp = self.realopener(name, 'a')
134 ofp.write(chunk)
134 for chunk in util.filechunkiter(ifp):
135 ifp.close()
135 ofp.write(chunk)
136 os.unlink(self.tmpname)
136 ifp.close()
137 ofp.close()
137 os.unlink(tmpname)
138 del self.tmpnames[name]
139 ofp.close()
140
138
141 def cleanup(self):
139 def cleanup(self):
142 '''delete temp files (this discards unwritten data!)'''
140 '''delete temp files (this discards unwritten data!)'''
143 for tmpname in self.tmpnames.values():
141 if self.tmpname:
144 os.unlink(tmpname)
142 os.unlink(self.tmpname)
145
143
146 # files for changelog and manifest are in different appendopeners, so
144 # files for changelog and manifest are in different appendopeners, so
147 # not mixed up together.
145 # not mixed up together.
148
146
149 class appendchangelog(changelog.changelog, appendopener):
147 class appendchangelog(changelog.changelog, appendopener):
150 def __init__(self, opener):
148 def __init__(self, opener):
151 appendopener.__init__(self, opener)
149 appendopener.__init__(self, opener)
152 changelog.changelog.__init__(self, self)
150 changelog.changelog.__init__(self, self)
153 def checkinlinesize(self, fp, tr):
151 def checkinlinesize(self, fp, tr):
154 return
152 return
General Comments 0
You need to be logged in to leave comments. Login now