##// END OF EJS Templates
remove appendfile data as soon as not needed.
Vadim Gelfer -
r2102:c6211281 default
parent child Browse files
Show More
@@ -1,157 +1,154 b''
1 # appendfile.py - special classes to make repo updates atomic
1 # appendfile.py - special classes to make repo updates atomic
2 #
2 #
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms
5 # This software may be used and distributed according to the terms
6 # of the GNU General Public License, incorporated herein by reference.
6 # of the GNU General Public License, incorporated herein by reference.
7
7
8 from demandload import *
8 from demandload import *
9 demandload(globals(), "cStringIO changelog errno manifest os tempfile")
9 demandload(globals(), "cStringIO changelog errno manifest os tempfile")
10
10
11 # writes to metadata files are ordered. reads: changelog, manifest,
11 # writes to metadata files are ordered. reads: changelog, manifest,
12 # normal files. writes: normal files, manifest, changelog.
12 # normal files. writes: normal files, manifest, changelog.
13
13
14 # manifest contains pointers to offsets in normal files. changelog
14 # manifest contains pointers to offsets in normal files. changelog
15 # contains pointers to offsets in manifest. if reader reads old
15 # contains pointers to offsets in manifest. if reader reads old
16 # changelog while manifest or normal files are written, it has no
16 # changelog while manifest or normal files are written, it has no
17 # pointers into new parts of those files that are maybe not consistent
17 # pointers into new parts of those files that are maybe not consistent
18 # yet, so will not read them.
18 # yet, so will not read them.
19
19
20 # localrepo.addchangegroup thinks it writes changelog first, then
20 # localrepo.addchangegroup thinks it writes changelog first, then
21 # manifest, then normal files (this is order they are available, and
21 # manifest, then normal files (this is order they are available, and
22 # needed for computing linkrev fields), but uses appendfile to hide
22 # needed for computing linkrev fields), but uses appendfile to hide
23 # updates from readers. data not written to manifest or changelog
23 # updates from readers. data not written to manifest or changelog
24 # until all normal files updated. write manifest first, then
24 # until all normal files updated. write manifest first, then
25 # changelog.
25 # changelog.
26
26
27 # with this write ordering, readers cannot see inconsistent view of
27 # with this write ordering, readers cannot see inconsistent view of
28 # repo during update.
28 # repo during update.
29
29
30 class appendfile(object):
30 class appendfile(object):
31 '''implement enough of file protocol to append to revlog file.
31 '''implement enough of file protocol to append to revlog file.
32 appended data is written to temp file. reads and seeks span real
32 appended data is written to temp file. reads and seeks span real
33 file and temp file. readers cannot see appended data until
33 file and temp file. readers cannot see appended data until
34 writedata called.'''
34 writedata called.'''
35
35
36 def __init__(self, fp, tmpname):
36 def __init__(self, fp, tmpname):
37 if tmpname:
37 if tmpname:
38 self.tmpname = tmpname
38 self.tmpname = tmpname
39 self.tmpfp = open(self.tmpname, 'ab+')
39 self.tmpfp = open(self.tmpname, 'ab+')
40 else:
40 else:
41 fd, self.tmpname = tempfile.mkstemp()
41 fd, self.tmpname = tempfile.mkstemp()
42 self.tmpfp = os.fdopen(fd, 'ab+')
42 self.tmpfp = os.fdopen(fd, 'ab+')
43 self.realfp = fp
43 self.realfp = fp
44 self.offset = fp.tell()
44 self.offset = fp.tell()
45 # real file is not written by anyone else. cache its size so
45 # real file is not written by anyone else. cache its size so
46 # seek and read can be fast.
46 # seek and read can be fast.
47 self.realsize = os.fstat(fp.fileno()).st_size
47 self.realsize = os.fstat(fp.fileno()).st_size
48
48
49 def end(self):
49 def end(self):
50 self.tmpfp.flush() # make sure the stat is correct
50 self.tmpfp.flush() # make sure the stat is correct
51 return self.realsize + os.fstat(self.tmpfp.fileno()).st_size
51 return self.realsize + os.fstat(self.tmpfp.fileno()).st_size
52
52
53 def tell(self):
53 def tell(self):
54 return self.offset
54 return self.offset
55
55
56 def flush(self):
56 def flush(self):
57 self.tmpfp.flush()
57 self.tmpfp.flush()
58
58
59 def close(self):
59 def close(self):
60 self.realfp.close()
60 self.realfp.close()
61 self.tmpfp.close()
61 self.tmpfp.close()
62
62
63 def seek(self, offset, whence=0):
63 def seek(self, offset, whence=0):
64 '''virtual file offset spans real file and temp file.'''
64 '''virtual file offset spans real file and temp file.'''
65 if whence == 0:
65 if whence == 0:
66 self.offset = offset
66 self.offset = offset
67 elif whence == 1:
67 elif whence == 1:
68 self.offset += offset
68 self.offset += offset
69 elif whence == 2:
69 elif whence == 2:
70 self.offset = self.end() + offset
70 self.offset = self.end() + offset
71
71
72 if self.offset < self.realsize:
72 if self.offset < self.realsize:
73 self.realfp.seek(self.offset)
73 self.realfp.seek(self.offset)
74 else:
74 else:
75 self.tmpfp.seek(self.offset - self.realsize)
75 self.tmpfp.seek(self.offset - self.realsize)
76
76
77 def read(self, count=-1):
77 def read(self, count=-1):
78 '''only trick here is reads that span real file and temp file.'''
78 '''only trick here is reads that span real file and temp file.'''
79 fp = cStringIO.StringIO()
79 fp = cStringIO.StringIO()
80 old_offset = self.offset
80 old_offset = self.offset
81 if self.offset < self.realsize:
81 if self.offset < self.realsize:
82 s = self.realfp.read(count)
82 s = self.realfp.read(count)
83 fp.write(s)
83 fp.write(s)
84 self.offset += len(s)
84 self.offset += len(s)
85 if count > 0:
85 if count > 0:
86 count -= len(s)
86 count -= len(s)
87 if count != 0:
87 if count != 0:
88 if old_offset != self.offset:
88 if old_offset != self.offset:
89 self.tmpfp.seek(self.offset - self.realsize)
89 self.tmpfp.seek(self.offset - self.realsize)
90 s = self.tmpfp.read(count)
90 s = self.tmpfp.read(count)
91 fp.write(s)
91 fp.write(s)
92 self.offset += len(s)
92 self.offset += len(s)
93 return fp.getvalue()
93 return fp.getvalue()
94
94
95 def write(self, s):
95 def write(self, s):
96 '''append to temp file.'''
96 '''append to temp file.'''
97 self.tmpfp.seek(0, 2)
97 self.tmpfp.seek(0, 2)
98 self.tmpfp.write(s)
98 self.tmpfp.write(s)
99 # all writes are appends, so offset must go to end of file.
99 # all writes are appends, so offset must go to end of file.
100 self.offset = self.realsize + self.tmpfp.tell()
100 self.offset = self.realsize + self.tmpfp.tell()
101
101
102 class appendopener(object):
102 class appendopener(object):
103 '''special opener for files that only read or append.'''
103 '''special opener for files that only read or append.'''
104
104
105 def __init__(self, opener):
105 def __init__(self, opener):
106 self.realopener = opener
106 self.realopener = opener
107 # key: file name, value: appendfile name
107 # key: file name, value: appendfile name
108 self.tmpnames = {}
108 self.tmpnames = {}
109
109
110 def __call__(self, name, mode='r'):
110 def __call__(self, name, mode='r'):
111 '''open file.'''
111 '''open file.'''
112
112
113 assert mode in 'ra+'
113 assert mode in 'ra+'
114 try:
114 try:
115 realfp = self.realopener(name, 'r')
115 realfp = self.realopener(name, 'r')
116 except IOError, err:
116 except IOError, err:
117 if err.errno != errno.ENOENT: raise
117 if err.errno != errno.ENOENT: raise
118 realfp = self.realopener(name, 'w+')
118 realfp = self.realopener(name, 'w+')
119 tmpname = self.tmpnames.get(name)
119 tmpname = self.tmpnames.get(name)
120 fp = appendfile(realfp, tmpname)
120 fp = appendfile(realfp, tmpname)
121 if tmpname is None:
121 if tmpname is None:
122 self.tmpnames[name] = fp.tmpname
122 self.tmpnames[name] = fp.tmpname
123 return fp
123 return fp
124
124
125 def writedata(self):
125 def writedata(self):
126 '''copy data from temp files to real files.'''
126 '''copy data from temp files to real files.'''
127 # write .d file before .i file.
127 # write .d file before .i file.
128 tmpnames = self.tmpnames.items()
128 tmpnames = self.tmpnames.items()
129 tmpnames.sort()
129 tmpnames.sort()
130 for name, tmpname in tmpnames:
130 for name, tmpname in tmpnames:
131 fp = open(tmpname, 'rb')
131 fp = open(tmpname, 'rb')
132 s = fp.read()
132 s = fp.read()
133 fp.close()
133 fp.close()
134 os.unlink(tmpname)
134 fp = self.realopener(name, 'a')
135 fp = self.realopener(name, 'a')
135 fp.write(s)
136 fp.write(s)
136 fp.close()
137 fp.close()
137
138
138 def __del__(self):
139 for tmpname in self.tmpnames.itervalues():
140 os.unlink(tmpname)
141
142 # files for changelog and manifest are in different appendopeners, so
139 # files for changelog and manifest are in different appendopeners, so
143 # not mixed up together.
140 # not mixed up together.
144
141
145 class appendchangelog(changelog.changelog, appendopener):
142 class appendchangelog(changelog.changelog, appendopener):
146 def __init__(self, opener, version):
143 def __init__(self, opener, version):
147 appendopener.__init__(self, opener)
144 appendopener.__init__(self, opener)
148 changelog.changelog.__init__(self, self, version)
145 changelog.changelog.__init__(self, self, version)
149 def checkinlinesize(self, fp, tr):
146 def checkinlinesize(self, fp, tr):
150 return
147 return
151
148
152 class appendmanifest(manifest.manifest, appendopener):
149 class appendmanifest(manifest.manifest, appendopener):
153 def __init__(self, opener, version):
150 def __init__(self, opener, version):
154 appendopener.__init__(self, opener)
151 appendopener.__init__(self, opener)
155 manifest.manifest.__init__(self, self, version)
152 manifest.manifest.__init__(self, self, version)
156 def checkinlinesize(self, fp, tr):
153 def checkinlinesize(self, fp, tr):
157 return
154 return
General Comments 0
You need to be logged in to leave comments. Login now