##// END OF EJS Templates
fix appendfile problem on macos....
Vadim Gelfer -
r2010:db1eb0de default
parent child Browse files
Show More
@@ -1,171 +1,171
1 # appendfile.py - special classes to make repo updates atomic
1 # appendfile.py - special classes to make repo updates atomic
2 #
2 #
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms
5 # This software may be used and distributed according to the terms
6 # of the GNU General Public License, incorporated herein by reference.
6 # of the GNU General Public License, incorporated herein by reference.
7
7
8 from demandload import *
8 from demandload import *
9 demandload(globals(), "cStringIO changelog manifest os tempfile")
9 demandload(globals(), "cStringIO changelog manifest os tempfile")
10
10
11 # writes to metadata files are ordered. reads: changelog, manifest,
11 # writes to metadata files are ordered. reads: changelog, manifest,
12 # normal files. writes: normal files, manifest, changelog.
12 # normal files. writes: normal files, manifest, changelog.
13
13
14 # manifest contains pointers to offsets in normal files. changelog
14 # manifest contains pointers to offsets in normal files. changelog
15 # contains pointers to offsets in manifest. if reader reads old
15 # contains pointers to offsets in manifest. if reader reads old
16 # changelog while manifest or normal files are written, it has no
16 # changelog while manifest or normal files are written, it has no
17 # pointers into new parts of those files that are maybe not consistent
17 # pointers into new parts of those files that are maybe not consistent
18 # yet, so will not read them.
18 # yet, so will not read them.
19
19
20 # localrepo.addchangegroup thinks it writes changelog first, then
20 # localrepo.addchangegroup thinks it writes changelog first, then
21 # manifest, then normal files (this is order they are available, and
21 # manifest, then normal files (this is order they are available, and
22 # needed for computing linkrev fields), but uses appendfile to hide
22 # needed for computing linkrev fields), but uses appendfile to hide
23 # updates from readers. data not written to manifest or changelog
23 # updates from readers. data not written to manifest or changelog
24 # until all normal files updated. write manifest first, then
24 # until all normal files updated. write manifest first, then
25 # changelog.
25 # changelog.
26
26
27 # with this write ordering, readers cannot see inconsistent view of
27 # with this write ordering, readers cannot see inconsistent view of
28 # repo during update.
28 # repo during update.
29
29
30 class appendfile(object):
30 class appendfile(object):
31 '''implement enough of file protocol to append to revlog file.
31 '''implement enough of file protocol to append to revlog file.
32 appended data is written to temp file. reads and seeks span real
32 appended data is written to temp file. reads and seeks span real
33 file and temp file. readers cannot see appended data until
33 file and temp file. readers cannot see appended data until
34 writedata called.'''
34 writedata called.'''
35
35
36 def __init__(self, fp):
36 def __init__(self, fp):
37 fd, self.tmpname = tempfile.mkstemp()
37 fd, self.tmpname = tempfile.mkstemp()
38 self.tmpfp = os.fdopen(fd, 'ab+')
38 self.tmpfp = os.fdopen(fd, 'ab+')
39 self.realfp = fp
39 self.realfp = fp
40 self.offset = 0
40 self.offset = fp.tell()
41 # real file is not written by anyone else. cache its size so
41 # real file is not written by anyone else. cache its size so
42 # seek and read can be fast.
42 # seek and read can be fast.
43 self.fpsize = os.fstat(fp.fileno()).st_size
43 self.fpsize = os.fstat(fp.fileno()).st_size
44
44
45 def seek(self, offset):
45 def seek(self, offset):
46 '''virtual file offset spans real file and temp file.'''
46 '''virtual file offset spans real file and temp file.'''
47 self.offset = offset
47 self.offset = offset
48 if self.offset < self.fpsize:
48 if self.offset < self.fpsize:
49 self.realfp.seek(self.offset)
49 self.realfp.seek(self.offset)
50 else:
50 else:
51 self.tmpfp.seek(self.offset - self.fpsize)
51 self.tmpfp.seek(self.offset - self.fpsize)
52
52
53 def read(self, count=-1):
53 def read(self, count=-1):
54 '''only trick here is reads that span real file and temp file.'''
54 '''only trick here is reads that span real file and temp file.'''
55 fp = cStringIO.StringIO()
55 fp = cStringIO.StringIO()
56 old_offset = self.offset
56 old_offset = self.offset
57 if self.offset < self.fpsize:
57 if self.offset < self.fpsize:
58 s = self.realfp.read(count)
58 s = self.realfp.read(count)
59 fp.write(s)
59 fp.write(s)
60 self.offset += len(s)
60 self.offset += len(s)
61 if count > 0:
61 if count > 0:
62 count -= len(s)
62 count -= len(s)
63 if count != 0:
63 if count != 0:
64 if old_offset != self.offset:
64 if old_offset != self.offset:
65 self.tmpfp.seek(self.offset - self.fpsize)
65 self.tmpfp.seek(self.offset - self.fpsize)
66 s = self.tmpfp.read(count)
66 s = self.tmpfp.read(count)
67 fp.write(s)
67 fp.write(s)
68 self.offset += len(s)
68 self.offset += len(s)
69 return fp.getvalue()
69 return fp.getvalue()
70
70
71 def write(self, s):
71 def write(self, s):
72 '''append to temp file.'''
72 '''append to temp file.'''
73 self.tmpfp.write(s)
73 self.tmpfp.write(s)
74 # all writes are appends, so offset must go to end of file.
74 # all writes are appends, so offset must go to end of file.
75 self.offset = self.fpsize + self.tmpfp.tell()
75 self.offset = self.fpsize + self.tmpfp.tell()
76
76
77 def writedata(self):
77 def writedata(self):
78 '''copy data from temp file to real file.'''
78 '''copy data from temp file to real file.'''
79 self.tmpfp.seek(0)
79 self.tmpfp.seek(0)
80 s = self.tmpfp.read()
80 s = self.tmpfp.read()
81 self.tmpfp.close()
81 self.tmpfp.close()
82 self.realfp.seek(0, 2)
82 self.realfp.seek(0, 2)
83 # small race here. we write all new data in one call, but
83 # small race here. we write all new data in one call, but
84 # reader can see partial update due to python or os. file
84 # reader can see partial update due to python or os. file
85 # locking no help: slow, not portable, not reliable over nfs.
85 # locking no help: slow, not portable, not reliable over nfs.
86 # only safe thing is write to temp file every time and rename,
86 # only safe thing is write to temp file every time and rename,
87 # but performance bad when manifest or changelog gets big.
87 # but performance bad when manifest or changelog gets big.
88 self.realfp.write(s)
88 self.realfp.write(s)
89 self.realfp.close()
89 self.realfp.close()
90
90
91 def __del__(self):
91 def __del__(self):
92 '''delete temp file even if exception raised.'''
92 '''delete temp file even if exception raised.'''
93 try: os.unlink(self.tmpname)
93 try: os.unlink(self.tmpname)
94 except: pass
94 except: pass
95
95
96 class sharedfile(object):
96 class sharedfile(object):
97 '''let file objects share a single appendfile safely. each
97 '''let file objects share a single appendfile safely. each
98 sharedfile has own offset, syncs up with appendfile offset before
98 sharedfile has own offset, syncs up with appendfile offset before
99 read and after read and write.'''
99 read and after read and write.'''
100
100
101 def __init__(self, fp):
101 def __init__(self, fp):
102 self.fp = fp
102 self.fp = fp
103 self.offset = 0
103 self.offset = 0
104
104
105 def seek(self, offset):
105 def seek(self, offset):
106 self.offset = offset
106 self.offset = offset
107
107
108 def read(self, count=-1):
108 def read(self, count=-1):
109 try:
109 try:
110 if self.offset != self.fp.offset:
110 if self.offset != self.fp.offset:
111 self.fp.seek(self.offset)
111 self.fp.seek(self.offset)
112 return self.fp.read(count)
112 return self.fp.read(count)
113 finally:
113 finally:
114 self.offset = self.fp.offset
114 self.offset = self.fp.offset
115
115
116 def write(self, s):
116 def write(self, s):
117 try:
117 try:
118 return self.fp.write(s)
118 return self.fp.write(s)
119 finally:
119 finally:
120 self.offset = self.fp.offset
120 self.offset = self.fp.offset
121
121
122 def close(self):
122 def close(self):
123 # revlog wants this.
123 # revlog wants this.
124 pass
124 pass
125
125
126 def flush(self):
126 def flush(self):
127 # revlog wants this.
127 # revlog wants this.
128 pass
128 pass
129
129
130 def writedata(self):
130 def writedata(self):
131 self.fp.writedata()
131 self.fp.writedata()
132
132
133 class appendopener(object):
133 class appendopener(object):
134 '''special opener for files that only read or append.'''
134 '''special opener for files that only read or append.'''
135
135
136 def __init__(self, opener):
136 def __init__(self, opener):
137 self.realopener = opener
137 self.realopener = opener
138 # key: file name, value: appendfile object
138 # key: file name, value: appendfile object
139 self.fps = {}
139 self.fps = {}
140
140
141 def __call__(self, name, mode='r'):
141 def __call__(self, name, mode='r'):
142 '''open file. return same cached appendfile object for every
142 '''open file. return same cached appendfile object for every
143 later call.'''
143 later call.'''
144
144
145 assert mode in 'ra'
145 assert mode in 'ra'
146 fp = self.fps.get(name)
146 fp = self.fps.get(name)
147 if fp is None:
147 if fp is None:
148 fp = appendfile(self.realopener(name, 'a+'))
148 fp = appendfile(self.realopener(name, 'a+'))
149 self.fps[name] = fp
149 self.fps[name] = fp
150 return sharedfile(fp)
150 return sharedfile(fp)
151
151
152 def writedata(self):
152 def writedata(self):
153 '''copy data from temp files to real files.'''
153 '''copy data from temp files to real files.'''
154 # write .d file before .i file.
154 # write .d file before .i file.
155 fps = self.fps.items()
155 fps = self.fps.items()
156 fps.sort()
156 fps.sort()
157 for name, fp in fps:
157 for name, fp in fps:
158 fp.writedata()
158 fp.writedata()
159
159
160 # files for changelog and manifest are in different appendopeners, so
160 # files for changelog and manifest are in different appendopeners, so
161 # not mixed up together.
161 # not mixed up together.
162
162
163 class appendchangelog(changelog.changelog, appendopener):
163 class appendchangelog(changelog.changelog, appendopener):
164 def __init__(self, opener):
164 def __init__(self, opener):
165 appendopener.__init__(self, opener)
165 appendopener.__init__(self, opener)
166 changelog.changelog.__init__(self, self)
166 changelog.changelog.__init__(self, self)
167
167
168 class appendmanifest(manifest.manifest, appendopener):
168 class appendmanifest(manifest.manifest, appendopener):
169 def __init__(self, opener):
169 def __init__(self, opener):
170 appendopener.__init__(self, opener)
170 appendopener.__init__(self, opener)
171 manifest.manifest.__init__(self, self)
171 manifest.manifest.__init__(self, self)
General Comments 0
You need to be logged in to leave comments. Login now