##// END OF EJS Templates
vfs: rename auditvfs to proxyvfs...
Yuya Nishihara -
r33412:a42369e0 default
parent child Browse files
Show More
@@ -1,575 +1,575 b''
1 # store.py - repository store handling for Mercurial
1 # store.py - repository store handling for Mercurial
2 #
2 #
3 # Copyright 2008 Matt Mackall <mpm@selenic.com>
3 # Copyright 2008 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import errno
10 import errno
11 import hashlib
11 import hashlib
12 import os
12 import os
13 import stat
13 import stat
14
14
15 from .i18n import _
15 from .i18n import _
16 from . import (
16 from . import (
17 error,
17 error,
18 policy,
18 policy,
19 pycompat,
19 pycompat,
20 util,
20 util,
21 vfs as vfsmod,
21 vfs as vfsmod,
22 )
22 )
23
23
24 parsers = policy.importmod(r'parsers')
24 parsers = policy.importmod(r'parsers')
25
25
26 # This avoids a collision between a file named foo and a dir named
26 # This avoids a collision between a file named foo and a dir named
27 # foo.i or foo.d
27 # foo.i or foo.d
28 def _encodedir(path):
28 def _encodedir(path):
29 '''
29 '''
30 >>> _encodedir('data/foo.i')
30 >>> _encodedir('data/foo.i')
31 'data/foo.i'
31 'data/foo.i'
32 >>> _encodedir('data/foo.i/bla.i')
32 >>> _encodedir('data/foo.i/bla.i')
33 'data/foo.i.hg/bla.i'
33 'data/foo.i.hg/bla.i'
34 >>> _encodedir('data/foo.i.hg/bla.i')
34 >>> _encodedir('data/foo.i.hg/bla.i')
35 'data/foo.i.hg.hg/bla.i'
35 'data/foo.i.hg.hg/bla.i'
36 >>> _encodedir('data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n')
36 >>> _encodedir('data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n')
37 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n'
37 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n'
38 '''
38 '''
39 return (path
39 return (path
40 .replace(".hg/", ".hg.hg/")
40 .replace(".hg/", ".hg.hg/")
41 .replace(".i/", ".i.hg/")
41 .replace(".i/", ".i.hg/")
42 .replace(".d/", ".d.hg/"))
42 .replace(".d/", ".d.hg/"))
43
43
44 encodedir = getattr(parsers, 'encodedir', _encodedir)
44 encodedir = getattr(parsers, 'encodedir', _encodedir)
45
45
46 def decodedir(path):
46 def decodedir(path):
47 '''
47 '''
48 >>> decodedir('data/foo.i')
48 >>> decodedir('data/foo.i')
49 'data/foo.i'
49 'data/foo.i'
50 >>> decodedir('data/foo.i.hg/bla.i')
50 >>> decodedir('data/foo.i.hg/bla.i')
51 'data/foo.i/bla.i'
51 'data/foo.i/bla.i'
52 >>> decodedir('data/foo.i.hg.hg/bla.i')
52 >>> decodedir('data/foo.i.hg.hg/bla.i')
53 'data/foo.i.hg/bla.i'
53 'data/foo.i.hg/bla.i'
54 '''
54 '''
55 if ".hg/" not in path:
55 if ".hg/" not in path:
56 return path
56 return path
57 return (path
57 return (path
58 .replace(".d.hg/", ".d/")
58 .replace(".d.hg/", ".d/")
59 .replace(".i.hg/", ".i/")
59 .replace(".i.hg/", ".i/")
60 .replace(".hg.hg/", ".hg/"))
60 .replace(".hg.hg/", ".hg/"))
61
61
62 def _reserved():
62 def _reserved():
63 ''' characters that are problematic for filesystems
63 ''' characters that are problematic for filesystems
64
64
65 * ascii escapes (0..31)
65 * ascii escapes (0..31)
66 * ascii hi (126..255)
66 * ascii hi (126..255)
67 * windows specials
67 * windows specials
68
68
69 these characters will be escaped by encodefunctions
69 these characters will be escaped by encodefunctions
70 '''
70 '''
71 winreserved = [ord(x) for x in u'\\:*?"<>|']
71 winreserved = [ord(x) for x in u'\\:*?"<>|']
72 for x in range(32):
72 for x in range(32):
73 yield x
73 yield x
74 for x in range(126, 256):
74 for x in range(126, 256):
75 yield x
75 yield x
76 for x in winreserved:
76 for x in winreserved:
77 yield x
77 yield x
78
78
79 def _buildencodefun():
79 def _buildencodefun():
80 '''
80 '''
81 >>> enc, dec = _buildencodefun()
81 >>> enc, dec = _buildencodefun()
82
82
83 >>> enc('nothing/special.txt')
83 >>> enc('nothing/special.txt')
84 'nothing/special.txt'
84 'nothing/special.txt'
85 >>> dec('nothing/special.txt')
85 >>> dec('nothing/special.txt')
86 'nothing/special.txt'
86 'nothing/special.txt'
87
87
88 >>> enc('HELLO')
88 >>> enc('HELLO')
89 '_h_e_l_l_o'
89 '_h_e_l_l_o'
90 >>> dec('_h_e_l_l_o')
90 >>> dec('_h_e_l_l_o')
91 'HELLO'
91 'HELLO'
92
92
93 >>> enc('hello:world?')
93 >>> enc('hello:world?')
94 'hello~3aworld~3f'
94 'hello~3aworld~3f'
95 >>> dec('hello~3aworld~3f')
95 >>> dec('hello~3aworld~3f')
96 'hello:world?'
96 'hello:world?'
97
97
98 >>> enc('the\x07quick\xADshot')
98 >>> enc('the\x07quick\xADshot')
99 'the~07quick~adshot'
99 'the~07quick~adshot'
100 >>> dec('the~07quick~adshot')
100 >>> dec('the~07quick~adshot')
101 'the\\x07quick\\xadshot'
101 'the\\x07quick\\xadshot'
102 '''
102 '''
103 e = '_'
103 e = '_'
104 xchr = pycompat.bytechr
104 xchr = pycompat.bytechr
105 asciistr = list(map(xchr, range(127)))
105 asciistr = list(map(xchr, range(127)))
106 capitals = list(range(ord("A"), ord("Z") + 1))
106 capitals = list(range(ord("A"), ord("Z") + 1))
107
107
108 cmap = dict((x, x) for x in asciistr)
108 cmap = dict((x, x) for x in asciistr)
109 for x in _reserved():
109 for x in _reserved():
110 cmap[xchr(x)] = "~%02x" % x
110 cmap[xchr(x)] = "~%02x" % x
111 for x in capitals + [ord(e)]:
111 for x in capitals + [ord(e)]:
112 cmap[xchr(x)] = e + xchr(x).lower()
112 cmap[xchr(x)] = e + xchr(x).lower()
113
113
114 dmap = {}
114 dmap = {}
115 for k, v in cmap.iteritems():
115 for k, v in cmap.iteritems():
116 dmap[v] = k
116 dmap[v] = k
117 def decode(s):
117 def decode(s):
118 i = 0
118 i = 0
119 while i < len(s):
119 while i < len(s):
120 for l in xrange(1, 4):
120 for l in xrange(1, 4):
121 try:
121 try:
122 yield dmap[s[i:i + l]]
122 yield dmap[s[i:i + l]]
123 i += l
123 i += l
124 break
124 break
125 except KeyError:
125 except KeyError:
126 pass
126 pass
127 else:
127 else:
128 raise KeyError
128 raise KeyError
129 return (lambda s: ''.join([cmap[s[c:c + 1]] for c in xrange(len(s))]),
129 return (lambda s: ''.join([cmap[s[c:c + 1]] for c in xrange(len(s))]),
130 lambda s: ''.join(list(decode(s))))
130 lambda s: ''.join(list(decode(s))))
131
131
132 _encodefname, _decodefname = _buildencodefun()
132 _encodefname, _decodefname = _buildencodefun()
133
133
134 def encodefilename(s):
134 def encodefilename(s):
135 '''
135 '''
136 >>> encodefilename('foo.i/bar.d/bla.hg/hi:world?/HELLO')
136 >>> encodefilename('foo.i/bar.d/bla.hg/hi:world?/HELLO')
137 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o'
137 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o'
138 '''
138 '''
139 return _encodefname(encodedir(s))
139 return _encodefname(encodedir(s))
140
140
141 def decodefilename(s):
141 def decodefilename(s):
142 '''
142 '''
143 >>> decodefilename('foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o')
143 >>> decodefilename('foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o')
144 'foo.i/bar.d/bla.hg/hi:world?/HELLO'
144 'foo.i/bar.d/bla.hg/hi:world?/HELLO'
145 '''
145 '''
146 return decodedir(_decodefname(s))
146 return decodedir(_decodefname(s))
147
147
148 def _buildlowerencodefun():
148 def _buildlowerencodefun():
149 '''
149 '''
150 >>> f = _buildlowerencodefun()
150 >>> f = _buildlowerencodefun()
151 >>> f('nothing/special.txt')
151 >>> f('nothing/special.txt')
152 'nothing/special.txt'
152 'nothing/special.txt'
153 >>> f('HELLO')
153 >>> f('HELLO')
154 'hello'
154 'hello'
155 >>> f('hello:world?')
155 >>> f('hello:world?')
156 'hello~3aworld~3f'
156 'hello~3aworld~3f'
157 >>> f('the\x07quick\xADshot')
157 >>> f('the\x07quick\xADshot')
158 'the~07quick~adshot'
158 'the~07quick~adshot'
159 '''
159 '''
160 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
160 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
161 for x in _reserved():
161 for x in _reserved():
162 cmap[chr(x)] = "~%02x" % x
162 cmap[chr(x)] = "~%02x" % x
163 for x in range(ord("A"), ord("Z") + 1):
163 for x in range(ord("A"), ord("Z") + 1):
164 cmap[chr(x)] = chr(x).lower()
164 cmap[chr(x)] = chr(x).lower()
165 return lambda s: "".join([cmap[c] for c in s])
165 return lambda s: "".join([cmap[c] for c in s])
166
166
167 lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun()
167 lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun()
168
168
169 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9
169 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9
170 _winres3 = ('aux', 'con', 'prn', 'nul') # length 3
170 _winres3 = ('aux', 'con', 'prn', 'nul') # length 3
171 _winres4 = ('com', 'lpt') # length 4 (with trailing 1..9)
171 _winres4 = ('com', 'lpt') # length 4 (with trailing 1..9)
172 def _auxencode(path, dotencode):
172 def _auxencode(path, dotencode):
173 '''
173 '''
174 Encodes filenames containing names reserved by Windows or which end in
174 Encodes filenames containing names reserved by Windows or which end in
175 period or space. Does not touch other single reserved characters c.
175 period or space. Does not touch other single reserved characters c.
176 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here.
176 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here.
177 Additionally encodes space or period at the beginning, if dotencode is
177 Additionally encodes space or period at the beginning, if dotencode is
178 True. Parameter path is assumed to be all lowercase.
178 True. Parameter path is assumed to be all lowercase.
179 A segment only needs encoding if a reserved name appears as a
179 A segment only needs encoding if a reserved name appears as a
180 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux"
180 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux"
181 doesn't need encoding.
181 doesn't need encoding.
182
182
183 >>> s = '.foo/aux.txt/txt.aux/con/prn/nul/foo.'
183 >>> s = '.foo/aux.txt/txt.aux/con/prn/nul/foo.'
184 >>> _auxencode(s.split('/'), True)
184 >>> _auxencode(s.split('/'), True)
185 ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e']
185 ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e']
186 >>> s = '.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.'
186 >>> s = '.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.'
187 >>> _auxencode(s.split('/'), False)
187 >>> _auxencode(s.split('/'), False)
188 ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e']
188 ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e']
189 >>> _auxencode(['foo. '], True)
189 >>> _auxencode(['foo. '], True)
190 ['foo.~20']
190 ['foo.~20']
191 >>> _auxencode([' .foo'], True)
191 >>> _auxencode([' .foo'], True)
192 ['~20.foo']
192 ['~20.foo']
193 '''
193 '''
194 for i, n in enumerate(path):
194 for i, n in enumerate(path):
195 if not n:
195 if not n:
196 continue
196 continue
197 if dotencode and n[0] in '. ':
197 if dotencode and n[0] in '. ':
198 n = "~%02x" % ord(n[0:1]) + n[1:]
198 n = "~%02x" % ord(n[0:1]) + n[1:]
199 path[i] = n
199 path[i] = n
200 else:
200 else:
201 l = n.find('.')
201 l = n.find('.')
202 if l == -1:
202 if l == -1:
203 l = len(n)
203 l = len(n)
204 if ((l == 3 and n[:3] in _winres3) or
204 if ((l == 3 and n[:3] in _winres3) or
205 (l == 4 and n[3:4] <= '9' and n[3:4] >= '1'
205 (l == 4 and n[3:4] <= '9' and n[3:4] >= '1'
206 and n[:3] in _winres4)):
206 and n[:3] in _winres4)):
207 # encode third letter ('aux' -> 'au~78')
207 # encode third letter ('aux' -> 'au~78')
208 ec = "~%02x" % ord(n[2:3])
208 ec = "~%02x" % ord(n[2:3])
209 n = n[0:2] + ec + n[3:]
209 n = n[0:2] + ec + n[3:]
210 path[i] = n
210 path[i] = n
211 if n[-1] in '. ':
211 if n[-1] in '. ':
212 # encode last period or space ('foo...' -> 'foo..~2e')
212 # encode last period or space ('foo...' -> 'foo..~2e')
213 path[i] = n[:-1] + "~%02x" % ord(n[-1:])
213 path[i] = n[:-1] + "~%02x" % ord(n[-1:])
214 return path
214 return path
215
215
216 _maxstorepathlen = 120
216 _maxstorepathlen = 120
217 _dirprefixlen = 8
217 _dirprefixlen = 8
218 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
218 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
219
219
220 def _hashencode(path, dotencode):
220 def _hashencode(path, dotencode):
221 digest = hashlib.sha1(path).hexdigest()
221 digest = hashlib.sha1(path).hexdigest()
222 le = lowerencode(path[5:]).split('/') # skips prefix 'data/' or 'meta/'
222 le = lowerencode(path[5:]).split('/') # skips prefix 'data/' or 'meta/'
223 parts = _auxencode(le, dotencode)
223 parts = _auxencode(le, dotencode)
224 basename = parts[-1]
224 basename = parts[-1]
225 _root, ext = os.path.splitext(basename)
225 _root, ext = os.path.splitext(basename)
226 sdirs = []
226 sdirs = []
227 sdirslen = 0
227 sdirslen = 0
228 for p in parts[:-1]:
228 for p in parts[:-1]:
229 d = p[:_dirprefixlen]
229 d = p[:_dirprefixlen]
230 if d[-1] in '. ':
230 if d[-1] in '. ':
231 # Windows can't access dirs ending in period or space
231 # Windows can't access dirs ending in period or space
232 d = d[:-1] + '_'
232 d = d[:-1] + '_'
233 if sdirslen == 0:
233 if sdirslen == 0:
234 t = len(d)
234 t = len(d)
235 else:
235 else:
236 t = sdirslen + 1 + len(d)
236 t = sdirslen + 1 + len(d)
237 if t > _maxshortdirslen:
237 if t > _maxshortdirslen:
238 break
238 break
239 sdirs.append(d)
239 sdirs.append(d)
240 sdirslen = t
240 sdirslen = t
241 dirs = '/'.join(sdirs)
241 dirs = '/'.join(sdirs)
242 if len(dirs) > 0:
242 if len(dirs) > 0:
243 dirs += '/'
243 dirs += '/'
244 res = 'dh/' + dirs + digest + ext
244 res = 'dh/' + dirs + digest + ext
245 spaceleft = _maxstorepathlen - len(res)
245 spaceleft = _maxstorepathlen - len(res)
246 if spaceleft > 0:
246 if spaceleft > 0:
247 filler = basename[:spaceleft]
247 filler = basename[:spaceleft]
248 res = 'dh/' + dirs + filler + digest + ext
248 res = 'dh/' + dirs + filler + digest + ext
249 return res
249 return res
250
250
251 def _hybridencode(path, dotencode):
251 def _hybridencode(path, dotencode):
252 '''encodes path with a length limit
252 '''encodes path with a length limit
253
253
254 Encodes all paths that begin with 'data/', according to the following.
254 Encodes all paths that begin with 'data/', according to the following.
255
255
256 Default encoding (reversible):
256 Default encoding (reversible):
257
257
258 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
258 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
259 characters are encoded as '~xx', where xx is the two digit hex code
259 characters are encoded as '~xx', where xx is the two digit hex code
260 of the character (see encodefilename).
260 of the character (see encodefilename).
261 Relevant path components consisting of Windows reserved filenames are
261 Relevant path components consisting of Windows reserved filenames are
262 masked by encoding the third character ('aux' -> 'au~78', see _auxencode).
262 masked by encoding the third character ('aux' -> 'au~78', see _auxencode).
263
263
264 Hashed encoding (not reversible):
264 Hashed encoding (not reversible):
265
265
266 If the default-encoded path is longer than _maxstorepathlen, a
266 If the default-encoded path is longer than _maxstorepathlen, a
267 non-reversible hybrid hashing of the path is done instead.
267 non-reversible hybrid hashing of the path is done instead.
268 This encoding uses up to _dirprefixlen characters of all directory
268 This encoding uses up to _dirprefixlen characters of all directory
269 levels of the lowerencoded path, but not more levels than can fit into
269 levels of the lowerencoded path, but not more levels than can fit into
270 _maxshortdirslen.
270 _maxshortdirslen.
271 Then follows the filler followed by the sha digest of the full path.
271 Then follows the filler followed by the sha digest of the full path.
272 The filler is the beginning of the basename of the lowerencoded path
272 The filler is the beginning of the basename of the lowerencoded path
273 (the basename is everything after the last path separator). The filler
273 (the basename is everything after the last path separator). The filler
274 is as long as possible, filling in characters from the basename until
274 is as long as possible, filling in characters from the basename until
275 the encoded path has _maxstorepathlen characters (or all chars of the
275 the encoded path has _maxstorepathlen characters (or all chars of the
276 basename have been taken).
276 basename have been taken).
277 The extension (e.g. '.i' or '.d') is preserved.
277 The extension (e.g. '.i' or '.d') is preserved.
278
278
279 The string 'data/' at the beginning is replaced with 'dh/', if the hashed
279 The string 'data/' at the beginning is replaced with 'dh/', if the hashed
280 encoding was used.
280 encoding was used.
281 '''
281 '''
282 path = encodedir(path)
282 path = encodedir(path)
283 ef = _encodefname(path).split('/')
283 ef = _encodefname(path).split('/')
284 res = '/'.join(_auxencode(ef, dotencode))
284 res = '/'.join(_auxencode(ef, dotencode))
285 if len(res) > _maxstorepathlen:
285 if len(res) > _maxstorepathlen:
286 res = _hashencode(path, dotencode)
286 res = _hashencode(path, dotencode)
287 return res
287 return res
288
288
289 def _pathencode(path):
289 def _pathencode(path):
290 de = encodedir(path)
290 de = encodedir(path)
291 if len(path) > _maxstorepathlen:
291 if len(path) > _maxstorepathlen:
292 return _hashencode(de, True)
292 return _hashencode(de, True)
293 ef = _encodefname(de).split('/')
293 ef = _encodefname(de).split('/')
294 res = '/'.join(_auxencode(ef, True))
294 res = '/'.join(_auxencode(ef, True))
295 if len(res) > _maxstorepathlen:
295 if len(res) > _maxstorepathlen:
296 return _hashencode(de, True)
296 return _hashencode(de, True)
297 return res
297 return res
298
298
299 _pathencode = getattr(parsers, 'pathencode', _pathencode)
299 _pathencode = getattr(parsers, 'pathencode', _pathencode)
300
300
301 def _plainhybridencode(f):
301 def _plainhybridencode(f):
302 return _hybridencode(f, False)
302 return _hybridencode(f, False)
303
303
304 def _calcmode(vfs):
304 def _calcmode(vfs):
305 try:
305 try:
306 # files in .hg/ will be created using this mode
306 # files in .hg/ will be created using this mode
307 mode = vfs.stat().st_mode
307 mode = vfs.stat().st_mode
308 # avoid some useless chmods
308 # avoid some useless chmods
309 if (0o777 & ~util.umask) == (0o777 & mode):
309 if (0o777 & ~util.umask) == (0o777 & mode):
310 mode = None
310 mode = None
311 except OSError:
311 except OSError:
312 mode = None
312 mode = None
313 return mode
313 return mode
314
314
315 _data = ('data meta 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
315 _data = ('data meta 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
316 ' phaseroots obsstore')
316 ' phaseroots obsstore')
317
317
318 class basicstore(object):
318 class basicstore(object):
319 '''base class for local repository stores'''
319 '''base class for local repository stores'''
320 def __init__(self, path, vfstype):
320 def __init__(self, path, vfstype):
321 vfs = vfstype(path)
321 vfs = vfstype(path)
322 self.path = vfs.base
322 self.path = vfs.base
323 self.createmode = _calcmode(vfs)
323 self.createmode = _calcmode(vfs)
324 vfs.createmode = self.createmode
324 vfs.createmode = self.createmode
325 self.rawvfs = vfs
325 self.rawvfs = vfs
326 self.vfs = vfsmod.filtervfs(vfs, encodedir)
326 self.vfs = vfsmod.filtervfs(vfs, encodedir)
327 self.opener = self.vfs
327 self.opener = self.vfs
328
328
329 def join(self, f):
329 def join(self, f):
330 return self.path + '/' + encodedir(f)
330 return self.path + '/' + encodedir(f)
331
331
332 def _walk(self, relpath, recurse):
332 def _walk(self, relpath, recurse):
333 '''yields (unencoded, encoded, size)'''
333 '''yields (unencoded, encoded, size)'''
334 path = self.path
334 path = self.path
335 if relpath:
335 if relpath:
336 path += '/' + relpath
336 path += '/' + relpath
337 striplen = len(self.path) + 1
337 striplen = len(self.path) + 1
338 l = []
338 l = []
339 if self.rawvfs.isdir(path):
339 if self.rawvfs.isdir(path):
340 visit = [path]
340 visit = [path]
341 readdir = self.rawvfs.readdir
341 readdir = self.rawvfs.readdir
342 while visit:
342 while visit:
343 p = visit.pop()
343 p = visit.pop()
344 for f, kind, st in readdir(p, stat=True):
344 for f, kind, st in readdir(p, stat=True):
345 fp = p + '/' + f
345 fp = p + '/' + f
346 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'):
346 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'):
347 n = util.pconvert(fp[striplen:])
347 n = util.pconvert(fp[striplen:])
348 l.append((decodedir(n), n, st.st_size))
348 l.append((decodedir(n), n, st.st_size))
349 elif kind == stat.S_IFDIR and recurse:
349 elif kind == stat.S_IFDIR and recurse:
350 visit.append(fp)
350 visit.append(fp)
351 l.sort()
351 l.sort()
352 return l
352 return l
353
353
354 def datafiles(self):
354 def datafiles(self):
355 return self._walk('data', True) + self._walk('meta', True)
355 return self._walk('data', True) + self._walk('meta', True)
356
356
357 def topfiles(self):
357 def topfiles(self):
358 # yield manifest before changelog
358 # yield manifest before changelog
359 return reversed(self._walk('', False))
359 return reversed(self._walk('', False))
360
360
361 def walk(self):
361 def walk(self):
362 '''yields (unencoded, encoded, size)'''
362 '''yields (unencoded, encoded, size)'''
363 # yield data files first
363 # yield data files first
364 for x in self.datafiles():
364 for x in self.datafiles():
365 yield x
365 yield x
366 for x in self.topfiles():
366 for x in self.topfiles():
367 yield x
367 yield x
368
368
369 def copylist(self):
369 def copylist(self):
370 return ['requires'] + _data.split()
370 return ['requires'] + _data.split()
371
371
372 def write(self, tr):
372 def write(self, tr):
373 pass
373 pass
374
374
375 def invalidatecaches(self):
375 def invalidatecaches(self):
376 pass
376 pass
377
377
378 def markremoved(self, fn):
378 def markremoved(self, fn):
379 pass
379 pass
380
380
381 def __contains__(self, path):
381 def __contains__(self, path):
382 '''Checks if the store contains path'''
382 '''Checks if the store contains path'''
383 path = "/".join(("data", path))
383 path = "/".join(("data", path))
384 # file?
384 # file?
385 if self.vfs.exists(path + ".i"):
385 if self.vfs.exists(path + ".i"):
386 return True
386 return True
387 # dir?
387 # dir?
388 if not path.endswith("/"):
388 if not path.endswith("/"):
389 path = path + "/"
389 path = path + "/"
390 return self.vfs.exists(path)
390 return self.vfs.exists(path)
391
391
392 class encodedstore(basicstore):
392 class encodedstore(basicstore):
393 def __init__(self, path, vfstype):
393 def __init__(self, path, vfstype):
394 vfs = vfstype(path + '/store')
394 vfs = vfstype(path + '/store')
395 self.path = vfs.base
395 self.path = vfs.base
396 self.createmode = _calcmode(vfs)
396 self.createmode = _calcmode(vfs)
397 vfs.createmode = self.createmode
397 vfs.createmode = self.createmode
398 self.rawvfs = vfs
398 self.rawvfs = vfs
399 self.vfs = vfsmod.filtervfs(vfs, encodefilename)
399 self.vfs = vfsmod.filtervfs(vfs, encodefilename)
400 self.opener = self.vfs
400 self.opener = self.vfs
401
401
402 def datafiles(self):
402 def datafiles(self):
403 for a, b, size in super(encodedstore, self).datafiles():
403 for a, b, size in super(encodedstore, self).datafiles():
404 try:
404 try:
405 a = decodefilename(a)
405 a = decodefilename(a)
406 except KeyError:
406 except KeyError:
407 a = None
407 a = None
408 yield a, b, size
408 yield a, b, size
409
409
410 def join(self, f):
410 def join(self, f):
411 return self.path + '/' + encodefilename(f)
411 return self.path + '/' + encodefilename(f)
412
412
413 def copylist(self):
413 def copylist(self):
414 return (['requires', '00changelog.i'] +
414 return (['requires', '00changelog.i'] +
415 ['store/' + f for f in _data.split()])
415 ['store/' + f for f in _data.split()])
416
416
417 class fncache(object):
417 class fncache(object):
418 # the filename used to be partially encoded
418 # the filename used to be partially encoded
419 # hence the encodedir/decodedir dance
419 # hence the encodedir/decodedir dance
420 def __init__(self, vfs):
420 def __init__(self, vfs):
421 self.vfs = vfs
421 self.vfs = vfs
422 self.entries = None
422 self.entries = None
423 self._dirty = False
423 self._dirty = False
424
424
425 def _load(self):
425 def _load(self):
426 '''fill the entries from the fncache file'''
426 '''fill the entries from the fncache file'''
427 self._dirty = False
427 self._dirty = False
428 try:
428 try:
429 fp = self.vfs('fncache', mode='rb')
429 fp = self.vfs('fncache', mode='rb')
430 except IOError:
430 except IOError:
431 # skip nonexistent file
431 # skip nonexistent file
432 self.entries = set()
432 self.entries = set()
433 return
433 return
434 self.entries = set(decodedir(fp.read()).splitlines())
434 self.entries = set(decodedir(fp.read()).splitlines())
435 if '' in self.entries:
435 if '' in self.entries:
436 fp.seek(0)
436 fp.seek(0)
437 for n, line in enumerate(util.iterfile(fp)):
437 for n, line in enumerate(util.iterfile(fp)):
438 if not line.rstrip('\n'):
438 if not line.rstrip('\n'):
439 t = _('invalid entry in fncache, line %d') % (n + 1)
439 t = _('invalid entry in fncache, line %d') % (n + 1)
440 raise error.Abort(t)
440 raise error.Abort(t)
441 fp.close()
441 fp.close()
442
442
443 def write(self, tr):
443 def write(self, tr):
444 if self._dirty:
444 if self._dirty:
445 tr.addbackup('fncache')
445 tr.addbackup('fncache')
446 fp = self.vfs('fncache', mode='wb', atomictemp=True)
446 fp = self.vfs('fncache', mode='wb', atomictemp=True)
447 if self.entries:
447 if self.entries:
448 fp.write(encodedir('\n'.join(self.entries) + '\n'))
448 fp.write(encodedir('\n'.join(self.entries) + '\n'))
449 fp.close()
449 fp.close()
450 self._dirty = False
450 self._dirty = False
451
451
452 def add(self, fn):
452 def add(self, fn):
453 if self.entries is None:
453 if self.entries is None:
454 self._load()
454 self._load()
455 if fn not in self.entries:
455 if fn not in self.entries:
456 self._dirty = True
456 self._dirty = True
457 self.entries.add(fn)
457 self.entries.add(fn)
458
458
459 def remove(self, fn):
459 def remove(self, fn):
460 if self.entries is None:
460 if self.entries is None:
461 self._load()
461 self._load()
462 try:
462 try:
463 self.entries.remove(fn)
463 self.entries.remove(fn)
464 self._dirty = True
464 self._dirty = True
465 except KeyError:
465 except KeyError:
466 pass
466 pass
467
467
468 def __contains__(self, fn):
468 def __contains__(self, fn):
469 if self.entries is None:
469 if self.entries is None:
470 self._load()
470 self._load()
471 return fn in self.entries
471 return fn in self.entries
472
472
473 def __iter__(self):
473 def __iter__(self):
474 if self.entries is None:
474 if self.entries is None:
475 self._load()
475 self._load()
476 return iter(self.entries)
476 return iter(self.entries)
477
477
478 class _fncachevfs(vfsmod.abstractvfs, vfsmod.auditvfs):
478 class _fncachevfs(vfsmod.abstractvfs, vfsmod.proxyvfs):
479 def __init__(self, vfs, fnc, encode):
479 def __init__(self, vfs, fnc, encode):
480 vfsmod.auditvfs.__init__(self, vfs)
480 vfsmod.proxyvfs.__init__(self, vfs)
481 self.fncache = fnc
481 self.fncache = fnc
482 self.encode = encode
482 self.encode = encode
483
483
484 def __call__(self, path, mode='r', *args, **kw):
484 def __call__(self, path, mode='r', *args, **kw):
485 if mode not in ('r', 'rb') and (path.startswith('data/') or
485 if mode not in ('r', 'rb') and (path.startswith('data/') or
486 path.startswith('meta/')):
486 path.startswith('meta/')):
487 self.fncache.add(path)
487 self.fncache.add(path)
488 return self.vfs(self.encode(path), mode, *args, **kw)
488 return self.vfs(self.encode(path), mode, *args, **kw)
489
489
490 def join(self, path):
490 def join(self, path):
491 if path:
491 if path:
492 return self.vfs.join(self.encode(path))
492 return self.vfs.join(self.encode(path))
493 else:
493 else:
494 return self.vfs.join(path)
494 return self.vfs.join(path)
495
495
496 class fncachestore(basicstore):
496 class fncachestore(basicstore):
497 def __init__(self, path, vfstype, dotencode):
497 def __init__(self, path, vfstype, dotencode):
498 if dotencode:
498 if dotencode:
499 encode = _pathencode
499 encode = _pathencode
500 else:
500 else:
501 encode = _plainhybridencode
501 encode = _plainhybridencode
502 self.encode = encode
502 self.encode = encode
503 vfs = vfstype(path + '/store')
503 vfs = vfstype(path + '/store')
504 self.path = vfs.base
504 self.path = vfs.base
505 self.pathsep = self.path + '/'
505 self.pathsep = self.path + '/'
506 self.createmode = _calcmode(vfs)
506 self.createmode = _calcmode(vfs)
507 vfs.createmode = self.createmode
507 vfs.createmode = self.createmode
508 self.rawvfs = vfs
508 self.rawvfs = vfs
509 fnc = fncache(vfs)
509 fnc = fncache(vfs)
510 self.fncache = fnc
510 self.fncache = fnc
511 self.vfs = _fncachevfs(vfs, fnc, encode)
511 self.vfs = _fncachevfs(vfs, fnc, encode)
512 self.opener = self.vfs
512 self.opener = self.vfs
513
513
514 def join(self, f):
514 def join(self, f):
515 return self.pathsep + self.encode(f)
515 return self.pathsep + self.encode(f)
516
516
517 def getsize(self, path):
517 def getsize(self, path):
518 return self.rawvfs.stat(path).st_size
518 return self.rawvfs.stat(path).st_size
519
519
520 def datafiles(self):
520 def datafiles(self):
521 for f in sorted(self.fncache):
521 for f in sorted(self.fncache):
522 ef = self.encode(f)
522 ef = self.encode(f)
523 try:
523 try:
524 yield f, ef, self.getsize(ef)
524 yield f, ef, self.getsize(ef)
525 except OSError as err:
525 except OSError as err:
526 if err.errno != errno.ENOENT:
526 if err.errno != errno.ENOENT:
527 raise
527 raise
528
528
529 def copylist(self):
529 def copylist(self):
530 d = ('data meta dh fncache phaseroots obsstore'
530 d = ('data meta dh fncache phaseroots obsstore'
531 ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i')
531 ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i')
532 return (['requires', '00changelog.i'] +
532 return (['requires', '00changelog.i'] +
533 ['store/' + f for f in d.split()])
533 ['store/' + f for f in d.split()])
534
534
535 def write(self, tr):
535 def write(self, tr):
536 self.fncache.write(tr)
536 self.fncache.write(tr)
537
537
538 def invalidatecaches(self):
538 def invalidatecaches(self):
539 self.fncache.entries = None
539 self.fncache.entries = None
540
540
541 def markremoved(self, fn):
541 def markremoved(self, fn):
542 self.fncache.remove(fn)
542 self.fncache.remove(fn)
543
543
544 def _exists(self, f):
544 def _exists(self, f):
545 ef = self.encode(f)
545 ef = self.encode(f)
546 try:
546 try:
547 self.getsize(ef)
547 self.getsize(ef)
548 return True
548 return True
549 except OSError as err:
549 except OSError as err:
550 if err.errno != errno.ENOENT:
550 if err.errno != errno.ENOENT:
551 raise
551 raise
552 # nonexistent entry
552 # nonexistent entry
553 return False
553 return False
554
554
555 def __contains__(self, path):
555 def __contains__(self, path):
556 '''Checks if the store contains path'''
556 '''Checks if the store contains path'''
557 path = "/".join(("data", path))
557 path = "/".join(("data", path))
558 # check for files (exact match)
558 # check for files (exact match)
559 e = path + '.i'
559 e = path + '.i'
560 if e in self.fncache and self._exists(e):
560 if e in self.fncache and self._exists(e):
561 return True
561 return True
562 # now check for directories (prefix match)
562 # now check for directories (prefix match)
563 if not path.endswith('/'):
563 if not path.endswith('/'):
564 path += '/'
564 path += '/'
565 for e in self.fncache:
565 for e in self.fncache:
566 if e.startswith(path) and self._exists(e):
566 if e.startswith(path) and self._exists(e):
567 return True
567 return True
568 return False
568 return False
569
569
570 def store(requirements, path, vfstype):
570 def store(requirements, path, vfstype):
571 if 'store' in requirements:
571 if 'store' in requirements:
572 if 'fncache' in requirements:
572 if 'fncache' in requirements:
573 return fncachestore(path, vfstype, 'dotencode' in requirements)
573 return fncachestore(path, vfstype, 'dotencode' in requirements)
574 return encodedstore(path, vfstype)
574 return encodedstore(path, vfstype)
575 return basicstore(path, vfstype)
575 return basicstore(path, vfstype)
@@ -1,642 +1,642 b''
1 # vfs.py - Mercurial 'vfs' classes
1 # vfs.py - Mercurial 'vfs' classes
2 #
2 #
3 # Copyright Matt Mackall <mpm@selenic.com>
3 # Copyright Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7 from __future__ import absolute_import
7 from __future__ import absolute_import
8
8
9 import contextlib
9 import contextlib
10 import errno
10 import errno
11 import os
11 import os
12 import shutil
12 import shutil
13 import stat
13 import stat
14 import tempfile
14 import tempfile
15 import threading
15 import threading
16
16
17 from .i18n import _
17 from .i18n import _
18 from . import (
18 from . import (
19 error,
19 error,
20 pathutil,
20 pathutil,
21 pycompat,
21 pycompat,
22 util,
22 util,
23 )
23 )
24
24
25 def _avoidambig(path, oldstat):
25 def _avoidambig(path, oldstat):
26 """Avoid file stat ambiguity forcibly
26 """Avoid file stat ambiguity forcibly
27
27
28 This function causes copying ``path`` file, if it is owned by
28 This function causes copying ``path`` file, if it is owned by
29 another (see issue5418 and issue5584 for detail).
29 another (see issue5418 and issue5584 for detail).
30 """
30 """
31 def checkandavoid():
31 def checkandavoid():
32 newstat = util.filestat.frompath(path)
32 newstat = util.filestat.frompath(path)
33 # return whether file stat ambiguity is (already) avoided
33 # return whether file stat ambiguity is (already) avoided
34 return (not newstat.isambig(oldstat) or
34 return (not newstat.isambig(oldstat) or
35 newstat.avoidambig(path, oldstat))
35 newstat.avoidambig(path, oldstat))
36 if not checkandavoid():
36 if not checkandavoid():
37 # simply copy to change owner of path to get privilege to
37 # simply copy to change owner of path to get privilege to
38 # advance mtime (see issue5418)
38 # advance mtime (see issue5418)
39 util.rename(util.mktempcopy(path), path)
39 util.rename(util.mktempcopy(path), path)
40 checkandavoid()
40 checkandavoid()
41
41
42 class abstractvfs(object):
42 class abstractvfs(object):
43 """Abstract base class; cannot be instantiated"""
43 """Abstract base class; cannot be instantiated"""
44
44
45 def __init__(self, *args, **kwargs):
45 def __init__(self, *args, **kwargs):
46 '''Prevent instantiation; don't call this from subclasses.'''
46 '''Prevent instantiation; don't call this from subclasses.'''
47 raise NotImplementedError('attempted instantiating ' + str(type(self)))
47 raise NotImplementedError('attempted instantiating ' + str(type(self)))
48
48
49 def tryread(self, path):
49 def tryread(self, path):
50 '''gracefully return an empty string for missing files'''
50 '''gracefully return an empty string for missing files'''
51 try:
51 try:
52 return self.read(path)
52 return self.read(path)
53 except IOError as inst:
53 except IOError as inst:
54 if inst.errno != errno.ENOENT:
54 if inst.errno != errno.ENOENT:
55 raise
55 raise
56 return ""
56 return ""
57
57
58 def tryreadlines(self, path, mode='rb'):
58 def tryreadlines(self, path, mode='rb'):
59 '''gracefully return an empty array for missing files'''
59 '''gracefully return an empty array for missing files'''
60 try:
60 try:
61 return self.readlines(path, mode=mode)
61 return self.readlines(path, mode=mode)
62 except IOError as inst:
62 except IOError as inst:
63 if inst.errno != errno.ENOENT:
63 if inst.errno != errno.ENOENT:
64 raise
64 raise
65 return []
65 return []
66
66
67 @util.propertycache
67 @util.propertycache
68 def open(self):
68 def open(self):
69 '''Open ``path`` file, which is relative to vfs root.
69 '''Open ``path`` file, which is relative to vfs root.
70
70
71 Newly created directories are marked as "not to be indexed by
71 Newly created directories are marked as "not to be indexed by
72 the content indexing service", if ``notindexed`` is specified
72 the content indexing service", if ``notindexed`` is specified
73 for "write" mode access.
73 for "write" mode access.
74 '''
74 '''
75 return self.__call__
75 return self.__call__
76
76
77 def read(self, path):
77 def read(self, path):
78 with self(path, 'rb') as fp:
78 with self(path, 'rb') as fp:
79 return fp.read()
79 return fp.read()
80
80
81 def readlines(self, path, mode='rb'):
81 def readlines(self, path, mode='rb'):
82 with self(path, mode=mode) as fp:
82 with self(path, mode=mode) as fp:
83 return fp.readlines()
83 return fp.readlines()
84
84
85 def write(self, path, data, backgroundclose=False):
85 def write(self, path, data, backgroundclose=False):
86 with self(path, 'wb', backgroundclose=backgroundclose) as fp:
86 with self(path, 'wb', backgroundclose=backgroundclose) as fp:
87 return fp.write(data)
87 return fp.write(data)
88
88
89 def writelines(self, path, data, mode='wb', notindexed=False):
89 def writelines(self, path, data, mode='wb', notindexed=False):
90 with self(path, mode=mode, notindexed=notindexed) as fp:
90 with self(path, mode=mode, notindexed=notindexed) as fp:
91 return fp.writelines(data)
91 return fp.writelines(data)
92
92
93 def append(self, path, data):
93 def append(self, path, data):
94 with self(path, 'ab') as fp:
94 with self(path, 'ab') as fp:
95 return fp.write(data)
95 return fp.write(data)
96
96
97 def basename(self, path):
97 def basename(self, path):
98 """return base element of a path (as os.path.basename would do)
98 """return base element of a path (as os.path.basename would do)
99
99
100 This exists to allow handling of strange encoding if needed."""
100 This exists to allow handling of strange encoding if needed."""
101 return os.path.basename(path)
101 return os.path.basename(path)
102
102
103 def chmod(self, path, mode):
103 def chmod(self, path, mode):
104 return os.chmod(self.join(path), mode)
104 return os.chmod(self.join(path), mode)
105
105
106 def dirname(self, path):
106 def dirname(self, path):
107 """return dirname element of a path (as os.path.dirname would do)
107 """return dirname element of a path (as os.path.dirname would do)
108
108
109 This exists to allow handling of strange encoding if needed."""
109 This exists to allow handling of strange encoding if needed."""
110 return os.path.dirname(path)
110 return os.path.dirname(path)
111
111
112 def exists(self, path=None):
112 def exists(self, path=None):
113 return os.path.exists(self.join(path))
113 return os.path.exists(self.join(path))
114
114
115 def fstat(self, fp):
115 def fstat(self, fp):
116 return util.fstat(fp)
116 return util.fstat(fp)
117
117
118 def isdir(self, path=None):
118 def isdir(self, path=None):
119 return os.path.isdir(self.join(path))
119 return os.path.isdir(self.join(path))
120
120
121 def isfile(self, path=None):
121 def isfile(self, path=None):
122 return os.path.isfile(self.join(path))
122 return os.path.isfile(self.join(path))
123
123
124 def islink(self, path=None):
124 def islink(self, path=None):
125 return os.path.islink(self.join(path))
125 return os.path.islink(self.join(path))
126
126
127 def isfileorlink(self, path=None):
127 def isfileorlink(self, path=None):
128 '''return whether path is a regular file or a symlink
128 '''return whether path is a regular file or a symlink
129
129
130 Unlike isfile, this doesn't follow symlinks.'''
130 Unlike isfile, this doesn't follow symlinks.'''
131 try:
131 try:
132 st = self.lstat(path)
132 st = self.lstat(path)
133 except OSError:
133 except OSError:
134 return False
134 return False
135 mode = st.st_mode
135 mode = st.st_mode
136 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
136 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
137
137
138 def reljoin(self, *paths):
138 def reljoin(self, *paths):
139 """join various elements of a path together (as os.path.join would do)
139 """join various elements of a path together (as os.path.join would do)
140
140
141 The vfs base is not injected so that path stay relative. This exists
141 The vfs base is not injected so that path stay relative. This exists
142 to allow handling of strange encoding if needed."""
142 to allow handling of strange encoding if needed."""
143 return os.path.join(*paths)
143 return os.path.join(*paths)
144
144
145 def split(self, path):
145 def split(self, path):
146 """split top-most element of a path (as os.path.split would do)
146 """split top-most element of a path (as os.path.split would do)
147
147
148 This exists to allow handling of strange encoding if needed."""
148 This exists to allow handling of strange encoding if needed."""
149 return os.path.split(path)
149 return os.path.split(path)
150
150
151 def lexists(self, path=None):
151 def lexists(self, path=None):
152 return os.path.lexists(self.join(path))
152 return os.path.lexists(self.join(path))
153
153
154 def lstat(self, path=None):
154 def lstat(self, path=None):
155 return os.lstat(self.join(path))
155 return os.lstat(self.join(path))
156
156
157 def listdir(self, path=None):
157 def listdir(self, path=None):
158 return os.listdir(self.join(path))
158 return os.listdir(self.join(path))
159
159
160 def makedir(self, path=None, notindexed=True):
160 def makedir(self, path=None, notindexed=True):
161 return util.makedir(self.join(path), notindexed)
161 return util.makedir(self.join(path), notindexed)
162
162
163 def makedirs(self, path=None, mode=None):
163 def makedirs(self, path=None, mode=None):
164 return util.makedirs(self.join(path), mode)
164 return util.makedirs(self.join(path), mode)
165
165
166 def makelock(self, info, path):
166 def makelock(self, info, path):
167 return util.makelock(info, self.join(path))
167 return util.makelock(info, self.join(path))
168
168
169 def mkdir(self, path=None):
169 def mkdir(self, path=None):
170 return os.mkdir(self.join(path))
170 return os.mkdir(self.join(path))
171
171
172 def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
172 def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
173 fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
173 fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
174 dir=self.join(dir), text=text)
174 dir=self.join(dir), text=text)
175 dname, fname = util.split(name)
175 dname, fname = util.split(name)
176 if dir:
176 if dir:
177 return fd, os.path.join(dir, fname)
177 return fd, os.path.join(dir, fname)
178 else:
178 else:
179 return fd, fname
179 return fd, fname
180
180
181 def readdir(self, path=None, stat=None, skip=None):
181 def readdir(self, path=None, stat=None, skip=None):
182 return util.listdir(self.join(path), stat, skip)
182 return util.listdir(self.join(path), stat, skip)
183
183
184 def readlock(self, path):
184 def readlock(self, path):
185 return util.readlock(self.join(path))
185 return util.readlock(self.join(path))
186
186
187 def rename(self, src, dst, checkambig=False):
187 def rename(self, src, dst, checkambig=False):
188 """Rename from src to dst
188 """Rename from src to dst
189
189
190 checkambig argument is used with util.filestat, and is useful
190 checkambig argument is used with util.filestat, and is useful
191 only if destination file is guarded by any lock
191 only if destination file is guarded by any lock
192 (e.g. repo.lock or repo.wlock).
192 (e.g. repo.lock or repo.wlock).
193
193
194 To avoid file stat ambiguity forcibly, checkambig=True involves
194 To avoid file stat ambiguity forcibly, checkambig=True involves
195 copying ``src`` file, if it is owned by another. Therefore, use
195 copying ``src`` file, if it is owned by another. Therefore, use
196 checkambig=True only in limited cases (see also issue5418 and
196 checkambig=True only in limited cases (see also issue5418 and
197 issue5584 for detail).
197 issue5584 for detail).
198 """
198 """
199 srcpath = self.join(src)
199 srcpath = self.join(src)
200 dstpath = self.join(dst)
200 dstpath = self.join(dst)
201 oldstat = checkambig and util.filestat.frompath(dstpath)
201 oldstat = checkambig and util.filestat.frompath(dstpath)
202 if oldstat and oldstat.stat:
202 if oldstat and oldstat.stat:
203 ret = util.rename(srcpath, dstpath)
203 ret = util.rename(srcpath, dstpath)
204 _avoidambig(dstpath, oldstat)
204 _avoidambig(dstpath, oldstat)
205 return ret
205 return ret
206 return util.rename(srcpath, dstpath)
206 return util.rename(srcpath, dstpath)
207
207
208 def readlink(self, path):
208 def readlink(self, path):
209 return os.readlink(self.join(path))
209 return os.readlink(self.join(path))
210
210
211 def removedirs(self, path=None):
211 def removedirs(self, path=None):
212 """Remove a leaf directory and all empty intermediate ones
212 """Remove a leaf directory and all empty intermediate ones
213 """
213 """
214 return util.removedirs(self.join(path))
214 return util.removedirs(self.join(path))
215
215
216 def rmtree(self, path=None, ignore_errors=False, forcibly=False):
216 def rmtree(self, path=None, ignore_errors=False, forcibly=False):
217 """Remove a directory tree recursively
217 """Remove a directory tree recursively
218
218
219 If ``forcibly``, this tries to remove READ-ONLY files, too.
219 If ``forcibly``, this tries to remove READ-ONLY files, too.
220 """
220 """
221 if forcibly:
221 if forcibly:
222 def onerror(function, path, excinfo):
222 def onerror(function, path, excinfo):
223 if function is not os.remove:
223 if function is not os.remove:
224 raise
224 raise
225 # read-only files cannot be unlinked under Windows
225 # read-only files cannot be unlinked under Windows
226 s = os.stat(path)
226 s = os.stat(path)
227 if (s.st_mode & stat.S_IWRITE) != 0:
227 if (s.st_mode & stat.S_IWRITE) != 0:
228 raise
228 raise
229 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
229 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
230 os.remove(path)
230 os.remove(path)
231 else:
231 else:
232 onerror = None
232 onerror = None
233 return shutil.rmtree(self.join(path),
233 return shutil.rmtree(self.join(path),
234 ignore_errors=ignore_errors, onerror=onerror)
234 ignore_errors=ignore_errors, onerror=onerror)
235
235
236 def setflags(self, path, l, x):
236 def setflags(self, path, l, x):
237 return util.setflags(self.join(path), l, x)
237 return util.setflags(self.join(path), l, x)
238
238
239 def stat(self, path=None):
239 def stat(self, path=None):
240 return os.stat(self.join(path))
240 return os.stat(self.join(path))
241
241
242 def unlink(self, path=None):
242 def unlink(self, path=None):
243 return util.unlink(self.join(path))
243 return util.unlink(self.join(path))
244
244
245 def tryunlink(self, path=None):
245 def tryunlink(self, path=None):
246 """Attempt to remove a file, ignoring missing file errors."""
246 """Attempt to remove a file, ignoring missing file errors."""
247 util.tryunlink(self.join(path))
247 util.tryunlink(self.join(path))
248
248
249 def unlinkpath(self, path=None, ignoremissing=False):
249 def unlinkpath(self, path=None, ignoremissing=False):
250 return util.unlinkpath(self.join(path), ignoremissing=ignoremissing)
250 return util.unlinkpath(self.join(path), ignoremissing=ignoremissing)
251
251
252 def utime(self, path=None, t=None):
252 def utime(self, path=None, t=None):
253 return os.utime(self.join(path), t)
253 return os.utime(self.join(path), t)
254
254
255 def walk(self, path=None, onerror=None):
255 def walk(self, path=None, onerror=None):
256 """Yield (dirpath, dirs, files) tuple for each directories under path
256 """Yield (dirpath, dirs, files) tuple for each directories under path
257
257
258 ``dirpath`` is relative one from the root of this vfs. This
258 ``dirpath`` is relative one from the root of this vfs. This
259 uses ``os.sep`` as path separator, even you specify POSIX
259 uses ``os.sep`` as path separator, even you specify POSIX
260 style ``path``.
260 style ``path``.
261
261
262 "The root of this vfs" is represented as empty ``dirpath``.
262 "The root of this vfs" is represented as empty ``dirpath``.
263 """
263 """
264 root = os.path.normpath(self.join(None))
264 root = os.path.normpath(self.join(None))
265 # when dirpath == root, dirpath[prefixlen:] becomes empty
265 # when dirpath == root, dirpath[prefixlen:] becomes empty
266 # because len(dirpath) < prefixlen.
266 # because len(dirpath) < prefixlen.
267 prefixlen = len(pathutil.normasprefix(root))
267 prefixlen = len(pathutil.normasprefix(root))
268 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
268 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
269 yield (dirpath[prefixlen:], dirs, files)
269 yield (dirpath[prefixlen:], dirs, files)
270
270
271 @contextlib.contextmanager
271 @contextlib.contextmanager
272 def backgroundclosing(self, ui, expectedcount=-1):
272 def backgroundclosing(self, ui, expectedcount=-1):
273 """Allow files to be closed asynchronously.
273 """Allow files to be closed asynchronously.
274
274
275 When this context manager is active, ``backgroundclose`` can be passed
275 When this context manager is active, ``backgroundclose`` can be passed
276 to ``__call__``/``open`` to result in the file possibly being closed
276 to ``__call__``/``open`` to result in the file possibly being closed
277 asynchronously, on a background thread.
277 asynchronously, on a background thread.
278 """
278 """
279 # This is an arbitrary restriction and could be changed if we ever
279 # This is an arbitrary restriction and could be changed if we ever
280 # have a use case.
280 # have a use case.
281 vfs = getattr(self, 'vfs', self)
281 vfs = getattr(self, 'vfs', self)
282 if getattr(vfs, '_backgroundfilecloser', None):
282 if getattr(vfs, '_backgroundfilecloser', None):
283 raise error.Abort(
283 raise error.Abort(
284 _('can only have 1 active background file closer'))
284 _('can only have 1 active background file closer'))
285
285
286 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
286 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
287 try:
287 try:
288 vfs._backgroundfilecloser = bfc
288 vfs._backgroundfilecloser = bfc
289 yield bfc
289 yield bfc
290 finally:
290 finally:
291 vfs._backgroundfilecloser = None
291 vfs._backgroundfilecloser = None
292
292
293 class vfs(abstractvfs):
293 class vfs(abstractvfs):
294 '''Operate files relative to a base directory
294 '''Operate files relative to a base directory
295
295
296 This class is used to hide the details of COW semantics and
296 This class is used to hide the details of COW semantics and
297 remote file access from higher level code.
297 remote file access from higher level code.
298 '''
298 '''
299 def __init__(self, base, audit=True, expandpath=False, realpath=False):
299 def __init__(self, base, audit=True, expandpath=False, realpath=False):
300 if expandpath:
300 if expandpath:
301 base = util.expandpath(base)
301 base = util.expandpath(base)
302 if realpath:
302 if realpath:
303 base = os.path.realpath(base)
303 base = os.path.realpath(base)
304 self.base = base
304 self.base = base
305 self._audit = audit
305 self._audit = audit
306 if audit:
306 if audit:
307 self.audit = pathutil.pathauditor(self.base)
307 self.audit = pathutil.pathauditor(self.base)
308 else:
308 else:
309 self.audit = util.always
309 self.audit = util.always
310 self.createmode = None
310 self.createmode = None
311 self._trustnlink = None
311 self._trustnlink = None
312
312
313 @util.propertycache
313 @util.propertycache
314 def _cansymlink(self):
314 def _cansymlink(self):
315 return util.checklink(self.base)
315 return util.checklink(self.base)
316
316
317 @util.propertycache
317 @util.propertycache
318 def _chmod(self):
318 def _chmod(self):
319 return util.checkexec(self.base)
319 return util.checkexec(self.base)
320
320
321 def _fixfilemode(self, name):
321 def _fixfilemode(self, name):
322 if self.createmode is None or not self._chmod:
322 if self.createmode is None or not self._chmod:
323 return
323 return
324 os.chmod(name, self.createmode & 0o666)
324 os.chmod(name, self.createmode & 0o666)
325
325
326 def __call__(self, path, mode="r", text=False, atomictemp=False,
326 def __call__(self, path, mode="r", text=False, atomictemp=False,
327 notindexed=False, backgroundclose=False, checkambig=False,
327 notindexed=False, backgroundclose=False, checkambig=False,
328 auditpath=True):
328 auditpath=True):
329 '''Open ``path`` file, which is relative to vfs root.
329 '''Open ``path`` file, which is relative to vfs root.
330
330
331 Newly created directories are marked as "not to be indexed by
331 Newly created directories are marked as "not to be indexed by
332 the content indexing service", if ``notindexed`` is specified
332 the content indexing service", if ``notindexed`` is specified
333 for "write" mode access.
333 for "write" mode access.
334
334
335 If ``backgroundclose`` is passed, the file may be closed asynchronously.
335 If ``backgroundclose`` is passed, the file may be closed asynchronously.
336 It can only be used if the ``self.backgroundclosing()`` context manager
336 It can only be used if the ``self.backgroundclosing()`` context manager
337 is active. This should only be specified if the following criteria hold:
337 is active. This should only be specified if the following criteria hold:
338
338
339 1. There is a potential for writing thousands of files. Unless you
339 1. There is a potential for writing thousands of files. Unless you
340 are writing thousands of files, the performance benefits of
340 are writing thousands of files, the performance benefits of
341 asynchronously closing files is not realized.
341 asynchronously closing files is not realized.
342 2. Files are opened exactly once for the ``backgroundclosing``
342 2. Files are opened exactly once for the ``backgroundclosing``
343 active duration and are therefore free of race conditions between
343 active duration and are therefore free of race conditions between
344 closing a file on a background thread and reopening it. (If the
344 closing a file on a background thread and reopening it. (If the
345 file were opened multiple times, there could be unflushed data
345 file were opened multiple times, there could be unflushed data
346 because the original file handle hasn't been flushed/closed yet.)
346 because the original file handle hasn't been flushed/closed yet.)
347
347
348 ``checkambig`` argument is passed to atomictemplfile (valid
348 ``checkambig`` argument is passed to atomictemplfile (valid
349 only for writing), and is useful only if target file is
349 only for writing), and is useful only if target file is
350 guarded by any lock (e.g. repo.lock or repo.wlock).
350 guarded by any lock (e.g. repo.lock or repo.wlock).
351
351
352 To avoid file stat ambiguity forcibly, checkambig=True involves
352 To avoid file stat ambiguity forcibly, checkambig=True involves
353 copying ``path`` file opened in "append" mode (e.g. for
353 copying ``path`` file opened in "append" mode (e.g. for
354 truncation), if it is owned by another. Therefore, use
354 truncation), if it is owned by another. Therefore, use
355 combination of append mode and checkambig=True only in limited
355 combination of append mode and checkambig=True only in limited
356 cases (see also issue5418 and issue5584 for detail).
356 cases (see also issue5418 and issue5584 for detail).
357 '''
357 '''
358 if auditpath:
358 if auditpath:
359 if self._audit:
359 if self._audit:
360 r = util.checkosfilename(path)
360 r = util.checkosfilename(path)
361 if r:
361 if r:
362 raise error.Abort("%s: %r" % (r, path))
362 raise error.Abort("%s: %r" % (r, path))
363 self.audit(path)
363 self.audit(path)
364 f = self.join(path)
364 f = self.join(path)
365
365
366 if not text and "b" not in mode:
366 if not text and "b" not in mode:
367 mode += "b" # for that other OS
367 mode += "b" # for that other OS
368
368
369 nlink = -1
369 nlink = -1
370 if mode not in ('r', 'rb'):
370 if mode not in ('r', 'rb'):
371 dirname, basename = util.split(f)
371 dirname, basename = util.split(f)
372 # If basename is empty, then the path is malformed because it points
372 # If basename is empty, then the path is malformed because it points
373 # to a directory. Let the posixfile() call below raise IOError.
373 # to a directory. Let the posixfile() call below raise IOError.
374 if basename:
374 if basename:
375 if atomictemp:
375 if atomictemp:
376 util.makedirs(dirname, self.createmode, notindexed)
376 util.makedirs(dirname, self.createmode, notindexed)
377 return util.atomictempfile(f, mode, self.createmode,
377 return util.atomictempfile(f, mode, self.createmode,
378 checkambig=checkambig)
378 checkambig=checkambig)
379 try:
379 try:
380 if 'w' in mode:
380 if 'w' in mode:
381 util.unlink(f)
381 util.unlink(f)
382 nlink = 0
382 nlink = 0
383 else:
383 else:
384 # nlinks() may behave differently for files on Windows
384 # nlinks() may behave differently for files on Windows
385 # shares if the file is open.
385 # shares if the file is open.
386 with util.posixfile(f):
386 with util.posixfile(f):
387 nlink = util.nlinks(f)
387 nlink = util.nlinks(f)
388 if nlink < 1:
388 if nlink < 1:
389 nlink = 2 # force mktempcopy (issue1922)
389 nlink = 2 # force mktempcopy (issue1922)
390 except (OSError, IOError) as e:
390 except (OSError, IOError) as e:
391 if e.errno != errno.ENOENT:
391 if e.errno != errno.ENOENT:
392 raise
392 raise
393 nlink = 0
393 nlink = 0
394 util.makedirs(dirname, self.createmode, notindexed)
394 util.makedirs(dirname, self.createmode, notindexed)
395 if nlink > 0:
395 if nlink > 0:
396 if self._trustnlink is None:
396 if self._trustnlink is None:
397 self._trustnlink = nlink > 1 or util.checknlink(f)
397 self._trustnlink = nlink > 1 or util.checknlink(f)
398 if nlink > 1 or not self._trustnlink:
398 if nlink > 1 or not self._trustnlink:
399 util.rename(util.mktempcopy(f), f)
399 util.rename(util.mktempcopy(f), f)
400 fp = util.posixfile(f, mode)
400 fp = util.posixfile(f, mode)
401 if nlink == 0:
401 if nlink == 0:
402 self._fixfilemode(f)
402 self._fixfilemode(f)
403
403
404 if checkambig:
404 if checkambig:
405 if mode in ('r', 'rb'):
405 if mode in ('r', 'rb'):
406 raise error.Abort(_('implementation error: mode %s is not'
406 raise error.Abort(_('implementation error: mode %s is not'
407 ' valid for checkambig=True') % mode)
407 ' valid for checkambig=True') % mode)
408 fp = checkambigatclosing(fp)
408 fp = checkambigatclosing(fp)
409
409
410 if backgroundclose:
410 if backgroundclose:
411 if not self._backgroundfilecloser:
411 if not self._backgroundfilecloser:
412 raise error.Abort(_('backgroundclose can only be used when a '
412 raise error.Abort(_('backgroundclose can only be used when a '
413 'backgroundclosing context manager is active')
413 'backgroundclosing context manager is active')
414 )
414 )
415
415
416 fp = delayclosedfile(fp, self._backgroundfilecloser)
416 fp = delayclosedfile(fp, self._backgroundfilecloser)
417
417
418 return fp
418 return fp
419
419
420 def symlink(self, src, dst):
420 def symlink(self, src, dst):
421 self.audit(dst)
421 self.audit(dst)
422 linkname = self.join(dst)
422 linkname = self.join(dst)
423 util.tryunlink(linkname)
423 util.tryunlink(linkname)
424
424
425 util.makedirs(os.path.dirname(linkname), self.createmode)
425 util.makedirs(os.path.dirname(linkname), self.createmode)
426
426
427 if self._cansymlink:
427 if self._cansymlink:
428 try:
428 try:
429 os.symlink(src, linkname)
429 os.symlink(src, linkname)
430 except OSError as err:
430 except OSError as err:
431 raise OSError(err.errno, _('could not symlink to %r: %s') %
431 raise OSError(err.errno, _('could not symlink to %r: %s') %
432 (src, err.strerror), linkname)
432 (src, err.strerror), linkname)
433 else:
433 else:
434 self.write(dst, src)
434 self.write(dst, src)
435
435
436 def join(self, path, *insidef):
436 def join(self, path, *insidef):
437 if path:
437 if path:
438 return os.path.join(self.base, path, *insidef)
438 return os.path.join(self.base, path, *insidef)
439 else:
439 else:
440 return self.base
440 return self.base
441
441
442 opener = vfs
442 opener = vfs
443
443
444 class auditvfs(object):
444 class proxyvfs(object):
445 def __init__(self, vfs):
445 def __init__(self, vfs):
446 self.vfs = vfs
446 self.vfs = vfs
447
447
448 @property
448 @property
449 def options(self):
449 def options(self):
450 return self.vfs.options
450 return self.vfs.options
451
451
452 @options.setter
452 @options.setter
453 def options(self, value):
453 def options(self, value):
454 self.vfs.options = value
454 self.vfs.options = value
455
455
456 class filtervfs(abstractvfs, auditvfs):
456 class filtervfs(abstractvfs, proxyvfs):
457 '''Wrapper vfs for filtering filenames with a function.'''
457 '''Wrapper vfs for filtering filenames with a function.'''
458
458
459 def __init__(self, vfs, filter):
459 def __init__(self, vfs, filter):
460 auditvfs.__init__(self, vfs)
460 proxyvfs.__init__(self, vfs)
461 self._filter = filter
461 self._filter = filter
462
462
463 def __call__(self, path, *args, **kwargs):
463 def __call__(self, path, *args, **kwargs):
464 return self.vfs(self._filter(path), *args, **kwargs)
464 return self.vfs(self._filter(path), *args, **kwargs)
465
465
466 def join(self, path, *insidef):
466 def join(self, path, *insidef):
467 if path:
467 if path:
468 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
468 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
469 else:
469 else:
470 return self.vfs.join(path)
470 return self.vfs.join(path)
471
471
472 filteropener = filtervfs
472 filteropener = filtervfs
473
473
474 class readonlyvfs(abstractvfs, auditvfs):
474 class readonlyvfs(abstractvfs, proxyvfs):
475 '''Wrapper vfs preventing any writing.'''
475 '''Wrapper vfs preventing any writing.'''
476
476
477 def __init__(self, vfs):
477 def __init__(self, vfs):
478 auditvfs.__init__(self, vfs)
478 proxyvfs.__init__(self, vfs)
479
479
480 def __call__(self, path, mode='r', *args, **kw):
480 def __call__(self, path, mode='r', *args, **kw):
481 if mode not in ('r', 'rb'):
481 if mode not in ('r', 'rb'):
482 raise error.Abort(_('this vfs is read only'))
482 raise error.Abort(_('this vfs is read only'))
483 return self.vfs(path, mode, *args, **kw)
483 return self.vfs(path, mode, *args, **kw)
484
484
485 def join(self, path, *insidef):
485 def join(self, path, *insidef):
486 return self.vfs.join(path, *insidef)
486 return self.vfs.join(path, *insidef)
487
487
488 class closewrapbase(object):
488 class closewrapbase(object):
489 """Base class of wrapper, which hooks closing
489 """Base class of wrapper, which hooks closing
490
490
491 Do not instantiate outside of the vfs layer.
491 Do not instantiate outside of the vfs layer.
492 """
492 """
493 def __init__(self, fh):
493 def __init__(self, fh):
494 object.__setattr__(self, r'_origfh', fh)
494 object.__setattr__(self, r'_origfh', fh)
495
495
496 def __getattr__(self, attr):
496 def __getattr__(self, attr):
497 return getattr(self._origfh, attr)
497 return getattr(self._origfh, attr)
498
498
499 def __setattr__(self, attr, value):
499 def __setattr__(self, attr, value):
500 return setattr(self._origfh, attr, value)
500 return setattr(self._origfh, attr, value)
501
501
502 def __delattr__(self, attr):
502 def __delattr__(self, attr):
503 return delattr(self._origfh, attr)
503 return delattr(self._origfh, attr)
504
504
505 def __enter__(self):
505 def __enter__(self):
506 return self._origfh.__enter__()
506 return self._origfh.__enter__()
507
507
508 def __exit__(self, exc_type, exc_value, exc_tb):
508 def __exit__(self, exc_type, exc_value, exc_tb):
509 raise NotImplementedError('attempted instantiating ' + str(type(self)))
509 raise NotImplementedError('attempted instantiating ' + str(type(self)))
510
510
511 def close(self):
511 def close(self):
512 raise NotImplementedError('attempted instantiating ' + str(type(self)))
512 raise NotImplementedError('attempted instantiating ' + str(type(self)))
513
513
514 class delayclosedfile(closewrapbase):
514 class delayclosedfile(closewrapbase):
515 """Proxy for a file object whose close is delayed.
515 """Proxy for a file object whose close is delayed.
516
516
517 Do not instantiate outside of the vfs layer.
517 Do not instantiate outside of the vfs layer.
518 """
518 """
519 def __init__(self, fh, closer):
519 def __init__(self, fh, closer):
520 super(delayclosedfile, self).__init__(fh)
520 super(delayclosedfile, self).__init__(fh)
521 object.__setattr__(self, r'_closer', closer)
521 object.__setattr__(self, r'_closer', closer)
522
522
523 def __exit__(self, exc_type, exc_value, exc_tb):
523 def __exit__(self, exc_type, exc_value, exc_tb):
524 self._closer.close(self._origfh)
524 self._closer.close(self._origfh)
525
525
526 def close(self):
526 def close(self):
527 self._closer.close(self._origfh)
527 self._closer.close(self._origfh)
528
528
529 class backgroundfilecloser(object):
529 class backgroundfilecloser(object):
530 """Coordinates background closing of file handles on multiple threads."""
530 """Coordinates background closing of file handles on multiple threads."""
531 def __init__(self, ui, expectedcount=-1):
531 def __init__(self, ui, expectedcount=-1):
532 self._running = False
532 self._running = False
533 self._entered = False
533 self._entered = False
534 self._threads = []
534 self._threads = []
535 self._threadexception = None
535 self._threadexception = None
536
536
537 # Only Windows/NTFS has slow file closing. So only enable by default
537 # Only Windows/NTFS has slow file closing. So only enable by default
538 # on that platform. But allow to be enabled elsewhere for testing.
538 # on that platform. But allow to be enabled elsewhere for testing.
539 defaultenabled = pycompat.osname == 'nt'
539 defaultenabled = pycompat.osname == 'nt'
540 enabled = ui.configbool('worker', 'backgroundclose', defaultenabled)
540 enabled = ui.configbool('worker', 'backgroundclose', defaultenabled)
541
541
542 if not enabled:
542 if not enabled:
543 return
543 return
544
544
545 # There is overhead to starting and stopping the background threads.
545 # There is overhead to starting and stopping the background threads.
546 # Don't do background processing unless the file count is large enough
546 # Don't do background processing unless the file count is large enough
547 # to justify it.
547 # to justify it.
548 minfilecount = ui.configint('worker', 'backgroundcloseminfilecount')
548 minfilecount = ui.configint('worker', 'backgroundcloseminfilecount')
549 # FUTURE dynamically start background threads after minfilecount closes.
549 # FUTURE dynamically start background threads after minfilecount closes.
550 # (We don't currently have any callers that don't know their file count)
550 # (We don't currently have any callers that don't know their file count)
551 if expectedcount > 0 and expectedcount < minfilecount:
551 if expectedcount > 0 and expectedcount < minfilecount:
552 return
552 return
553
553
554 maxqueue = ui.configint('worker', 'backgroundclosemaxqueue')
554 maxqueue = ui.configint('worker', 'backgroundclosemaxqueue')
555 threadcount = ui.configint('worker', 'backgroundclosethreadcount')
555 threadcount = ui.configint('worker', 'backgroundclosethreadcount')
556
556
557 ui.debug('starting %d threads for background file closing\n' %
557 ui.debug('starting %d threads for background file closing\n' %
558 threadcount)
558 threadcount)
559
559
560 self._queue = util.queue(maxsize=maxqueue)
560 self._queue = util.queue(maxsize=maxqueue)
561 self._running = True
561 self._running = True
562
562
563 for i in range(threadcount):
563 for i in range(threadcount):
564 t = threading.Thread(target=self._worker, name='backgroundcloser')
564 t = threading.Thread(target=self._worker, name='backgroundcloser')
565 self._threads.append(t)
565 self._threads.append(t)
566 t.start()
566 t.start()
567
567
568 def __enter__(self):
568 def __enter__(self):
569 self._entered = True
569 self._entered = True
570 return self
570 return self
571
571
572 def __exit__(self, exc_type, exc_value, exc_tb):
572 def __exit__(self, exc_type, exc_value, exc_tb):
573 self._running = False
573 self._running = False
574
574
575 # Wait for threads to finish closing so open files don't linger for
575 # Wait for threads to finish closing so open files don't linger for
576 # longer than lifetime of context manager.
576 # longer than lifetime of context manager.
577 for t in self._threads:
577 for t in self._threads:
578 t.join()
578 t.join()
579
579
580 def _worker(self):
580 def _worker(self):
581 """Main routine for worker thread."""
581 """Main routine for worker thread."""
582 while True:
582 while True:
583 try:
583 try:
584 fh = self._queue.get(block=True, timeout=0.100)
584 fh = self._queue.get(block=True, timeout=0.100)
585 # Need to catch or the thread will terminate and
585 # Need to catch or the thread will terminate and
586 # we could orphan file descriptors.
586 # we could orphan file descriptors.
587 try:
587 try:
588 fh.close()
588 fh.close()
589 except Exception as e:
589 except Exception as e:
590 # Stash so can re-raise from main thread later.
590 # Stash so can re-raise from main thread later.
591 self._threadexception = e
591 self._threadexception = e
592 except util.empty:
592 except util.empty:
593 if not self._running:
593 if not self._running:
594 break
594 break
595
595
596 def close(self, fh):
596 def close(self, fh):
597 """Schedule a file for closing."""
597 """Schedule a file for closing."""
598 if not self._entered:
598 if not self._entered:
599 raise error.Abort(_('can only call close() when context manager '
599 raise error.Abort(_('can only call close() when context manager '
600 'active'))
600 'active'))
601
601
602 # If a background thread encountered an exception, raise now so we fail
602 # If a background thread encountered an exception, raise now so we fail
603 # fast. Otherwise we may potentially go on for minutes until the error
603 # fast. Otherwise we may potentially go on for minutes until the error
604 # is acted on.
604 # is acted on.
605 if self._threadexception:
605 if self._threadexception:
606 e = self._threadexception
606 e = self._threadexception
607 self._threadexception = None
607 self._threadexception = None
608 raise e
608 raise e
609
609
610 # If we're not actively running, close synchronously.
610 # If we're not actively running, close synchronously.
611 if not self._running:
611 if not self._running:
612 fh.close()
612 fh.close()
613 return
613 return
614
614
615 self._queue.put(fh, block=True, timeout=None)
615 self._queue.put(fh, block=True, timeout=None)
616
616
617 class checkambigatclosing(closewrapbase):
617 class checkambigatclosing(closewrapbase):
618 """Proxy for a file object, to avoid ambiguity of file stat
618 """Proxy for a file object, to avoid ambiguity of file stat
619
619
620 See also util.filestat for detail about "ambiguity of file stat".
620 See also util.filestat for detail about "ambiguity of file stat".
621
621
622 This proxy is useful only if the target file is guarded by any
622 This proxy is useful only if the target file is guarded by any
623 lock (e.g. repo.lock or repo.wlock)
623 lock (e.g. repo.lock or repo.wlock)
624
624
625 Do not instantiate outside of the vfs layer.
625 Do not instantiate outside of the vfs layer.
626 """
626 """
627 def __init__(self, fh):
627 def __init__(self, fh):
628 super(checkambigatclosing, self).__init__(fh)
628 super(checkambigatclosing, self).__init__(fh)
629 object.__setattr__(self, r'_oldstat', util.filestat.frompath(fh.name))
629 object.__setattr__(self, r'_oldstat', util.filestat.frompath(fh.name))
630
630
631 def _checkambig(self):
631 def _checkambig(self):
632 oldstat = self._oldstat
632 oldstat = self._oldstat
633 if oldstat.stat:
633 if oldstat.stat:
634 _avoidambig(self._origfh.name, oldstat)
634 _avoidambig(self._origfh.name, oldstat)
635
635
636 def __exit__(self, exc_type, exc_value, exc_tb):
636 def __exit__(self, exc_type, exc_value, exc_tb):
637 self._origfh.__exit__(exc_type, exc_value, exc_tb)
637 self._origfh.__exit__(exc_type, exc_value, exc_tb)
638 self._checkambig()
638 self._checkambig()
639
639
640 def close(self):
640 def close(self):
641 self._origfh.close()
641 self._origfh.close()
642 self._checkambig()
642 self._checkambig()
General Comments 0
You need to be logged in to leave comments. Login now