##// END OF EJS Templates
vfs: rename auditvfs to proxyvfs...
Yuya Nishihara -
r33412:a42369e0 default
parent child Browse files
Show More
@@ -1,575 +1,575
1 1 # store.py - repository store handling for Mercurial
2 2 #
3 3 # Copyright 2008 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import errno
11 11 import hashlib
12 12 import os
13 13 import stat
14 14
15 15 from .i18n import _
16 16 from . import (
17 17 error,
18 18 policy,
19 19 pycompat,
20 20 util,
21 21 vfs as vfsmod,
22 22 )
23 23
24 24 parsers = policy.importmod(r'parsers')
25 25
26 26 # This avoids a collision between a file named foo and a dir named
27 27 # foo.i or foo.d
28 28 def _encodedir(path):
29 29 '''
30 30 >>> _encodedir('data/foo.i')
31 31 'data/foo.i'
32 32 >>> _encodedir('data/foo.i/bla.i')
33 33 'data/foo.i.hg/bla.i'
34 34 >>> _encodedir('data/foo.i.hg/bla.i')
35 35 'data/foo.i.hg.hg/bla.i'
36 36 >>> _encodedir('data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n')
37 37 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n'
38 38 '''
39 39 return (path
40 40 .replace(".hg/", ".hg.hg/")
41 41 .replace(".i/", ".i.hg/")
42 42 .replace(".d/", ".d.hg/"))
43 43
44 44 encodedir = getattr(parsers, 'encodedir', _encodedir)
45 45
46 46 def decodedir(path):
47 47 '''
48 48 >>> decodedir('data/foo.i')
49 49 'data/foo.i'
50 50 >>> decodedir('data/foo.i.hg/bla.i')
51 51 'data/foo.i/bla.i'
52 52 >>> decodedir('data/foo.i.hg.hg/bla.i')
53 53 'data/foo.i.hg/bla.i'
54 54 '''
55 55 if ".hg/" not in path:
56 56 return path
57 57 return (path
58 58 .replace(".d.hg/", ".d/")
59 59 .replace(".i.hg/", ".i/")
60 60 .replace(".hg.hg/", ".hg/"))
61 61
62 62 def _reserved():
63 63 ''' characters that are problematic for filesystems
64 64
65 65 * ascii escapes (0..31)
66 66 * ascii hi (126..255)
67 67 * windows specials
68 68
69 69 these characters will be escaped by encodefunctions
70 70 '''
71 71 winreserved = [ord(x) for x in u'\\:*?"<>|']
72 72 for x in range(32):
73 73 yield x
74 74 for x in range(126, 256):
75 75 yield x
76 76 for x in winreserved:
77 77 yield x
78 78
79 79 def _buildencodefun():
80 80 '''
81 81 >>> enc, dec = _buildencodefun()
82 82
83 83 >>> enc('nothing/special.txt')
84 84 'nothing/special.txt'
85 85 >>> dec('nothing/special.txt')
86 86 'nothing/special.txt'
87 87
88 88 >>> enc('HELLO')
89 89 '_h_e_l_l_o'
90 90 >>> dec('_h_e_l_l_o')
91 91 'HELLO'
92 92
93 93 >>> enc('hello:world?')
94 94 'hello~3aworld~3f'
95 95 >>> dec('hello~3aworld~3f')
96 96 'hello:world?'
97 97
98 98 >>> enc('the\x07quick\xADshot')
99 99 'the~07quick~adshot'
100 100 >>> dec('the~07quick~adshot')
101 101 'the\\x07quick\\xadshot'
102 102 '''
103 103 e = '_'
104 104 xchr = pycompat.bytechr
105 105 asciistr = list(map(xchr, range(127)))
106 106 capitals = list(range(ord("A"), ord("Z") + 1))
107 107
108 108 cmap = dict((x, x) for x in asciistr)
109 109 for x in _reserved():
110 110 cmap[xchr(x)] = "~%02x" % x
111 111 for x in capitals + [ord(e)]:
112 112 cmap[xchr(x)] = e + xchr(x).lower()
113 113
114 114 dmap = {}
115 115 for k, v in cmap.iteritems():
116 116 dmap[v] = k
117 117 def decode(s):
118 118 i = 0
119 119 while i < len(s):
120 120 for l in xrange(1, 4):
121 121 try:
122 122 yield dmap[s[i:i + l]]
123 123 i += l
124 124 break
125 125 except KeyError:
126 126 pass
127 127 else:
128 128 raise KeyError
129 129 return (lambda s: ''.join([cmap[s[c:c + 1]] for c in xrange(len(s))]),
130 130 lambda s: ''.join(list(decode(s))))
131 131
132 132 _encodefname, _decodefname = _buildencodefun()
133 133
134 134 def encodefilename(s):
135 135 '''
136 136 >>> encodefilename('foo.i/bar.d/bla.hg/hi:world?/HELLO')
137 137 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o'
138 138 '''
139 139 return _encodefname(encodedir(s))
140 140
141 141 def decodefilename(s):
142 142 '''
143 143 >>> decodefilename('foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o')
144 144 'foo.i/bar.d/bla.hg/hi:world?/HELLO'
145 145 '''
146 146 return decodedir(_decodefname(s))
147 147
148 148 def _buildlowerencodefun():
149 149 '''
150 150 >>> f = _buildlowerencodefun()
151 151 >>> f('nothing/special.txt')
152 152 'nothing/special.txt'
153 153 >>> f('HELLO')
154 154 'hello'
155 155 >>> f('hello:world?')
156 156 'hello~3aworld~3f'
157 157 >>> f('the\x07quick\xADshot')
158 158 'the~07quick~adshot'
159 159 '''
160 160 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
161 161 for x in _reserved():
162 162 cmap[chr(x)] = "~%02x" % x
163 163 for x in range(ord("A"), ord("Z") + 1):
164 164 cmap[chr(x)] = chr(x).lower()
165 165 return lambda s: "".join([cmap[c] for c in s])
166 166
167 167 lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun()
168 168
169 169 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9
170 170 _winres3 = ('aux', 'con', 'prn', 'nul') # length 3
171 171 _winres4 = ('com', 'lpt') # length 4 (with trailing 1..9)
172 172 def _auxencode(path, dotencode):
173 173 '''
174 174 Encodes filenames containing names reserved by Windows or which end in
175 175 period or space. Does not touch other single reserved characters c.
176 176 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here.
177 177 Additionally encodes space or period at the beginning, if dotencode is
178 178 True. Parameter path is assumed to be all lowercase.
179 179 A segment only needs encoding if a reserved name appears as a
180 180 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux"
181 181 doesn't need encoding.
182 182
183 183 >>> s = '.foo/aux.txt/txt.aux/con/prn/nul/foo.'
184 184 >>> _auxencode(s.split('/'), True)
185 185 ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e']
186 186 >>> s = '.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.'
187 187 >>> _auxencode(s.split('/'), False)
188 188 ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e']
189 189 >>> _auxencode(['foo. '], True)
190 190 ['foo.~20']
191 191 >>> _auxencode([' .foo'], True)
192 192 ['~20.foo']
193 193 '''
194 194 for i, n in enumerate(path):
195 195 if not n:
196 196 continue
197 197 if dotencode and n[0] in '. ':
198 198 n = "~%02x" % ord(n[0:1]) + n[1:]
199 199 path[i] = n
200 200 else:
201 201 l = n.find('.')
202 202 if l == -1:
203 203 l = len(n)
204 204 if ((l == 3 and n[:3] in _winres3) or
205 205 (l == 4 and n[3:4] <= '9' and n[3:4] >= '1'
206 206 and n[:3] in _winres4)):
207 207 # encode third letter ('aux' -> 'au~78')
208 208 ec = "~%02x" % ord(n[2:3])
209 209 n = n[0:2] + ec + n[3:]
210 210 path[i] = n
211 211 if n[-1] in '. ':
212 212 # encode last period or space ('foo...' -> 'foo..~2e')
213 213 path[i] = n[:-1] + "~%02x" % ord(n[-1:])
214 214 return path
215 215
216 216 _maxstorepathlen = 120
217 217 _dirprefixlen = 8
218 218 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
219 219
220 220 def _hashencode(path, dotencode):
221 221 digest = hashlib.sha1(path).hexdigest()
222 222 le = lowerencode(path[5:]).split('/') # skips prefix 'data/' or 'meta/'
223 223 parts = _auxencode(le, dotencode)
224 224 basename = parts[-1]
225 225 _root, ext = os.path.splitext(basename)
226 226 sdirs = []
227 227 sdirslen = 0
228 228 for p in parts[:-1]:
229 229 d = p[:_dirprefixlen]
230 230 if d[-1] in '. ':
231 231 # Windows can't access dirs ending in period or space
232 232 d = d[:-1] + '_'
233 233 if sdirslen == 0:
234 234 t = len(d)
235 235 else:
236 236 t = sdirslen + 1 + len(d)
237 237 if t > _maxshortdirslen:
238 238 break
239 239 sdirs.append(d)
240 240 sdirslen = t
241 241 dirs = '/'.join(sdirs)
242 242 if len(dirs) > 0:
243 243 dirs += '/'
244 244 res = 'dh/' + dirs + digest + ext
245 245 spaceleft = _maxstorepathlen - len(res)
246 246 if spaceleft > 0:
247 247 filler = basename[:spaceleft]
248 248 res = 'dh/' + dirs + filler + digest + ext
249 249 return res
250 250
251 251 def _hybridencode(path, dotencode):
252 252 '''encodes path with a length limit
253 253
254 254 Encodes all paths that begin with 'data/', according to the following.
255 255
256 256 Default encoding (reversible):
257 257
258 258 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
259 259 characters are encoded as '~xx', where xx is the two digit hex code
260 260 of the character (see encodefilename).
261 261 Relevant path components consisting of Windows reserved filenames are
262 262 masked by encoding the third character ('aux' -> 'au~78', see _auxencode).
263 263
264 264 Hashed encoding (not reversible):
265 265
266 266 If the default-encoded path is longer than _maxstorepathlen, a
267 267 non-reversible hybrid hashing of the path is done instead.
268 268 This encoding uses up to _dirprefixlen characters of all directory
269 269 levels of the lowerencoded path, but not more levels than can fit into
270 270 _maxshortdirslen.
271 271 Then follows the filler followed by the sha digest of the full path.
272 272 The filler is the beginning of the basename of the lowerencoded path
273 273 (the basename is everything after the last path separator). The filler
274 274 is as long as possible, filling in characters from the basename until
275 275 the encoded path has _maxstorepathlen characters (or all chars of the
276 276 basename have been taken).
277 277 The extension (e.g. '.i' or '.d') is preserved.
278 278
279 279 The string 'data/' at the beginning is replaced with 'dh/', if the hashed
280 280 encoding was used.
281 281 '''
282 282 path = encodedir(path)
283 283 ef = _encodefname(path).split('/')
284 284 res = '/'.join(_auxencode(ef, dotencode))
285 285 if len(res) > _maxstorepathlen:
286 286 res = _hashencode(path, dotencode)
287 287 return res
288 288
289 289 def _pathencode(path):
290 290 de = encodedir(path)
291 291 if len(path) > _maxstorepathlen:
292 292 return _hashencode(de, True)
293 293 ef = _encodefname(de).split('/')
294 294 res = '/'.join(_auxencode(ef, True))
295 295 if len(res) > _maxstorepathlen:
296 296 return _hashencode(de, True)
297 297 return res
298 298
299 299 _pathencode = getattr(parsers, 'pathencode', _pathencode)
300 300
301 301 def _plainhybridencode(f):
302 302 return _hybridencode(f, False)
303 303
304 304 def _calcmode(vfs):
305 305 try:
306 306 # files in .hg/ will be created using this mode
307 307 mode = vfs.stat().st_mode
308 308 # avoid some useless chmods
309 309 if (0o777 & ~util.umask) == (0o777 & mode):
310 310 mode = None
311 311 except OSError:
312 312 mode = None
313 313 return mode
314 314
315 315 _data = ('data meta 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
316 316 ' phaseroots obsstore')
317 317
318 318 class basicstore(object):
319 319 '''base class for local repository stores'''
320 320 def __init__(self, path, vfstype):
321 321 vfs = vfstype(path)
322 322 self.path = vfs.base
323 323 self.createmode = _calcmode(vfs)
324 324 vfs.createmode = self.createmode
325 325 self.rawvfs = vfs
326 326 self.vfs = vfsmod.filtervfs(vfs, encodedir)
327 327 self.opener = self.vfs
328 328
329 329 def join(self, f):
330 330 return self.path + '/' + encodedir(f)
331 331
332 332 def _walk(self, relpath, recurse):
333 333 '''yields (unencoded, encoded, size)'''
334 334 path = self.path
335 335 if relpath:
336 336 path += '/' + relpath
337 337 striplen = len(self.path) + 1
338 338 l = []
339 339 if self.rawvfs.isdir(path):
340 340 visit = [path]
341 341 readdir = self.rawvfs.readdir
342 342 while visit:
343 343 p = visit.pop()
344 344 for f, kind, st in readdir(p, stat=True):
345 345 fp = p + '/' + f
346 346 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'):
347 347 n = util.pconvert(fp[striplen:])
348 348 l.append((decodedir(n), n, st.st_size))
349 349 elif kind == stat.S_IFDIR and recurse:
350 350 visit.append(fp)
351 351 l.sort()
352 352 return l
353 353
354 354 def datafiles(self):
355 355 return self._walk('data', True) + self._walk('meta', True)
356 356
357 357 def topfiles(self):
358 358 # yield manifest before changelog
359 359 return reversed(self._walk('', False))
360 360
361 361 def walk(self):
362 362 '''yields (unencoded, encoded, size)'''
363 363 # yield data files first
364 364 for x in self.datafiles():
365 365 yield x
366 366 for x in self.topfiles():
367 367 yield x
368 368
369 369 def copylist(self):
370 370 return ['requires'] + _data.split()
371 371
372 372 def write(self, tr):
373 373 pass
374 374
375 375 def invalidatecaches(self):
376 376 pass
377 377
378 378 def markremoved(self, fn):
379 379 pass
380 380
381 381 def __contains__(self, path):
382 382 '''Checks if the store contains path'''
383 383 path = "/".join(("data", path))
384 384 # file?
385 385 if self.vfs.exists(path + ".i"):
386 386 return True
387 387 # dir?
388 388 if not path.endswith("/"):
389 389 path = path + "/"
390 390 return self.vfs.exists(path)
391 391
392 392 class encodedstore(basicstore):
393 393 def __init__(self, path, vfstype):
394 394 vfs = vfstype(path + '/store')
395 395 self.path = vfs.base
396 396 self.createmode = _calcmode(vfs)
397 397 vfs.createmode = self.createmode
398 398 self.rawvfs = vfs
399 399 self.vfs = vfsmod.filtervfs(vfs, encodefilename)
400 400 self.opener = self.vfs
401 401
402 402 def datafiles(self):
403 403 for a, b, size in super(encodedstore, self).datafiles():
404 404 try:
405 405 a = decodefilename(a)
406 406 except KeyError:
407 407 a = None
408 408 yield a, b, size
409 409
410 410 def join(self, f):
411 411 return self.path + '/' + encodefilename(f)
412 412
413 413 def copylist(self):
414 414 return (['requires', '00changelog.i'] +
415 415 ['store/' + f for f in _data.split()])
416 416
417 417 class fncache(object):
418 418 # the filename used to be partially encoded
419 419 # hence the encodedir/decodedir dance
420 420 def __init__(self, vfs):
421 421 self.vfs = vfs
422 422 self.entries = None
423 423 self._dirty = False
424 424
425 425 def _load(self):
426 426 '''fill the entries from the fncache file'''
427 427 self._dirty = False
428 428 try:
429 429 fp = self.vfs('fncache', mode='rb')
430 430 except IOError:
431 431 # skip nonexistent file
432 432 self.entries = set()
433 433 return
434 434 self.entries = set(decodedir(fp.read()).splitlines())
435 435 if '' in self.entries:
436 436 fp.seek(0)
437 437 for n, line in enumerate(util.iterfile(fp)):
438 438 if not line.rstrip('\n'):
439 439 t = _('invalid entry in fncache, line %d') % (n + 1)
440 440 raise error.Abort(t)
441 441 fp.close()
442 442
443 443 def write(self, tr):
444 444 if self._dirty:
445 445 tr.addbackup('fncache')
446 446 fp = self.vfs('fncache', mode='wb', atomictemp=True)
447 447 if self.entries:
448 448 fp.write(encodedir('\n'.join(self.entries) + '\n'))
449 449 fp.close()
450 450 self._dirty = False
451 451
452 452 def add(self, fn):
453 453 if self.entries is None:
454 454 self._load()
455 455 if fn not in self.entries:
456 456 self._dirty = True
457 457 self.entries.add(fn)
458 458
459 459 def remove(self, fn):
460 460 if self.entries is None:
461 461 self._load()
462 462 try:
463 463 self.entries.remove(fn)
464 464 self._dirty = True
465 465 except KeyError:
466 466 pass
467 467
468 468 def __contains__(self, fn):
469 469 if self.entries is None:
470 470 self._load()
471 471 return fn in self.entries
472 472
473 473 def __iter__(self):
474 474 if self.entries is None:
475 475 self._load()
476 476 return iter(self.entries)
477 477
478 class _fncachevfs(vfsmod.abstractvfs, vfsmod.auditvfs):
478 class _fncachevfs(vfsmod.abstractvfs, vfsmod.proxyvfs):
479 479 def __init__(self, vfs, fnc, encode):
480 vfsmod.auditvfs.__init__(self, vfs)
480 vfsmod.proxyvfs.__init__(self, vfs)
481 481 self.fncache = fnc
482 482 self.encode = encode
483 483
484 484 def __call__(self, path, mode='r', *args, **kw):
485 485 if mode not in ('r', 'rb') and (path.startswith('data/') or
486 486 path.startswith('meta/')):
487 487 self.fncache.add(path)
488 488 return self.vfs(self.encode(path), mode, *args, **kw)
489 489
490 490 def join(self, path):
491 491 if path:
492 492 return self.vfs.join(self.encode(path))
493 493 else:
494 494 return self.vfs.join(path)
495 495
496 496 class fncachestore(basicstore):
497 497 def __init__(self, path, vfstype, dotencode):
498 498 if dotencode:
499 499 encode = _pathencode
500 500 else:
501 501 encode = _plainhybridencode
502 502 self.encode = encode
503 503 vfs = vfstype(path + '/store')
504 504 self.path = vfs.base
505 505 self.pathsep = self.path + '/'
506 506 self.createmode = _calcmode(vfs)
507 507 vfs.createmode = self.createmode
508 508 self.rawvfs = vfs
509 509 fnc = fncache(vfs)
510 510 self.fncache = fnc
511 511 self.vfs = _fncachevfs(vfs, fnc, encode)
512 512 self.opener = self.vfs
513 513
514 514 def join(self, f):
515 515 return self.pathsep + self.encode(f)
516 516
517 517 def getsize(self, path):
518 518 return self.rawvfs.stat(path).st_size
519 519
520 520 def datafiles(self):
521 521 for f in sorted(self.fncache):
522 522 ef = self.encode(f)
523 523 try:
524 524 yield f, ef, self.getsize(ef)
525 525 except OSError as err:
526 526 if err.errno != errno.ENOENT:
527 527 raise
528 528
529 529 def copylist(self):
530 530 d = ('data meta dh fncache phaseroots obsstore'
531 531 ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i')
532 532 return (['requires', '00changelog.i'] +
533 533 ['store/' + f for f in d.split()])
534 534
535 535 def write(self, tr):
536 536 self.fncache.write(tr)
537 537
538 538 def invalidatecaches(self):
539 539 self.fncache.entries = None
540 540
541 541 def markremoved(self, fn):
542 542 self.fncache.remove(fn)
543 543
544 544 def _exists(self, f):
545 545 ef = self.encode(f)
546 546 try:
547 547 self.getsize(ef)
548 548 return True
549 549 except OSError as err:
550 550 if err.errno != errno.ENOENT:
551 551 raise
552 552 # nonexistent entry
553 553 return False
554 554
555 555 def __contains__(self, path):
556 556 '''Checks if the store contains path'''
557 557 path = "/".join(("data", path))
558 558 # check for files (exact match)
559 559 e = path + '.i'
560 560 if e in self.fncache and self._exists(e):
561 561 return True
562 562 # now check for directories (prefix match)
563 563 if not path.endswith('/'):
564 564 path += '/'
565 565 for e in self.fncache:
566 566 if e.startswith(path) and self._exists(e):
567 567 return True
568 568 return False
569 569
570 570 def store(requirements, path, vfstype):
571 571 if 'store' in requirements:
572 572 if 'fncache' in requirements:
573 573 return fncachestore(path, vfstype, 'dotencode' in requirements)
574 574 return encodedstore(path, vfstype)
575 575 return basicstore(path, vfstype)
@@ -1,642 +1,642
1 1 # vfs.py - Mercurial 'vfs' classes
2 2 #
3 3 # Copyright Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7 from __future__ import absolute_import
8 8
9 9 import contextlib
10 10 import errno
11 11 import os
12 12 import shutil
13 13 import stat
14 14 import tempfile
15 15 import threading
16 16
17 17 from .i18n import _
18 18 from . import (
19 19 error,
20 20 pathutil,
21 21 pycompat,
22 22 util,
23 23 )
24 24
25 25 def _avoidambig(path, oldstat):
26 26 """Avoid file stat ambiguity forcibly
27 27
28 28 This function causes copying ``path`` file, if it is owned by
29 29 another (see issue5418 and issue5584 for detail).
30 30 """
31 31 def checkandavoid():
32 32 newstat = util.filestat.frompath(path)
33 33 # return whether file stat ambiguity is (already) avoided
34 34 return (not newstat.isambig(oldstat) or
35 35 newstat.avoidambig(path, oldstat))
36 36 if not checkandavoid():
37 37 # simply copy to change owner of path to get privilege to
38 38 # advance mtime (see issue5418)
39 39 util.rename(util.mktempcopy(path), path)
40 40 checkandavoid()
41 41
42 42 class abstractvfs(object):
43 43 """Abstract base class; cannot be instantiated"""
44 44
45 45 def __init__(self, *args, **kwargs):
46 46 '''Prevent instantiation; don't call this from subclasses.'''
47 47 raise NotImplementedError('attempted instantiating ' + str(type(self)))
48 48
49 49 def tryread(self, path):
50 50 '''gracefully return an empty string for missing files'''
51 51 try:
52 52 return self.read(path)
53 53 except IOError as inst:
54 54 if inst.errno != errno.ENOENT:
55 55 raise
56 56 return ""
57 57
58 58 def tryreadlines(self, path, mode='rb'):
59 59 '''gracefully return an empty array for missing files'''
60 60 try:
61 61 return self.readlines(path, mode=mode)
62 62 except IOError as inst:
63 63 if inst.errno != errno.ENOENT:
64 64 raise
65 65 return []
66 66
67 67 @util.propertycache
68 68 def open(self):
69 69 '''Open ``path`` file, which is relative to vfs root.
70 70
71 71 Newly created directories are marked as "not to be indexed by
72 72 the content indexing service", if ``notindexed`` is specified
73 73 for "write" mode access.
74 74 '''
75 75 return self.__call__
76 76
77 77 def read(self, path):
78 78 with self(path, 'rb') as fp:
79 79 return fp.read()
80 80
81 81 def readlines(self, path, mode='rb'):
82 82 with self(path, mode=mode) as fp:
83 83 return fp.readlines()
84 84
85 85 def write(self, path, data, backgroundclose=False):
86 86 with self(path, 'wb', backgroundclose=backgroundclose) as fp:
87 87 return fp.write(data)
88 88
89 89 def writelines(self, path, data, mode='wb', notindexed=False):
90 90 with self(path, mode=mode, notindexed=notindexed) as fp:
91 91 return fp.writelines(data)
92 92
93 93 def append(self, path, data):
94 94 with self(path, 'ab') as fp:
95 95 return fp.write(data)
96 96
97 97 def basename(self, path):
98 98 """return base element of a path (as os.path.basename would do)
99 99
100 100 This exists to allow handling of strange encoding if needed."""
101 101 return os.path.basename(path)
102 102
103 103 def chmod(self, path, mode):
104 104 return os.chmod(self.join(path), mode)
105 105
106 106 def dirname(self, path):
107 107 """return dirname element of a path (as os.path.dirname would do)
108 108
109 109 This exists to allow handling of strange encoding if needed."""
110 110 return os.path.dirname(path)
111 111
112 112 def exists(self, path=None):
113 113 return os.path.exists(self.join(path))
114 114
115 115 def fstat(self, fp):
116 116 return util.fstat(fp)
117 117
118 118 def isdir(self, path=None):
119 119 return os.path.isdir(self.join(path))
120 120
121 121 def isfile(self, path=None):
122 122 return os.path.isfile(self.join(path))
123 123
124 124 def islink(self, path=None):
125 125 return os.path.islink(self.join(path))
126 126
127 127 def isfileorlink(self, path=None):
128 128 '''return whether path is a regular file or a symlink
129 129
130 130 Unlike isfile, this doesn't follow symlinks.'''
131 131 try:
132 132 st = self.lstat(path)
133 133 except OSError:
134 134 return False
135 135 mode = st.st_mode
136 136 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
137 137
138 138 def reljoin(self, *paths):
139 139 """join various elements of a path together (as os.path.join would do)
140 140
141 141 The vfs base is not injected so that path stay relative. This exists
142 142 to allow handling of strange encoding if needed."""
143 143 return os.path.join(*paths)
144 144
145 145 def split(self, path):
146 146 """split top-most element of a path (as os.path.split would do)
147 147
148 148 This exists to allow handling of strange encoding if needed."""
149 149 return os.path.split(path)
150 150
151 151 def lexists(self, path=None):
152 152 return os.path.lexists(self.join(path))
153 153
154 154 def lstat(self, path=None):
155 155 return os.lstat(self.join(path))
156 156
157 157 def listdir(self, path=None):
158 158 return os.listdir(self.join(path))
159 159
160 160 def makedir(self, path=None, notindexed=True):
161 161 return util.makedir(self.join(path), notindexed)
162 162
163 163 def makedirs(self, path=None, mode=None):
164 164 return util.makedirs(self.join(path), mode)
165 165
166 166 def makelock(self, info, path):
167 167 return util.makelock(info, self.join(path))
168 168
169 169 def mkdir(self, path=None):
170 170 return os.mkdir(self.join(path))
171 171
172 172 def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
173 173 fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
174 174 dir=self.join(dir), text=text)
175 175 dname, fname = util.split(name)
176 176 if dir:
177 177 return fd, os.path.join(dir, fname)
178 178 else:
179 179 return fd, fname
180 180
181 181 def readdir(self, path=None, stat=None, skip=None):
182 182 return util.listdir(self.join(path), stat, skip)
183 183
184 184 def readlock(self, path):
185 185 return util.readlock(self.join(path))
186 186
187 187 def rename(self, src, dst, checkambig=False):
188 188 """Rename from src to dst
189 189
190 190 checkambig argument is used with util.filestat, and is useful
191 191 only if destination file is guarded by any lock
192 192 (e.g. repo.lock or repo.wlock).
193 193
194 194 To avoid file stat ambiguity forcibly, checkambig=True involves
195 195 copying ``src`` file, if it is owned by another. Therefore, use
196 196 checkambig=True only in limited cases (see also issue5418 and
197 197 issue5584 for detail).
198 198 """
199 199 srcpath = self.join(src)
200 200 dstpath = self.join(dst)
201 201 oldstat = checkambig and util.filestat.frompath(dstpath)
202 202 if oldstat and oldstat.stat:
203 203 ret = util.rename(srcpath, dstpath)
204 204 _avoidambig(dstpath, oldstat)
205 205 return ret
206 206 return util.rename(srcpath, dstpath)
207 207
208 208 def readlink(self, path):
209 209 return os.readlink(self.join(path))
210 210
211 211 def removedirs(self, path=None):
212 212 """Remove a leaf directory and all empty intermediate ones
213 213 """
214 214 return util.removedirs(self.join(path))
215 215
216 216 def rmtree(self, path=None, ignore_errors=False, forcibly=False):
217 217 """Remove a directory tree recursively
218 218
219 219 If ``forcibly``, this tries to remove READ-ONLY files, too.
220 220 """
221 221 if forcibly:
222 222 def onerror(function, path, excinfo):
223 223 if function is not os.remove:
224 224 raise
225 225 # read-only files cannot be unlinked under Windows
226 226 s = os.stat(path)
227 227 if (s.st_mode & stat.S_IWRITE) != 0:
228 228 raise
229 229 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
230 230 os.remove(path)
231 231 else:
232 232 onerror = None
233 233 return shutil.rmtree(self.join(path),
234 234 ignore_errors=ignore_errors, onerror=onerror)
235 235
236 236 def setflags(self, path, l, x):
237 237 return util.setflags(self.join(path), l, x)
238 238
239 239 def stat(self, path=None):
240 240 return os.stat(self.join(path))
241 241
242 242 def unlink(self, path=None):
243 243 return util.unlink(self.join(path))
244 244
245 245 def tryunlink(self, path=None):
246 246 """Attempt to remove a file, ignoring missing file errors."""
247 247 util.tryunlink(self.join(path))
248 248
249 249 def unlinkpath(self, path=None, ignoremissing=False):
250 250 return util.unlinkpath(self.join(path), ignoremissing=ignoremissing)
251 251
252 252 def utime(self, path=None, t=None):
253 253 return os.utime(self.join(path), t)
254 254
255 255 def walk(self, path=None, onerror=None):
256 256 """Yield (dirpath, dirs, files) tuple for each directories under path
257 257
258 258 ``dirpath`` is relative one from the root of this vfs. This
259 259 uses ``os.sep`` as path separator, even you specify POSIX
260 260 style ``path``.
261 261
262 262 "The root of this vfs" is represented as empty ``dirpath``.
263 263 """
264 264 root = os.path.normpath(self.join(None))
265 265 # when dirpath == root, dirpath[prefixlen:] becomes empty
266 266 # because len(dirpath) < prefixlen.
267 267 prefixlen = len(pathutil.normasprefix(root))
268 268 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
269 269 yield (dirpath[prefixlen:], dirs, files)
270 270
271 271 @contextlib.contextmanager
272 272 def backgroundclosing(self, ui, expectedcount=-1):
273 273 """Allow files to be closed asynchronously.
274 274
275 275 When this context manager is active, ``backgroundclose`` can be passed
276 276 to ``__call__``/``open`` to result in the file possibly being closed
277 277 asynchronously, on a background thread.
278 278 """
279 279 # This is an arbitrary restriction and could be changed if we ever
280 280 # have a use case.
281 281 vfs = getattr(self, 'vfs', self)
282 282 if getattr(vfs, '_backgroundfilecloser', None):
283 283 raise error.Abort(
284 284 _('can only have 1 active background file closer'))
285 285
286 286 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
287 287 try:
288 288 vfs._backgroundfilecloser = bfc
289 289 yield bfc
290 290 finally:
291 291 vfs._backgroundfilecloser = None
292 292
293 293 class vfs(abstractvfs):
294 294 '''Operate files relative to a base directory
295 295
296 296 This class is used to hide the details of COW semantics and
297 297 remote file access from higher level code.
298 298 '''
299 299 def __init__(self, base, audit=True, expandpath=False, realpath=False):
300 300 if expandpath:
301 301 base = util.expandpath(base)
302 302 if realpath:
303 303 base = os.path.realpath(base)
304 304 self.base = base
305 305 self._audit = audit
306 306 if audit:
307 307 self.audit = pathutil.pathauditor(self.base)
308 308 else:
309 309 self.audit = util.always
310 310 self.createmode = None
311 311 self._trustnlink = None
312 312
313 313 @util.propertycache
314 314 def _cansymlink(self):
315 315 return util.checklink(self.base)
316 316
317 317 @util.propertycache
318 318 def _chmod(self):
319 319 return util.checkexec(self.base)
320 320
321 321 def _fixfilemode(self, name):
322 322 if self.createmode is None or not self._chmod:
323 323 return
324 324 os.chmod(name, self.createmode & 0o666)
325 325
326 326 def __call__(self, path, mode="r", text=False, atomictemp=False,
327 327 notindexed=False, backgroundclose=False, checkambig=False,
328 328 auditpath=True):
329 329 '''Open ``path`` file, which is relative to vfs root.
330 330
331 331 Newly created directories are marked as "not to be indexed by
332 332 the content indexing service", if ``notindexed`` is specified
333 333 for "write" mode access.
334 334
335 335 If ``backgroundclose`` is passed, the file may be closed asynchronously.
336 336 It can only be used if the ``self.backgroundclosing()`` context manager
337 337 is active. This should only be specified if the following criteria hold:
338 338
339 339 1. There is a potential for writing thousands of files. Unless you
340 340 are writing thousands of files, the performance benefits of
341 341 asynchronously closing files is not realized.
342 342 2. Files are opened exactly once for the ``backgroundclosing``
343 343 active duration and are therefore free of race conditions between
344 344 closing a file on a background thread and reopening it. (If the
345 345 file were opened multiple times, there could be unflushed data
346 346 because the original file handle hasn't been flushed/closed yet.)
347 347
348 348 ``checkambig`` argument is passed to atomictemplfile (valid
349 349 only for writing), and is useful only if target file is
350 350 guarded by any lock (e.g. repo.lock or repo.wlock).
351 351
352 352 To avoid file stat ambiguity forcibly, checkambig=True involves
353 353 copying ``path`` file opened in "append" mode (e.g. for
354 354 truncation), if it is owned by another. Therefore, use
355 355 combination of append mode and checkambig=True only in limited
356 356 cases (see also issue5418 and issue5584 for detail).
357 357 '''
358 358 if auditpath:
359 359 if self._audit:
360 360 r = util.checkosfilename(path)
361 361 if r:
362 362 raise error.Abort("%s: %r" % (r, path))
363 363 self.audit(path)
364 364 f = self.join(path)
365 365
366 366 if not text and "b" not in mode:
367 367 mode += "b" # for that other OS
368 368
369 369 nlink = -1
370 370 if mode not in ('r', 'rb'):
371 371 dirname, basename = util.split(f)
372 372 # If basename is empty, then the path is malformed because it points
373 373 # to a directory. Let the posixfile() call below raise IOError.
374 374 if basename:
375 375 if atomictemp:
376 376 util.makedirs(dirname, self.createmode, notindexed)
377 377 return util.atomictempfile(f, mode, self.createmode,
378 378 checkambig=checkambig)
379 379 try:
380 380 if 'w' in mode:
381 381 util.unlink(f)
382 382 nlink = 0
383 383 else:
384 384 # nlinks() may behave differently for files on Windows
385 385 # shares if the file is open.
386 386 with util.posixfile(f):
387 387 nlink = util.nlinks(f)
388 388 if nlink < 1:
389 389 nlink = 2 # force mktempcopy (issue1922)
390 390 except (OSError, IOError) as e:
391 391 if e.errno != errno.ENOENT:
392 392 raise
393 393 nlink = 0
394 394 util.makedirs(dirname, self.createmode, notindexed)
395 395 if nlink > 0:
396 396 if self._trustnlink is None:
397 397 self._trustnlink = nlink > 1 or util.checknlink(f)
398 398 if nlink > 1 or not self._trustnlink:
399 399 util.rename(util.mktempcopy(f), f)
400 400 fp = util.posixfile(f, mode)
401 401 if nlink == 0:
402 402 self._fixfilemode(f)
403 403
404 404 if checkambig:
405 405 if mode in ('r', 'rb'):
406 406 raise error.Abort(_('implementation error: mode %s is not'
407 407 ' valid for checkambig=True') % mode)
408 408 fp = checkambigatclosing(fp)
409 409
410 410 if backgroundclose:
411 411 if not self._backgroundfilecloser:
412 412 raise error.Abort(_('backgroundclose can only be used when a '
413 413 'backgroundclosing context manager is active')
414 414 )
415 415
416 416 fp = delayclosedfile(fp, self._backgroundfilecloser)
417 417
418 418 return fp
419 419
420 420 def symlink(self, src, dst):
421 421 self.audit(dst)
422 422 linkname = self.join(dst)
423 423 util.tryunlink(linkname)
424 424
425 425 util.makedirs(os.path.dirname(linkname), self.createmode)
426 426
427 427 if self._cansymlink:
428 428 try:
429 429 os.symlink(src, linkname)
430 430 except OSError as err:
431 431 raise OSError(err.errno, _('could not symlink to %r: %s') %
432 432 (src, err.strerror), linkname)
433 433 else:
434 434 self.write(dst, src)
435 435
436 436 def join(self, path, *insidef):
437 437 if path:
438 438 return os.path.join(self.base, path, *insidef)
439 439 else:
440 440 return self.base
441 441
442 442 opener = vfs
443 443
444 class auditvfs(object):
444 class proxyvfs(object):
445 445 def __init__(self, vfs):
446 446 self.vfs = vfs
447 447
448 448 @property
449 449 def options(self):
450 450 return self.vfs.options
451 451
452 452 @options.setter
453 453 def options(self, value):
454 454 self.vfs.options = value
455 455
456 class filtervfs(abstractvfs, auditvfs):
456 class filtervfs(abstractvfs, proxyvfs):
457 457 '''Wrapper vfs for filtering filenames with a function.'''
458 458
459 459 def __init__(self, vfs, filter):
460 auditvfs.__init__(self, vfs)
460 proxyvfs.__init__(self, vfs)
461 461 self._filter = filter
462 462
463 463 def __call__(self, path, *args, **kwargs):
464 464 return self.vfs(self._filter(path), *args, **kwargs)
465 465
466 466 def join(self, path, *insidef):
467 467 if path:
468 468 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
469 469 else:
470 470 return self.vfs.join(path)
471 471
472 472 filteropener = filtervfs
473 473
474 class readonlyvfs(abstractvfs, auditvfs):
474 class readonlyvfs(abstractvfs, proxyvfs):
475 475 '''Wrapper vfs preventing any writing.'''
476 476
477 477 def __init__(self, vfs):
478 auditvfs.__init__(self, vfs)
478 proxyvfs.__init__(self, vfs)
479 479
480 480 def __call__(self, path, mode='r', *args, **kw):
481 481 if mode not in ('r', 'rb'):
482 482 raise error.Abort(_('this vfs is read only'))
483 483 return self.vfs(path, mode, *args, **kw)
484 484
485 485 def join(self, path, *insidef):
486 486 return self.vfs.join(path, *insidef)
487 487
488 488 class closewrapbase(object):
489 489 """Base class of wrapper, which hooks closing
490 490
491 491 Do not instantiate outside of the vfs layer.
492 492 """
493 493 def __init__(self, fh):
494 494 object.__setattr__(self, r'_origfh', fh)
495 495
496 496 def __getattr__(self, attr):
497 497 return getattr(self._origfh, attr)
498 498
499 499 def __setattr__(self, attr, value):
500 500 return setattr(self._origfh, attr, value)
501 501
502 502 def __delattr__(self, attr):
503 503 return delattr(self._origfh, attr)
504 504
505 505 def __enter__(self):
506 506 return self._origfh.__enter__()
507 507
508 508 def __exit__(self, exc_type, exc_value, exc_tb):
509 509 raise NotImplementedError('attempted instantiating ' + str(type(self)))
510 510
511 511 def close(self):
512 512 raise NotImplementedError('attempted instantiating ' + str(type(self)))
513 513
514 514 class delayclosedfile(closewrapbase):
515 515 """Proxy for a file object whose close is delayed.
516 516
517 517 Do not instantiate outside of the vfs layer.
518 518 """
519 519 def __init__(self, fh, closer):
520 520 super(delayclosedfile, self).__init__(fh)
521 521 object.__setattr__(self, r'_closer', closer)
522 522
523 523 def __exit__(self, exc_type, exc_value, exc_tb):
524 524 self._closer.close(self._origfh)
525 525
526 526 def close(self):
527 527 self._closer.close(self._origfh)
528 528
529 529 class backgroundfilecloser(object):
530 530 """Coordinates background closing of file handles on multiple threads."""
531 531 def __init__(self, ui, expectedcount=-1):
532 532 self._running = False
533 533 self._entered = False
534 534 self._threads = []
535 535 self._threadexception = None
536 536
537 537 # Only Windows/NTFS has slow file closing. So only enable by default
538 538 # on that platform. But allow to be enabled elsewhere for testing.
539 539 defaultenabled = pycompat.osname == 'nt'
540 540 enabled = ui.configbool('worker', 'backgroundclose', defaultenabled)
541 541
542 542 if not enabled:
543 543 return
544 544
545 545 # There is overhead to starting and stopping the background threads.
546 546 # Don't do background processing unless the file count is large enough
547 547 # to justify it.
548 548 minfilecount = ui.configint('worker', 'backgroundcloseminfilecount')
549 549 # FUTURE dynamically start background threads after minfilecount closes.
550 550 # (We don't currently have any callers that don't know their file count)
551 551 if expectedcount > 0 and expectedcount < minfilecount:
552 552 return
553 553
554 554 maxqueue = ui.configint('worker', 'backgroundclosemaxqueue')
555 555 threadcount = ui.configint('worker', 'backgroundclosethreadcount')
556 556
557 557 ui.debug('starting %d threads for background file closing\n' %
558 558 threadcount)
559 559
560 560 self._queue = util.queue(maxsize=maxqueue)
561 561 self._running = True
562 562
563 563 for i in range(threadcount):
564 564 t = threading.Thread(target=self._worker, name='backgroundcloser')
565 565 self._threads.append(t)
566 566 t.start()
567 567
568 568 def __enter__(self):
569 569 self._entered = True
570 570 return self
571 571
572 572 def __exit__(self, exc_type, exc_value, exc_tb):
573 573 self._running = False
574 574
575 575 # Wait for threads to finish closing so open files don't linger for
576 576 # longer than lifetime of context manager.
577 577 for t in self._threads:
578 578 t.join()
579 579
580 580 def _worker(self):
581 581 """Main routine for worker thread."""
582 582 while True:
583 583 try:
584 584 fh = self._queue.get(block=True, timeout=0.100)
585 585 # Need to catch or the thread will terminate and
586 586 # we could orphan file descriptors.
587 587 try:
588 588 fh.close()
589 589 except Exception as e:
590 590 # Stash so can re-raise from main thread later.
591 591 self._threadexception = e
592 592 except util.empty:
593 593 if not self._running:
594 594 break
595 595
596 596 def close(self, fh):
597 597 """Schedule a file for closing."""
598 598 if not self._entered:
599 599 raise error.Abort(_('can only call close() when context manager '
600 600 'active'))
601 601
602 602 # If a background thread encountered an exception, raise now so we fail
603 603 # fast. Otherwise we may potentially go on for minutes until the error
604 604 # is acted on.
605 605 if self._threadexception:
606 606 e = self._threadexception
607 607 self._threadexception = None
608 608 raise e
609 609
610 610 # If we're not actively running, close synchronously.
611 611 if not self._running:
612 612 fh.close()
613 613 return
614 614
615 615 self._queue.put(fh, block=True, timeout=None)
616 616
617 617 class checkambigatclosing(closewrapbase):
618 618 """Proxy for a file object, to avoid ambiguity of file stat
619 619
620 620 See also util.filestat for detail about "ambiguity of file stat".
621 621
622 622 This proxy is useful only if the target file is guarded by any
623 623 lock (e.g. repo.lock or repo.wlock)
624 624
625 625 Do not instantiate outside of the vfs layer.
626 626 """
627 627 def __init__(self, fh):
628 628 super(checkambigatclosing, self).__init__(fh)
629 629 object.__setattr__(self, r'_oldstat', util.filestat.frompath(fh.name))
630 630
631 631 def _checkambig(self):
632 632 oldstat = self._oldstat
633 633 if oldstat.stat:
634 634 _avoidambig(self._origfh.name, oldstat)
635 635
636 636 def __exit__(self, exc_type, exc_value, exc_tb):
637 637 self._origfh.__exit__(exc_type, exc_value, exc_tb)
638 638 self._checkambig()
639 639
640 640 def close(self):
641 641 self._origfh.close()
642 642 self._checkambig()
General Comments 0
You need to be logged in to leave comments. Login now