##// END OF EJS Templates
vfsproxy: inherit the `createmode` attribute too...
marmoute -
r51347:d1d458fb stable
parent child Browse files
Show More
@@ -1,813 +1,817 b''
1 1 # vfs.py - Mercurial 'vfs' classes
2 2 #
3 3 # Copyright Olivia Mackall <olivia@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 import contextlib
9 9 import os
10 10 import shutil
11 11 import stat
12 12 import threading
13 13
14 14 from typing import (
15 15 Optional,
16 16 )
17 17
18 18 from .i18n import _
19 19 from .pycompat import (
20 20 delattr,
21 21 getattr,
22 22 setattr,
23 23 )
24 24 from . import (
25 25 encoding,
26 26 error,
27 27 pathutil,
28 28 pycompat,
29 29 util,
30 30 )
31 31
32 32
33 33 def _avoidambig(path: bytes, oldstat):
34 34 """Avoid file stat ambiguity forcibly
35 35
36 36 This function causes copying ``path`` file, if it is owned by
37 37 another (see issue5418 and issue5584 for detail).
38 38 """
39 39
40 40 def checkandavoid():
41 41 newstat = util.filestat.frompath(path)
42 42 # return whether file stat ambiguity is (already) avoided
43 43 return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat)
44 44
45 45 if not checkandavoid():
46 46 # simply copy to change owner of path to get privilege to
47 47 # advance mtime (see issue5418)
48 48 util.rename(util.mktempcopy(path), path)
49 49 checkandavoid()
50 50
51 51
52 52 class abstractvfs:
53 53 """Abstract base class; cannot be instantiated"""
54 54
55 55 # default directory separator for vfs
56 56 #
57 57 # Other vfs code always use `/` and this works fine because python file API
58 58 # abstract the use of `/` and make it work transparently. For consistency
59 59 # vfs will always use `/` when joining. This avoid some confusion in
60 60 # encoded vfs (see issue6546)
61 61 _dir_sep = b'/'
62 62
63 63 def __init__(self, *args, **kwargs):
64 64 '''Prevent instantiation; don't call this from subclasses.'''
65 65 raise NotImplementedError('attempted instantiating ' + str(type(self)))
66 66
67 67 # TODO: type return, which is util.posixfile wrapped by a proxy
68 68 def __call__(self, path: bytes, mode: bytes = b'rb', **kwargs):
69 69 raise NotImplementedError
70 70
71 71 def _auditpath(self, path: bytes, mode: bytes):
72 72 raise NotImplementedError
73 73
74 74 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
75 75 raise NotImplementedError
76 76
77 77 def tryread(self, path: bytes) -> bytes:
78 78 '''gracefully return an empty string for missing files'''
79 79 try:
80 80 return self.read(path)
81 81 except FileNotFoundError:
82 82 pass
83 83 return b""
84 84
85 85 def tryreadlines(self, path: bytes, mode: bytes = b'rb'):
86 86 '''gracefully return an empty array for missing files'''
87 87 try:
88 88 return self.readlines(path, mode=mode)
89 89 except FileNotFoundError:
90 90 pass
91 91 return []
92 92
93 93 @util.propertycache
94 94 def open(self):
95 95 """Open ``path`` file, which is relative to vfs root.
96 96
97 97 Newly created directories are marked as "not to be indexed by
98 98 the content indexing service", if ``notindexed`` is specified
99 99 for "write" mode access.
100 100 """
101 101 return self.__call__
102 102
103 103 def read(self, path: bytes) -> bytes:
104 104 with self(path, b'rb') as fp:
105 105 return fp.read()
106 106
107 107 def readlines(self, path: bytes, mode: bytes = b'rb'):
108 108 with self(path, mode=mode) as fp:
109 109 return fp.readlines()
110 110
111 111 def write(
112 112 self, path: bytes, data: bytes, backgroundclose=False, **kwargs
113 113 ) -> int:
114 114 with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp:
115 115 return fp.write(data)
116 116
117 117 def writelines(
118 118 self, path: bytes, data: bytes, mode: bytes = b'wb', notindexed=False
119 119 ) -> None:
120 120 with self(path, mode=mode, notindexed=notindexed) as fp:
121 121 return fp.writelines(data)
122 122
123 123 def append(self, path: bytes, data: bytes) -> int:
124 124 with self(path, b'ab') as fp:
125 125 return fp.write(data)
126 126
127 127 def basename(self, path: bytes) -> bytes:
128 128 """return base element of a path (as os.path.basename would do)
129 129
130 130 This exists to allow handling of strange encoding if needed."""
131 131 return os.path.basename(path)
132 132
133 133 def chmod(self, path: bytes, mode: int) -> None:
134 134 return os.chmod(self.join(path), mode)
135 135
136 136 def dirname(self, path: bytes) -> bytes:
137 137 """return dirname element of a path (as os.path.dirname would do)
138 138
139 139 This exists to allow handling of strange encoding if needed."""
140 140 return os.path.dirname(path)
141 141
142 142 def exists(self, path: Optional[bytes] = None) -> bool:
143 143 return os.path.exists(self.join(path))
144 144
145 145 def fstat(self, fp):
146 146 return util.fstat(fp)
147 147
148 148 def isdir(self, path: Optional[bytes] = None) -> bool:
149 149 return os.path.isdir(self.join(path))
150 150
151 151 def isfile(self, path: Optional[bytes] = None) -> bool:
152 152 return os.path.isfile(self.join(path))
153 153
154 154 def islink(self, path: Optional[bytes] = None) -> bool:
155 155 return os.path.islink(self.join(path))
156 156
157 157 def isfileorlink(self, path: Optional[bytes] = None) -> bool:
158 158 """return whether path is a regular file or a symlink
159 159
160 160 Unlike isfile, this doesn't follow symlinks."""
161 161 try:
162 162 st = self.lstat(path)
163 163 except OSError:
164 164 return False
165 165 mode = st.st_mode
166 166 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
167 167
168 168 def _join(self, *paths: bytes) -> bytes:
169 169 root_idx = 0
170 170 for idx, p in enumerate(paths):
171 171 if os.path.isabs(p) or p.startswith(self._dir_sep):
172 172 root_idx = idx
173 173 if root_idx != 0:
174 174 paths = paths[root_idx:]
175 175 paths = [p for p in paths if p]
176 176 return self._dir_sep.join(paths)
177 177
178 178 def reljoin(self, *paths: bytes) -> bytes:
179 179 """join various elements of a path together (as os.path.join would do)
180 180
181 181 The vfs base is not injected so that path stay relative. This exists
182 182 to allow handling of strange encoding if needed."""
183 183 return self._join(*paths)
184 184
185 185 def split(self, path: bytes):
186 186 """split top-most element of a path (as os.path.split would do)
187 187
188 188 This exists to allow handling of strange encoding if needed."""
189 189 return os.path.split(path)
190 190
191 191 def lexists(self, path: Optional[bytes] = None) -> bool:
192 192 return os.path.lexists(self.join(path))
193 193
194 194 def lstat(self, path: Optional[bytes] = None):
195 195 return os.lstat(self.join(path))
196 196
197 197 def listdir(self, path: Optional[bytes] = None):
198 198 return os.listdir(self.join(path))
199 199
200 200 def makedir(self, path: Optional[bytes] = None, notindexed=True):
201 201 return util.makedir(self.join(path), notindexed)
202 202
203 203 def makedirs(
204 204 self, path: Optional[bytes] = None, mode: Optional[int] = None
205 205 ):
206 206 return util.makedirs(self.join(path), mode)
207 207
208 208 def makelock(self, info, path: bytes):
209 209 return util.makelock(info, self.join(path))
210 210
211 211 def mkdir(self, path: Optional[bytes] = None):
212 212 return os.mkdir(self.join(path))
213 213
214 214 def mkstemp(
215 215 self,
216 216 suffix: bytes = b'',
217 217 prefix: bytes = b'tmp',
218 218 dir: Optional[bytes] = None,
219 219 ):
220 220 fd, name = pycompat.mkstemp(
221 221 suffix=suffix, prefix=prefix, dir=self.join(dir)
222 222 )
223 223 dname, fname = util.split(name)
224 224 if dir:
225 225 return fd, os.path.join(dir, fname)
226 226 else:
227 227 return fd, fname
228 228
229 229 def readdir(self, path: Optional[bytes] = None, stat=None, skip=None):
230 230 return util.listdir(self.join(path), stat, skip)
231 231
232 232 def readlock(self, path: bytes) -> bytes:
233 233 return util.readlock(self.join(path))
234 234
235 235 def rename(self, src: bytes, dst: bytes, checkambig=False):
236 236 """Rename from src to dst
237 237
238 238 checkambig argument is used with util.filestat, and is useful
239 239 only if destination file is guarded by any lock
240 240 (e.g. repo.lock or repo.wlock).
241 241
242 242 To avoid file stat ambiguity forcibly, checkambig=True involves
243 243 copying ``src`` file, if it is owned by another. Therefore, use
244 244 checkambig=True only in limited cases (see also issue5418 and
245 245 issue5584 for detail).
246 246 """
247 247 self._auditpath(dst, b'w')
248 248 srcpath = self.join(src)
249 249 dstpath = self.join(dst)
250 250 oldstat = checkambig and util.filestat.frompath(dstpath)
251 251 if oldstat and oldstat.stat:
252 252 ret = util.rename(srcpath, dstpath)
253 253 _avoidambig(dstpath, oldstat)
254 254 return ret
255 255 return util.rename(srcpath, dstpath)
256 256
257 257 def readlink(self, path: bytes) -> bytes:
258 258 return util.readlink(self.join(path))
259 259
260 260 def removedirs(self, path: Optional[bytes] = None):
261 261 """Remove a leaf directory and all empty intermediate ones"""
262 262 return util.removedirs(self.join(path))
263 263
264 264 def rmdir(self, path: Optional[bytes] = None):
265 265 """Remove an empty directory."""
266 266 return os.rmdir(self.join(path))
267 267
268 268 def rmtree(
269 269 self, path: Optional[bytes] = None, ignore_errors=False, forcibly=False
270 270 ):
271 271 """Remove a directory tree recursively
272 272
273 273 If ``forcibly``, this tries to remove READ-ONLY files, too.
274 274 """
275 275 if forcibly:
276 276
277 277 def onerror(function, path, excinfo):
278 278 if function is not os.remove:
279 279 raise
280 280 # read-only files cannot be unlinked under Windows
281 281 s = os.stat(path)
282 282 if (s.st_mode & stat.S_IWRITE) != 0:
283 283 raise
284 284 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
285 285 os.remove(path)
286 286
287 287 else:
288 288 onerror = None
289 289 return shutil.rmtree(
290 290 self.join(path), ignore_errors=ignore_errors, onerror=onerror
291 291 )
292 292
293 293 def setflags(self, path: bytes, l: bool, x: bool):
294 294 return util.setflags(self.join(path), l, x)
295 295
296 296 def stat(self, path: Optional[bytes] = None):
297 297 return os.stat(self.join(path))
298 298
299 299 def unlink(self, path: Optional[bytes] = None):
300 300 return util.unlink(self.join(path))
301 301
302 302 def tryunlink(self, path: Optional[bytes] = None):
303 303 """Attempt to remove a file, ignoring missing file errors."""
304 304 util.tryunlink(self.join(path))
305 305
306 306 def unlinkpath(
307 307 self, path: Optional[bytes] = None, ignoremissing=False, rmdir=True
308 308 ):
309 309 return util.unlinkpath(
310 310 self.join(path), ignoremissing=ignoremissing, rmdir=rmdir
311 311 )
312 312
313 313 def utime(self, path: Optional[bytes] = None, t=None):
314 314 return os.utime(self.join(path), t)
315 315
316 316 def walk(self, path: Optional[bytes] = None, onerror=None):
317 317 """Yield (dirpath, dirs, files) tuple for each directories under path
318 318
319 319 ``dirpath`` is relative one from the root of this vfs. This
320 320 uses ``os.sep`` as path separator, even you specify POSIX
321 321 style ``path``.
322 322
323 323 "The root of this vfs" is represented as empty ``dirpath``.
324 324 """
325 325 root = os.path.normpath(self.join(None))
326 326 # when dirpath == root, dirpath[prefixlen:] becomes empty
327 327 # because len(dirpath) < prefixlen.
328 328 prefixlen = len(pathutil.normasprefix(root))
329 329 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
330 330 yield (dirpath[prefixlen:], dirs, files)
331 331
332 332 @contextlib.contextmanager
333 333 def backgroundclosing(self, ui, expectedcount=-1):
334 334 """Allow files to be closed asynchronously.
335 335
336 336 When this context manager is active, ``backgroundclose`` can be passed
337 337 to ``__call__``/``open`` to result in the file possibly being closed
338 338 asynchronously, on a background thread.
339 339 """
340 340 # Sharing backgroundfilecloser between threads is complex and using
341 341 # multiple instances puts us at risk of running out of file descriptors
342 342 # only allow to use backgroundfilecloser when in main thread.
343 343 if not isinstance(
344 344 threading.current_thread(),
345 345 threading._MainThread, # pytype: disable=module-attr
346 346 ):
347 347 yield
348 348 return
349 349 vfs = getattr(self, 'vfs', self)
350 350 if getattr(vfs, '_backgroundfilecloser', None):
351 351 raise error.Abort(
352 352 _(b'can only have 1 active background file closer')
353 353 )
354 354
355 355 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
356 356 try:
357 357 vfs._backgroundfilecloser = (
358 358 bfc # pytype: disable=attribute-error
359 359 )
360 360 yield bfc
361 361 finally:
362 362 vfs._backgroundfilecloser = (
363 363 None # pytype: disable=attribute-error
364 364 )
365 365
366 366 def register_file(self, path):
367 367 """generic hook point to lets fncache steer its stew"""
368 368
369 369
370 370 class vfs(abstractvfs):
371 371 """Operate files relative to a base directory
372 372
373 373 This class is used to hide the details of COW semantics and
374 374 remote file access from higher level code.
375 375
376 376 'cacheaudited' should be enabled only if (a) vfs object is short-lived, or
377 377 (b) the base directory is managed by hg and considered sort-of append-only.
378 378 See pathutil.pathauditor() for details.
379 379 """
380 380
381 381 def __init__(
382 382 self,
383 383 base: bytes,
384 384 audit=True,
385 385 cacheaudited=False,
386 386 expandpath=False,
387 387 realpath=False,
388 388 ):
389 389 if expandpath:
390 390 base = util.expandpath(base)
391 391 if realpath:
392 392 base = os.path.realpath(base)
393 393 self.base = base
394 394 self._audit = audit
395 395 if audit:
396 396 self.audit = pathutil.pathauditor(self.base, cached=cacheaudited)
397 397 else:
398 398 self.audit = lambda path, mode=None: True
399 399 self.createmode = None
400 400 self._trustnlink = None
401 401 self.options = {}
402 402
403 403 @util.propertycache
404 404 def _cansymlink(self) -> bool:
405 405 return util.checklink(self.base)
406 406
407 407 @util.propertycache
408 408 def _chmod(self):
409 409 return util.checkexec(self.base)
410 410
411 411 def _fixfilemode(self, name):
412 412 if self.createmode is None or not self._chmod:
413 413 return
414 414 os.chmod(name, self.createmode & 0o666)
415 415
416 416 def _auditpath(self, path, mode) -> None:
417 417 if self._audit:
418 418 if os.path.isabs(path) and path.startswith(self.base):
419 419 path = os.path.relpath(path, self.base)
420 420 r = util.checkosfilename(path)
421 421 if r:
422 422 raise error.Abort(b"%s: %r" % (r, path))
423 423 self.audit(path, mode=mode)
424 424
425 425 def isfileorlink_checkdir(
426 426 self, dircache, path: Optional[bytes] = None
427 427 ) -> bool:
428 428 """return True if the path is a regular file or a symlink and
429 429 the directories along the path are "normal", that is
430 430 not symlinks or nested hg repositories.
431 431
432 432 Ignores the `_audit` setting, and checks the directories regardless.
433 433 `dircache` is used to cache the directory checks.
434 434 """
435 435 try:
436 436 for prefix in pathutil.finddirs_rev_noroot(util.localpath(path)):
437 437 if prefix in dircache:
438 438 res = dircache[prefix]
439 439 else:
440 440 res = pathutil.pathauditor._checkfs_exists(
441 441 self.base, prefix, path
442 442 )
443 443 dircache[prefix] = res
444 444 if not res:
445 445 return False
446 446 except (OSError, error.Abort):
447 447 return False
448 448 return self.isfileorlink(path)
449 449
450 450 def __call__(
451 451 self,
452 452 path: bytes,
453 453 mode: bytes = b"rb",
454 454 atomictemp=False,
455 455 notindexed=False,
456 456 backgroundclose=False,
457 457 checkambig=False,
458 458 auditpath=True,
459 459 makeparentdirs=True,
460 460 ):
461 461 """Open ``path`` file, which is relative to vfs root.
462 462
463 463 By default, parent directories are created as needed. Newly created
464 464 directories are marked as "not to be indexed by the content indexing
465 465 service", if ``notindexed`` is specified for "write" mode access.
466 466 Set ``makeparentdirs=False`` to not create directories implicitly.
467 467
468 468 If ``backgroundclose`` is passed, the file may be closed asynchronously.
469 469 It can only be used if the ``self.backgroundclosing()`` context manager
470 470 is active. This should only be specified if the following criteria hold:
471 471
472 472 1. There is a potential for writing thousands of files. Unless you
473 473 are writing thousands of files, the performance benefits of
474 474 asynchronously closing files is not realized.
475 475 2. Files are opened exactly once for the ``backgroundclosing``
476 476 active duration and are therefore free of race conditions between
477 477 closing a file on a background thread and reopening it. (If the
478 478 file were opened multiple times, there could be unflushed data
479 479 because the original file handle hasn't been flushed/closed yet.)
480 480
481 481 ``checkambig`` argument is passed to atomictempfile (valid
482 482 only for writing), and is useful only if target file is
483 483 guarded by any lock (e.g. repo.lock or repo.wlock).
484 484
485 485 To avoid file stat ambiguity forcibly, checkambig=True involves
486 486 copying ``path`` file opened in "append" mode (e.g. for
487 487 truncation), if it is owned by another. Therefore, use
488 488 combination of append mode and checkambig=True only in limited
489 489 cases (see also issue5418 and issue5584 for detail).
490 490 """
491 491 if auditpath:
492 492 self._auditpath(path, mode)
493 493 f = self.join(path)
494 494
495 495 if b"b" not in mode:
496 496 mode += b"b" # for that other OS
497 497
498 498 nlink = -1
499 499 if mode not in (b'r', b'rb'):
500 500 dirname, basename = util.split(f)
501 501 # If basename is empty, then the path is malformed because it points
502 502 # to a directory. Let the posixfile() call below raise IOError.
503 503 if basename:
504 504 if atomictemp:
505 505 if makeparentdirs:
506 506 util.makedirs(dirname, self.createmode, notindexed)
507 507 return util.atomictempfile(
508 508 f, mode, self.createmode, checkambig=checkambig
509 509 )
510 510 try:
511 511 if b'w' in mode:
512 512 util.unlink(f)
513 513 nlink = 0
514 514 else:
515 515 # nlinks() may behave differently for files on Windows
516 516 # shares if the file is open.
517 517 with util.posixfile(f):
518 518 nlink = util.nlinks(f)
519 519 if nlink < 1:
520 520 nlink = 2 # force mktempcopy (issue1922)
521 521 except FileNotFoundError:
522 522 nlink = 0
523 523 if makeparentdirs:
524 524 util.makedirs(dirname, self.createmode, notindexed)
525 525 if nlink > 0:
526 526 if self._trustnlink is None:
527 527 self._trustnlink = nlink > 1 or util.checknlink(f)
528 528 if nlink > 1 or not self._trustnlink:
529 529 util.rename(util.mktempcopy(f), f)
530 530 fp = util.posixfile(f, mode)
531 531 if nlink == 0:
532 532 self._fixfilemode(f)
533 533
534 534 if checkambig:
535 535 if mode in (b'r', b'rb'):
536 536 raise error.Abort(
537 537 _(
538 538 b'implementation error: mode %s is not'
539 539 b' valid for checkambig=True'
540 540 )
541 541 % mode
542 542 )
543 543 fp = checkambigatclosing(fp)
544 544
545 545 if backgroundclose and isinstance(
546 546 threading.current_thread(),
547 547 threading._MainThread, # pytype: disable=module-attr
548 548 ):
549 549 if (
550 550 not self._backgroundfilecloser # pytype: disable=attribute-error
551 551 ):
552 552 raise error.Abort(
553 553 _(
554 554 b'backgroundclose can only be used when a '
555 555 b'backgroundclosing context manager is active'
556 556 )
557 557 )
558 558
559 559 fp = delayclosedfile(
560 560 fp,
561 561 self._backgroundfilecloser, # pytype: disable=attribute-error
562 562 )
563 563
564 564 return fp
565 565
566 566 def symlink(self, src: bytes, dst: bytes) -> None:
567 567 self.audit(dst)
568 568 linkname = self.join(dst)
569 569 util.tryunlink(linkname)
570 570
571 571 util.makedirs(os.path.dirname(linkname), self.createmode)
572 572
573 573 if self._cansymlink:
574 574 try:
575 575 os.symlink(src, linkname)
576 576 except OSError as err:
577 577 raise OSError(
578 578 err.errno,
579 579 _(b'could not symlink to %r: %s')
580 580 % (src, encoding.strtolocal(err.strerror)),
581 581 linkname,
582 582 )
583 583 else:
584 584 self.write(dst, src)
585 585
586 586 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
587 587 if path:
588 588 parts = [self.base, path]
589 589 parts.extend(insidef)
590 590 return self._join(*parts)
591 591 else:
592 592 return self.base
593 593
594 594
595 595 opener = vfs
596 596
597 597
598 598 class proxyvfs(abstractvfs):
599 599 def __init__(self, vfs: "vfs"):
600 600 self.vfs = vfs
601 601
602 @property
603 def createmode(self):
604 return self.vfs.createmode
605
602 606 def _auditpath(self, path, mode):
603 607 return self.vfs._auditpath(path, mode)
604 608
605 609 @property
606 610 def options(self):
607 611 return self.vfs.options
608 612
609 613 @options.setter
610 614 def options(self, value):
611 615 self.vfs.options = value
612 616
613 617
614 618 class filtervfs(proxyvfs, abstractvfs):
615 619 '''Wrapper vfs for filtering filenames with a function.'''
616 620
617 621 def __init__(self, vfs: "vfs", filter):
618 622 proxyvfs.__init__(self, vfs)
619 623 self._filter = filter
620 624
621 625 def __call__(self, path: bytes, *args, **kwargs):
622 626 return self.vfs(self._filter(path), *args, **kwargs)
623 627
624 628 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
625 629 if path:
626 630 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
627 631 else:
628 632 return self.vfs.join(path)
629 633
630 634
631 635 filteropener = filtervfs
632 636
633 637
634 638 class readonlyvfs(proxyvfs):
635 639 '''Wrapper vfs preventing any writing.'''
636 640
637 641 def __init__(self, vfs: "vfs"):
638 642 proxyvfs.__init__(self, vfs)
639 643
640 644 def __call__(self, path: bytes, mode: bytes = b'rb', *args, **kw):
641 645 if mode not in (b'r', b'rb'):
642 646 raise error.Abort(_(b'this vfs is read only'))
643 647 return self.vfs(path, mode, *args, **kw)
644 648
645 649 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
646 650 return self.vfs.join(path, *insidef)
647 651
648 652
649 653 class closewrapbase:
650 654 """Base class of wrapper, which hooks closing
651 655
652 656 Do not instantiate outside of the vfs layer.
653 657 """
654 658
655 659 def __init__(self, fh):
656 660 object.__setattr__(self, '_origfh', fh)
657 661
658 662 def __getattr__(self, attr):
659 663 return getattr(self._origfh, attr)
660 664
661 665 def __setattr__(self, attr, value):
662 666 return setattr(self._origfh, attr, value)
663 667
664 668 def __delattr__(self, attr):
665 669 return delattr(self._origfh, attr)
666 670
667 671 def __enter__(self):
668 672 self._origfh.__enter__()
669 673 return self
670 674
671 675 def __exit__(self, exc_type, exc_value, exc_tb):
672 676 raise NotImplementedError('attempted instantiating ' + str(type(self)))
673 677
674 678 def close(self):
675 679 raise NotImplementedError('attempted instantiating ' + str(type(self)))
676 680
677 681
678 682 class delayclosedfile(closewrapbase):
679 683 """Proxy for a file object whose close is delayed.
680 684
681 685 Do not instantiate outside of the vfs layer.
682 686 """
683 687
684 688 def __init__(self, fh, closer):
685 689 super(delayclosedfile, self).__init__(fh)
686 690 object.__setattr__(self, '_closer', closer)
687 691
688 692 def __exit__(self, exc_type, exc_value, exc_tb):
689 693 self._closer.close(self._origfh)
690 694
691 695 def close(self):
692 696 self._closer.close(self._origfh)
693 697
694 698
695 699 class backgroundfilecloser:
696 700 """Coordinates background closing of file handles on multiple threads."""
697 701
698 702 def __init__(self, ui, expectedcount=-1):
699 703 self._running = False
700 704 self._entered = False
701 705 self._threads = []
702 706 self._threadexception = None
703 707
704 708 # Only Windows/NTFS has slow file closing. So only enable by default
705 709 # on that platform. But allow to be enabled elsewhere for testing.
706 710 defaultenabled = pycompat.iswindows
707 711 enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled)
708 712
709 713 if not enabled:
710 714 return
711 715
712 716 # There is overhead to starting and stopping the background threads.
713 717 # Don't do background processing unless the file count is large enough
714 718 # to justify it.
715 719 minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount')
716 720 # FUTURE dynamically start background threads after minfilecount closes.
717 721 # (We don't currently have any callers that don't know their file count)
718 722 if expectedcount > 0 and expectedcount < minfilecount:
719 723 return
720 724
721 725 maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue')
722 726 threadcount = ui.configint(b'worker', b'backgroundclosethreadcount')
723 727
724 728 ui.debug(
725 729 b'starting %d threads for background file closing\n' % threadcount
726 730 )
727 731
728 732 self._queue = pycompat.queue.Queue(maxsize=maxqueue)
729 733 self._running = True
730 734
731 735 for i in range(threadcount):
732 736 t = threading.Thread(target=self._worker, name='backgroundcloser')
733 737 self._threads.append(t)
734 738 t.start()
735 739
736 740 def __enter__(self):
737 741 self._entered = True
738 742 return self
739 743
740 744 def __exit__(self, exc_type, exc_value, exc_tb):
741 745 self._running = False
742 746
743 747 # Wait for threads to finish closing so open files don't linger for
744 748 # longer than lifetime of context manager.
745 749 for t in self._threads:
746 750 t.join()
747 751
748 752 def _worker(self):
749 753 """Main routine for worker thread."""
750 754 while True:
751 755 try:
752 756 fh = self._queue.get(block=True, timeout=0.100)
753 757 # Need to catch or the thread will terminate and
754 758 # we could orphan file descriptors.
755 759 try:
756 760 fh.close()
757 761 except Exception as e:
758 762 # Stash so can re-raise from main thread later.
759 763 self._threadexception = e
760 764 except pycompat.queue.Empty:
761 765 if not self._running:
762 766 break
763 767
764 768 def close(self, fh):
765 769 """Schedule a file for closing."""
766 770 if not self._entered:
767 771 raise error.Abort(
768 772 _(b'can only call close() when context manager active')
769 773 )
770 774
771 775 # If a background thread encountered an exception, raise now so we fail
772 776 # fast. Otherwise we may potentially go on for minutes until the error
773 777 # is acted on.
774 778 if self._threadexception:
775 779 e = self._threadexception
776 780 self._threadexception = None
777 781 raise e
778 782
779 783 # If we're not actively running, close synchronously.
780 784 if not self._running:
781 785 fh.close()
782 786 return
783 787
784 788 self._queue.put(fh, block=True, timeout=None)
785 789
786 790
787 791 class checkambigatclosing(closewrapbase):
788 792 """Proxy for a file object, to avoid ambiguity of file stat
789 793
790 794 See also util.filestat for detail about "ambiguity of file stat".
791 795
792 796 This proxy is useful only if the target file is guarded by any
793 797 lock (e.g. repo.lock or repo.wlock)
794 798
795 799 Do not instantiate outside of the vfs layer.
796 800 """
797 801
798 802 def __init__(self, fh):
799 803 super(checkambigatclosing, self).__init__(fh)
800 804 object.__setattr__(self, '_oldstat', util.filestat.frompath(fh.name))
801 805
802 806 def _checkambig(self):
803 807 oldstat = self._oldstat
804 808 if oldstat.stat:
805 809 _avoidambig(self._origfh.name, oldstat)
806 810
807 811 def __exit__(self, exc_type, exc_value, exc_tb):
808 812 self._origfh.__exit__(exc_type, exc_value, exc_tb)
809 813 self._checkambig()
810 814
811 815 def close(self):
812 816 self._origfh.close()
813 817 self._checkambig()
General Comments 0
You need to be logged in to leave comments. Login now