##// END OF EJS Templates
typing: manually add type annotations to `mercurial/vfs.py`...
Matt Harbison -
r52785:fa9e8a65 default
parent child Browse files
Show More
@@ -1,877 +1,908 b''
1 1 # vfs.py - Mercurial 'vfs' classes
2 2 #
3 3 # Copyright Olivia Mackall <olivia@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import annotations
9 9
10 10 import abc
11 11 import contextlib
12 12 import os
13 13 import shutil
14 14 import stat
15 15 import threading
16 16 import typing
17 17
18 18 from typing import (
19 19 Any,
20 BinaryIO,
21 Callable,
20 22 Iterable,
21 23 Iterator,
22 24 List,
25 MutableMapping,
23 26 Optional,
24 27 Tuple,
25 28 Type,
26 29 TypeVar,
27 30 )
28 31
29 32 from .i18n import _
30 33 from . import (
31 34 encoding,
32 35 error,
33 36 pathutil,
34 37 pycompat,
35 38 util,
36 39 )
37 40
38 41 if typing.TYPE_CHECKING:
42 from . import (
43 ui as uimod,
44 )
45
39 46 _Tbackgroundfilecloser = TypeVar(
40 47 '_Tbackgroundfilecloser', bound='backgroundfilecloser'
41 48 )
42 49 _Tclosewrapbase = TypeVar('_Tclosewrapbase', bound='closewrapbase')
50 _OnErrorFn = Callable[[Exception], Optional[object]]
43 51
44 52
45 def _avoidambig(path: bytes, oldstat) -> None:
53 def _avoidambig(path: bytes, oldstat: util.filestat) -> None:
46 54 """Avoid file stat ambiguity forcibly
47 55
48 56 This function causes copying ``path`` file, if it is owned by
49 57 another (see issue5418 and issue5584 for detail).
50 58 """
51 59
52 60 def checkandavoid():
53 61 newstat = util.filestat.frompath(path)
54 62 # return whether file stat ambiguity is (already) avoided
55 63 return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat)
56 64
57 65 if not checkandavoid():
58 66 # simply copy to change owner of path to get privilege to
59 67 # advance mtime (see issue5418)
60 68 util.rename(util.mktempcopy(path), path)
61 69 checkandavoid()
62 70
63 71
64 72 class abstractvfs(abc.ABC):
65 73 """Abstract base class; cannot be instantiated"""
66 74
67 75 # default directory separator for vfs
68 76 #
69 77 # Other vfs code always use `/` and this works fine because python file API
70 78 # abstract the use of `/` and make it work transparently. For consistency
71 79 # vfs will always use `/` when joining. This avoids some confusion in
72 80 # encoded vfs (see issue6546)
73 81 _dir_sep: bytes = b'/'
74 82
75 83 # TODO: type return, which is util.posixfile wrapped by a proxy
76 84 @abc.abstractmethod
77 85 def __call__(self, path: bytes, mode: bytes = b'rb', **kwargs) -> Any:
78 86 ...
79 87
80 88 @abc.abstractmethod
81 def _auditpath(self, path: bytes, mode: bytes) -> Any:
89 def _auditpath(self, path: bytes, mode: bytes) -> None:
82 90 ...
83 91
84 92 @abc.abstractmethod
85 93 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
86 94 ...
87 95
88 96 def tryread(self, path: bytes) -> bytes:
89 97 '''gracefully return an empty string for missing files'''
90 98 try:
91 99 return self.read(path)
92 100 except FileNotFoundError:
93 101 pass
94 102 return b""
95 103
96 def tryreadlines(self, path: bytes, mode: bytes = b'rb') -> Any:
104 def tryreadlines(self, path: bytes, mode: bytes = b'rb') -> List[bytes]:
97 105 '''gracefully return an empty array for missing files'''
98 106 try:
99 107 return self.readlines(path, mode=mode)
100 108 except FileNotFoundError:
101 109 pass
102 110 return []
103 111
104 112 @util.propertycache
105 113 def open(self):
106 114 """Open ``path`` file, which is relative to vfs root.
107 115
108 116 Newly created directories are marked as "not to be indexed by
109 117 the content indexing service", if ``notindexed`` is specified
110 118 for "write" mode access.
111 119 """
112 120 return self.__call__
113 121
114 122 def read(self, path: bytes) -> bytes:
115 123 with self(path, b'rb') as fp:
116 124 return fp.read()
117 125
118 def readlines(self, path: bytes, mode: bytes = b'rb') -> Any:
126 def readlines(self, path: bytes, mode: bytes = b'rb') -> List[bytes]:
119 127 with self(path, mode=mode) as fp:
120 128 return fp.readlines()
121 129
122 130 def write(
123 self, path: bytes, data: bytes, backgroundclose=False, **kwargs
131 self, path: bytes, data: bytes, backgroundclose: bool = False, **kwargs
124 132 ) -> int:
125 133 with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp:
126 134 return fp.write(data)
127 135
128 136 def writelines(
129 137 self,
130 138 path: bytes,
131 139 data: Iterable[bytes],
132 140 mode: bytes = b'wb',
133 notindexed=False,
141 notindexed: bool = False,
134 142 ) -> None:
135 143 with self(path, mode=mode, notindexed=notindexed) as fp:
136 144 return fp.writelines(data)
137 145
138 146 def append(self, path: bytes, data: bytes) -> int:
139 147 with self(path, b'ab') as fp:
140 148 return fp.write(data)
141 149
142 150 def basename(self, path: bytes) -> bytes:
143 151 """return base element of a path (as os.path.basename would do)
144 152
145 153 This exists to allow handling of strange encoding if needed."""
146 154 return os.path.basename(path)
147 155
148 156 def chmod(self, path: bytes, mode: int) -> None:
149 157 return os.chmod(self.join(path), mode)
150 158
151 159 def dirname(self, path: bytes) -> bytes:
152 160 """return dirname element of a path (as os.path.dirname would do)
153 161
154 162 This exists to allow handling of strange encoding if needed."""
155 163 return os.path.dirname(path)
156 164
157 165 def exists(self, path: Optional[bytes] = None) -> bool:
158 166 return os.path.exists(self.join(path))
159 167
160 def fstat(self, fp) -> os.stat_result:
168 def fstat(self, fp: BinaryIO) -> os.stat_result:
161 169 return util.fstat(fp)
162 170
163 171 def isdir(self, path: Optional[bytes] = None) -> bool:
164 172 return os.path.isdir(self.join(path))
165 173
166 174 def isfile(self, path: Optional[bytes] = None) -> bool:
167 175 return os.path.isfile(self.join(path))
168 176
169 177 def islink(self, path: Optional[bytes] = None) -> bool:
170 178 return os.path.islink(self.join(path))
171 179
172 180 def isfileorlink(self, path: Optional[bytes] = None) -> bool:
173 181 """return whether path is a regular file or a symlink
174 182
175 183 Unlike isfile, this doesn't follow symlinks."""
176 184 try:
177 185 st = self.lstat(path)
178 186 except OSError:
179 187 return False
180 188 mode = st.st_mode
181 189 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
182 190
183 191 def _join(self, *paths: bytes) -> bytes:
184 192 root_idx = 0
185 193 for idx, p in enumerate(paths):
186 194 if os.path.isabs(p) or p.startswith(self._dir_sep):
187 195 root_idx = idx
188 196 if root_idx != 0:
189 197 paths = paths[root_idx:]
190 198 paths = [p for p in paths if p]
191 199 return self._dir_sep.join(paths)
192 200
193 201 def reljoin(self, *paths: bytes) -> bytes:
194 202 """join various elements of a path together (as os.path.join would do)
195 203
196 204 The vfs base is not injected so that path stay relative. This exists
197 205 to allow handling of strange encoding if needed."""
198 206 return self._join(*paths)
199 207
200 208 def split(self, path: bytes) -> Tuple[bytes, bytes]:
201 209 """split top-most element of a path (as os.path.split would do)
202 210
203 211 This exists to allow handling of strange encoding if needed."""
204 212 return os.path.split(path)
205 213
206 214 def lexists(self, path: Optional[bytes] = None) -> bool:
207 215 return os.path.lexists(self.join(path))
208 216
209 217 def lstat(self, path: Optional[bytes] = None) -> os.stat_result:
210 218 return os.lstat(self.join(path))
211 219
212 220 def is_mmap_safe(self, path: Optional[bytes] = None) -> bool:
213 221 """return True if it is safe to read a file content as mmap
214 222
215 223 This focus on the file system aspect of such safety, the application
216 224 logic around that file is not taken into account, so caller need to
217 225 make sure the file won't be truncated in a way that will create SIGBUS
218 226 on access.
219 227
220 228
221 229 The initial motivation for this logic is that if mmap is used on NFS
222 230 and somebody deletes the mapped file (e.g. by renaming on top of it),
223 231 then you get SIGBUS, which can be pretty disruptive: we get core dump
224 232 reports, and the process terminates without writing to the blackbox.
225 233
226 234 Instead, in this situation we prefer to read the file normally.
227 235 The risk of ESTALE in the middle of the read remains, but it's
228 236 smaller because we read sooner and the error should be reported
229 237 just as any other error.
230 238
231 239 Note that python standard library does not offer the necessary function
232 240 to detect the file stem bits. So this detection rely on compiled bits
233 241 and is not available in pure python.
234 242 """
235 243 # XXX Since we already assume a vfs to address a consistent file system
236 244 # in other location, we could determine the fstype once for the root
237 245 # and cache that value.
238 246 fstype = util.getfstype(self.join(path))
239 247 return fstype is not None and fstype != b'nfs'
240 248
241 249 def listdir(self, path: Optional[bytes] = None) -> List[bytes]:
242 250 return os.listdir(self.join(path))
243 251
244 252 def makedir(self, path: Optional[bytes] = None, notindexed=True) -> None:
245 253 return util.makedir(self.join(path), notindexed)
246 254
247 255 def makedirs(
248 256 self, path: Optional[bytes] = None, mode: Optional[int] = None
249 257 ) -> None:
250 258 return util.makedirs(self.join(path), mode)
251 259
252 def makelock(self, info, path: bytes) -> None:
260 def makelock(self, info: bytes, path: bytes) -> None:
253 261 return util.makelock(info, self.join(path))
254 262
255 263 def mkdir(self, path: Optional[bytes] = None) -> None:
256 264 return os.mkdir(self.join(path))
257 265
258 266 def mkstemp(
259 267 self,
260 268 suffix: bytes = b'',
261 269 prefix: bytes = b'tmp',
262 270 dir: Optional[bytes] = None,
263 271 ) -> Tuple[int, bytes]:
264 272 fd, name = pycompat.mkstemp(
265 273 suffix=suffix, prefix=prefix, dir=self.join(dir)
266 274 )
267 275 dname, fname = util.split(name)
268 276 if dir:
269 277 return fd, os.path.join(dir, fname)
270 278 else:
271 279 return fd, fname
272 280
281 # TODO: This doesn't match osutil.listdir(). stat=False in pure;
282 # non-optional bool in cext. 'skip' is bool if we trust cext, or bytes
283 # going by how pure uses it. Also, cext returns a custom stat structure.
284 # from cext.osutil.pyi:
285 #
286 # path: bytes, st: bool, skip: Optional[bool]
273 287 def readdir(
274 288 self, path: Optional[bytes] = None, stat=None, skip=None
275 289 ) -> Any:
276 290 return util.listdir(self.join(path), stat, skip)
277 291
278 292 def readlock(self, path: bytes) -> bytes:
279 293 return util.readlock(self.join(path))
280 294
281 def rename(self, src: bytes, dst: bytes, checkambig=False) -> None:
295 def rename(self, src: bytes, dst: bytes, checkambig: bool = False) -> None:
282 296 """Rename from src to dst
283 297
284 298 checkambig argument is used with util.filestat, and is useful
285 299 only if destination file is guarded by any lock
286 300 (e.g. repo.lock or repo.wlock).
287 301
288 302 To avoid file stat ambiguity forcibly, checkambig=True involves
289 303 copying ``src`` file, if it is owned by another. Therefore, use
290 304 checkambig=True only in limited cases (see also issue5418 and
291 305 issue5584 for detail).
292 306 """
293 307 self._auditpath(dst, b'w')
294 308 srcpath = self.join(src)
295 309 dstpath = self.join(dst)
296 310 oldstat = util.filestat.frompath(dstpath) if checkambig else None
297 311
298 312 util.rename(srcpath, dstpath)
299 313
300 314 if oldstat and oldstat.stat:
301 315 _avoidambig(dstpath, oldstat)
302 316
303 317 def readlink(self, path: bytes) -> bytes:
304 318 return util.readlink(self.join(path))
305 319
306 320 def removedirs(self, path: Optional[bytes] = None) -> None:
307 321 """Remove a leaf directory and all empty intermediate ones"""
308 322 return util.removedirs(self.join(path))
309 323
310 324 def rmdir(self, path: Optional[bytes] = None) -> None:
311 325 """Remove an empty directory."""
312 326 return os.rmdir(self.join(path))
313 327
314 328 def rmtree(
315 self, path: Optional[bytes] = None, ignore_errors=False, forcibly=False
329 self,
330 path: Optional[bytes] = None,
331 ignore_errors: bool = False,
332 forcibly: bool = False,
316 333 ) -> None:
317 334 """Remove a directory tree recursively
318 335
319 336 If ``forcibly``, this tries to remove READ-ONLY files, too.
320 337 """
321 338 if forcibly:
322 339
323 def onexc(function, path, excinfo):
340 def onexc(function, path: bytes, excinfo):
324 341 if function is not os.remove:
325 342 raise
326 343 # read-only files cannot be unlinked under Windows
327 344 s = os.stat(path)
328 345 if (s.st_mode & stat.S_IWRITE) != 0:
329 346 raise
330 347 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
331 348 os.remove(path)
332 349
333 350 else:
334 351 onexc = None
335 352 try:
336 353 # pytype: disable=wrong-keyword-args
337 354 return shutil.rmtree(
338 355 self.join(path), ignore_errors=ignore_errors, onexc=onexc
339 356 )
340 357 # pytype: enable=wrong-keyword-args
341 358 except TypeError: # onexc was introduced in Python 3.12
342 359 return shutil.rmtree(
343 360 self.join(path), ignore_errors=ignore_errors, onerror=onexc
344 361 )
345 362
346 363 def setflags(self, path: bytes, l: bool, x: bool) -> None:
347 364 return util.setflags(self.join(path), l, x)
348 365
349 366 def stat(self, path: Optional[bytes] = None) -> os.stat_result:
350 367 return os.stat(self.join(path))
351 368
352 369 def unlink(self, path: Optional[bytes] = None) -> None:
353 370 return util.unlink(self.join(path))
354 371
355 372 def tryunlink(self, path: Optional[bytes] = None) -> bool:
356 373 """Attempt to remove a file, ignoring missing file errors."""
357 374 return util.tryunlink(self.join(path))
358 375
359 376 def unlinkpath(
360 self, path: Optional[bytes] = None, ignoremissing=False, rmdir=True
377 self,
378 path: Optional[bytes] = None,
379 ignoremissing: bool = False,
380 rmdir: bool = True,
361 381 ) -> None:
362 382 return util.unlinkpath(
363 383 self.join(path), ignoremissing=ignoremissing, rmdir=rmdir
364 384 )
365 385
366 def utime(self, path: Optional[bytes] = None, t=None) -> None:
386 # TODO: could be Tuple[float, float] too.
387 def utime(
388 self, path: Optional[bytes] = None, t: Optional[Tuple[int, int]] = None
389 ) -> None:
367 390 return os.utime(self.join(path), t)
368 391
369 392 def walk(
370 self, path: Optional[bytes] = None, onerror=None
393 self, path: Optional[bytes] = None, onerror: Optional[_OnErrorFn] = None
371 394 ) -> Iterator[Tuple[bytes, List[bytes], List[bytes]]]:
372 395 """Yield (dirpath, dirs, files) tuple for each directory under path
373 396
374 397 ``dirpath`` is relative one from the root of this vfs. This
375 398 uses ``os.sep`` as path separator, even you specify POSIX
376 399 style ``path``.
377 400
378 401 "The root of this vfs" is represented as empty ``dirpath``.
379 402 """
380 403 root = os.path.normpath(self.join(None))
381 404 # when dirpath == root, dirpath[prefixlen:] becomes empty
382 405 # because len(dirpath) < prefixlen.
383 406 prefixlen = len(pathutil.normasprefix(root))
384 407 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
385 408 yield (dirpath[prefixlen:], dirs, files)
386 409
387 410 @contextlib.contextmanager
388 411 def backgroundclosing(
389 self, ui, expectedcount=-1
412 self, ui: uimod.ui, expectedcount: int = -1
390 413 ) -> Iterator[Optional[backgroundfilecloser]]:
391 414 """Allow files to be closed asynchronously.
392 415
393 416 When this context manager is active, ``backgroundclose`` can be passed
394 417 to ``__call__``/``open`` to result in the file possibly being closed
395 418 asynchronously, on a background thread.
396 419 """
397 420 # Sharing backgroundfilecloser between threads is complex and using
398 421 # multiple instances puts us at risk of running out of file descriptors
399 422 # only allow to use backgroundfilecloser when in main thread.
400 423 if threading.current_thread() is not threading.main_thread():
401 424 yield
402 425 return
403 426 vfs = getattr(self, 'vfs', self)
404 427 if getattr(vfs, '_backgroundfilecloser', None):
405 428 raise error.Abort(
406 429 _(b'can only have 1 active background file closer')
407 430 )
408 431
409 432 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
410 433 try:
411 434 vfs._backgroundfilecloser = (
412 435 bfc # pytype: disable=attribute-error
413 436 )
414 437 yield bfc
415 438 finally:
416 439 vfs._backgroundfilecloser = (
417 440 None # pytype: disable=attribute-error
418 441 )
419 442
420 def register_file(self, path) -> None:
443 def register_file(self, path: bytes) -> None:
421 444 """generic hook point to lets fncache steer its stew"""
422 445
423 446
424 447 class vfs(abstractvfs):
425 448 """Operate files relative to a base directory
426 449
427 450 This class is used to hide the details of COW semantics and
428 451 remote file access from higher level code.
429 452
430 453 'cacheaudited' should be enabled only if (a) vfs object is short-lived, or
431 454 (b) the base directory is managed by hg and considered sort-of append-only.
432 455 See pathutil.pathauditor() for details.
433 456 """
434 457
458 createmode: Optional[int]
459
435 460 def __init__(
436 461 self,
437 462 base: bytes,
438 audit=True,
439 cacheaudited=False,
440 expandpath=False,
441 realpath=False,
463 audit: bool = True,
464 cacheaudited: bool = False,
465 expandpath: bool = False,
466 realpath: bool = False,
442 467 ) -> None:
443 468 if expandpath:
444 469 base = util.expandpath(base)
445 470 if realpath:
446 471 base = os.path.realpath(base)
447 472 self.base = base
448 473 self._audit = audit
449 474 if audit:
450 475 self.audit = pathutil.pathauditor(self.base, cached=cacheaudited)
451 476 else:
452 477 self.audit = lambda path, mode=None: True
453 478 self.createmode = None
454 479 self._trustnlink = None
455 480 self.options = {}
456 481
457 482 @util.propertycache
458 483 def _cansymlink(self) -> bool:
459 484 return util.checklink(self.base)
460 485
461 486 @util.propertycache
462 def _chmod(self):
487 def _chmod(self) -> bool:
463 488 return util.checkexec(self.base)
464 489
465 def _fixfilemode(self, name) -> None:
490 def _fixfilemode(self, name: bytes) -> None:
466 491 if self.createmode is None or not self._chmod:
467 492 return
468 493 os.chmod(name, self.createmode & 0o666)
469 494
470 def _auditpath(self, path, mode) -> None:
495 def _auditpath(self, path: bytes, mode: bytes) -> None:
471 496 if self._audit:
472 497 if os.path.isabs(path) and path.startswith(self.base):
473 498 path = os.path.relpath(path, self.base)
474 499 r = util.checkosfilename(path)
475 500 if r:
476 501 raise error.Abort(b"%s: %r" % (r, path))
477 502 self.audit(path, mode=mode)
478 503
479 504 def isfileorlink_checkdir(
480 self, dircache, path: Optional[bytes] = None
505 self,
506 dircache: MutableMapping[bytes, bool],
507 path: Optional[bytes] = None,
481 508 ) -> bool:
482 509 """return True if the path is a regular file or a symlink and
483 510 the directories along the path are "normal", that is
484 511 not symlinks or nested hg repositories.
485 512
486 513 Ignores the `_audit` setting, and checks the directories regardless.
487 514 `dircache` is used to cache the directory checks.
488 515 """
516 # TODO: Should be a None check on 'path', or shouldn't default to None
517 # because of the immediate call to util.localpath().
489 518 try:
490 519 for prefix in pathutil.finddirs_rev_noroot(util.localpath(path)):
491 520 if prefix in dircache:
492 521 res = dircache[prefix]
493 522 else:
494 523 res = pathutil.pathauditor._checkfs_exists(
495 524 self.base, prefix, path
496 525 )
497 526 dircache[prefix] = res
498 527 if not res:
499 528 return False
500 529 except (OSError, error.Abort):
501 530 return False
502 531 return self.isfileorlink(path)
503 532
504 533 def __call__(
505 534 self,
506 535 path: bytes,
507 536 mode: bytes = b"rb",
508 atomictemp=False,
509 notindexed=False,
510 backgroundclose=False,
511 checkambig=False,
512 auditpath=True,
513 makeparentdirs=True,
514 ) -> Any:
537 atomictemp: bool = False,
538 notindexed: bool = False,
539 backgroundclose: bool = False,
540 checkambig: bool = False,
541 auditpath: bool = True,
542 makeparentdirs: bool = True,
543 ) -> Any: # TODO: should be BinaryIO if util.atomictempfile can be coersed
515 544 """Open ``path`` file, which is relative to vfs root.
516 545
517 546 By default, parent directories are created as needed. Newly created
518 547 directories are marked as "not to be indexed by the content indexing
519 548 service", if ``notindexed`` is specified for "write" mode access.
520 549 Set ``makeparentdirs=False`` to not create directories implicitly.
521 550
522 551 If ``backgroundclose`` is passed, the file may be closed asynchronously.
523 552 It can only be used if the ``self.backgroundclosing()`` context manager
524 553 is active. This should only be specified if the following criteria hold:
525 554
526 555 1. There is a potential for writing thousands of files. Unless you
527 556 are writing thousands of files, the performance benefits of
528 557 asynchronously closing files is not realized.
529 558 2. Files are opened exactly once for the ``backgroundclosing``
530 559 active duration and are therefore free of race conditions between
531 560 closing a file on a background thread and reopening it. (If the
532 561 file were opened multiple times, there could be unflushed data
533 562 because the original file handle hasn't been flushed/closed yet.)
534 563
535 564 ``checkambig`` argument is passed to atomictempfile (valid
536 565 only for writing), and is useful only if target file is
537 566 guarded by any lock (e.g. repo.lock or repo.wlock).
538 567
539 568 To avoid file stat ambiguity forcibly, checkambig=True involves
540 569 copying ``path`` file opened in "append" mode (e.g. for
541 570 truncation), if it is owned by another. Therefore, use
542 571 combination of append mode and checkambig=True only in limited
543 572 cases (see also issue5418 and issue5584 for detail).
544 573 """
545 574 if auditpath:
546 575 self._auditpath(path, mode)
547 576 f = self.join(path)
548 577
549 578 if b"b" not in mode:
550 579 mode += b"b" # for that other OS
551 580
552 581 nlink = -1
553 582 if mode not in (b'r', b'rb'):
554 583 dirname, basename = util.split(f)
555 584 # If basename is empty, then the path is malformed because it points
556 585 # to a directory. Let the posixfile() call below raise IOError.
557 586 if basename:
558 587 if atomictemp:
559 588 if makeparentdirs:
560 589 util.makedirs(dirname, self.createmode, notindexed)
561 590 return util.atomictempfile(
562 591 f, mode, self.createmode, checkambig=checkambig
563 592 )
564 593 try:
565 594 if b'w' in mode:
566 595 util.unlink(f)
567 596 nlink = 0
568 597 else:
569 598 # nlinks() may behave differently for files on Windows
570 599 # shares if the file is open.
571 600 with util.posixfile(f):
572 601 nlink = util.nlinks(f)
573 602 if nlink < 1:
574 603 nlink = 2 # force mktempcopy (issue1922)
575 604 except FileNotFoundError:
576 605 nlink = 0
577 606 if makeparentdirs:
578 607 util.makedirs(dirname, self.createmode, notindexed)
579 608 if nlink > 0:
580 609 if self._trustnlink is None:
581 610 self._trustnlink = nlink > 1 or util.checknlink(f)
582 611 if nlink > 1 or not self._trustnlink:
583 612 util.rename(util.mktempcopy(f), f)
584 613 fp = util.posixfile(f, mode)
585 614 if nlink == 0:
586 615 self._fixfilemode(f)
587 616
588 617 if checkambig:
589 618 if mode in (b'r', b'rb'):
590 619 raise error.Abort(
591 620 _(
592 621 b'implementation error: mode %s is not'
593 622 b' valid for checkambig=True'
594 623 )
595 624 % mode
596 625 )
597 626 fp = checkambigatclosing(fp)
598 627
599 628 if (
600 629 backgroundclose
601 630 and threading.current_thread() is threading.main_thread()
602 631 ):
603 632 if (
604 633 not self._backgroundfilecloser # pytype: disable=attribute-error
605 634 ):
606 635 raise error.Abort(
607 636 _(
608 637 b'backgroundclose can only be used when a '
609 638 b'backgroundclosing context manager is active'
610 639 )
611 640 )
612 641
613 642 fp = delayclosedfile(
614 643 fp,
615 644 self._backgroundfilecloser, # pytype: disable=attribute-error
616 645 )
617 646
618 647 return fp
619 648
620 649 def symlink(self, src: bytes, dst: bytes) -> None:
621 650 self.audit(dst)
622 651 linkname = self.join(dst)
623 652 util.tryunlink(linkname)
624 653
625 654 util.makedirs(os.path.dirname(linkname), self.createmode)
626 655
627 656 if self._cansymlink:
628 657 try:
629 658 os.symlink(src, linkname)
630 659 except OSError as err:
631 660 raise OSError(
632 661 err.errno,
633 662 _(b'could not symlink to %r: %s')
634 663 % (src, encoding.strtolocal(err.strerror)),
635 664 linkname,
636 665 )
637 666 else:
638 667 self.write(dst, src)
639 668
640 669 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
641 670 if path:
642 671 parts = [self.base, path]
643 672 parts.extend(insidef)
644 673 return self._join(*parts)
645 674 else:
646 675 return self.base
647 676
648 677
649 678 opener: Type[vfs] = vfs
650 679
651 680
652 681 class proxyvfs(abstractvfs, abc.ABC):
653 def __init__(self, vfs: "vfs"):
682 def __init__(self, vfs: vfs) -> None:
654 683 self.vfs = vfs
655 684
656 685 @property
657 def createmode(self):
686 def createmode(self) -> Optional[int]:
658 687 return self.vfs.createmode
659 688
660 def _auditpath(self, path, mode) -> None:
689 def _auditpath(self, path: bytes, mode: bytes) -> None:
661 690 return self.vfs._auditpath(path, mode)
662 691
663 692 @property
664 693 def options(self):
665 694 return self.vfs.options
666 695
667 696 @options.setter
668 697 def options(self, value):
669 698 self.vfs.options = value
670 699
671 700 @property
672 701 def audit(self):
673 702 return self.vfs.audit
674 703
675 704
676 705 class filtervfs(proxyvfs, abstractvfs):
677 706 '''Wrapper vfs for filtering filenames with a function.'''
678 707
679 def __init__(self, vfs: "vfs", filter):
708 def __init__(self, vfs: vfs, filter) -> None:
680 709 proxyvfs.__init__(self, vfs)
681 710 self._filter = filter
682 711
712 # TODO: The return type should be BinaryIO
683 713 def __call__(self, path: bytes, *args, **kwargs) -> Any:
684 714 return self.vfs(self._filter(path), *args, **kwargs)
685 715
686 716 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
687 717 if path:
688 718 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
689 719 else:
690 720 return self.vfs.join(path)
691 721
692 722
693 723 filteropener: Type[filtervfs] = filtervfs
694 724
695 725
696 726 class readonlyvfs(proxyvfs):
697 727 '''Wrapper vfs preventing any writing.'''
698 728
699 def __init__(self, vfs: "vfs"):
729 def __init__(self, vfs: vfs) -> None:
700 730 proxyvfs.__init__(self, vfs)
701 731
732 # TODO: The return type should be BinaryIO
702 733 def __call__(self, path: bytes, mode: bytes = b'rb', *args, **kw) -> Any:
703 734 if mode not in (b'r', b'rb'):
704 735 raise error.Abort(_(b'this vfs is read only'))
705 736 return self.vfs(path, mode, *args, **kw)
706 737
707 738 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
708 739 return self.vfs.join(path, *insidef)
709 740
710 741
711 742 class closewrapbase(abc.ABC):
712 743 """Base class of wrapper, which hooks closing
713 744
714 745 Do not instantiate outside the vfs layer.
715 746 """
716 747
717 748 def __init__(self, fh) -> None:
718 749 object.__setattr__(self, '_origfh', fh)
719 750
720 def __getattr__(self, attr) -> Any:
751 def __getattr__(self, attr: str) -> Any:
721 752 return getattr(self._origfh, attr)
722 753
723 def __setattr__(self, attr, value) -> None:
754 def __setattr__(self, attr: str, value: Any) -> None:
724 755 return setattr(self._origfh, attr, value)
725 756
726 def __delattr__(self, attr) -> None:
757 def __delattr__(self, attr: str) -> None:
727 758 return delattr(self._origfh, attr)
728 759
729 760 def __enter__(self: _Tclosewrapbase) -> _Tclosewrapbase:
730 761 self._origfh.__enter__()
731 762 return self
732 763
733 764 @abc.abstractmethod
734 765 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
735 766 ...
736 767
737 768 @abc.abstractmethod
738 769 def close(self) -> None:
739 770 ...
740 771
741 772
742 773 class delayclosedfile(closewrapbase):
743 774 """Proxy for a file object whose close is delayed.
744 775
745 776 Do not instantiate outside the vfs layer.
746 777 """
747 778
748 779 def __init__(self, fh, closer) -> None:
749 780 super(delayclosedfile, self).__init__(fh)
750 781 object.__setattr__(self, '_closer', closer)
751 782
752 783 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
753 784 self._closer.close(self._origfh)
754 785
755 786 def close(self) -> None:
756 787 self._closer.close(self._origfh)
757 788
758 789
759 790 class backgroundfilecloser:
760 791 """Coordinates background closing of file handles on multiple threads."""
761 792
762 def __init__(self, ui, expectedcount=-1) -> None:
793 def __init__(self, ui: uimod.ui, expectedcount: int = -1) -> None:
763 794 self._running = False
764 795 self._entered = False
765 796 self._threads = []
766 797 self._threadexception = None
767 798
768 799 # Only Windows/NTFS has slow file closing. So only enable by default
769 800 # on that platform. But allow to be enabled elsewhere for testing.
770 801 defaultenabled = pycompat.iswindows
771 802 enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled)
772 803
773 804 if not enabled:
774 805 return
775 806
776 807 # There is overhead to starting and stopping the background threads.
777 808 # Don't do background processing unless the file count is large enough
778 809 # to justify it.
779 810 minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount')
780 811 # FUTURE dynamically start background threads after minfilecount closes.
781 812 # (We don't currently have any callers that don't know their file count)
782 813 if expectedcount > 0 and expectedcount < minfilecount:
783 814 return
784 815
785 816 maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue')
786 817 threadcount = ui.configint(b'worker', b'backgroundclosethreadcount')
787 818
788 819 ui.debug(
789 820 b'starting %d threads for background file closing\n' % threadcount
790 821 )
791 822
792 823 self._queue = pycompat.queue.Queue(maxsize=maxqueue)
793 824 self._running = True
794 825
795 826 for i in range(threadcount):
796 827 t = threading.Thread(target=self._worker, name='backgroundcloser')
797 828 self._threads.append(t)
798 829 t.start()
799 830
800 831 def __enter__(self: _Tbackgroundfilecloser) -> _Tbackgroundfilecloser:
801 832 self._entered = True
802 833 return self
803 834
804 835 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
805 836 self._running = False
806 837
807 838 # Wait for threads to finish closing so open files don't linger for
808 839 # longer than lifetime of context manager.
809 840 for t in self._threads:
810 841 t.join()
811 842
812 843 def _worker(self) -> None:
813 844 """Main routine for worker thread."""
814 845 while True:
815 846 try:
816 847 fh = self._queue.get(block=True, timeout=0.100)
817 848 # Need to catch or the thread will terminate and
818 849 # we could orphan file descriptors.
819 850 try:
820 851 fh.close()
821 852 except Exception as e:
822 853 # Stash so can re-raise from main thread later.
823 854 self._threadexception = e
824 855 except pycompat.queue.Empty:
825 856 if not self._running:
826 857 break
827 858
828 859 def close(self, fh) -> None:
829 860 """Schedule a file for closing."""
830 861 if not self._entered:
831 862 raise error.Abort(
832 863 _(b'can only call close() when context manager active')
833 864 )
834 865
835 866 # If a background thread encountered an exception, raise now so we fail
836 867 # fast. Otherwise, we may potentially go on for minutes until the error
837 868 # is acted on.
838 869 if self._threadexception:
839 870 e = self._threadexception
840 871 self._threadexception = None
841 872 raise e
842 873
843 874 # If we're not actively running, close synchronously.
844 875 if not self._running:
845 876 fh.close()
846 877 return
847 878
848 879 self._queue.put(fh, block=True, timeout=None)
849 880
850 881
851 882 class checkambigatclosing(closewrapbase):
852 883 """Proxy for a file object, to avoid ambiguity of file stat
853 884
854 885 See also util.filestat for detail about "ambiguity of file stat".
855 886
856 887 This proxy is useful only if the target file is guarded by any
857 888 lock (e.g. repo.lock or repo.wlock)
858 889
859 890 Do not instantiate outside the vfs layer.
860 891 """
861 892
862 893 def __init__(self, fh) -> None:
863 894 super(checkambigatclosing, self).__init__(fh)
864 895 object.__setattr__(self, '_oldstat', util.filestat.frompath(fh.name))
865 896
866 897 def _checkambig(self) -> None:
867 898 oldstat = self._oldstat
868 899 if oldstat.stat:
869 900 _avoidambig(self._origfh.name, oldstat)
870 901
871 902 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
872 903 self._origfh.__exit__(exc_type, exc_value, exc_tb)
873 904 self._checkambig()
874 905
875 906 def close(self) -> None:
876 907 self._origfh.close()
877 908 self._checkambig()
General Comments 0
You need to be logged in to leave comments. Login now