vfs.py
852 lines
| 28.7 KiB
| text/x-python
|
PythonLexer
/ mercurial / vfs.py
Pierre-Yves David
|
r31217 | # vfs.py - Mercurial 'vfs' classes | ||
# | ||||
Raphaël Gomès
|
r47575 | # Copyright Olivia Mackall <olivia@selenic.com> | ||
Pierre-Yves David
|
r31217 | # | ||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
import contextlib | ||||
import os | ||||
import shutil | ||||
import stat | ||||
import threading | ||||
Matt Harbison
|
r50468 | from typing import ( | ||
Optional, | ||||
) | ||||
Pierre-Yves David
|
r31217 | from .i18n import _ | ||
from . import ( | ||||
Augie Fackler
|
r34024 | encoding, | ||
Pierre-Yves David
|
r31217 | error, | ||
pathutil, | ||||
pycompat, | ||||
util, | ||||
) | ||||
Augie Fackler
|
r43346 | |||
Matt Harbison
|
r50468 | def _avoidambig(path: bytes, oldstat): | ||
FUJIWARA Katsunori
|
r33280 | """Avoid file stat ambiguity forcibly | ||
This function causes copying ``path`` file, if it is owned by | ||||
another (see issue5418 and issue5584 for detail). | ||||
""" | ||||
Augie Fackler
|
r43346 | |||
FUJIWARA Katsunori
|
r33280 | def checkandavoid(): | ||
newstat = util.filestat.frompath(path) | ||||
# return whether file stat ambiguity is (already) avoided | ||||
Augie Fackler
|
r43346 | return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat) | ||
FUJIWARA Katsunori
|
r33280 | if not checkandavoid(): | ||
# simply copy to change owner of path to get privilege to | ||||
# advance mtime (see issue5418) | ||||
util.rename(util.mktempcopy(path), path) | ||||
checkandavoid() | ||||
Augie Fackler
|
r43346 | |||
Gregory Szorc
|
r49801 | class abstractvfs: | ||
Pierre-Yves David
|
r31217 | """Abstract base class; cannot be instantiated""" | ||
r48617 | # default directory separator for vfs | |||
# | ||||
# Other vfs code always use `/` and this works fine because python file API | ||||
# abstract the use of `/` and make it work transparently. For consistency | ||||
# vfs will always use `/` when joining. This avoid some confusion in | ||||
# encoded vfs (see issue6546) | ||||
_dir_sep = b'/' | ||||
Pierre-Yves David
|
r31217 | def __init__(self, *args, **kwargs): | ||
'''Prevent instantiation; don't call this from subclasses.''' | ||||
Augie Fackler
|
r43767 | raise NotImplementedError('attempted instantiating ' + str(type(self))) | ||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | # TODO: type return, which is util.posixfile wrapped by a proxy | ||
def __call__(self, path: bytes, mode: bytes = b'rb', **kwargs): | ||||
Augie Fackler
|
r43768 | raise NotImplementedError | ||
Matt Harbison
|
r50468 | def _auditpath(self, path: bytes, mode: bytes): | ||
Boris Feld
|
r41127 | raise NotImplementedError | ||
Boris Feld
|
r41123 | |||
Matt Harbison
|
r50468 | def join(self, path: Optional[bytes], *insidef: bytes) -> bytes: | ||
Augie Fackler
|
r43769 | raise NotImplementedError | ||
Matt Harbison
|
r50468 | def tryread(self, path: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | '''gracefully return an empty string for missing files''' | ||
try: | ||||
return self.read(path) | ||||
Manuel Jacob
|
r50201 | except FileNotFoundError: | ||
pass | ||||
Augie Fackler
|
r43347 | return b"" | ||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def tryreadlines(self, path: bytes, mode: bytes = b'rb'): | ||
Pierre-Yves David
|
r31217 | '''gracefully return an empty array for missing files''' | ||
try: | ||||
return self.readlines(path, mode=mode) | ||||
Manuel Jacob
|
r50201 | except FileNotFoundError: | ||
pass | ||||
Pierre-Yves David
|
r31217 | return [] | ||
@util.propertycache | ||||
def open(self): | ||||
Augie Fackler
|
r46554 | """Open ``path`` file, which is relative to vfs root. | ||
Pierre-Yves David
|
r31217 | |||
Newly created directories are marked as "not to be indexed by | ||||
the content indexing service", if ``notindexed`` is specified | ||||
for "write" mode access. | ||||
Augie Fackler
|
r46554 | """ | ||
Pierre-Yves David
|
r31217 | return self.__call__ | ||
Matt Harbison
|
r50468 | def read(self, path: bytes) -> bytes: | ||
Augie Fackler
|
r43347 | with self(path, b'rb') as fp: | ||
Pierre-Yves David
|
r31217 | return fp.read() | ||
Matt Harbison
|
r50468 | def readlines(self, path: bytes, mode: bytes = b'rb'): | ||
Pierre-Yves David
|
r31217 | with self(path, mode=mode) as fp: | ||
return fp.readlines() | ||||
Matt Harbison
|
r50468 | def write( | ||
self, path: bytes, data: bytes, backgroundclose=False, **kwargs | ||||
) -> int: | ||||
Augie Fackler
|
r43347 | with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp: | ||
Pierre-Yves David
|
r31217 | return fp.write(data) | ||
Matt Harbison
|
r50468 | def writelines( | ||
self, path: bytes, data: bytes, mode: bytes = b'wb', notindexed=False | ||||
) -> None: | ||||
Pierre-Yves David
|
r31217 | with self(path, mode=mode, notindexed=notindexed) as fp: | ||
return fp.writelines(data) | ||||
Matt Harbison
|
r50468 | def append(self, path: bytes, data: bytes) -> int: | ||
Augie Fackler
|
r43347 | with self(path, b'ab') as fp: | ||
Pierre-Yves David
|
r31217 | return fp.write(data) | ||
Matt Harbison
|
r50468 | def basename(self, path: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | """return base element of a path (as os.path.basename would do) | ||
This exists to allow handling of strange encoding if needed.""" | ||||
return os.path.basename(path) | ||||
Matt Harbison
|
r50468 | def chmod(self, path: bytes, mode: int) -> None: | ||
Pierre-Yves David
|
r31217 | return os.chmod(self.join(path), mode) | ||
Matt Harbison
|
r50468 | def dirname(self, path: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | """return dirname element of a path (as os.path.dirname would do) | ||
This exists to allow handling of strange encoding if needed.""" | ||||
return os.path.dirname(path) | ||||
Matt Harbison
|
r50468 | def exists(self, path: Optional[bytes] = None) -> bool: | ||
Pierre-Yves David
|
r31217 | return os.path.exists(self.join(path)) | ||
def fstat(self, fp): | ||||
return util.fstat(fp) | ||||
Matt Harbison
|
r50468 | def isdir(self, path: Optional[bytes] = None) -> bool: | ||
Pierre-Yves David
|
r31217 | return os.path.isdir(self.join(path)) | ||
Matt Harbison
|
r50468 | def isfile(self, path: Optional[bytes] = None) -> bool: | ||
Pierre-Yves David
|
r31217 | return os.path.isfile(self.join(path)) | ||
Matt Harbison
|
r50468 | def islink(self, path: Optional[bytes] = None) -> bool: | ||
Pierre-Yves David
|
r31217 | return os.path.islink(self.join(path)) | ||
Matt Harbison
|
r50468 | def isfileorlink(self, path: Optional[bytes] = None) -> bool: | ||
Augie Fackler
|
r46554 | """return whether path is a regular file or a symlink | ||
Pierre-Yves David
|
r31217 | |||
Augie Fackler
|
r46554 | Unlike isfile, this doesn't follow symlinks.""" | ||
Pierre-Yves David
|
r31217 | try: | ||
st = self.lstat(path) | ||||
except OSError: | ||||
return False | ||||
mode = st.st_mode | ||||
return stat.S_ISREG(mode) or stat.S_ISLNK(mode) | ||||
Matt Harbison
|
r50468 | def _join(self, *paths: bytes) -> bytes: | ||
r48617 | root_idx = 0 | |||
for idx, p in enumerate(paths): | ||||
if os.path.isabs(p) or p.startswith(self._dir_sep): | ||||
root_idx = idx | ||||
if root_idx != 0: | ||||
paths = paths[root_idx:] | ||||
paths = [p for p in paths if p] | ||||
return self._dir_sep.join(paths) | ||||
Matt Harbison
|
r50468 | def reljoin(self, *paths: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | """join various elements of a path together (as os.path.join would do) | ||
The vfs base is not injected so that path stay relative. This exists | ||||
to allow handling of strange encoding if needed.""" | ||||
r48617 | return self._join(*paths) | |||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def split(self, path: bytes): | ||
Pierre-Yves David
|
r31217 | """split top-most element of a path (as os.path.split would do) | ||
This exists to allow handling of strange encoding if needed.""" | ||||
return os.path.split(path) | ||||
Matt Harbison
|
r50468 | def lexists(self, path: Optional[bytes] = None) -> bool: | ||
Pierre-Yves David
|
r31217 | return os.path.lexists(self.join(path)) | ||
Matt Harbison
|
r50468 | def lstat(self, path: Optional[bytes] = None): | ||
Pierre-Yves David
|
r31217 | return os.lstat(self.join(path)) | ||
r52545 | def is_mmap_safe(self, path: Optional[bytes] = None) -> bool: | |||
"""return True if it is safe to read a file content as mmap | ||||
This focus on the file system aspect of such safety, the application | ||||
logic around that file is not taken into account, so caller need to | ||||
make sure the file won't be truncated in a way that will create SIGBUS | ||||
on access. | ||||
The initial motivation for this logic is that if mmap is used on NFS | ||||
and somebody deletes the mapped file (e.g. by renaming on top of it), | ||||
then you get SIGBUS, which can be pretty disruptive: we get core dump | ||||
reports, and the process terminates without writing to the blackbox. | ||||
Instead in this situation we prefer to read the file normally. | ||||
The risk of ESTALE in the middle of the read remains, but it's | ||||
smaller because we read sooner and the error should be reported | ||||
just as any other error. | ||||
Note that python standard library does not offer the necessary function | ||||
to detect the file stem bits. So this detection rely on compiled bits | ||||
and is not available in pure python. | ||||
""" | ||||
# XXX Since we already assume a vfs to address a consistent file system | ||||
# in other location, we could determine the fstype once for the root | ||||
# and cache that value. | ||||
fstype = util.getfstype(self.join(path)) | ||||
return fstype is not None and fstype != b'nfs' | ||||
Matt Harbison
|
r50468 | def listdir(self, path: Optional[bytes] = None): | ||
Pierre-Yves David
|
r31217 | return os.listdir(self.join(path)) | ||
Matt Harbison
|
r50468 | def makedir(self, path: Optional[bytes] = None, notindexed=True): | ||
Pierre-Yves David
|
r31217 | return util.makedir(self.join(path), notindexed) | ||
Matt Harbison
|
r50468 | def makedirs( | ||
self, path: Optional[bytes] = None, mode: Optional[int] = None | ||||
): | ||||
Pierre-Yves David
|
r31217 | return util.makedirs(self.join(path), mode) | ||
Matt Harbison
|
r50468 | def makelock(self, info, path: bytes): | ||
Pierre-Yves David
|
r31217 | return util.makelock(info, self.join(path)) | ||
Matt Harbison
|
r50468 | def mkdir(self, path: Optional[bytes] = None): | ||
Pierre-Yves David
|
r31217 | return os.mkdir(self.join(path)) | ||
Matt Harbison
|
r50468 | def mkstemp( | ||
self, | ||||
suffix: bytes = b'', | ||||
prefix: bytes = b'tmp', | ||||
dir: Optional[bytes] = None, | ||||
): | ||||
Augie Fackler
|
r43346 | fd, name = pycompat.mkstemp( | ||
suffix=suffix, prefix=prefix, dir=self.join(dir) | ||||
) | ||||
Pierre-Yves David
|
r31217 | dname, fname = util.split(name) | ||
if dir: | ||||
return fd, os.path.join(dir, fname) | ||||
else: | ||||
return fd, fname | ||||
Matt Harbison
|
r50468 | def readdir(self, path: Optional[bytes] = None, stat=None, skip=None): | ||
Yuya Nishihara
|
r32203 | return util.listdir(self.join(path), stat, skip) | ||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def readlock(self, path: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | return util.readlock(self.join(path)) | ||
Matt Harbison
|
r50468 | def rename(self, src: bytes, dst: bytes, checkambig=False): | ||
Pierre-Yves David
|
r31217 | """Rename from src to dst | ||
checkambig argument is used with util.filestat, and is useful | ||||
only if destination file is guarded by any lock | ||||
(e.g. repo.lock or repo.wlock). | ||||
FUJIWARA Katsunori
|
r33282 | |||
To avoid file stat ambiguity forcibly, checkambig=True involves | ||||
copying ``src`` file, if it is owned by another. Therefore, use | ||||
checkambig=True only in limited cases (see also issue5418 and | ||||
issue5584 for detail). | ||||
Pierre-Yves David
|
r31217 | """ | ||
Augie Fackler
|
r43347 | self._auditpath(dst, b'w') | ||
FUJIWARA Katsunori
|
r32748 | srcpath = self.join(src) | ||
Pierre-Yves David
|
r31217 | dstpath = self.join(dst) | ||
Siddharth Agarwal
|
r32772 | oldstat = checkambig and util.filestat.frompath(dstpath) | ||
Pierre-Yves David
|
r31217 | if oldstat and oldstat.stat: | ||
FUJIWARA Katsunori
|
r33281 | ret = util.rename(srcpath, dstpath) | ||
_avoidambig(dstpath, oldstat) | ||||
Pierre-Yves David
|
r31217 | return ret | ||
FUJIWARA Katsunori
|
r32748 | return util.rename(srcpath, dstpath) | ||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def readlink(self, path: bytes) -> bytes: | ||
Matt Harbison
|
r39940 | return util.readlink(self.join(path)) | ||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def removedirs(self, path: Optional[bytes] = None): | ||
Augie Fackler
|
r46554 | """Remove a leaf directory and all empty intermediate ones""" | ||
Pierre-Yves David
|
r31217 | return util.removedirs(self.join(path)) | ||
Matt Harbison
|
r50468 | def rmdir(self, path: Optional[bytes] = None): | ||
Gregory Szorc
|
r39500 | """Remove an empty directory.""" | ||
return os.rmdir(self.join(path)) | ||||
Matt Harbison
|
r50468 | def rmtree( | ||
self, path: Optional[bytes] = None, ignore_errors=False, forcibly=False | ||||
): | ||||
Pierre-Yves David
|
r31217 | """Remove a directory tree recursively | ||
If ``forcibly``, this tries to remove READ-ONLY files, too. | ||||
""" | ||||
if forcibly: | ||||
Augie Fackler
|
r43346 | |||
Mads Kiilerich
|
r51647 | def onexc(function, path, excinfo): | ||
Pierre-Yves David
|
r31217 | if function is not os.remove: | ||
raise | ||||
# read-only files cannot be unlinked under Windows | ||||
s = os.stat(path) | ||||
if (s.st_mode & stat.S_IWRITE) != 0: | ||||
raise | ||||
os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE) | ||||
os.remove(path) | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | else: | ||
Mads Kiilerich
|
r51647 | onexc = None | ||
try: | ||||
# pytype: disable=wrong-keyword-args | ||||
return shutil.rmtree( | ||||
self.join(path), ignore_errors=ignore_errors, onexc=onexc | ||||
) | ||||
# pytype: enable=wrong-keyword-args | ||||
except TypeError: # onexc was introduced in Python 3.12 | ||||
return shutil.rmtree( | ||||
self.join(path), ignore_errors=ignore_errors, onerror=onexc | ||||
) | ||||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def setflags(self, path: bytes, l: bool, x: bool): | ||
Pierre-Yves David
|
r31217 | return util.setflags(self.join(path), l, x) | ||
Matt Harbison
|
r50468 | def stat(self, path: Optional[bytes] = None): | ||
Pierre-Yves David
|
r31217 | return os.stat(self.join(path)) | ||
Matt Harbison
|
r50468 | def unlink(self, path: Optional[bytes] = None): | ||
Pierre-Yves David
|
r31217 | return util.unlink(self.join(path)) | ||
Matt Harbison
|
r50468 | def tryunlink(self, path: Optional[bytes] = None): | ||
Ryan McElroy
|
r31542 | """Attempt to remove a file, ignoring missing file errors.""" | ||
Georges Racinet
|
r52323 | return util.tryunlink(self.join(path)) | ||
Ryan McElroy
|
r31542 | |||
Matt Harbison
|
r50468 | def unlinkpath( | ||
self, path: Optional[bytes] = None, ignoremissing=False, rmdir=True | ||||
): | ||||
Augie Fackler
|
r43346 | return util.unlinkpath( | ||
self.join(path), ignoremissing=ignoremissing, rmdir=rmdir | ||||
) | ||||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50468 | def utime(self, path: Optional[bytes] = None, t=None): | ||
Pierre-Yves David
|
r31217 | return os.utime(self.join(path), t) | ||
Matt Harbison
|
r50468 | def walk(self, path: Optional[bytes] = None, onerror=None): | ||
Pierre-Yves David
|
r31217 | """Yield (dirpath, dirs, files) tuple for each directories under path | ||
``dirpath`` is relative one from the root of this vfs. This | ||||
uses ``os.sep`` as path separator, even you specify POSIX | ||||
style ``path``. | ||||
"The root of this vfs" is represented as empty ``dirpath``. | ||||
""" | ||||
root = os.path.normpath(self.join(None)) | ||||
# when dirpath == root, dirpath[prefixlen:] becomes empty | ||||
# because len(dirpath) < prefixlen. | ||||
prefixlen = len(pathutil.normasprefix(root)) | ||||
for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror): | ||||
yield (dirpath[prefixlen:], dirs, files) | ||||
@contextlib.contextmanager | ||||
def backgroundclosing(self, ui, expectedcount=-1): | ||||
"""Allow files to be closed asynchronously. | ||||
When this context manager is active, ``backgroundclose`` can be passed | ||||
to ``__call__``/``open`` to result in the file possibly being closed | ||||
asynchronously, on a background thread. | ||||
""" | ||||
Wojciech Lis
|
r35426 | # Sharing backgroundfilecloser between threads is complex and using | ||
# multiple instances puts us at risk of running out of file descriptors | ||||
# only allow to use backgroundfilecloser when in main thread. | ||||
Augie Fackler
|
r43783 | if not isinstance( | ||
Karthikeyan Singaravelan
|
r47952 | threading.current_thread(), | ||
Augie Fackler
|
r43783 | threading._MainThread, # pytype: disable=module-attr | ||
): | ||||
Wojciech Lis
|
r35426 | yield | ||
return | ||||
Pierre-Yves David
|
r31217 | vfs = getattr(self, 'vfs', self) | ||
if getattr(vfs, '_backgroundfilecloser', None): | ||||
raise error.Abort( | ||||
Augie Fackler
|
r43347 | _(b'can only have 1 active background file closer') | ||
Augie Fackler
|
r43346 | ) | ||
Pierre-Yves David
|
r31217 | |||
with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc: | ||||
try: | ||||
Augie Fackler
|
r43783 | vfs._backgroundfilecloser = ( | ||
bfc # pytype: disable=attribute-error | ||||
) | ||||
Pierre-Yves David
|
r31217 | yield bfc | ||
finally: | ||||
Augie Fackler
|
r43783 | vfs._backgroundfilecloser = ( | ||
None # pytype: disable=attribute-error | ||||
) | ||||
Pierre-Yves David
|
r31217 | |||
r48236 | def register_file(self, path): | |||
"""generic hook point to lets fncache steer its stew""" | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | class vfs(abstractvfs): | ||
Augie Fackler
|
r46554 | """Operate files relative to a base directory | ||
Pierre-Yves David
|
r31217 | |||
This class is used to hide the details of COW semantics and | ||||
remote file access from higher level code. | ||||
Yuya Nishihara
|
r33722 | |||
'cacheaudited' should be enabled only if (a) vfs object is short-lived, or | ||||
(b) the base directory is managed by hg and considered sort-of append-only. | ||||
See pathutil.pathauditor() for details. | ||||
Augie Fackler
|
r46554 | """ | ||
Augie Fackler
|
r43346 | |||
def __init__( | ||||
self, | ||||
Matt Harbison
|
r50468 | base: bytes, | ||
Augie Fackler
|
r43346 | audit=True, | ||
cacheaudited=False, | ||||
expandpath=False, | ||||
realpath=False, | ||||
): | ||||
Pierre-Yves David
|
r31217 | if expandpath: | ||
base = util.expandpath(base) | ||||
if realpath: | ||||
base = os.path.realpath(base) | ||||
self.base = base | ||||
r33257 | self._audit = audit | |||
if audit: | ||||
Yuya Nishihara
|
r33722 | self.audit = pathutil.pathauditor(self.base, cached=cacheaudited) | ||
Pierre-Yves David
|
r31217 | else: | ||
Augie Fackler
|
r43346 | self.audit = lambda path, mode=None: True | ||
r33257 | self.createmode = None | |||
self._trustnlink = None | ||||
r43295 | self.options = {} | |||
Pierre-Yves David
|
r31217 | |||
@util.propertycache | ||||
Matt Harbison
|
r50468 | def _cansymlink(self) -> bool: | ||
Pierre-Yves David
|
r31217 | return util.checklink(self.base) | ||
@util.propertycache | ||||
def _chmod(self): | ||||
return util.checkexec(self.base) | ||||
def _fixfilemode(self, name): | ||||
if self.createmode is None or not self._chmod: | ||||
return | ||||
os.chmod(name, self.createmode & 0o666) | ||||
Matt Harbison
|
r50468 | def _auditpath(self, path, mode) -> None: | ||
Boris Feld
|
r40785 | if self._audit: | ||
Boris Feld
|
r41122 | if os.path.isabs(path) and path.startswith(self.base): | ||
path = os.path.relpath(path, self.base) | ||||
Boris Feld
|
r40785 | r = util.checkosfilename(path) | ||
if r: | ||||
Augie Fackler
|
r43347 | raise error.Abort(b"%s: %r" % (r, path)) | ||
Boris Feld
|
r40785 | self.audit(path, mode=mode) | ||
Arseniy Alekseyev
|
r50804 | def isfileorlink_checkdir( | ||
self, dircache, path: Optional[bytes] = None | ||||
) -> bool: | ||||
"""return True if the path is a regular file or a symlink and | ||||
the directories along the path are "normal", that is | ||||
Arseniy Alekseyev
|
r50807 | not symlinks or nested hg repositories. | ||
Ignores the `_audit` setting, and checks the directories regardless. | ||||
`dircache` is used to cache the directory checks. | ||||
""" | ||||
Arseniy Alekseyev
|
r50804 | try: | ||
for prefix in pathutil.finddirs_rev_noroot(util.localpath(path)): | ||||
if prefix in dircache: | ||||
res = dircache[prefix] | ||||
else: | ||||
Arseniy Alekseyev
|
r50807 | res = pathutil.pathauditor._checkfs_exists( | ||
self.base, prefix, path | ||||
) | ||||
Arseniy Alekseyev
|
r50804 | dircache[prefix] = res | ||
if not res: | ||||
return False | ||||
except (OSError, error.Abort): | ||||
return False | ||||
return self.isfileorlink(path) | ||||
Augie Fackler
|
r43346 | def __call__( | ||
self, | ||||
Matt Harbison
|
r50468 | path: bytes, | ||
Matt Harbison
|
r50469 | mode: bytes = b"rb", | ||
Augie Fackler
|
r43346 | atomictemp=False, | ||
notindexed=False, | ||||
backgroundclose=False, | ||||
checkambig=False, | ||||
auditpath=True, | ||||
makeparentdirs=True, | ||||
): | ||||
Augie Fackler
|
r46554 | """Open ``path`` file, which is relative to vfs root. | ||
Pierre-Yves David
|
r31217 | |||
Yuya Nishihara
|
r40827 | By default, parent directories are created as needed. Newly created | ||
directories are marked as "not to be indexed by the content indexing | ||||
service", if ``notindexed`` is specified for "write" mode access. | ||||
Set ``makeparentdirs=False`` to not create directories implicitly. | ||||
Pierre-Yves David
|
r31217 | |||
If ``backgroundclose`` is passed, the file may be closed asynchronously. | ||||
It can only be used if the ``self.backgroundclosing()`` context manager | ||||
is active. This should only be specified if the following criteria hold: | ||||
1. There is a potential for writing thousands of files. Unless you | ||||
are writing thousands of files, the performance benefits of | ||||
asynchronously closing files is not realized. | ||||
2. Files are opened exactly once for the ``backgroundclosing`` | ||||
active duration and are therefore free of race conditions between | ||||
closing a file on a background thread and reopening it. (If the | ||||
file were opened multiple times, there could be unflushed data | ||||
because the original file handle hasn't been flushed/closed yet.) | ||||
Kyle Lippincott
|
r45056 | ``checkambig`` argument is passed to atomictempfile (valid | ||
Pierre-Yves David
|
r31217 | only for writing), and is useful only if target file is | ||
guarded by any lock (e.g. repo.lock or repo.wlock). | ||||
FUJIWARA Katsunori
|
r33282 | |||
To avoid file stat ambiguity forcibly, checkambig=True involves | ||||
copying ``path`` file opened in "append" mode (e.g. for | ||||
truncation), if it is owned by another. Therefore, use | ||||
combination of append mode and checkambig=True only in limited | ||||
cases (see also issue5418 and issue5584 for detail). | ||||
Augie Fackler
|
r46554 | """ | ||
r33255 | if auditpath: | |||
Boris Feld
|
r40785 | self._auditpath(path, mode) | ||
Pierre-Yves David
|
r31217 | f = self.join(path) | ||
Augie Fackler
|
r43347 | if b"b" not in mode: | ||
mode += b"b" # for that other OS | ||||
Pierre-Yves David
|
r31217 | |||
nlink = -1 | ||||
Augie Fackler
|
r43347 | if mode not in (b'r', b'rb'): | ||
Pierre-Yves David
|
r31217 | dirname, basename = util.split(f) | ||
# If basename is empty, then the path is malformed because it points | ||||
# to a directory. Let the posixfile() call below raise IOError. | ||||
if basename: | ||||
if atomictemp: | ||||
Yuya Nishihara
|
r40827 | if makeparentdirs: | ||
util.makedirs(dirname, self.createmode, notindexed) | ||||
Augie Fackler
|
r43346 | return util.atomictempfile( | ||
f, mode, self.createmode, checkambig=checkambig | ||||
) | ||||
Pierre-Yves David
|
r31217 | try: | ||
Augie Fackler
|
r43347 | if b'w' in mode: | ||
Pierre-Yves David
|
r31217 | util.unlink(f) | ||
nlink = 0 | ||||
else: | ||||
# nlinks() may behave differently for files on Windows | ||||
# shares if the file is open. | ||||
with util.posixfile(f): | ||||
nlink = util.nlinks(f) | ||||
if nlink < 1: | ||||
Augie Fackler
|
r43346 | nlink = 2 # force mktempcopy (issue1922) | ||
Manuel Jacob
|
r50201 | except FileNotFoundError: | ||
Pierre-Yves David
|
r31217 | nlink = 0 | ||
Yuya Nishihara
|
r40827 | if makeparentdirs: | ||
util.makedirs(dirname, self.createmode, notindexed) | ||||
Pierre-Yves David
|
r31217 | if nlink > 0: | ||
if self._trustnlink is None: | ||||
self._trustnlink = nlink > 1 or util.checknlink(f) | ||||
if nlink > 1 or not self._trustnlink: | ||||
util.rename(util.mktempcopy(f), f) | ||||
fp = util.posixfile(f, mode) | ||||
if nlink == 0: | ||||
self._fixfilemode(f) | ||||
if checkambig: | ||||
Augie Fackler
|
r43347 | if mode in (b'r', b'rb'): | ||
Augie Fackler
|
r43346 | raise error.Abort( | ||
_( | ||||
Augie Fackler
|
r43347 | b'implementation error: mode %s is not' | ||
b' valid for checkambig=True' | ||||
Augie Fackler
|
r43346 | ) | ||
% mode | ||||
) | ||||
Pierre-Yves David
|
r31217 | fp = checkambigatclosing(fp) | ||
Augie Fackler
|
r43346 | if backgroundclose and isinstance( | ||
Karthikeyan Singaravelan
|
r47952 | threading.current_thread(), | ||
Augie Fackler
|
r43783 | threading._MainThread, # pytype: disable=module-attr | ||
Augie Fackler
|
r43346 | ): | ||
Augie Fackler
|
r43785 | if ( | ||
not self._backgroundfilecloser # pytype: disable=attribute-error | ||||
): | ||||
Augie Fackler
|
r43346 | raise error.Abort( | ||
_( | ||||
Augie Fackler
|
r43347 | b'backgroundclose can only be used when a ' | ||
b'backgroundclosing context manager is active' | ||||
Augie Fackler
|
r43346 | ) | ||
) | ||||
Pierre-Yves David
|
r31217 | |||
Augie Fackler
|
r43785 | fp = delayclosedfile( | ||
fp, | ||||
self._backgroundfilecloser, # pytype: disable=attribute-error | ||||
) | ||||
Pierre-Yves David
|
r31217 | |||
return fp | ||||
Matt Harbison
|
r50468 | def symlink(self, src: bytes, dst: bytes) -> None: | ||
Pierre-Yves David
|
r31217 | self.audit(dst) | ||
linkname = self.join(dst) | ||||
Ryan McElroy
|
r31549 | util.tryunlink(linkname) | ||
Pierre-Yves David
|
r31217 | |||
util.makedirs(os.path.dirname(linkname), self.createmode) | ||||
if self._cansymlink: | ||||
try: | ||||
os.symlink(src, linkname) | ||||
except OSError as err: | ||||
Augie Fackler
|
r43346 | raise OSError( | ||
err.errno, | ||||
Augie Fackler
|
r43347 | _(b'could not symlink to %r: %s') | ||
Augie Fackler
|
r43346 | % (src, encoding.strtolocal(err.strerror)), | ||
linkname, | ||||
) | ||||
Pierre-Yves David
|
r31217 | else: | ||
self.write(dst, src) | ||||
Matt Harbison
|
r50468 | def join(self, path: Optional[bytes], *insidef: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | if path: | ||
r48617 | parts = [self.base, path] | |||
parts.extend(insidef) | ||||
return self._join(*parts) | ||||
Pierre-Yves David
|
r31217 | else: | ||
return self.base | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | opener = vfs | ||
Augie Fackler
|
r43346 | |||
Boris Feld
|
r41125 | class proxyvfs(abstractvfs): | ||
Matt Harbison
|
r50468 | def __init__(self, vfs: "vfs"): | ||
Pierre-Yves David
|
r31217 | self.vfs = vfs | ||
r51347 | @property | |||
def createmode(self): | ||||
return self.vfs.createmode | ||||
Boris Feld
|
r41126 | def _auditpath(self, path, mode): | ||
return self.vfs._auditpath(path, mode) | ||||
Pierre-Yves David
|
r31217 | @property | ||
def options(self): | ||||
return self.vfs.options | ||||
@options.setter | ||||
def options(self, value): | ||||
self.vfs.options = value | ||||
r52486 | @property | |||
def audit(self): | ||||
return self.vfs.audit | ||||
Augie Fackler
|
r43346 | |||
Boris Feld
|
r41125 | class filtervfs(proxyvfs, abstractvfs): | ||
Pierre-Yves David
|
r31217 | '''Wrapper vfs for filtering filenames with a function.''' | ||
Matt Harbison
|
r50468 | def __init__(self, vfs: "vfs", filter): | ||
Yuya Nishihara
|
r33412 | proxyvfs.__init__(self, vfs) | ||
Pierre-Yves David
|
r31217 | self._filter = filter | ||
Matt Harbison
|
r50468 | def __call__(self, path: bytes, *args, **kwargs): | ||
Pierre-Yves David
|
r31217 | return self.vfs(self._filter(path), *args, **kwargs) | ||
Matt Harbison
|
r50468 | def join(self, path: Optional[bytes], *insidef: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | if path: | ||
return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef))) | ||||
else: | ||||
return self.vfs.join(path) | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | filteropener = filtervfs | ||
Augie Fackler
|
r43346 | |||
Boris Feld
|
r41125 | class readonlyvfs(proxyvfs): | ||
Pierre-Yves David
|
r31217 | '''Wrapper vfs preventing any writing.''' | ||
Matt Harbison
|
r50468 | def __init__(self, vfs: "vfs"): | ||
Yuya Nishihara
|
r33412 | proxyvfs.__init__(self, vfs) | ||
Pierre-Yves David
|
r31217 | |||
Matt Harbison
|
r50469 | def __call__(self, path: bytes, mode: bytes = b'rb', *args, **kw): | ||
Augie Fackler
|
r43347 | if mode not in (b'r', b'rb'): | ||
raise error.Abort(_(b'this vfs is read only')) | ||||
Pierre-Yves David
|
r31217 | return self.vfs(path, mode, *args, **kw) | ||
Matt Harbison
|
r50468 | def join(self, path: Optional[bytes], *insidef: bytes) -> bytes: | ||
Pierre-Yves David
|
r31217 | return self.vfs.join(path, *insidef) | ||
Augie Fackler
|
r43346 | |||
Gregory Szorc
|
r49801 | class closewrapbase: | ||
Pierre-Yves David
|
r31217 | """Base class of wrapper, which hooks closing | ||
Do not instantiate outside of the vfs layer. | ||||
""" | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | def __init__(self, fh): | ||
Augie Fackler
|
r43906 | object.__setattr__(self, '_origfh', fh) | ||
Pierre-Yves David
|
r31217 | |||
def __getattr__(self, attr): | ||||
return getattr(self._origfh, attr) | ||||
def __setattr__(self, attr, value): | ||||
return setattr(self._origfh, attr, value) | ||||
def __delattr__(self, attr): | ||||
return delattr(self._origfh, attr) | ||||
def __enter__(self): | ||||
Matt Harbison
|
r40975 | self._origfh.__enter__() | ||
return self | ||||
Pierre-Yves David
|
r31217 | |||
def __exit__(self, exc_type, exc_value, exc_tb): | ||||
Augie Fackler
|
r43767 | raise NotImplementedError('attempted instantiating ' + str(type(self))) | ||
Pierre-Yves David
|
r31217 | |||
def close(self): | ||||
Augie Fackler
|
r43767 | raise NotImplementedError('attempted instantiating ' + str(type(self))) | ||
Pierre-Yves David
|
r31217 | |||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | class delayclosedfile(closewrapbase): | ||
"""Proxy for a file object whose close is delayed. | ||||
Do not instantiate outside of the vfs layer. | ||||
""" | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | def __init__(self, fh, closer): | ||
super(delayclosedfile, self).__init__(fh) | ||||
Augie Fackler
|
r43906 | object.__setattr__(self, '_closer', closer) | ||
Pierre-Yves David
|
r31217 | |||
def __exit__(self, exc_type, exc_value, exc_tb): | ||||
self._closer.close(self._origfh) | ||||
def close(self): | ||||
self._closer.close(self._origfh) | ||||
Augie Fackler
|
r43346 | |||
Gregory Szorc
|
r49801 | class backgroundfilecloser: | ||
Pierre-Yves David
|
r31217 | """Coordinates background closing of file handles on multiple threads.""" | ||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | def __init__(self, ui, expectedcount=-1): | ||
self._running = False | ||||
self._entered = False | ||||
self._threads = [] | ||||
self._threadexception = None | ||||
# Only Windows/NTFS has slow file closing. So only enable by default | ||||
# on that platform. But allow to be enabled elsewhere for testing. | ||||
Jun Wu
|
r34646 | defaultenabled = pycompat.iswindows | ||
Augie Fackler
|
r43347 | enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled) | ||
Pierre-Yves David
|
r31217 | |||
if not enabled: | ||||
return | ||||
# There is overhead to starting and stopping the background threads. | ||||
# Don't do background processing unless the file count is large enough | ||||
# to justify it. | ||||
Augie Fackler
|
r43347 | minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount') | ||
Pierre-Yves David
|
r31217 | # FUTURE dynamically start background threads after minfilecount closes. | ||
# (We don't currently have any callers that don't know their file count) | ||||
if expectedcount > 0 and expectedcount < minfilecount: | ||||
return | ||||
Augie Fackler
|
r43347 | maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue') | ||
threadcount = ui.configint(b'worker', b'backgroundclosethreadcount') | ||||
Pierre-Yves David
|
r31217 | |||
Augie Fackler
|
r43346 | ui.debug( | ||
Augie Fackler
|
r43347 | b'starting %d threads for background file closing\n' % threadcount | ||
Augie Fackler
|
r43346 | ) | ||
Pierre-Yves David
|
r31217 | |||
Gregory Szorc
|
r37863 | self._queue = pycompat.queue.Queue(maxsize=maxqueue) | ||
Pierre-Yves David
|
r31217 | self._running = True | ||
for i in range(threadcount): | ||||
Augie Fackler
|
r43763 | t = threading.Thread(target=self._worker, name='backgroundcloser') | ||
Pierre-Yves David
|
r31217 | self._threads.append(t) | ||
t.start() | ||||
def __enter__(self): | ||||
self._entered = True | ||||
return self | ||||
def __exit__(self, exc_type, exc_value, exc_tb): | ||||
self._running = False | ||||
# Wait for threads to finish closing so open files don't linger for | ||||
# longer than lifetime of context manager. | ||||
for t in self._threads: | ||||
t.join() | ||||
def _worker(self): | ||||
"""Main routine for worker thread.""" | ||||
while True: | ||||
try: | ||||
fh = self._queue.get(block=True, timeout=0.100) | ||||
# Need to catch or the thread will terminate and | ||||
# we could orphan file descriptors. | ||||
try: | ||||
fh.close() | ||||
except Exception as e: | ||||
# Stash so can re-raise from main thread later. | ||||
self._threadexception = e | ||||
Gregory Szorc
|
r37863 | except pycompat.queue.Empty: | ||
Pierre-Yves David
|
r31217 | if not self._running: | ||
break | ||||
def close(self, fh): | ||||
"""Schedule a file for closing.""" | ||||
if not self._entered: | ||||
Augie Fackler
|
r43346 | raise error.Abort( | ||
Martin von Zweigbergk
|
r43387 | _(b'can only call close() when context manager active') | ||
Augie Fackler
|
r43346 | ) | ||
Pierre-Yves David
|
r31217 | |||
# If a background thread encountered an exception, raise now so we fail | ||||
# fast. Otherwise we may potentially go on for minutes until the error | ||||
# is acted on. | ||||
if self._threadexception: | ||||
e = self._threadexception | ||||
self._threadexception = None | ||||
raise e | ||||
# If we're not actively running, close synchronously. | ||||
if not self._running: | ||||
fh.close() | ||||
return | ||||
self._queue.put(fh, block=True, timeout=None) | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | class checkambigatclosing(closewrapbase): | ||
"""Proxy for a file object, to avoid ambiguity of file stat | ||||
See also util.filestat for detail about "ambiguity of file stat". | ||||
This proxy is useful only if the target file is guarded by any | ||||
lock (e.g. repo.lock or repo.wlock) | ||||
Do not instantiate outside of the vfs layer. | ||||
""" | ||||
Augie Fackler
|
r43346 | |||
Pierre-Yves David
|
r31217 | def __init__(self, fh): | ||
super(checkambigatclosing, self).__init__(fh) | ||||
Augie Fackler
|
r43906 | object.__setattr__(self, '_oldstat', util.filestat.frompath(fh.name)) | ||
Pierre-Yves David
|
r31217 | |||
def _checkambig(self): | ||||
oldstat = self._oldstat | ||||
if oldstat.stat: | ||||
FUJIWARA Katsunori
|
r33280 | _avoidambig(self._origfh.name, oldstat) | ||
Pierre-Yves David
|
r31217 | |||
def __exit__(self, exc_type, exc_value, exc_tb): | ||||
self._origfh.__exit__(exc_type, exc_value, exc_tb) | ||||
self._checkambig() | ||||
def close(self): | ||||
self._origfh.close() | ||||
self._checkambig() | ||||