##// END OF EJS Templates
typing: manually add type annotations to `mercurial/vfs.py`...
Matt Harbison -
r52785:fa9e8a65 default
parent child Browse files
Show More
@@ -1,877 +1,908 b''
1 # vfs.py - Mercurial 'vfs' classes
1 # vfs.py - Mercurial 'vfs' classes
2 #
2 #
3 # Copyright Olivia Mackall <olivia@selenic.com>
3 # Copyright Olivia Mackall <olivia@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import annotations
8 from __future__ import annotations
9
9
10 import abc
10 import abc
11 import contextlib
11 import contextlib
12 import os
12 import os
13 import shutil
13 import shutil
14 import stat
14 import stat
15 import threading
15 import threading
16 import typing
16 import typing
17
17
18 from typing import (
18 from typing import (
19 Any,
19 Any,
20 BinaryIO,
21 Callable,
20 Iterable,
22 Iterable,
21 Iterator,
23 Iterator,
22 List,
24 List,
25 MutableMapping,
23 Optional,
26 Optional,
24 Tuple,
27 Tuple,
25 Type,
28 Type,
26 TypeVar,
29 TypeVar,
27 )
30 )
28
31
29 from .i18n import _
32 from .i18n import _
30 from . import (
33 from . import (
31 encoding,
34 encoding,
32 error,
35 error,
33 pathutil,
36 pathutil,
34 pycompat,
37 pycompat,
35 util,
38 util,
36 )
39 )
37
40
38 if typing.TYPE_CHECKING:
41 if typing.TYPE_CHECKING:
42 from . import (
43 ui as uimod,
44 )
45
39 _Tbackgroundfilecloser = TypeVar(
46 _Tbackgroundfilecloser = TypeVar(
40 '_Tbackgroundfilecloser', bound='backgroundfilecloser'
47 '_Tbackgroundfilecloser', bound='backgroundfilecloser'
41 )
48 )
42 _Tclosewrapbase = TypeVar('_Tclosewrapbase', bound='closewrapbase')
49 _Tclosewrapbase = TypeVar('_Tclosewrapbase', bound='closewrapbase')
50 _OnErrorFn = Callable[[Exception], Optional[object]]
43
51
44
52
45 def _avoidambig(path: bytes, oldstat) -> None:
53 def _avoidambig(path: bytes, oldstat: util.filestat) -> None:
46 """Avoid file stat ambiguity forcibly
54 """Avoid file stat ambiguity forcibly
47
55
48 This function causes copying ``path`` file, if it is owned by
56 This function causes copying ``path`` file, if it is owned by
49 another (see issue5418 and issue5584 for detail).
57 another (see issue5418 and issue5584 for detail).
50 """
58 """
51
59
52 def checkandavoid():
60 def checkandavoid():
53 newstat = util.filestat.frompath(path)
61 newstat = util.filestat.frompath(path)
54 # return whether file stat ambiguity is (already) avoided
62 # return whether file stat ambiguity is (already) avoided
55 return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat)
63 return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat)
56
64
57 if not checkandavoid():
65 if not checkandavoid():
58 # simply copy to change owner of path to get privilege to
66 # simply copy to change owner of path to get privilege to
59 # advance mtime (see issue5418)
67 # advance mtime (see issue5418)
60 util.rename(util.mktempcopy(path), path)
68 util.rename(util.mktempcopy(path), path)
61 checkandavoid()
69 checkandavoid()
62
70
63
71
64 class abstractvfs(abc.ABC):
72 class abstractvfs(abc.ABC):
65 """Abstract base class; cannot be instantiated"""
73 """Abstract base class; cannot be instantiated"""
66
74
67 # default directory separator for vfs
75 # default directory separator for vfs
68 #
76 #
69 # Other vfs code always use `/` and this works fine because python file API
77 # Other vfs code always use `/` and this works fine because python file API
70 # abstract the use of `/` and make it work transparently. For consistency
78 # abstract the use of `/` and make it work transparently. For consistency
71 # vfs will always use `/` when joining. This avoids some confusion in
79 # vfs will always use `/` when joining. This avoids some confusion in
72 # encoded vfs (see issue6546)
80 # encoded vfs (see issue6546)
73 _dir_sep: bytes = b'/'
81 _dir_sep: bytes = b'/'
74
82
75 # TODO: type return, which is util.posixfile wrapped by a proxy
83 # TODO: type return, which is util.posixfile wrapped by a proxy
76 @abc.abstractmethod
84 @abc.abstractmethod
77 def __call__(self, path: bytes, mode: bytes = b'rb', **kwargs) -> Any:
85 def __call__(self, path: bytes, mode: bytes = b'rb', **kwargs) -> Any:
78 ...
86 ...
79
87
80 @abc.abstractmethod
88 @abc.abstractmethod
81 def _auditpath(self, path: bytes, mode: bytes) -> Any:
89 def _auditpath(self, path: bytes, mode: bytes) -> None:
82 ...
90 ...
83
91
84 @abc.abstractmethod
92 @abc.abstractmethod
85 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
93 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
86 ...
94 ...
87
95
88 def tryread(self, path: bytes) -> bytes:
96 def tryread(self, path: bytes) -> bytes:
89 '''gracefully return an empty string for missing files'''
97 '''gracefully return an empty string for missing files'''
90 try:
98 try:
91 return self.read(path)
99 return self.read(path)
92 except FileNotFoundError:
100 except FileNotFoundError:
93 pass
101 pass
94 return b""
102 return b""
95
103
96 def tryreadlines(self, path: bytes, mode: bytes = b'rb') -> Any:
104 def tryreadlines(self, path: bytes, mode: bytes = b'rb') -> List[bytes]:
97 '''gracefully return an empty array for missing files'''
105 '''gracefully return an empty array for missing files'''
98 try:
106 try:
99 return self.readlines(path, mode=mode)
107 return self.readlines(path, mode=mode)
100 except FileNotFoundError:
108 except FileNotFoundError:
101 pass
109 pass
102 return []
110 return []
103
111
104 @util.propertycache
112 @util.propertycache
105 def open(self):
113 def open(self):
106 """Open ``path`` file, which is relative to vfs root.
114 """Open ``path`` file, which is relative to vfs root.
107
115
108 Newly created directories are marked as "not to be indexed by
116 Newly created directories are marked as "not to be indexed by
109 the content indexing service", if ``notindexed`` is specified
117 the content indexing service", if ``notindexed`` is specified
110 for "write" mode access.
118 for "write" mode access.
111 """
119 """
112 return self.__call__
120 return self.__call__
113
121
114 def read(self, path: bytes) -> bytes:
122 def read(self, path: bytes) -> bytes:
115 with self(path, b'rb') as fp:
123 with self(path, b'rb') as fp:
116 return fp.read()
124 return fp.read()
117
125
118 def readlines(self, path: bytes, mode: bytes = b'rb') -> Any:
126 def readlines(self, path: bytes, mode: bytes = b'rb') -> List[bytes]:
119 with self(path, mode=mode) as fp:
127 with self(path, mode=mode) as fp:
120 return fp.readlines()
128 return fp.readlines()
121
129
122 def write(
130 def write(
123 self, path: bytes, data: bytes, backgroundclose=False, **kwargs
131 self, path: bytes, data: bytes, backgroundclose: bool = False, **kwargs
124 ) -> int:
132 ) -> int:
125 with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp:
133 with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp:
126 return fp.write(data)
134 return fp.write(data)
127
135
128 def writelines(
136 def writelines(
129 self,
137 self,
130 path: bytes,
138 path: bytes,
131 data: Iterable[bytes],
139 data: Iterable[bytes],
132 mode: bytes = b'wb',
140 mode: bytes = b'wb',
133 notindexed=False,
141 notindexed: bool = False,
134 ) -> None:
142 ) -> None:
135 with self(path, mode=mode, notindexed=notindexed) as fp:
143 with self(path, mode=mode, notindexed=notindexed) as fp:
136 return fp.writelines(data)
144 return fp.writelines(data)
137
145
138 def append(self, path: bytes, data: bytes) -> int:
146 def append(self, path: bytes, data: bytes) -> int:
139 with self(path, b'ab') as fp:
147 with self(path, b'ab') as fp:
140 return fp.write(data)
148 return fp.write(data)
141
149
142 def basename(self, path: bytes) -> bytes:
150 def basename(self, path: bytes) -> bytes:
143 """return base element of a path (as os.path.basename would do)
151 """return base element of a path (as os.path.basename would do)
144
152
145 This exists to allow handling of strange encoding if needed."""
153 This exists to allow handling of strange encoding if needed."""
146 return os.path.basename(path)
154 return os.path.basename(path)
147
155
148 def chmod(self, path: bytes, mode: int) -> None:
156 def chmod(self, path: bytes, mode: int) -> None:
149 return os.chmod(self.join(path), mode)
157 return os.chmod(self.join(path), mode)
150
158
151 def dirname(self, path: bytes) -> bytes:
159 def dirname(self, path: bytes) -> bytes:
152 """return dirname element of a path (as os.path.dirname would do)
160 """return dirname element of a path (as os.path.dirname would do)
153
161
154 This exists to allow handling of strange encoding if needed."""
162 This exists to allow handling of strange encoding if needed."""
155 return os.path.dirname(path)
163 return os.path.dirname(path)
156
164
157 def exists(self, path: Optional[bytes] = None) -> bool:
165 def exists(self, path: Optional[bytes] = None) -> bool:
158 return os.path.exists(self.join(path))
166 return os.path.exists(self.join(path))
159
167
160 def fstat(self, fp) -> os.stat_result:
168 def fstat(self, fp: BinaryIO) -> os.stat_result:
161 return util.fstat(fp)
169 return util.fstat(fp)
162
170
163 def isdir(self, path: Optional[bytes] = None) -> bool:
171 def isdir(self, path: Optional[bytes] = None) -> bool:
164 return os.path.isdir(self.join(path))
172 return os.path.isdir(self.join(path))
165
173
166 def isfile(self, path: Optional[bytes] = None) -> bool:
174 def isfile(self, path: Optional[bytes] = None) -> bool:
167 return os.path.isfile(self.join(path))
175 return os.path.isfile(self.join(path))
168
176
169 def islink(self, path: Optional[bytes] = None) -> bool:
177 def islink(self, path: Optional[bytes] = None) -> bool:
170 return os.path.islink(self.join(path))
178 return os.path.islink(self.join(path))
171
179
172 def isfileorlink(self, path: Optional[bytes] = None) -> bool:
180 def isfileorlink(self, path: Optional[bytes] = None) -> bool:
173 """return whether path is a regular file or a symlink
181 """return whether path is a regular file or a symlink
174
182
175 Unlike isfile, this doesn't follow symlinks."""
183 Unlike isfile, this doesn't follow symlinks."""
176 try:
184 try:
177 st = self.lstat(path)
185 st = self.lstat(path)
178 except OSError:
186 except OSError:
179 return False
187 return False
180 mode = st.st_mode
188 mode = st.st_mode
181 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
189 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
182
190
183 def _join(self, *paths: bytes) -> bytes:
191 def _join(self, *paths: bytes) -> bytes:
184 root_idx = 0
192 root_idx = 0
185 for idx, p in enumerate(paths):
193 for idx, p in enumerate(paths):
186 if os.path.isabs(p) or p.startswith(self._dir_sep):
194 if os.path.isabs(p) or p.startswith(self._dir_sep):
187 root_idx = idx
195 root_idx = idx
188 if root_idx != 0:
196 if root_idx != 0:
189 paths = paths[root_idx:]
197 paths = paths[root_idx:]
190 paths = [p for p in paths if p]
198 paths = [p for p in paths if p]
191 return self._dir_sep.join(paths)
199 return self._dir_sep.join(paths)
192
200
193 def reljoin(self, *paths: bytes) -> bytes:
201 def reljoin(self, *paths: bytes) -> bytes:
194 """join various elements of a path together (as os.path.join would do)
202 """join various elements of a path together (as os.path.join would do)
195
203
196 The vfs base is not injected so that path stay relative. This exists
204 The vfs base is not injected so that path stay relative. This exists
197 to allow handling of strange encoding if needed."""
205 to allow handling of strange encoding if needed."""
198 return self._join(*paths)
206 return self._join(*paths)
199
207
200 def split(self, path: bytes) -> Tuple[bytes, bytes]:
208 def split(self, path: bytes) -> Tuple[bytes, bytes]:
201 """split top-most element of a path (as os.path.split would do)
209 """split top-most element of a path (as os.path.split would do)
202
210
203 This exists to allow handling of strange encoding if needed."""
211 This exists to allow handling of strange encoding if needed."""
204 return os.path.split(path)
212 return os.path.split(path)
205
213
206 def lexists(self, path: Optional[bytes] = None) -> bool:
214 def lexists(self, path: Optional[bytes] = None) -> bool:
207 return os.path.lexists(self.join(path))
215 return os.path.lexists(self.join(path))
208
216
209 def lstat(self, path: Optional[bytes] = None) -> os.stat_result:
217 def lstat(self, path: Optional[bytes] = None) -> os.stat_result:
210 return os.lstat(self.join(path))
218 return os.lstat(self.join(path))
211
219
212 def is_mmap_safe(self, path: Optional[bytes] = None) -> bool:
220 def is_mmap_safe(self, path: Optional[bytes] = None) -> bool:
213 """return True if it is safe to read a file content as mmap
221 """return True if it is safe to read a file content as mmap
214
222
215 This focus on the file system aspect of such safety, the application
223 This focus on the file system aspect of such safety, the application
216 logic around that file is not taken into account, so caller need to
224 logic around that file is not taken into account, so caller need to
217 make sure the file won't be truncated in a way that will create SIGBUS
225 make sure the file won't be truncated in a way that will create SIGBUS
218 on access.
226 on access.
219
227
220
228
221 The initial motivation for this logic is that if mmap is used on NFS
229 The initial motivation for this logic is that if mmap is used on NFS
222 and somebody deletes the mapped file (e.g. by renaming on top of it),
230 and somebody deletes the mapped file (e.g. by renaming on top of it),
223 then you get SIGBUS, which can be pretty disruptive: we get core dump
231 then you get SIGBUS, which can be pretty disruptive: we get core dump
224 reports, and the process terminates without writing to the blackbox.
232 reports, and the process terminates without writing to the blackbox.
225
233
226 Instead, in this situation we prefer to read the file normally.
234 Instead, in this situation we prefer to read the file normally.
227 The risk of ESTALE in the middle of the read remains, but it's
235 The risk of ESTALE in the middle of the read remains, but it's
228 smaller because we read sooner and the error should be reported
236 smaller because we read sooner and the error should be reported
229 just as any other error.
237 just as any other error.
230
238
231 Note that python standard library does not offer the necessary function
239 Note that python standard library does not offer the necessary function
232 to detect the file stem bits. So this detection rely on compiled bits
240 to detect the file stem bits. So this detection rely on compiled bits
233 and is not available in pure python.
241 and is not available in pure python.
234 """
242 """
235 # XXX Since we already assume a vfs to address a consistent file system
243 # XXX Since we already assume a vfs to address a consistent file system
236 # in other location, we could determine the fstype once for the root
244 # in other location, we could determine the fstype once for the root
237 # and cache that value.
245 # and cache that value.
238 fstype = util.getfstype(self.join(path))
246 fstype = util.getfstype(self.join(path))
239 return fstype is not None and fstype != b'nfs'
247 return fstype is not None and fstype != b'nfs'
240
248
241 def listdir(self, path: Optional[bytes] = None) -> List[bytes]:
249 def listdir(self, path: Optional[bytes] = None) -> List[bytes]:
242 return os.listdir(self.join(path))
250 return os.listdir(self.join(path))
243
251
244 def makedir(self, path: Optional[bytes] = None, notindexed=True) -> None:
252 def makedir(self, path: Optional[bytes] = None, notindexed=True) -> None:
245 return util.makedir(self.join(path), notindexed)
253 return util.makedir(self.join(path), notindexed)
246
254
247 def makedirs(
255 def makedirs(
248 self, path: Optional[bytes] = None, mode: Optional[int] = None
256 self, path: Optional[bytes] = None, mode: Optional[int] = None
249 ) -> None:
257 ) -> None:
250 return util.makedirs(self.join(path), mode)
258 return util.makedirs(self.join(path), mode)
251
259
252 def makelock(self, info, path: bytes) -> None:
260 def makelock(self, info: bytes, path: bytes) -> None:
253 return util.makelock(info, self.join(path))
261 return util.makelock(info, self.join(path))
254
262
255 def mkdir(self, path: Optional[bytes] = None) -> None:
263 def mkdir(self, path: Optional[bytes] = None) -> None:
256 return os.mkdir(self.join(path))
264 return os.mkdir(self.join(path))
257
265
258 def mkstemp(
266 def mkstemp(
259 self,
267 self,
260 suffix: bytes = b'',
268 suffix: bytes = b'',
261 prefix: bytes = b'tmp',
269 prefix: bytes = b'tmp',
262 dir: Optional[bytes] = None,
270 dir: Optional[bytes] = None,
263 ) -> Tuple[int, bytes]:
271 ) -> Tuple[int, bytes]:
264 fd, name = pycompat.mkstemp(
272 fd, name = pycompat.mkstemp(
265 suffix=suffix, prefix=prefix, dir=self.join(dir)
273 suffix=suffix, prefix=prefix, dir=self.join(dir)
266 )
274 )
267 dname, fname = util.split(name)
275 dname, fname = util.split(name)
268 if dir:
276 if dir:
269 return fd, os.path.join(dir, fname)
277 return fd, os.path.join(dir, fname)
270 else:
278 else:
271 return fd, fname
279 return fd, fname
272
280
281 # TODO: This doesn't match osutil.listdir(). stat=False in pure;
282 # non-optional bool in cext. 'skip' is bool if we trust cext, or bytes
283 # going by how pure uses it. Also, cext returns a custom stat structure.
284 # from cext.osutil.pyi:
285 #
286 # path: bytes, st: bool, skip: Optional[bool]
273 def readdir(
287 def readdir(
274 self, path: Optional[bytes] = None, stat=None, skip=None
288 self, path: Optional[bytes] = None, stat=None, skip=None
275 ) -> Any:
289 ) -> Any:
276 return util.listdir(self.join(path), stat, skip)
290 return util.listdir(self.join(path), stat, skip)
277
291
278 def readlock(self, path: bytes) -> bytes:
292 def readlock(self, path: bytes) -> bytes:
279 return util.readlock(self.join(path))
293 return util.readlock(self.join(path))
280
294
281 def rename(self, src: bytes, dst: bytes, checkambig=False) -> None:
295 def rename(self, src: bytes, dst: bytes, checkambig: bool = False) -> None:
282 """Rename from src to dst
296 """Rename from src to dst
283
297
284 checkambig argument is used with util.filestat, and is useful
298 checkambig argument is used with util.filestat, and is useful
285 only if destination file is guarded by any lock
299 only if destination file is guarded by any lock
286 (e.g. repo.lock or repo.wlock).
300 (e.g. repo.lock or repo.wlock).
287
301
288 To avoid file stat ambiguity forcibly, checkambig=True involves
302 To avoid file stat ambiguity forcibly, checkambig=True involves
289 copying ``src`` file, if it is owned by another. Therefore, use
303 copying ``src`` file, if it is owned by another. Therefore, use
290 checkambig=True only in limited cases (see also issue5418 and
304 checkambig=True only in limited cases (see also issue5418 and
291 issue5584 for detail).
305 issue5584 for detail).
292 """
306 """
293 self._auditpath(dst, b'w')
307 self._auditpath(dst, b'w')
294 srcpath = self.join(src)
308 srcpath = self.join(src)
295 dstpath = self.join(dst)
309 dstpath = self.join(dst)
296 oldstat = util.filestat.frompath(dstpath) if checkambig else None
310 oldstat = util.filestat.frompath(dstpath) if checkambig else None
297
311
298 util.rename(srcpath, dstpath)
312 util.rename(srcpath, dstpath)
299
313
300 if oldstat and oldstat.stat:
314 if oldstat and oldstat.stat:
301 _avoidambig(dstpath, oldstat)
315 _avoidambig(dstpath, oldstat)
302
316
303 def readlink(self, path: bytes) -> bytes:
317 def readlink(self, path: bytes) -> bytes:
304 return util.readlink(self.join(path))
318 return util.readlink(self.join(path))
305
319
306 def removedirs(self, path: Optional[bytes] = None) -> None:
320 def removedirs(self, path: Optional[bytes] = None) -> None:
307 """Remove a leaf directory and all empty intermediate ones"""
321 """Remove a leaf directory and all empty intermediate ones"""
308 return util.removedirs(self.join(path))
322 return util.removedirs(self.join(path))
309
323
310 def rmdir(self, path: Optional[bytes] = None) -> None:
324 def rmdir(self, path: Optional[bytes] = None) -> None:
311 """Remove an empty directory."""
325 """Remove an empty directory."""
312 return os.rmdir(self.join(path))
326 return os.rmdir(self.join(path))
313
327
314 def rmtree(
328 def rmtree(
315 self, path: Optional[bytes] = None, ignore_errors=False, forcibly=False
329 self,
330 path: Optional[bytes] = None,
331 ignore_errors: bool = False,
332 forcibly: bool = False,
316 ) -> None:
333 ) -> None:
317 """Remove a directory tree recursively
334 """Remove a directory tree recursively
318
335
319 If ``forcibly``, this tries to remove READ-ONLY files, too.
336 If ``forcibly``, this tries to remove READ-ONLY files, too.
320 """
337 """
321 if forcibly:
338 if forcibly:
322
339
323 def onexc(function, path, excinfo):
340 def onexc(function, path: bytes, excinfo):
324 if function is not os.remove:
341 if function is not os.remove:
325 raise
342 raise
326 # read-only files cannot be unlinked under Windows
343 # read-only files cannot be unlinked under Windows
327 s = os.stat(path)
344 s = os.stat(path)
328 if (s.st_mode & stat.S_IWRITE) != 0:
345 if (s.st_mode & stat.S_IWRITE) != 0:
329 raise
346 raise
330 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
347 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
331 os.remove(path)
348 os.remove(path)
332
349
333 else:
350 else:
334 onexc = None
351 onexc = None
335 try:
352 try:
336 # pytype: disable=wrong-keyword-args
353 # pytype: disable=wrong-keyword-args
337 return shutil.rmtree(
354 return shutil.rmtree(
338 self.join(path), ignore_errors=ignore_errors, onexc=onexc
355 self.join(path), ignore_errors=ignore_errors, onexc=onexc
339 )
356 )
340 # pytype: enable=wrong-keyword-args
357 # pytype: enable=wrong-keyword-args
341 except TypeError: # onexc was introduced in Python 3.12
358 except TypeError: # onexc was introduced in Python 3.12
342 return shutil.rmtree(
359 return shutil.rmtree(
343 self.join(path), ignore_errors=ignore_errors, onerror=onexc
360 self.join(path), ignore_errors=ignore_errors, onerror=onexc
344 )
361 )
345
362
346 def setflags(self, path: bytes, l: bool, x: bool) -> None:
363 def setflags(self, path: bytes, l: bool, x: bool) -> None:
347 return util.setflags(self.join(path), l, x)
364 return util.setflags(self.join(path), l, x)
348
365
349 def stat(self, path: Optional[bytes] = None) -> os.stat_result:
366 def stat(self, path: Optional[bytes] = None) -> os.stat_result:
350 return os.stat(self.join(path))
367 return os.stat(self.join(path))
351
368
352 def unlink(self, path: Optional[bytes] = None) -> None:
369 def unlink(self, path: Optional[bytes] = None) -> None:
353 return util.unlink(self.join(path))
370 return util.unlink(self.join(path))
354
371
355 def tryunlink(self, path: Optional[bytes] = None) -> bool:
372 def tryunlink(self, path: Optional[bytes] = None) -> bool:
356 """Attempt to remove a file, ignoring missing file errors."""
373 """Attempt to remove a file, ignoring missing file errors."""
357 return util.tryunlink(self.join(path))
374 return util.tryunlink(self.join(path))
358
375
359 def unlinkpath(
376 def unlinkpath(
360 self, path: Optional[bytes] = None, ignoremissing=False, rmdir=True
377 self,
378 path: Optional[bytes] = None,
379 ignoremissing: bool = False,
380 rmdir: bool = True,
361 ) -> None:
381 ) -> None:
362 return util.unlinkpath(
382 return util.unlinkpath(
363 self.join(path), ignoremissing=ignoremissing, rmdir=rmdir
383 self.join(path), ignoremissing=ignoremissing, rmdir=rmdir
364 )
384 )
365
385
366 def utime(self, path: Optional[bytes] = None, t=None) -> None:
386 # TODO: could be Tuple[float, float] too.
387 def utime(
388 self, path: Optional[bytes] = None, t: Optional[Tuple[int, int]] = None
389 ) -> None:
367 return os.utime(self.join(path), t)
390 return os.utime(self.join(path), t)
368
391
369 def walk(
392 def walk(
370 self, path: Optional[bytes] = None, onerror=None
393 self, path: Optional[bytes] = None, onerror: Optional[_OnErrorFn] = None
371 ) -> Iterator[Tuple[bytes, List[bytes], List[bytes]]]:
394 ) -> Iterator[Tuple[bytes, List[bytes], List[bytes]]]:
372 """Yield (dirpath, dirs, files) tuple for each directory under path
395 """Yield (dirpath, dirs, files) tuple for each directory under path
373
396
374 ``dirpath`` is relative one from the root of this vfs. This
397 ``dirpath`` is relative one from the root of this vfs. This
375 uses ``os.sep`` as path separator, even you specify POSIX
398 uses ``os.sep`` as path separator, even you specify POSIX
376 style ``path``.
399 style ``path``.
377
400
378 "The root of this vfs" is represented as empty ``dirpath``.
401 "The root of this vfs" is represented as empty ``dirpath``.
379 """
402 """
380 root = os.path.normpath(self.join(None))
403 root = os.path.normpath(self.join(None))
381 # when dirpath == root, dirpath[prefixlen:] becomes empty
404 # when dirpath == root, dirpath[prefixlen:] becomes empty
382 # because len(dirpath) < prefixlen.
405 # because len(dirpath) < prefixlen.
383 prefixlen = len(pathutil.normasprefix(root))
406 prefixlen = len(pathutil.normasprefix(root))
384 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
407 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
385 yield (dirpath[prefixlen:], dirs, files)
408 yield (dirpath[prefixlen:], dirs, files)
386
409
387 @contextlib.contextmanager
410 @contextlib.contextmanager
388 def backgroundclosing(
411 def backgroundclosing(
389 self, ui, expectedcount=-1
412 self, ui: uimod.ui, expectedcount: int = -1
390 ) -> Iterator[Optional[backgroundfilecloser]]:
413 ) -> Iterator[Optional[backgroundfilecloser]]:
391 """Allow files to be closed asynchronously.
414 """Allow files to be closed asynchronously.
392
415
393 When this context manager is active, ``backgroundclose`` can be passed
416 When this context manager is active, ``backgroundclose`` can be passed
394 to ``__call__``/``open`` to result in the file possibly being closed
417 to ``__call__``/``open`` to result in the file possibly being closed
395 asynchronously, on a background thread.
418 asynchronously, on a background thread.
396 """
419 """
397 # Sharing backgroundfilecloser between threads is complex and using
420 # Sharing backgroundfilecloser between threads is complex and using
398 # multiple instances puts us at risk of running out of file descriptors
421 # multiple instances puts us at risk of running out of file descriptors
399 # only allow to use backgroundfilecloser when in main thread.
422 # only allow to use backgroundfilecloser when in main thread.
400 if threading.current_thread() is not threading.main_thread():
423 if threading.current_thread() is not threading.main_thread():
401 yield
424 yield
402 return
425 return
403 vfs = getattr(self, 'vfs', self)
426 vfs = getattr(self, 'vfs', self)
404 if getattr(vfs, '_backgroundfilecloser', None):
427 if getattr(vfs, '_backgroundfilecloser', None):
405 raise error.Abort(
428 raise error.Abort(
406 _(b'can only have 1 active background file closer')
429 _(b'can only have 1 active background file closer')
407 )
430 )
408
431
409 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
432 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
410 try:
433 try:
411 vfs._backgroundfilecloser = (
434 vfs._backgroundfilecloser = (
412 bfc # pytype: disable=attribute-error
435 bfc # pytype: disable=attribute-error
413 )
436 )
414 yield bfc
437 yield bfc
415 finally:
438 finally:
416 vfs._backgroundfilecloser = (
439 vfs._backgroundfilecloser = (
417 None # pytype: disable=attribute-error
440 None # pytype: disable=attribute-error
418 )
441 )
419
442
420 def register_file(self, path) -> None:
443 def register_file(self, path: bytes) -> None:
421 """generic hook point to lets fncache steer its stew"""
444 """generic hook point to lets fncache steer its stew"""
422
445
423
446
424 class vfs(abstractvfs):
447 class vfs(abstractvfs):
425 """Operate files relative to a base directory
448 """Operate files relative to a base directory
426
449
427 This class is used to hide the details of COW semantics and
450 This class is used to hide the details of COW semantics and
428 remote file access from higher level code.
451 remote file access from higher level code.
429
452
430 'cacheaudited' should be enabled only if (a) vfs object is short-lived, or
453 'cacheaudited' should be enabled only if (a) vfs object is short-lived, or
431 (b) the base directory is managed by hg and considered sort-of append-only.
454 (b) the base directory is managed by hg and considered sort-of append-only.
432 See pathutil.pathauditor() for details.
455 See pathutil.pathauditor() for details.
433 """
456 """
434
457
458 createmode: Optional[int]
459
435 def __init__(
460 def __init__(
436 self,
461 self,
437 base: bytes,
462 base: bytes,
438 audit=True,
463 audit: bool = True,
439 cacheaudited=False,
464 cacheaudited: bool = False,
440 expandpath=False,
465 expandpath: bool = False,
441 realpath=False,
466 realpath: bool = False,
442 ) -> None:
467 ) -> None:
443 if expandpath:
468 if expandpath:
444 base = util.expandpath(base)
469 base = util.expandpath(base)
445 if realpath:
470 if realpath:
446 base = os.path.realpath(base)
471 base = os.path.realpath(base)
447 self.base = base
472 self.base = base
448 self._audit = audit
473 self._audit = audit
449 if audit:
474 if audit:
450 self.audit = pathutil.pathauditor(self.base, cached=cacheaudited)
475 self.audit = pathutil.pathauditor(self.base, cached=cacheaudited)
451 else:
476 else:
452 self.audit = lambda path, mode=None: True
477 self.audit = lambda path, mode=None: True
453 self.createmode = None
478 self.createmode = None
454 self._trustnlink = None
479 self._trustnlink = None
455 self.options = {}
480 self.options = {}
456
481
457 @util.propertycache
482 @util.propertycache
458 def _cansymlink(self) -> bool:
483 def _cansymlink(self) -> bool:
459 return util.checklink(self.base)
484 return util.checklink(self.base)
460
485
461 @util.propertycache
486 @util.propertycache
462 def _chmod(self):
487 def _chmod(self) -> bool:
463 return util.checkexec(self.base)
488 return util.checkexec(self.base)
464
489
465 def _fixfilemode(self, name) -> None:
490 def _fixfilemode(self, name: bytes) -> None:
466 if self.createmode is None or not self._chmod:
491 if self.createmode is None or not self._chmod:
467 return
492 return
468 os.chmod(name, self.createmode & 0o666)
493 os.chmod(name, self.createmode & 0o666)
469
494
470 def _auditpath(self, path, mode) -> None:
495 def _auditpath(self, path: bytes, mode: bytes) -> None:
471 if self._audit:
496 if self._audit:
472 if os.path.isabs(path) and path.startswith(self.base):
497 if os.path.isabs(path) and path.startswith(self.base):
473 path = os.path.relpath(path, self.base)
498 path = os.path.relpath(path, self.base)
474 r = util.checkosfilename(path)
499 r = util.checkosfilename(path)
475 if r:
500 if r:
476 raise error.Abort(b"%s: %r" % (r, path))
501 raise error.Abort(b"%s: %r" % (r, path))
477 self.audit(path, mode=mode)
502 self.audit(path, mode=mode)
478
503
479 def isfileorlink_checkdir(
504 def isfileorlink_checkdir(
480 self, dircache, path: Optional[bytes] = None
505 self,
506 dircache: MutableMapping[bytes, bool],
507 path: Optional[bytes] = None,
481 ) -> bool:
508 ) -> bool:
482 """return True if the path is a regular file or a symlink and
509 """return True if the path is a regular file or a symlink and
483 the directories along the path are "normal", that is
510 the directories along the path are "normal", that is
484 not symlinks or nested hg repositories.
511 not symlinks or nested hg repositories.
485
512
486 Ignores the `_audit` setting, and checks the directories regardless.
513 Ignores the `_audit` setting, and checks the directories regardless.
487 `dircache` is used to cache the directory checks.
514 `dircache` is used to cache the directory checks.
488 """
515 """
516 # TODO: Should be a None check on 'path', or shouldn't default to None
517 # because of the immediate call to util.localpath().
489 try:
518 try:
490 for prefix in pathutil.finddirs_rev_noroot(util.localpath(path)):
519 for prefix in pathutil.finddirs_rev_noroot(util.localpath(path)):
491 if prefix in dircache:
520 if prefix in dircache:
492 res = dircache[prefix]
521 res = dircache[prefix]
493 else:
522 else:
494 res = pathutil.pathauditor._checkfs_exists(
523 res = pathutil.pathauditor._checkfs_exists(
495 self.base, prefix, path
524 self.base, prefix, path
496 )
525 )
497 dircache[prefix] = res
526 dircache[prefix] = res
498 if not res:
527 if not res:
499 return False
528 return False
500 except (OSError, error.Abort):
529 except (OSError, error.Abort):
501 return False
530 return False
502 return self.isfileorlink(path)
531 return self.isfileorlink(path)
503
532
504 def __call__(
533 def __call__(
505 self,
534 self,
506 path: bytes,
535 path: bytes,
507 mode: bytes = b"rb",
536 mode: bytes = b"rb",
508 atomictemp=False,
537 atomictemp: bool = False,
509 notindexed=False,
538 notindexed: bool = False,
510 backgroundclose=False,
539 backgroundclose: bool = False,
511 checkambig=False,
540 checkambig: bool = False,
512 auditpath=True,
541 auditpath: bool = True,
513 makeparentdirs=True,
542 makeparentdirs: bool = True,
514 ) -> Any:
543 ) -> Any: # TODO: should be BinaryIO if util.atomictempfile can be coersed
515 """Open ``path`` file, which is relative to vfs root.
544 """Open ``path`` file, which is relative to vfs root.
516
545
517 By default, parent directories are created as needed. Newly created
546 By default, parent directories are created as needed. Newly created
518 directories are marked as "not to be indexed by the content indexing
547 directories are marked as "not to be indexed by the content indexing
519 service", if ``notindexed`` is specified for "write" mode access.
548 service", if ``notindexed`` is specified for "write" mode access.
520 Set ``makeparentdirs=False`` to not create directories implicitly.
549 Set ``makeparentdirs=False`` to not create directories implicitly.
521
550
522 If ``backgroundclose`` is passed, the file may be closed asynchronously.
551 If ``backgroundclose`` is passed, the file may be closed asynchronously.
523 It can only be used if the ``self.backgroundclosing()`` context manager
552 It can only be used if the ``self.backgroundclosing()`` context manager
524 is active. This should only be specified if the following criteria hold:
553 is active. This should only be specified if the following criteria hold:
525
554
526 1. There is a potential for writing thousands of files. Unless you
555 1. There is a potential for writing thousands of files. Unless you
527 are writing thousands of files, the performance benefits of
556 are writing thousands of files, the performance benefits of
528 asynchronously closing files is not realized.
557 asynchronously closing files is not realized.
529 2. Files are opened exactly once for the ``backgroundclosing``
558 2. Files are opened exactly once for the ``backgroundclosing``
530 active duration and are therefore free of race conditions between
559 active duration and are therefore free of race conditions between
531 closing a file on a background thread and reopening it. (If the
560 closing a file on a background thread and reopening it. (If the
532 file were opened multiple times, there could be unflushed data
561 file were opened multiple times, there could be unflushed data
533 because the original file handle hasn't been flushed/closed yet.)
562 because the original file handle hasn't been flushed/closed yet.)
534
563
535 ``checkambig`` argument is passed to atomictempfile (valid
564 ``checkambig`` argument is passed to atomictempfile (valid
536 only for writing), and is useful only if target file is
565 only for writing), and is useful only if target file is
537 guarded by any lock (e.g. repo.lock or repo.wlock).
566 guarded by any lock (e.g. repo.lock or repo.wlock).
538
567
539 To avoid file stat ambiguity forcibly, checkambig=True involves
568 To avoid file stat ambiguity forcibly, checkambig=True involves
540 copying ``path`` file opened in "append" mode (e.g. for
569 copying ``path`` file opened in "append" mode (e.g. for
541 truncation), if it is owned by another. Therefore, use
570 truncation), if it is owned by another. Therefore, use
542 combination of append mode and checkambig=True only in limited
571 combination of append mode and checkambig=True only in limited
543 cases (see also issue5418 and issue5584 for detail).
572 cases (see also issue5418 and issue5584 for detail).
544 """
573 """
545 if auditpath:
574 if auditpath:
546 self._auditpath(path, mode)
575 self._auditpath(path, mode)
547 f = self.join(path)
576 f = self.join(path)
548
577
549 if b"b" not in mode:
578 if b"b" not in mode:
550 mode += b"b" # for that other OS
579 mode += b"b" # for that other OS
551
580
552 nlink = -1
581 nlink = -1
553 if mode not in (b'r', b'rb'):
582 if mode not in (b'r', b'rb'):
554 dirname, basename = util.split(f)
583 dirname, basename = util.split(f)
555 # If basename is empty, then the path is malformed because it points
584 # If basename is empty, then the path is malformed because it points
556 # to a directory. Let the posixfile() call below raise IOError.
585 # to a directory. Let the posixfile() call below raise IOError.
557 if basename:
586 if basename:
558 if atomictemp:
587 if atomictemp:
559 if makeparentdirs:
588 if makeparentdirs:
560 util.makedirs(dirname, self.createmode, notindexed)
589 util.makedirs(dirname, self.createmode, notindexed)
561 return util.atomictempfile(
590 return util.atomictempfile(
562 f, mode, self.createmode, checkambig=checkambig
591 f, mode, self.createmode, checkambig=checkambig
563 )
592 )
564 try:
593 try:
565 if b'w' in mode:
594 if b'w' in mode:
566 util.unlink(f)
595 util.unlink(f)
567 nlink = 0
596 nlink = 0
568 else:
597 else:
569 # nlinks() may behave differently for files on Windows
598 # nlinks() may behave differently for files on Windows
570 # shares if the file is open.
599 # shares if the file is open.
571 with util.posixfile(f):
600 with util.posixfile(f):
572 nlink = util.nlinks(f)
601 nlink = util.nlinks(f)
573 if nlink < 1:
602 if nlink < 1:
574 nlink = 2 # force mktempcopy (issue1922)
603 nlink = 2 # force mktempcopy (issue1922)
575 except FileNotFoundError:
604 except FileNotFoundError:
576 nlink = 0
605 nlink = 0
577 if makeparentdirs:
606 if makeparentdirs:
578 util.makedirs(dirname, self.createmode, notindexed)
607 util.makedirs(dirname, self.createmode, notindexed)
579 if nlink > 0:
608 if nlink > 0:
580 if self._trustnlink is None:
609 if self._trustnlink is None:
581 self._trustnlink = nlink > 1 or util.checknlink(f)
610 self._trustnlink = nlink > 1 or util.checknlink(f)
582 if nlink > 1 or not self._trustnlink:
611 if nlink > 1 or not self._trustnlink:
583 util.rename(util.mktempcopy(f), f)
612 util.rename(util.mktempcopy(f), f)
584 fp = util.posixfile(f, mode)
613 fp = util.posixfile(f, mode)
585 if nlink == 0:
614 if nlink == 0:
586 self._fixfilemode(f)
615 self._fixfilemode(f)
587
616
588 if checkambig:
617 if checkambig:
589 if mode in (b'r', b'rb'):
618 if mode in (b'r', b'rb'):
590 raise error.Abort(
619 raise error.Abort(
591 _(
620 _(
592 b'implementation error: mode %s is not'
621 b'implementation error: mode %s is not'
593 b' valid for checkambig=True'
622 b' valid for checkambig=True'
594 )
623 )
595 % mode
624 % mode
596 )
625 )
597 fp = checkambigatclosing(fp)
626 fp = checkambigatclosing(fp)
598
627
599 if (
628 if (
600 backgroundclose
629 backgroundclose
601 and threading.current_thread() is threading.main_thread()
630 and threading.current_thread() is threading.main_thread()
602 ):
631 ):
603 if (
632 if (
604 not self._backgroundfilecloser # pytype: disable=attribute-error
633 not self._backgroundfilecloser # pytype: disable=attribute-error
605 ):
634 ):
606 raise error.Abort(
635 raise error.Abort(
607 _(
636 _(
608 b'backgroundclose can only be used when a '
637 b'backgroundclose can only be used when a '
609 b'backgroundclosing context manager is active'
638 b'backgroundclosing context manager is active'
610 )
639 )
611 )
640 )
612
641
613 fp = delayclosedfile(
642 fp = delayclosedfile(
614 fp,
643 fp,
615 self._backgroundfilecloser, # pytype: disable=attribute-error
644 self._backgroundfilecloser, # pytype: disable=attribute-error
616 )
645 )
617
646
618 return fp
647 return fp
619
648
620 def symlink(self, src: bytes, dst: bytes) -> None:
649 def symlink(self, src: bytes, dst: bytes) -> None:
621 self.audit(dst)
650 self.audit(dst)
622 linkname = self.join(dst)
651 linkname = self.join(dst)
623 util.tryunlink(linkname)
652 util.tryunlink(linkname)
624
653
625 util.makedirs(os.path.dirname(linkname), self.createmode)
654 util.makedirs(os.path.dirname(linkname), self.createmode)
626
655
627 if self._cansymlink:
656 if self._cansymlink:
628 try:
657 try:
629 os.symlink(src, linkname)
658 os.symlink(src, linkname)
630 except OSError as err:
659 except OSError as err:
631 raise OSError(
660 raise OSError(
632 err.errno,
661 err.errno,
633 _(b'could not symlink to %r: %s')
662 _(b'could not symlink to %r: %s')
634 % (src, encoding.strtolocal(err.strerror)),
663 % (src, encoding.strtolocal(err.strerror)),
635 linkname,
664 linkname,
636 )
665 )
637 else:
666 else:
638 self.write(dst, src)
667 self.write(dst, src)
639
668
640 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
669 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
641 if path:
670 if path:
642 parts = [self.base, path]
671 parts = [self.base, path]
643 parts.extend(insidef)
672 parts.extend(insidef)
644 return self._join(*parts)
673 return self._join(*parts)
645 else:
674 else:
646 return self.base
675 return self.base
647
676
648
677
649 opener: Type[vfs] = vfs
678 opener: Type[vfs] = vfs
650
679
651
680
652 class proxyvfs(abstractvfs, abc.ABC):
681 class proxyvfs(abstractvfs, abc.ABC):
653 def __init__(self, vfs: "vfs"):
682 def __init__(self, vfs: vfs) -> None:
654 self.vfs = vfs
683 self.vfs = vfs
655
684
656 @property
685 @property
657 def createmode(self):
686 def createmode(self) -> Optional[int]:
658 return self.vfs.createmode
687 return self.vfs.createmode
659
688
660 def _auditpath(self, path, mode) -> None:
689 def _auditpath(self, path: bytes, mode: bytes) -> None:
661 return self.vfs._auditpath(path, mode)
690 return self.vfs._auditpath(path, mode)
662
691
663 @property
692 @property
664 def options(self):
693 def options(self):
665 return self.vfs.options
694 return self.vfs.options
666
695
667 @options.setter
696 @options.setter
668 def options(self, value):
697 def options(self, value):
669 self.vfs.options = value
698 self.vfs.options = value
670
699
671 @property
700 @property
672 def audit(self):
701 def audit(self):
673 return self.vfs.audit
702 return self.vfs.audit
674
703
675
704
676 class filtervfs(proxyvfs, abstractvfs):
705 class filtervfs(proxyvfs, abstractvfs):
677 '''Wrapper vfs for filtering filenames with a function.'''
706 '''Wrapper vfs for filtering filenames with a function.'''
678
707
679 def __init__(self, vfs: "vfs", filter):
708 def __init__(self, vfs: vfs, filter) -> None:
680 proxyvfs.__init__(self, vfs)
709 proxyvfs.__init__(self, vfs)
681 self._filter = filter
710 self._filter = filter
682
711
712 # TODO: The return type should be BinaryIO
683 def __call__(self, path: bytes, *args, **kwargs) -> Any:
713 def __call__(self, path: bytes, *args, **kwargs) -> Any:
684 return self.vfs(self._filter(path), *args, **kwargs)
714 return self.vfs(self._filter(path), *args, **kwargs)
685
715
686 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
716 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
687 if path:
717 if path:
688 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
718 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
689 else:
719 else:
690 return self.vfs.join(path)
720 return self.vfs.join(path)
691
721
692
722
693 filteropener: Type[filtervfs] = filtervfs
723 filteropener: Type[filtervfs] = filtervfs
694
724
695
725
696 class readonlyvfs(proxyvfs):
726 class readonlyvfs(proxyvfs):
697 '''Wrapper vfs preventing any writing.'''
727 '''Wrapper vfs preventing any writing.'''
698
728
699 def __init__(self, vfs: "vfs"):
729 def __init__(self, vfs: vfs) -> None:
700 proxyvfs.__init__(self, vfs)
730 proxyvfs.__init__(self, vfs)
701
731
732 # TODO: The return type should be BinaryIO
702 def __call__(self, path: bytes, mode: bytes = b'rb', *args, **kw) -> Any:
733 def __call__(self, path: bytes, mode: bytes = b'rb', *args, **kw) -> Any:
703 if mode not in (b'r', b'rb'):
734 if mode not in (b'r', b'rb'):
704 raise error.Abort(_(b'this vfs is read only'))
735 raise error.Abort(_(b'this vfs is read only'))
705 return self.vfs(path, mode, *args, **kw)
736 return self.vfs(path, mode, *args, **kw)
706
737
707 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
738 def join(self, path: Optional[bytes], *insidef: bytes) -> bytes:
708 return self.vfs.join(path, *insidef)
739 return self.vfs.join(path, *insidef)
709
740
710
741
711 class closewrapbase(abc.ABC):
742 class closewrapbase(abc.ABC):
712 """Base class of wrapper, which hooks closing
743 """Base class of wrapper, which hooks closing
713
744
714 Do not instantiate outside the vfs layer.
745 Do not instantiate outside the vfs layer.
715 """
746 """
716
747
717 def __init__(self, fh) -> None:
748 def __init__(self, fh) -> None:
718 object.__setattr__(self, '_origfh', fh)
749 object.__setattr__(self, '_origfh', fh)
719
750
720 def __getattr__(self, attr) -> Any:
751 def __getattr__(self, attr: str) -> Any:
721 return getattr(self._origfh, attr)
752 return getattr(self._origfh, attr)
722
753
723 def __setattr__(self, attr, value) -> None:
754 def __setattr__(self, attr: str, value: Any) -> None:
724 return setattr(self._origfh, attr, value)
755 return setattr(self._origfh, attr, value)
725
756
726 def __delattr__(self, attr) -> None:
757 def __delattr__(self, attr: str) -> None:
727 return delattr(self._origfh, attr)
758 return delattr(self._origfh, attr)
728
759
729 def __enter__(self: _Tclosewrapbase) -> _Tclosewrapbase:
760 def __enter__(self: _Tclosewrapbase) -> _Tclosewrapbase:
730 self._origfh.__enter__()
761 self._origfh.__enter__()
731 return self
762 return self
732
763
733 @abc.abstractmethod
764 @abc.abstractmethod
734 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
765 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
735 ...
766 ...
736
767
737 @abc.abstractmethod
768 @abc.abstractmethod
738 def close(self) -> None:
769 def close(self) -> None:
739 ...
770 ...
740
771
741
772
742 class delayclosedfile(closewrapbase):
773 class delayclosedfile(closewrapbase):
743 """Proxy for a file object whose close is delayed.
774 """Proxy for a file object whose close is delayed.
744
775
745 Do not instantiate outside the vfs layer.
776 Do not instantiate outside the vfs layer.
746 """
777 """
747
778
748 def __init__(self, fh, closer) -> None:
779 def __init__(self, fh, closer) -> None:
749 super(delayclosedfile, self).__init__(fh)
780 super(delayclosedfile, self).__init__(fh)
750 object.__setattr__(self, '_closer', closer)
781 object.__setattr__(self, '_closer', closer)
751
782
752 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
783 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
753 self._closer.close(self._origfh)
784 self._closer.close(self._origfh)
754
785
755 def close(self) -> None:
786 def close(self) -> None:
756 self._closer.close(self._origfh)
787 self._closer.close(self._origfh)
757
788
758
789
759 class backgroundfilecloser:
790 class backgroundfilecloser:
760 """Coordinates background closing of file handles on multiple threads."""
791 """Coordinates background closing of file handles on multiple threads."""
761
792
762 def __init__(self, ui, expectedcount=-1) -> None:
793 def __init__(self, ui: uimod.ui, expectedcount: int = -1) -> None:
763 self._running = False
794 self._running = False
764 self._entered = False
795 self._entered = False
765 self._threads = []
796 self._threads = []
766 self._threadexception = None
797 self._threadexception = None
767
798
768 # Only Windows/NTFS has slow file closing. So only enable by default
799 # Only Windows/NTFS has slow file closing. So only enable by default
769 # on that platform. But allow to be enabled elsewhere for testing.
800 # on that platform. But allow to be enabled elsewhere for testing.
770 defaultenabled = pycompat.iswindows
801 defaultenabled = pycompat.iswindows
771 enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled)
802 enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled)
772
803
773 if not enabled:
804 if not enabled:
774 return
805 return
775
806
776 # There is overhead to starting and stopping the background threads.
807 # There is overhead to starting and stopping the background threads.
777 # Don't do background processing unless the file count is large enough
808 # Don't do background processing unless the file count is large enough
778 # to justify it.
809 # to justify it.
779 minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount')
810 minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount')
780 # FUTURE dynamically start background threads after minfilecount closes.
811 # FUTURE dynamically start background threads after minfilecount closes.
781 # (We don't currently have any callers that don't know their file count)
812 # (We don't currently have any callers that don't know their file count)
782 if expectedcount > 0 and expectedcount < minfilecount:
813 if expectedcount > 0 and expectedcount < minfilecount:
783 return
814 return
784
815
785 maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue')
816 maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue')
786 threadcount = ui.configint(b'worker', b'backgroundclosethreadcount')
817 threadcount = ui.configint(b'worker', b'backgroundclosethreadcount')
787
818
788 ui.debug(
819 ui.debug(
789 b'starting %d threads for background file closing\n' % threadcount
820 b'starting %d threads for background file closing\n' % threadcount
790 )
821 )
791
822
792 self._queue = pycompat.queue.Queue(maxsize=maxqueue)
823 self._queue = pycompat.queue.Queue(maxsize=maxqueue)
793 self._running = True
824 self._running = True
794
825
795 for i in range(threadcount):
826 for i in range(threadcount):
796 t = threading.Thread(target=self._worker, name='backgroundcloser')
827 t = threading.Thread(target=self._worker, name='backgroundcloser')
797 self._threads.append(t)
828 self._threads.append(t)
798 t.start()
829 t.start()
799
830
800 def __enter__(self: _Tbackgroundfilecloser) -> _Tbackgroundfilecloser:
831 def __enter__(self: _Tbackgroundfilecloser) -> _Tbackgroundfilecloser:
801 self._entered = True
832 self._entered = True
802 return self
833 return self
803
834
804 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
835 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
805 self._running = False
836 self._running = False
806
837
807 # Wait for threads to finish closing so open files don't linger for
838 # Wait for threads to finish closing so open files don't linger for
808 # longer than lifetime of context manager.
839 # longer than lifetime of context manager.
809 for t in self._threads:
840 for t in self._threads:
810 t.join()
841 t.join()
811
842
812 def _worker(self) -> None:
843 def _worker(self) -> None:
813 """Main routine for worker thread."""
844 """Main routine for worker thread."""
814 while True:
845 while True:
815 try:
846 try:
816 fh = self._queue.get(block=True, timeout=0.100)
847 fh = self._queue.get(block=True, timeout=0.100)
817 # Need to catch or the thread will terminate and
848 # Need to catch or the thread will terminate and
818 # we could orphan file descriptors.
849 # we could orphan file descriptors.
819 try:
850 try:
820 fh.close()
851 fh.close()
821 except Exception as e:
852 except Exception as e:
822 # Stash so can re-raise from main thread later.
853 # Stash so can re-raise from main thread later.
823 self._threadexception = e
854 self._threadexception = e
824 except pycompat.queue.Empty:
855 except pycompat.queue.Empty:
825 if not self._running:
856 if not self._running:
826 break
857 break
827
858
828 def close(self, fh) -> None:
859 def close(self, fh) -> None:
829 """Schedule a file for closing."""
860 """Schedule a file for closing."""
830 if not self._entered:
861 if not self._entered:
831 raise error.Abort(
862 raise error.Abort(
832 _(b'can only call close() when context manager active')
863 _(b'can only call close() when context manager active')
833 )
864 )
834
865
835 # If a background thread encountered an exception, raise now so we fail
866 # If a background thread encountered an exception, raise now so we fail
836 # fast. Otherwise, we may potentially go on for minutes until the error
867 # fast. Otherwise, we may potentially go on for minutes until the error
837 # is acted on.
868 # is acted on.
838 if self._threadexception:
869 if self._threadexception:
839 e = self._threadexception
870 e = self._threadexception
840 self._threadexception = None
871 self._threadexception = None
841 raise e
872 raise e
842
873
843 # If we're not actively running, close synchronously.
874 # If we're not actively running, close synchronously.
844 if not self._running:
875 if not self._running:
845 fh.close()
876 fh.close()
846 return
877 return
847
878
848 self._queue.put(fh, block=True, timeout=None)
879 self._queue.put(fh, block=True, timeout=None)
849
880
850
881
851 class checkambigatclosing(closewrapbase):
882 class checkambigatclosing(closewrapbase):
852 """Proxy for a file object, to avoid ambiguity of file stat
883 """Proxy for a file object, to avoid ambiguity of file stat
853
884
854 See also util.filestat for detail about "ambiguity of file stat".
885 See also util.filestat for detail about "ambiguity of file stat".
855
886
856 This proxy is useful only if the target file is guarded by any
887 This proxy is useful only if the target file is guarded by any
857 lock (e.g. repo.lock or repo.wlock)
888 lock (e.g. repo.lock or repo.wlock)
858
889
859 Do not instantiate outside the vfs layer.
890 Do not instantiate outside the vfs layer.
860 """
891 """
861
892
862 def __init__(self, fh) -> None:
893 def __init__(self, fh) -> None:
863 super(checkambigatclosing, self).__init__(fh)
894 super(checkambigatclosing, self).__init__(fh)
864 object.__setattr__(self, '_oldstat', util.filestat.frompath(fh.name))
895 object.__setattr__(self, '_oldstat', util.filestat.frompath(fh.name))
865
896
866 def _checkambig(self) -> None:
897 def _checkambig(self) -> None:
867 oldstat = self._oldstat
898 oldstat = self._oldstat
868 if oldstat.stat:
899 if oldstat.stat:
869 _avoidambig(self._origfh.name, oldstat)
900 _avoidambig(self._origfh.name, oldstat)
870
901
871 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
902 def __exit__(self, exc_type, exc_value, exc_tb) -> None:
872 self._origfh.__exit__(exc_type, exc_value, exc_tb)
903 self._origfh.__exit__(exc_type, exc_value, exc_tb)
873 self._checkambig()
904 self._checkambig()
874
905
875 def close(self) -> None:
906 def close(self) -> None:
876 self._origfh.close()
907 self._origfh.close()
877 self._checkambig()
908 self._checkambig()
General Comments 0
You need to be logged in to leave comments. Login now