##// END OF EJS Templates
vfs: fix erroneous bytes constants...
Augie Fackler -
r43767:2dcd4e77 default
parent child Browse files
Show More
@@ -1,733 +1,733 b''
1 # vfs.py - Mercurial 'vfs' classes
1 # vfs.py - Mercurial 'vfs' classes
2 #
2 #
3 # Copyright Matt Mackall <mpm@selenic.com>
3 # Copyright Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7 from __future__ import absolute_import
7 from __future__ import absolute_import
8
8
9 import contextlib
9 import contextlib
10 import errno
10 import errno
11 import os
11 import os
12 import shutil
12 import shutil
13 import stat
13 import stat
14 import threading
14 import threading
15
15
16 from .i18n import _
16 from .i18n import _
17 from .pycompat import (
17 from .pycompat import (
18 delattr,
18 delattr,
19 getattr,
19 getattr,
20 setattr,
20 setattr,
21 )
21 )
22 from . import (
22 from . import (
23 encoding,
23 encoding,
24 error,
24 error,
25 pathutil,
25 pathutil,
26 pycompat,
26 pycompat,
27 util,
27 util,
28 )
28 )
29
29
30
30
31 def _avoidambig(path, oldstat):
31 def _avoidambig(path, oldstat):
32 """Avoid file stat ambiguity forcibly
32 """Avoid file stat ambiguity forcibly
33
33
34 This function causes copying ``path`` file, if it is owned by
34 This function causes copying ``path`` file, if it is owned by
35 another (see issue5418 and issue5584 for detail).
35 another (see issue5418 and issue5584 for detail).
36 """
36 """
37
37
38 def checkandavoid():
38 def checkandavoid():
39 newstat = util.filestat.frompath(path)
39 newstat = util.filestat.frompath(path)
40 # return whether file stat ambiguity is (already) avoided
40 # return whether file stat ambiguity is (already) avoided
41 return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat)
41 return not newstat.isambig(oldstat) or newstat.avoidambig(path, oldstat)
42
42
43 if not checkandavoid():
43 if not checkandavoid():
44 # simply copy to change owner of path to get privilege to
44 # simply copy to change owner of path to get privilege to
45 # advance mtime (see issue5418)
45 # advance mtime (see issue5418)
46 util.rename(util.mktempcopy(path), path)
46 util.rename(util.mktempcopy(path), path)
47 checkandavoid()
47 checkandavoid()
48
48
49
49
50 class abstractvfs(object):
50 class abstractvfs(object):
51 """Abstract base class; cannot be instantiated"""
51 """Abstract base class; cannot be instantiated"""
52
52
53 def __init__(self, *args, **kwargs):
53 def __init__(self, *args, **kwargs):
54 '''Prevent instantiation; don't call this from subclasses.'''
54 '''Prevent instantiation; don't call this from subclasses.'''
55 raise NotImplementedError(b'attempted instantiating ' + str(type(self)))
55 raise NotImplementedError('attempted instantiating ' + str(type(self)))
56
56
57 def _auditpath(self, path, mode):
57 def _auditpath(self, path, mode):
58 raise NotImplementedError
58 raise NotImplementedError
59
59
60 def tryread(self, path):
60 def tryread(self, path):
61 '''gracefully return an empty string for missing files'''
61 '''gracefully return an empty string for missing files'''
62 try:
62 try:
63 return self.read(path)
63 return self.read(path)
64 except IOError as inst:
64 except IOError as inst:
65 if inst.errno != errno.ENOENT:
65 if inst.errno != errno.ENOENT:
66 raise
66 raise
67 return b""
67 return b""
68
68
69 def tryreadlines(self, path, mode=b'rb'):
69 def tryreadlines(self, path, mode=b'rb'):
70 '''gracefully return an empty array for missing files'''
70 '''gracefully return an empty array for missing files'''
71 try:
71 try:
72 return self.readlines(path, mode=mode)
72 return self.readlines(path, mode=mode)
73 except IOError as inst:
73 except IOError as inst:
74 if inst.errno != errno.ENOENT:
74 if inst.errno != errno.ENOENT:
75 raise
75 raise
76 return []
76 return []
77
77
78 @util.propertycache
78 @util.propertycache
79 def open(self):
79 def open(self):
80 '''Open ``path`` file, which is relative to vfs root.
80 '''Open ``path`` file, which is relative to vfs root.
81
81
82 Newly created directories are marked as "not to be indexed by
82 Newly created directories are marked as "not to be indexed by
83 the content indexing service", if ``notindexed`` is specified
83 the content indexing service", if ``notindexed`` is specified
84 for "write" mode access.
84 for "write" mode access.
85 '''
85 '''
86 return self.__call__
86 return self.__call__
87
87
88 def read(self, path):
88 def read(self, path):
89 with self(path, b'rb') as fp:
89 with self(path, b'rb') as fp:
90 return fp.read()
90 return fp.read()
91
91
92 def readlines(self, path, mode=b'rb'):
92 def readlines(self, path, mode=b'rb'):
93 with self(path, mode=mode) as fp:
93 with self(path, mode=mode) as fp:
94 return fp.readlines()
94 return fp.readlines()
95
95
96 def write(self, path, data, backgroundclose=False, **kwargs):
96 def write(self, path, data, backgroundclose=False, **kwargs):
97 with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp:
97 with self(path, b'wb', backgroundclose=backgroundclose, **kwargs) as fp:
98 return fp.write(data)
98 return fp.write(data)
99
99
100 def writelines(self, path, data, mode=b'wb', notindexed=False):
100 def writelines(self, path, data, mode=b'wb', notindexed=False):
101 with self(path, mode=mode, notindexed=notindexed) as fp:
101 with self(path, mode=mode, notindexed=notindexed) as fp:
102 return fp.writelines(data)
102 return fp.writelines(data)
103
103
104 def append(self, path, data):
104 def append(self, path, data):
105 with self(path, b'ab') as fp:
105 with self(path, b'ab') as fp:
106 return fp.write(data)
106 return fp.write(data)
107
107
108 def basename(self, path):
108 def basename(self, path):
109 """return base element of a path (as os.path.basename would do)
109 """return base element of a path (as os.path.basename would do)
110
110
111 This exists to allow handling of strange encoding if needed."""
111 This exists to allow handling of strange encoding if needed."""
112 return os.path.basename(path)
112 return os.path.basename(path)
113
113
114 def chmod(self, path, mode):
114 def chmod(self, path, mode):
115 return os.chmod(self.join(path), mode)
115 return os.chmod(self.join(path), mode)
116
116
117 def dirname(self, path):
117 def dirname(self, path):
118 """return dirname element of a path (as os.path.dirname would do)
118 """return dirname element of a path (as os.path.dirname would do)
119
119
120 This exists to allow handling of strange encoding if needed."""
120 This exists to allow handling of strange encoding if needed."""
121 return os.path.dirname(path)
121 return os.path.dirname(path)
122
122
123 def exists(self, path=None):
123 def exists(self, path=None):
124 return os.path.exists(self.join(path))
124 return os.path.exists(self.join(path))
125
125
126 def fstat(self, fp):
126 def fstat(self, fp):
127 return util.fstat(fp)
127 return util.fstat(fp)
128
128
129 def isdir(self, path=None):
129 def isdir(self, path=None):
130 return os.path.isdir(self.join(path))
130 return os.path.isdir(self.join(path))
131
131
132 def isfile(self, path=None):
132 def isfile(self, path=None):
133 return os.path.isfile(self.join(path))
133 return os.path.isfile(self.join(path))
134
134
135 def islink(self, path=None):
135 def islink(self, path=None):
136 return os.path.islink(self.join(path))
136 return os.path.islink(self.join(path))
137
137
138 def isfileorlink(self, path=None):
138 def isfileorlink(self, path=None):
139 '''return whether path is a regular file or a symlink
139 '''return whether path is a regular file or a symlink
140
140
141 Unlike isfile, this doesn't follow symlinks.'''
141 Unlike isfile, this doesn't follow symlinks.'''
142 try:
142 try:
143 st = self.lstat(path)
143 st = self.lstat(path)
144 except OSError:
144 except OSError:
145 return False
145 return False
146 mode = st.st_mode
146 mode = st.st_mode
147 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
147 return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
148
148
149 def reljoin(self, *paths):
149 def reljoin(self, *paths):
150 """join various elements of a path together (as os.path.join would do)
150 """join various elements of a path together (as os.path.join would do)
151
151
152 The vfs base is not injected so that path stay relative. This exists
152 The vfs base is not injected so that path stay relative. This exists
153 to allow handling of strange encoding if needed."""
153 to allow handling of strange encoding if needed."""
154 return os.path.join(*paths)
154 return os.path.join(*paths)
155
155
156 def split(self, path):
156 def split(self, path):
157 """split top-most element of a path (as os.path.split would do)
157 """split top-most element of a path (as os.path.split would do)
158
158
159 This exists to allow handling of strange encoding if needed."""
159 This exists to allow handling of strange encoding if needed."""
160 return os.path.split(path)
160 return os.path.split(path)
161
161
162 def lexists(self, path=None):
162 def lexists(self, path=None):
163 return os.path.lexists(self.join(path))
163 return os.path.lexists(self.join(path))
164
164
165 def lstat(self, path=None):
165 def lstat(self, path=None):
166 return os.lstat(self.join(path))
166 return os.lstat(self.join(path))
167
167
168 def listdir(self, path=None):
168 def listdir(self, path=None):
169 return os.listdir(self.join(path))
169 return os.listdir(self.join(path))
170
170
171 def makedir(self, path=None, notindexed=True):
171 def makedir(self, path=None, notindexed=True):
172 return util.makedir(self.join(path), notindexed)
172 return util.makedir(self.join(path), notindexed)
173
173
174 def makedirs(self, path=None, mode=None):
174 def makedirs(self, path=None, mode=None):
175 return util.makedirs(self.join(path), mode)
175 return util.makedirs(self.join(path), mode)
176
176
177 def makelock(self, info, path):
177 def makelock(self, info, path):
178 return util.makelock(info, self.join(path))
178 return util.makelock(info, self.join(path))
179
179
180 def mkdir(self, path=None):
180 def mkdir(self, path=None):
181 return os.mkdir(self.join(path))
181 return os.mkdir(self.join(path))
182
182
183 def mkstemp(self, suffix=b'', prefix=b'tmp', dir=None):
183 def mkstemp(self, suffix=b'', prefix=b'tmp', dir=None):
184 fd, name = pycompat.mkstemp(
184 fd, name = pycompat.mkstemp(
185 suffix=suffix, prefix=prefix, dir=self.join(dir)
185 suffix=suffix, prefix=prefix, dir=self.join(dir)
186 )
186 )
187 dname, fname = util.split(name)
187 dname, fname = util.split(name)
188 if dir:
188 if dir:
189 return fd, os.path.join(dir, fname)
189 return fd, os.path.join(dir, fname)
190 else:
190 else:
191 return fd, fname
191 return fd, fname
192
192
193 def readdir(self, path=None, stat=None, skip=None):
193 def readdir(self, path=None, stat=None, skip=None):
194 return util.listdir(self.join(path), stat, skip)
194 return util.listdir(self.join(path), stat, skip)
195
195
196 def readlock(self, path):
196 def readlock(self, path):
197 return util.readlock(self.join(path))
197 return util.readlock(self.join(path))
198
198
199 def rename(self, src, dst, checkambig=False):
199 def rename(self, src, dst, checkambig=False):
200 """Rename from src to dst
200 """Rename from src to dst
201
201
202 checkambig argument is used with util.filestat, and is useful
202 checkambig argument is used with util.filestat, and is useful
203 only if destination file is guarded by any lock
203 only if destination file is guarded by any lock
204 (e.g. repo.lock or repo.wlock).
204 (e.g. repo.lock or repo.wlock).
205
205
206 To avoid file stat ambiguity forcibly, checkambig=True involves
206 To avoid file stat ambiguity forcibly, checkambig=True involves
207 copying ``src`` file, if it is owned by another. Therefore, use
207 copying ``src`` file, if it is owned by another. Therefore, use
208 checkambig=True only in limited cases (see also issue5418 and
208 checkambig=True only in limited cases (see also issue5418 and
209 issue5584 for detail).
209 issue5584 for detail).
210 """
210 """
211 self._auditpath(dst, b'w')
211 self._auditpath(dst, b'w')
212 srcpath = self.join(src)
212 srcpath = self.join(src)
213 dstpath = self.join(dst)
213 dstpath = self.join(dst)
214 oldstat = checkambig and util.filestat.frompath(dstpath)
214 oldstat = checkambig and util.filestat.frompath(dstpath)
215 if oldstat and oldstat.stat:
215 if oldstat and oldstat.stat:
216 ret = util.rename(srcpath, dstpath)
216 ret = util.rename(srcpath, dstpath)
217 _avoidambig(dstpath, oldstat)
217 _avoidambig(dstpath, oldstat)
218 return ret
218 return ret
219 return util.rename(srcpath, dstpath)
219 return util.rename(srcpath, dstpath)
220
220
221 def readlink(self, path):
221 def readlink(self, path):
222 return util.readlink(self.join(path))
222 return util.readlink(self.join(path))
223
223
224 def removedirs(self, path=None):
224 def removedirs(self, path=None):
225 """Remove a leaf directory and all empty intermediate ones
225 """Remove a leaf directory and all empty intermediate ones
226 """
226 """
227 return util.removedirs(self.join(path))
227 return util.removedirs(self.join(path))
228
228
229 def rmdir(self, path=None):
229 def rmdir(self, path=None):
230 """Remove an empty directory."""
230 """Remove an empty directory."""
231 return os.rmdir(self.join(path))
231 return os.rmdir(self.join(path))
232
232
233 def rmtree(self, path=None, ignore_errors=False, forcibly=False):
233 def rmtree(self, path=None, ignore_errors=False, forcibly=False):
234 """Remove a directory tree recursively
234 """Remove a directory tree recursively
235
235
236 If ``forcibly``, this tries to remove READ-ONLY files, too.
236 If ``forcibly``, this tries to remove READ-ONLY files, too.
237 """
237 """
238 if forcibly:
238 if forcibly:
239
239
240 def onerror(function, path, excinfo):
240 def onerror(function, path, excinfo):
241 if function is not os.remove:
241 if function is not os.remove:
242 raise
242 raise
243 # read-only files cannot be unlinked under Windows
243 # read-only files cannot be unlinked under Windows
244 s = os.stat(path)
244 s = os.stat(path)
245 if (s.st_mode & stat.S_IWRITE) != 0:
245 if (s.st_mode & stat.S_IWRITE) != 0:
246 raise
246 raise
247 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
247 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
248 os.remove(path)
248 os.remove(path)
249
249
250 else:
250 else:
251 onerror = None
251 onerror = None
252 return shutil.rmtree(
252 return shutil.rmtree(
253 self.join(path), ignore_errors=ignore_errors, onerror=onerror
253 self.join(path), ignore_errors=ignore_errors, onerror=onerror
254 )
254 )
255
255
256 def setflags(self, path, l, x):
256 def setflags(self, path, l, x):
257 return util.setflags(self.join(path), l, x)
257 return util.setflags(self.join(path), l, x)
258
258
259 def stat(self, path=None):
259 def stat(self, path=None):
260 return os.stat(self.join(path))
260 return os.stat(self.join(path))
261
261
262 def unlink(self, path=None):
262 def unlink(self, path=None):
263 return util.unlink(self.join(path))
263 return util.unlink(self.join(path))
264
264
265 def tryunlink(self, path=None):
265 def tryunlink(self, path=None):
266 """Attempt to remove a file, ignoring missing file errors."""
266 """Attempt to remove a file, ignoring missing file errors."""
267 util.tryunlink(self.join(path))
267 util.tryunlink(self.join(path))
268
268
269 def unlinkpath(self, path=None, ignoremissing=False, rmdir=True):
269 def unlinkpath(self, path=None, ignoremissing=False, rmdir=True):
270 return util.unlinkpath(
270 return util.unlinkpath(
271 self.join(path), ignoremissing=ignoremissing, rmdir=rmdir
271 self.join(path), ignoremissing=ignoremissing, rmdir=rmdir
272 )
272 )
273
273
274 def utime(self, path=None, t=None):
274 def utime(self, path=None, t=None):
275 return os.utime(self.join(path), t)
275 return os.utime(self.join(path), t)
276
276
277 def walk(self, path=None, onerror=None):
277 def walk(self, path=None, onerror=None):
278 """Yield (dirpath, dirs, files) tuple for each directories under path
278 """Yield (dirpath, dirs, files) tuple for each directories under path
279
279
280 ``dirpath`` is relative one from the root of this vfs. This
280 ``dirpath`` is relative one from the root of this vfs. This
281 uses ``os.sep`` as path separator, even you specify POSIX
281 uses ``os.sep`` as path separator, even you specify POSIX
282 style ``path``.
282 style ``path``.
283
283
284 "The root of this vfs" is represented as empty ``dirpath``.
284 "The root of this vfs" is represented as empty ``dirpath``.
285 """
285 """
286 root = os.path.normpath(self.join(None))
286 root = os.path.normpath(self.join(None))
287 # when dirpath == root, dirpath[prefixlen:] becomes empty
287 # when dirpath == root, dirpath[prefixlen:] becomes empty
288 # because len(dirpath) < prefixlen.
288 # because len(dirpath) < prefixlen.
289 prefixlen = len(pathutil.normasprefix(root))
289 prefixlen = len(pathutil.normasprefix(root))
290 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
290 for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
291 yield (dirpath[prefixlen:], dirs, files)
291 yield (dirpath[prefixlen:], dirs, files)
292
292
293 @contextlib.contextmanager
293 @contextlib.contextmanager
294 def backgroundclosing(self, ui, expectedcount=-1):
294 def backgroundclosing(self, ui, expectedcount=-1):
295 """Allow files to be closed asynchronously.
295 """Allow files to be closed asynchronously.
296
296
297 When this context manager is active, ``backgroundclose`` can be passed
297 When this context manager is active, ``backgroundclose`` can be passed
298 to ``__call__``/``open`` to result in the file possibly being closed
298 to ``__call__``/``open`` to result in the file possibly being closed
299 asynchronously, on a background thread.
299 asynchronously, on a background thread.
300 """
300 """
301 # Sharing backgroundfilecloser between threads is complex and using
301 # Sharing backgroundfilecloser between threads is complex and using
302 # multiple instances puts us at risk of running out of file descriptors
302 # multiple instances puts us at risk of running out of file descriptors
303 # only allow to use backgroundfilecloser when in main thread.
303 # only allow to use backgroundfilecloser when in main thread.
304 if not isinstance(threading.currentThread(), threading._MainThread):
304 if not isinstance(threading.currentThread(), threading._MainThread):
305 yield
305 yield
306 return
306 return
307 vfs = getattr(self, 'vfs', self)
307 vfs = getattr(self, 'vfs', self)
308 if getattr(vfs, '_backgroundfilecloser', None):
308 if getattr(vfs, '_backgroundfilecloser', None):
309 raise error.Abort(
309 raise error.Abort(
310 _(b'can only have 1 active background file closer')
310 _(b'can only have 1 active background file closer')
311 )
311 )
312
312
313 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
313 with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
314 try:
314 try:
315 vfs._backgroundfilecloser = bfc
315 vfs._backgroundfilecloser = bfc
316 yield bfc
316 yield bfc
317 finally:
317 finally:
318 vfs._backgroundfilecloser = None
318 vfs._backgroundfilecloser = None
319
319
320
320
321 class vfs(abstractvfs):
321 class vfs(abstractvfs):
322 '''Operate files relative to a base directory
322 '''Operate files relative to a base directory
323
323
324 This class is used to hide the details of COW semantics and
324 This class is used to hide the details of COW semantics and
325 remote file access from higher level code.
325 remote file access from higher level code.
326
326
327 'cacheaudited' should be enabled only if (a) vfs object is short-lived, or
327 'cacheaudited' should be enabled only if (a) vfs object is short-lived, or
328 (b) the base directory is managed by hg and considered sort-of append-only.
328 (b) the base directory is managed by hg and considered sort-of append-only.
329 See pathutil.pathauditor() for details.
329 See pathutil.pathauditor() for details.
330 '''
330 '''
331
331
332 def __init__(
332 def __init__(
333 self,
333 self,
334 base,
334 base,
335 audit=True,
335 audit=True,
336 cacheaudited=False,
336 cacheaudited=False,
337 expandpath=False,
337 expandpath=False,
338 realpath=False,
338 realpath=False,
339 ):
339 ):
340 if expandpath:
340 if expandpath:
341 base = util.expandpath(base)
341 base = util.expandpath(base)
342 if realpath:
342 if realpath:
343 base = os.path.realpath(base)
343 base = os.path.realpath(base)
344 self.base = base
344 self.base = base
345 self._audit = audit
345 self._audit = audit
346 if audit:
346 if audit:
347 self.audit = pathutil.pathauditor(self.base, cached=cacheaudited)
347 self.audit = pathutil.pathauditor(self.base, cached=cacheaudited)
348 else:
348 else:
349 self.audit = lambda path, mode=None: True
349 self.audit = lambda path, mode=None: True
350 self.createmode = None
350 self.createmode = None
351 self._trustnlink = None
351 self._trustnlink = None
352 self.options = {}
352 self.options = {}
353
353
354 @util.propertycache
354 @util.propertycache
355 def _cansymlink(self):
355 def _cansymlink(self):
356 return util.checklink(self.base)
356 return util.checklink(self.base)
357
357
358 @util.propertycache
358 @util.propertycache
359 def _chmod(self):
359 def _chmod(self):
360 return util.checkexec(self.base)
360 return util.checkexec(self.base)
361
361
362 def _fixfilemode(self, name):
362 def _fixfilemode(self, name):
363 if self.createmode is None or not self._chmod:
363 if self.createmode is None or not self._chmod:
364 return
364 return
365 os.chmod(name, self.createmode & 0o666)
365 os.chmod(name, self.createmode & 0o666)
366
366
367 def _auditpath(self, path, mode):
367 def _auditpath(self, path, mode):
368 if self._audit:
368 if self._audit:
369 if os.path.isabs(path) and path.startswith(self.base):
369 if os.path.isabs(path) and path.startswith(self.base):
370 path = os.path.relpath(path, self.base)
370 path = os.path.relpath(path, self.base)
371 r = util.checkosfilename(path)
371 r = util.checkosfilename(path)
372 if r:
372 if r:
373 raise error.Abort(b"%s: %r" % (r, path))
373 raise error.Abort(b"%s: %r" % (r, path))
374 self.audit(path, mode=mode)
374 self.audit(path, mode=mode)
375
375
376 def __call__(
376 def __call__(
377 self,
377 self,
378 path,
378 path,
379 mode=b"r",
379 mode=b"r",
380 atomictemp=False,
380 atomictemp=False,
381 notindexed=False,
381 notindexed=False,
382 backgroundclose=False,
382 backgroundclose=False,
383 checkambig=False,
383 checkambig=False,
384 auditpath=True,
384 auditpath=True,
385 makeparentdirs=True,
385 makeparentdirs=True,
386 ):
386 ):
387 '''Open ``path`` file, which is relative to vfs root.
387 '''Open ``path`` file, which is relative to vfs root.
388
388
389 By default, parent directories are created as needed. Newly created
389 By default, parent directories are created as needed. Newly created
390 directories are marked as "not to be indexed by the content indexing
390 directories are marked as "not to be indexed by the content indexing
391 service", if ``notindexed`` is specified for "write" mode access.
391 service", if ``notindexed`` is specified for "write" mode access.
392 Set ``makeparentdirs=False`` to not create directories implicitly.
392 Set ``makeparentdirs=False`` to not create directories implicitly.
393
393
394 If ``backgroundclose`` is passed, the file may be closed asynchronously.
394 If ``backgroundclose`` is passed, the file may be closed asynchronously.
395 It can only be used if the ``self.backgroundclosing()`` context manager
395 It can only be used if the ``self.backgroundclosing()`` context manager
396 is active. This should only be specified if the following criteria hold:
396 is active. This should only be specified if the following criteria hold:
397
397
398 1. There is a potential for writing thousands of files. Unless you
398 1. There is a potential for writing thousands of files. Unless you
399 are writing thousands of files, the performance benefits of
399 are writing thousands of files, the performance benefits of
400 asynchronously closing files is not realized.
400 asynchronously closing files is not realized.
401 2. Files are opened exactly once for the ``backgroundclosing``
401 2. Files are opened exactly once for the ``backgroundclosing``
402 active duration and are therefore free of race conditions between
402 active duration and are therefore free of race conditions between
403 closing a file on a background thread and reopening it. (If the
403 closing a file on a background thread and reopening it. (If the
404 file were opened multiple times, there could be unflushed data
404 file were opened multiple times, there could be unflushed data
405 because the original file handle hasn't been flushed/closed yet.)
405 because the original file handle hasn't been flushed/closed yet.)
406
406
407 ``checkambig`` argument is passed to atomictemplfile (valid
407 ``checkambig`` argument is passed to atomictemplfile (valid
408 only for writing), and is useful only if target file is
408 only for writing), and is useful only if target file is
409 guarded by any lock (e.g. repo.lock or repo.wlock).
409 guarded by any lock (e.g. repo.lock or repo.wlock).
410
410
411 To avoid file stat ambiguity forcibly, checkambig=True involves
411 To avoid file stat ambiguity forcibly, checkambig=True involves
412 copying ``path`` file opened in "append" mode (e.g. for
412 copying ``path`` file opened in "append" mode (e.g. for
413 truncation), if it is owned by another. Therefore, use
413 truncation), if it is owned by another. Therefore, use
414 combination of append mode and checkambig=True only in limited
414 combination of append mode and checkambig=True only in limited
415 cases (see also issue5418 and issue5584 for detail).
415 cases (see also issue5418 and issue5584 for detail).
416 '''
416 '''
417 if auditpath:
417 if auditpath:
418 self._auditpath(path, mode)
418 self._auditpath(path, mode)
419 f = self.join(path)
419 f = self.join(path)
420
420
421 if b"b" not in mode:
421 if b"b" not in mode:
422 mode += b"b" # for that other OS
422 mode += b"b" # for that other OS
423
423
424 nlink = -1
424 nlink = -1
425 if mode not in (b'r', b'rb'):
425 if mode not in (b'r', b'rb'):
426 dirname, basename = util.split(f)
426 dirname, basename = util.split(f)
427 # If basename is empty, then the path is malformed because it points
427 # If basename is empty, then the path is malformed because it points
428 # to a directory. Let the posixfile() call below raise IOError.
428 # to a directory. Let the posixfile() call below raise IOError.
429 if basename:
429 if basename:
430 if atomictemp:
430 if atomictemp:
431 if makeparentdirs:
431 if makeparentdirs:
432 util.makedirs(dirname, self.createmode, notindexed)
432 util.makedirs(dirname, self.createmode, notindexed)
433 return util.atomictempfile(
433 return util.atomictempfile(
434 f, mode, self.createmode, checkambig=checkambig
434 f, mode, self.createmode, checkambig=checkambig
435 )
435 )
436 try:
436 try:
437 if b'w' in mode:
437 if b'w' in mode:
438 util.unlink(f)
438 util.unlink(f)
439 nlink = 0
439 nlink = 0
440 else:
440 else:
441 # nlinks() may behave differently for files on Windows
441 # nlinks() may behave differently for files on Windows
442 # shares if the file is open.
442 # shares if the file is open.
443 with util.posixfile(f):
443 with util.posixfile(f):
444 nlink = util.nlinks(f)
444 nlink = util.nlinks(f)
445 if nlink < 1:
445 if nlink < 1:
446 nlink = 2 # force mktempcopy (issue1922)
446 nlink = 2 # force mktempcopy (issue1922)
447 except (OSError, IOError) as e:
447 except (OSError, IOError) as e:
448 if e.errno != errno.ENOENT:
448 if e.errno != errno.ENOENT:
449 raise
449 raise
450 nlink = 0
450 nlink = 0
451 if makeparentdirs:
451 if makeparentdirs:
452 util.makedirs(dirname, self.createmode, notindexed)
452 util.makedirs(dirname, self.createmode, notindexed)
453 if nlink > 0:
453 if nlink > 0:
454 if self._trustnlink is None:
454 if self._trustnlink is None:
455 self._trustnlink = nlink > 1 or util.checknlink(f)
455 self._trustnlink = nlink > 1 or util.checknlink(f)
456 if nlink > 1 or not self._trustnlink:
456 if nlink > 1 or not self._trustnlink:
457 util.rename(util.mktempcopy(f), f)
457 util.rename(util.mktempcopy(f), f)
458 fp = util.posixfile(f, mode)
458 fp = util.posixfile(f, mode)
459 if nlink == 0:
459 if nlink == 0:
460 self._fixfilemode(f)
460 self._fixfilemode(f)
461
461
462 if checkambig:
462 if checkambig:
463 if mode in (b'r', b'rb'):
463 if mode in (b'r', b'rb'):
464 raise error.Abort(
464 raise error.Abort(
465 _(
465 _(
466 b'implementation error: mode %s is not'
466 b'implementation error: mode %s is not'
467 b' valid for checkambig=True'
467 b' valid for checkambig=True'
468 )
468 )
469 % mode
469 % mode
470 )
470 )
471 fp = checkambigatclosing(fp)
471 fp = checkambigatclosing(fp)
472
472
473 if backgroundclose and isinstance(
473 if backgroundclose and isinstance(
474 threading.currentThread(), threading._MainThread
474 threading.currentThread(), threading._MainThread
475 ):
475 ):
476 if not self._backgroundfilecloser:
476 if not self._backgroundfilecloser:
477 raise error.Abort(
477 raise error.Abort(
478 _(
478 _(
479 b'backgroundclose can only be used when a '
479 b'backgroundclose can only be used when a '
480 b'backgroundclosing context manager is active'
480 b'backgroundclosing context manager is active'
481 )
481 )
482 )
482 )
483
483
484 fp = delayclosedfile(fp, self._backgroundfilecloser)
484 fp = delayclosedfile(fp, self._backgroundfilecloser)
485
485
486 return fp
486 return fp
487
487
488 def symlink(self, src, dst):
488 def symlink(self, src, dst):
489 self.audit(dst)
489 self.audit(dst)
490 linkname = self.join(dst)
490 linkname = self.join(dst)
491 util.tryunlink(linkname)
491 util.tryunlink(linkname)
492
492
493 util.makedirs(os.path.dirname(linkname), self.createmode)
493 util.makedirs(os.path.dirname(linkname), self.createmode)
494
494
495 if self._cansymlink:
495 if self._cansymlink:
496 try:
496 try:
497 os.symlink(src, linkname)
497 os.symlink(src, linkname)
498 except OSError as err:
498 except OSError as err:
499 raise OSError(
499 raise OSError(
500 err.errno,
500 err.errno,
501 _(b'could not symlink to %r: %s')
501 _(b'could not symlink to %r: %s')
502 % (src, encoding.strtolocal(err.strerror)),
502 % (src, encoding.strtolocal(err.strerror)),
503 linkname,
503 linkname,
504 )
504 )
505 else:
505 else:
506 self.write(dst, src)
506 self.write(dst, src)
507
507
508 def join(self, path, *insidef):
508 def join(self, path, *insidef):
509 if path:
509 if path:
510 return os.path.join(self.base, path, *insidef)
510 return os.path.join(self.base, path, *insidef)
511 else:
511 else:
512 return self.base
512 return self.base
513
513
514
514
515 opener = vfs
515 opener = vfs
516
516
517
517
518 class proxyvfs(abstractvfs):
518 class proxyvfs(abstractvfs):
519 def __init__(self, vfs):
519 def __init__(self, vfs):
520 self.vfs = vfs
520 self.vfs = vfs
521
521
522 def _auditpath(self, path, mode):
522 def _auditpath(self, path, mode):
523 return self.vfs._auditpath(path, mode)
523 return self.vfs._auditpath(path, mode)
524
524
525 @property
525 @property
526 def options(self):
526 def options(self):
527 return self.vfs.options
527 return self.vfs.options
528
528
529 @options.setter
529 @options.setter
530 def options(self, value):
530 def options(self, value):
531 self.vfs.options = value
531 self.vfs.options = value
532
532
533
533
534 class filtervfs(proxyvfs, abstractvfs):
534 class filtervfs(proxyvfs, abstractvfs):
535 '''Wrapper vfs for filtering filenames with a function.'''
535 '''Wrapper vfs for filtering filenames with a function.'''
536
536
537 def __init__(self, vfs, filter):
537 def __init__(self, vfs, filter):
538 proxyvfs.__init__(self, vfs)
538 proxyvfs.__init__(self, vfs)
539 self._filter = filter
539 self._filter = filter
540
540
541 def __call__(self, path, *args, **kwargs):
541 def __call__(self, path, *args, **kwargs):
542 return self.vfs(self._filter(path), *args, **kwargs)
542 return self.vfs(self._filter(path), *args, **kwargs)
543
543
544 def join(self, path, *insidef):
544 def join(self, path, *insidef):
545 if path:
545 if path:
546 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
546 return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
547 else:
547 else:
548 return self.vfs.join(path)
548 return self.vfs.join(path)
549
549
550
550
551 filteropener = filtervfs
551 filteropener = filtervfs
552
552
553
553
554 class readonlyvfs(proxyvfs):
554 class readonlyvfs(proxyvfs):
555 '''Wrapper vfs preventing any writing.'''
555 '''Wrapper vfs preventing any writing.'''
556
556
557 def __init__(self, vfs):
557 def __init__(self, vfs):
558 proxyvfs.__init__(self, vfs)
558 proxyvfs.__init__(self, vfs)
559
559
560 def __call__(self, path, mode=b'r', *args, **kw):
560 def __call__(self, path, mode=b'r', *args, **kw):
561 if mode not in (b'r', b'rb'):
561 if mode not in (b'r', b'rb'):
562 raise error.Abort(_(b'this vfs is read only'))
562 raise error.Abort(_(b'this vfs is read only'))
563 return self.vfs(path, mode, *args, **kw)
563 return self.vfs(path, mode, *args, **kw)
564
564
565 def join(self, path, *insidef):
565 def join(self, path, *insidef):
566 return self.vfs.join(path, *insidef)
566 return self.vfs.join(path, *insidef)
567
567
568
568
569 class closewrapbase(object):
569 class closewrapbase(object):
570 """Base class of wrapper, which hooks closing
570 """Base class of wrapper, which hooks closing
571
571
572 Do not instantiate outside of the vfs layer.
572 Do not instantiate outside of the vfs layer.
573 """
573 """
574
574
575 def __init__(self, fh):
575 def __init__(self, fh):
576 object.__setattr__(self, r'_origfh', fh)
576 object.__setattr__(self, r'_origfh', fh)
577
577
578 def __getattr__(self, attr):
578 def __getattr__(self, attr):
579 return getattr(self._origfh, attr)
579 return getattr(self._origfh, attr)
580
580
581 def __setattr__(self, attr, value):
581 def __setattr__(self, attr, value):
582 return setattr(self._origfh, attr, value)
582 return setattr(self._origfh, attr, value)
583
583
584 def __delattr__(self, attr):
584 def __delattr__(self, attr):
585 return delattr(self._origfh, attr)
585 return delattr(self._origfh, attr)
586
586
587 def __enter__(self):
587 def __enter__(self):
588 self._origfh.__enter__()
588 self._origfh.__enter__()
589 return self
589 return self
590
590
591 def __exit__(self, exc_type, exc_value, exc_tb):
591 def __exit__(self, exc_type, exc_value, exc_tb):
592 raise NotImplementedError(b'attempted instantiating ' + str(type(self)))
592 raise NotImplementedError('attempted instantiating ' + str(type(self)))
593
593
594 def close(self):
594 def close(self):
595 raise NotImplementedError(b'attempted instantiating ' + str(type(self)))
595 raise NotImplementedError('attempted instantiating ' + str(type(self)))
596
596
597
597
598 class delayclosedfile(closewrapbase):
598 class delayclosedfile(closewrapbase):
599 """Proxy for a file object whose close is delayed.
599 """Proxy for a file object whose close is delayed.
600
600
601 Do not instantiate outside of the vfs layer.
601 Do not instantiate outside of the vfs layer.
602 """
602 """
603
603
604 def __init__(self, fh, closer):
604 def __init__(self, fh, closer):
605 super(delayclosedfile, self).__init__(fh)
605 super(delayclosedfile, self).__init__(fh)
606 object.__setattr__(self, r'_closer', closer)
606 object.__setattr__(self, r'_closer', closer)
607
607
608 def __exit__(self, exc_type, exc_value, exc_tb):
608 def __exit__(self, exc_type, exc_value, exc_tb):
609 self._closer.close(self._origfh)
609 self._closer.close(self._origfh)
610
610
611 def close(self):
611 def close(self):
612 self._closer.close(self._origfh)
612 self._closer.close(self._origfh)
613
613
614
614
615 class backgroundfilecloser(object):
615 class backgroundfilecloser(object):
616 """Coordinates background closing of file handles on multiple threads."""
616 """Coordinates background closing of file handles on multiple threads."""
617
617
618 def __init__(self, ui, expectedcount=-1):
618 def __init__(self, ui, expectedcount=-1):
619 self._running = False
619 self._running = False
620 self._entered = False
620 self._entered = False
621 self._threads = []
621 self._threads = []
622 self._threadexception = None
622 self._threadexception = None
623
623
624 # Only Windows/NTFS has slow file closing. So only enable by default
624 # Only Windows/NTFS has slow file closing. So only enable by default
625 # on that platform. But allow to be enabled elsewhere for testing.
625 # on that platform. But allow to be enabled elsewhere for testing.
626 defaultenabled = pycompat.iswindows
626 defaultenabled = pycompat.iswindows
627 enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled)
627 enabled = ui.configbool(b'worker', b'backgroundclose', defaultenabled)
628
628
629 if not enabled:
629 if not enabled:
630 return
630 return
631
631
632 # There is overhead to starting and stopping the background threads.
632 # There is overhead to starting and stopping the background threads.
633 # Don't do background processing unless the file count is large enough
633 # Don't do background processing unless the file count is large enough
634 # to justify it.
634 # to justify it.
635 minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount')
635 minfilecount = ui.configint(b'worker', b'backgroundcloseminfilecount')
636 # FUTURE dynamically start background threads after minfilecount closes.
636 # FUTURE dynamically start background threads after minfilecount closes.
637 # (We don't currently have any callers that don't know their file count)
637 # (We don't currently have any callers that don't know their file count)
638 if expectedcount > 0 and expectedcount < minfilecount:
638 if expectedcount > 0 and expectedcount < minfilecount:
639 return
639 return
640
640
641 maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue')
641 maxqueue = ui.configint(b'worker', b'backgroundclosemaxqueue')
642 threadcount = ui.configint(b'worker', b'backgroundclosethreadcount')
642 threadcount = ui.configint(b'worker', b'backgroundclosethreadcount')
643
643
644 ui.debug(
644 ui.debug(
645 b'starting %d threads for background file closing\n' % threadcount
645 b'starting %d threads for background file closing\n' % threadcount
646 )
646 )
647
647
648 self._queue = pycompat.queue.Queue(maxsize=maxqueue)
648 self._queue = pycompat.queue.Queue(maxsize=maxqueue)
649 self._running = True
649 self._running = True
650
650
651 for i in range(threadcount):
651 for i in range(threadcount):
652 t = threading.Thread(target=self._worker, name='backgroundcloser')
652 t = threading.Thread(target=self._worker, name='backgroundcloser')
653 self._threads.append(t)
653 self._threads.append(t)
654 t.start()
654 t.start()
655
655
656 def __enter__(self):
656 def __enter__(self):
657 self._entered = True
657 self._entered = True
658 return self
658 return self
659
659
660 def __exit__(self, exc_type, exc_value, exc_tb):
660 def __exit__(self, exc_type, exc_value, exc_tb):
661 self._running = False
661 self._running = False
662
662
663 # Wait for threads to finish closing so open files don't linger for
663 # Wait for threads to finish closing so open files don't linger for
664 # longer than lifetime of context manager.
664 # longer than lifetime of context manager.
665 for t in self._threads:
665 for t in self._threads:
666 t.join()
666 t.join()
667
667
668 def _worker(self):
668 def _worker(self):
669 """Main routine for worker thread."""
669 """Main routine for worker thread."""
670 while True:
670 while True:
671 try:
671 try:
672 fh = self._queue.get(block=True, timeout=0.100)
672 fh = self._queue.get(block=True, timeout=0.100)
673 # Need to catch or the thread will terminate and
673 # Need to catch or the thread will terminate and
674 # we could orphan file descriptors.
674 # we could orphan file descriptors.
675 try:
675 try:
676 fh.close()
676 fh.close()
677 except Exception as e:
677 except Exception as e:
678 # Stash so can re-raise from main thread later.
678 # Stash so can re-raise from main thread later.
679 self._threadexception = e
679 self._threadexception = e
680 except pycompat.queue.Empty:
680 except pycompat.queue.Empty:
681 if not self._running:
681 if not self._running:
682 break
682 break
683
683
684 def close(self, fh):
684 def close(self, fh):
685 """Schedule a file for closing."""
685 """Schedule a file for closing."""
686 if not self._entered:
686 if not self._entered:
687 raise error.Abort(
687 raise error.Abort(
688 _(b'can only call close() when context manager active')
688 _(b'can only call close() when context manager active')
689 )
689 )
690
690
691 # If a background thread encountered an exception, raise now so we fail
691 # If a background thread encountered an exception, raise now so we fail
692 # fast. Otherwise we may potentially go on for minutes until the error
692 # fast. Otherwise we may potentially go on for minutes until the error
693 # is acted on.
693 # is acted on.
694 if self._threadexception:
694 if self._threadexception:
695 e = self._threadexception
695 e = self._threadexception
696 self._threadexception = None
696 self._threadexception = None
697 raise e
697 raise e
698
698
699 # If we're not actively running, close synchronously.
699 # If we're not actively running, close synchronously.
700 if not self._running:
700 if not self._running:
701 fh.close()
701 fh.close()
702 return
702 return
703
703
704 self._queue.put(fh, block=True, timeout=None)
704 self._queue.put(fh, block=True, timeout=None)
705
705
706
706
707 class checkambigatclosing(closewrapbase):
707 class checkambigatclosing(closewrapbase):
708 """Proxy for a file object, to avoid ambiguity of file stat
708 """Proxy for a file object, to avoid ambiguity of file stat
709
709
710 See also util.filestat for detail about "ambiguity of file stat".
710 See also util.filestat for detail about "ambiguity of file stat".
711
711
712 This proxy is useful only if the target file is guarded by any
712 This proxy is useful only if the target file is guarded by any
713 lock (e.g. repo.lock or repo.wlock)
713 lock (e.g. repo.lock or repo.wlock)
714
714
715 Do not instantiate outside of the vfs layer.
715 Do not instantiate outside of the vfs layer.
716 """
716 """
717
717
718 def __init__(self, fh):
718 def __init__(self, fh):
719 super(checkambigatclosing, self).__init__(fh)
719 super(checkambigatclosing, self).__init__(fh)
720 object.__setattr__(self, r'_oldstat', util.filestat.frompath(fh.name))
720 object.__setattr__(self, r'_oldstat', util.filestat.frompath(fh.name))
721
721
722 def _checkambig(self):
722 def _checkambig(self):
723 oldstat = self._oldstat
723 oldstat = self._oldstat
724 if oldstat.stat:
724 if oldstat.stat:
725 _avoidambig(self._origfh.name, oldstat)
725 _avoidambig(self._origfh.name, oldstat)
726
726
727 def __exit__(self, exc_type, exc_value, exc_tb):
727 def __exit__(self, exc_type, exc_value, exc_tb):
728 self._origfh.__exit__(exc_type, exc_value, exc_tb)
728 self._origfh.__exit__(exc_type, exc_value, exc_tb)
729 self._checkambig()
729 self._checkambig()
730
730
731 def close(self):
731 def close(self):
732 self._origfh.close()
732 self._origfh.close()
733 self._checkambig()
733 self._checkambig()
General Comments 0
You need to be logged in to leave comments. Login now