##// END OF EJS Templates
py3: add b'' to make a triple quoted string bytes on Python 3...
Pulkit Goyal -
r33094:4c5af472 default
parent child Browse files
Show More
@@ -1,3743 +1,3743 b''
1 1 # util.py - Mercurial utility functions and platform specific implementations
2 2 #
3 3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
4 4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
5 5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
6 6 #
7 7 # This software may be used and distributed according to the terms of the
8 8 # GNU General Public License version 2 or any later version.
9 9
10 10 """Mercurial utility functions and platform specific implementations.
11 11
12 12 This contains helper routines that are independent of the SCM core and
13 13 hide platform-specific details from the core.
14 14 """
15 15
16 16 from __future__ import absolute_import
17 17
18 18 import bz2
19 19 import calendar
20 20 import codecs
21 21 import collections
22 22 import datetime
23 23 import errno
24 24 import gc
25 25 import hashlib
26 26 import imp
27 27 import os
28 28 import platform as pyplatform
29 29 import re as remod
30 30 import shutil
31 31 import signal
32 32 import socket
33 33 import stat
34 34 import string
35 35 import subprocess
36 36 import sys
37 37 import tempfile
38 38 import textwrap
39 39 import time
40 40 import traceback
41 41 import warnings
42 42 import zlib
43 43
44 44 from . import (
45 45 encoding,
46 46 error,
47 47 i18n,
48 48 policy,
49 49 pycompat,
50 50 )
51 51
52 52 base85 = policy.importmod(r'base85')
53 53 osutil = policy.importmod(r'osutil')
54 54 parsers = policy.importmod(r'parsers')
55 55
56 56 b85decode = base85.b85decode
57 57 b85encode = base85.b85encode
58 58
59 59 cookielib = pycompat.cookielib
60 60 empty = pycompat.empty
61 61 httplib = pycompat.httplib
62 62 httpserver = pycompat.httpserver
63 63 pickle = pycompat.pickle
64 64 queue = pycompat.queue
65 65 socketserver = pycompat.socketserver
66 66 stderr = pycompat.stderr
67 67 stdin = pycompat.stdin
68 68 stdout = pycompat.stdout
69 69 stringio = pycompat.stringio
70 70 urlerr = pycompat.urlerr
71 71 urlreq = pycompat.urlreq
72 72 xmlrpclib = pycompat.xmlrpclib
73 73
74 74 # workaround for win32mbcs
75 75 _filenamebytestr = pycompat.bytestr
76 76
77 77 def isatty(fp):
78 78 try:
79 79 return fp.isatty()
80 80 except AttributeError:
81 81 return False
82 82
83 83 # glibc determines buffering on first write to stdout - if we replace a TTY
84 84 # destined stdout with a pipe destined stdout (e.g. pager), we want line
85 85 # buffering
86 86 if isatty(stdout):
87 87 stdout = os.fdopen(stdout.fileno(), pycompat.sysstr('wb'), 1)
88 88
89 89 if pycompat.osname == 'nt':
90 90 from . import windows as platform
91 91 stdout = platform.winstdout(stdout)
92 92 else:
93 93 from . import posix as platform
94 94
95 95 _ = i18n._
96 96
97 97 bindunixsocket = platform.bindunixsocket
98 98 cachestat = platform.cachestat
99 99 checkexec = platform.checkexec
100 100 checklink = platform.checklink
101 101 copymode = platform.copymode
102 102 executablepath = platform.executablepath
103 103 expandglobs = platform.expandglobs
104 104 explainexit = platform.explainexit
105 105 findexe = platform.findexe
106 106 gethgcmd = platform.gethgcmd
107 107 getuser = platform.getuser
108 108 getpid = os.getpid
109 109 groupmembers = platform.groupmembers
110 110 groupname = platform.groupname
111 111 hidewindow = platform.hidewindow
112 112 isexec = platform.isexec
113 113 isowner = platform.isowner
114 114 listdir = osutil.listdir
115 115 localpath = platform.localpath
116 116 lookupreg = platform.lookupreg
117 117 makedir = platform.makedir
118 118 nlinks = platform.nlinks
119 119 normpath = platform.normpath
120 120 normcase = platform.normcase
121 121 normcasespec = platform.normcasespec
122 122 normcasefallback = platform.normcasefallback
123 123 openhardlinks = platform.openhardlinks
124 124 oslink = platform.oslink
125 125 parsepatchoutput = platform.parsepatchoutput
126 126 pconvert = platform.pconvert
127 127 poll = platform.poll
128 128 popen = platform.popen
129 129 posixfile = platform.posixfile
130 130 quotecommand = platform.quotecommand
131 131 readpipe = platform.readpipe
132 132 rename = platform.rename
133 133 removedirs = platform.removedirs
134 134 samedevice = platform.samedevice
135 135 samefile = platform.samefile
136 136 samestat = platform.samestat
137 137 setbinary = platform.setbinary
138 138 setflags = platform.setflags
139 139 setsignalhandler = platform.setsignalhandler
140 140 shellquote = platform.shellquote
141 141 spawndetached = platform.spawndetached
142 142 split = platform.split
143 143 sshargs = platform.sshargs
144 144 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
145 145 statisexec = platform.statisexec
146 146 statislink = platform.statislink
147 147 testpid = platform.testpid
148 148 umask = platform.umask
149 149 unlink = platform.unlink
150 150 username = platform.username
151 151
152 152 try:
153 153 recvfds = osutil.recvfds
154 154 except AttributeError:
155 155 pass
156 156 try:
157 157 setprocname = osutil.setprocname
158 158 except AttributeError:
159 159 pass
160 160
161 161 # Python compatibility
162 162
163 163 _notset = object()
164 164
165 165 # disable Python's problematic floating point timestamps (issue4836)
166 166 # (Python hypocritically says you shouldn't change this behavior in
167 167 # libraries, and sure enough Mercurial is not a library.)
168 168 os.stat_float_times(False)
169 169
170 170 def safehasattr(thing, attr):
171 171 return getattr(thing, attr, _notset) is not _notset
172 172
173 173 def bitsfrom(container):
174 174 bits = 0
175 175 for bit in container:
176 176 bits |= bit
177 177 return bits
178 178
179 179 # python 2.6 still have deprecation warning enabled by default. We do not want
180 180 # to display anything to standard user so detect if we are running test and
181 181 # only use python deprecation warning in this case.
182 182 _dowarn = bool(encoding.environ.get('HGEMITWARNINGS'))
183 183 if _dowarn:
184 184 # explicitly unfilter our warning for python 2.7
185 185 #
186 186 # The option of setting PYTHONWARNINGS in the test runner was investigated.
187 187 # However, module name set through PYTHONWARNINGS was exactly matched, so
188 188 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
189 189 # makes the whole PYTHONWARNINGS thing useless for our usecase.
190 190 warnings.filterwarnings(r'default', r'', DeprecationWarning, r'mercurial')
191 191 warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext')
192 192 warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext3rd')
193 193
194 194 def nouideprecwarn(msg, version, stacklevel=1):
195 195 """Issue an python native deprecation warning
196 196
197 197 This is a noop outside of tests, use 'ui.deprecwarn' when possible.
198 198 """
199 199 if _dowarn:
200 200 msg += ("\n(compatibility will be dropped after Mercurial-%s,"
201 201 " update your code.)") % version
202 202 warnings.warn(msg, DeprecationWarning, stacklevel + 1)
203 203
204 204 DIGESTS = {
205 205 'md5': hashlib.md5,
206 206 'sha1': hashlib.sha1,
207 207 'sha512': hashlib.sha512,
208 208 }
209 209 # List of digest types from strongest to weakest
210 210 DIGESTS_BY_STRENGTH = ['sha512', 'sha1', 'md5']
211 211
212 212 for k in DIGESTS_BY_STRENGTH:
213 213 assert k in DIGESTS
214 214
215 215 class digester(object):
216 216 """helper to compute digests.
217 217
218 218 This helper can be used to compute one or more digests given their name.
219 219
220 220 >>> d = digester(['md5', 'sha1'])
221 221 >>> d.update('foo')
222 222 >>> [k for k in sorted(d)]
223 223 ['md5', 'sha1']
224 224 >>> d['md5']
225 225 'acbd18db4cc2f85cedef654fccc4a4d8'
226 226 >>> d['sha1']
227 227 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
228 228 >>> digester.preferred(['md5', 'sha1'])
229 229 'sha1'
230 230 """
231 231
232 232 def __init__(self, digests, s=''):
233 233 self._hashes = {}
234 234 for k in digests:
235 235 if k not in DIGESTS:
236 236 raise Abort(_('unknown digest type: %s') % k)
237 237 self._hashes[k] = DIGESTS[k]()
238 238 if s:
239 239 self.update(s)
240 240
241 241 def update(self, data):
242 242 for h in self._hashes.values():
243 243 h.update(data)
244 244
245 245 def __getitem__(self, key):
246 246 if key not in DIGESTS:
247 247 raise Abort(_('unknown digest type: %s') % k)
248 248 return self._hashes[key].hexdigest()
249 249
250 250 def __iter__(self):
251 251 return iter(self._hashes)
252 252
253 253 @staticmethod
254 254 def preferred(supported):
255 255 """returns the strongest digest type in both supported and DIGESTS."""
256 256
257 257 for k in DIGESTS_BY_STRENGTH:
258 258 if k in supported:
259 259 return k
260 260 return None
261 261
262 262 class digestchecker(object):
263 263 """file handle wrapper that additionally checks content against a given
264 264 size and digests.
265 265
266 266 d = digestchecker(fh, size, {'md5': '...'})
267 267
268 268 When multiple digests are given, all of them are validated.
269 269 """
270 270
271 271 def __init__(self, fh, size, digests):
272 272 self._fh = fh
273 273 self._size = size
274 274 self._got = 0
275 275 self._digests = dict(digests)
276 276 self._digester = digester(self._digests.keys())
277 277
278 278 def read(self, length=-1):
279 279 content = self._fh.read(length)
280 280 self._digester.update(content)
281 281 self._got += len(content)
282 282 return content
283 283
284 284 def validate(self):
285 285 if self._size != self._got:
286 286 raise Abort(_('size mismatch: expected %d, got %d') %
287 287 (self._size, self._got))
288 288 for k, v in self._digests.items():
289 289 if v != self._digester[k]:
290 290 # i18n: first parameter is a digest name
291 291 raise Abort(_('%s mismatch: expected %s, got %s') %
292 292 (k, v, self._digester[k]))
293 293
294 294 try:
295 295 buffer = buffer
296 296 except NameError:
297 297 if not pycompat.ispy3:
298 298 def buffer(sliceable, offset=0, length=None):
299 299 if length is not None:
300 300 return sliceable[offset:offset + length]
301 301 return sliceable[offset:]
302 302 else:
303 303 def buffer(sliceable, offset=0, length=None):
304 304 if length is not None:
305 305 return memoryview(sliceable)[offset:offset + length]
306 306 return memoryview(sliceable)[offset:]
307 307
308 308 closefds = pycompat.osname == 'posix'
309 309
310 310 _chunksize = 4096
311 311
312 312 class bufferedinputpipe(object):
313 313 """a manually buffered input pipe
314 314
315 315 Python will not let us use buffered IO and lazy reading with 'polling' at
316 316 the same time. We cannot probe the buffer state and select will not detect
317 317 that data are ready to read if they are already buffered.
318 318
319 319 This class let us work around that by implementing its own buffering
320 320 (allowing efficient readline) while offering a way to know if the buffer is
321 321 empty from the output (allowing collaboration of the buffer with polling).
322 322
323 323 This class lives in the 'util' module because it makes use of the 'os'
324 324 module from the python stdlib.
325 325 """
326 326
327 327 def __init__(self, input):
328 328 self._input = input
329 329 self._buffer = []
330 330 self._eof = False
331 331 self._lenbuf = 0
332 332
333 333 @property
334 334 def hasbuffer(self):
335 335 """True is any data is currently buffered
336 336
337 337 This will be used externally a pre-step for polling IO. If there is
338 338 already data then no polling should be set in place."""
339 339 return bool(self._buffer)
340 340
341 341 @property
342 342 def closed(self):
343 343 return self._input.closed
344 344
345 345 def fileno(self):
346 346 return self._input.fileno()
347 347
348 348 def close(self):
349 349 return self._input.close()
350 350
351 351 def read(self, size):
352 352 while (not self._eof) and (self._lenbuf < size):
353 353 self._fillbuffer()
354 354 return self._frombuffer(size)
355 355
356 356 def readline(self, *args, **kwargs):
357 357 if 1 < len(self._buffer):
358 358 # this should not happen because both read and readline end with a
359 359 # _frombuffer call that collapse it.
360 360 self._buffer = [''.join(self._buffer)]
361 361 self._lenbuf = len(self._buffer[0])
362 362 lfi = -1
363 363 if self._buffer:
364 364 lfi = self._buffer[-1].find('\n')
365 365 while (not self._eof) and lfi < 0:
366 366 self._fillbuffer()
367 367 if self._buffer:
368 368 lfi = self._buffer[-1].find('\n')
369 369 size = lfi + 1
370 370 if lfi < 0: # end of file
371 371 size = self._lenbuf
372 372 elif 1 < len(self._buffer):
373 373 # we need to take previous chunks into account
374 374 size += self._lenbuf - len(self._buffer[-1])
375 375 return self._frombuffer(size)
376 376
377 377 def _frombuffer(self, size):
378 378 """return at most 'size' data from the buffer
379 379
380 380 The data are removed from the buffer."""
381 381 if size == 0 or not self._buffer:
382 382 return ''
383 383 buf = self._buffer[0]
384 384 if 1 < len(self._buffer):
385 385 buf = ''.join(self._buffer)
386 386
387 387 data = buf[:size]
388 388 buf = buf[len(data):]
389 389 if buf:
390 390 self._buffer = [buf]
391 391 self._lenbuf = len(buf)
392 392 else:
393 393 self._buffer = []
394 394 self._lenbuf = 0
395 395 return data
396 396
397 397 def _fillbuffer(self):
398 398 """read data to the buffer"""
399 399 data = os.read(self._input.fileno(), _chunksize)
400 400 if not data:
401 401 self._eof = True
402 402 else:
403 403 self._lenbuf += len(data)
404 404 self._buffer.append(data)
405 405
406 406 def popen2(cmd, env=None, newlines=False):
407 407 # Setting bufsize to -1 lets the system decide the buffer size.
408 408 # The default for bufsize is 0, meaning unbuffered. This leads to
409 409 # poor performance on Mac OS X: http://bugs.python.org/issue4194
410 410 p = subprocess.Popen(cmd, shell=True, bufsize=-1,
411 411 close_fds=closefds,
412 412 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
413 413 universal_newlines=newlines,
414 414 env=env)
415 415 return p.stdin, p.stdout
416 416
417 417 def popen3(cmd, env=None, newlines=False):
418 418 stdin, stdout, stderr, p = popen4(cmd, env, newlines)
419 419 return stdin, stdout, stderr
420 420
421 421 def popen4(cmd, env=None, newlines=False, bufsize=-1):
422 422 p = subprocess.Popen(cmd, shell=True, bufsize=bufsize,
423 423 close_fds=closefds,
424 424 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
425 425 stderr=subprocess.PIPE,
426 426 universal_newlines=newlines,
427 427 env=env)
428 428 return p.stdin, p.stdout, p.stderr, p
429 429
430 430 def version():
431 431 """Return version information if available."""
432 432 try:
433 433 from . import __version__
434 434 return __version__.version
435 435 except ImportError:
436 436 return 'unknown'
437 437
438 438 def versiontuple(v=None, n=4):
439 439 """Parses a Mercurial version string into an N-tuple.
440 440
441 441 The version string to be parsed is specified with the ``v`` argument.
442 442 If it isn't defined, the current Mercurial version string will be parsed.
443 443
444 444 ``n`` can be 2, 3, or 4. Here is how some version strings map to
445 445 returned values:
446 446
447 447 >>> v = '3.6.1+190-df9b73d2d444'
448 448 >>> versiontuple(v, 2)
449 449 (3, 6)
450 450 >>> versiontuple(v, 3)
451 451 (3, 6, 1)
452 452 >>> versiontuple(v, 4)
453 453 (3, 6, 1, '190-df9b73d2d444')
454 454
455 455 >>> versiontuple('3.6.1+190-df9b73d2d444+20151118')
456 456 (3, 6, 1, '190-df9b73d2d444+20151118')
457 457
458 458 >>> v = '3.6'
459 459 >>> versiontuple(v, 2)
460 460 (3, 6)
461 461 >>> versiontuple(v, 3)
462 462 (3, 6, None)
463 463 >>> versiontuple(v, 4)
464 464 (3, 6, None, None)
465 465
466 466 >>> v = '3.9-rc'
467 467 >>> versiontuple(v, 2)
468 468 (3, 9)
469 469 >>> versiontuple(v, 3)
470 470 (3, 9, None)
471 471 >>> versiontuple(v, 4)
472 472 (3, 9, None, 'rc')
473 473
474 474 >>> v = '3.9-rc+2-02a8fea4289b'
475 475 >>> versiontuple(v, 2)
476 476 (3, 9)
477 477 >>> versiontuple(v, 3)
478 478 (3, 9, None)
479 479 >>> versiontuple(v, 4)
480 480 (3, 9, None, 'rc+2-02a8fea4289b')
481 481 """
482 482 if not v:
483 483 v = version()
484 484 parts = remod.split('[\+-]', v, 1)
485 485 if len(parts) == 1:
486 486 vparts, extra = parts[0], None
487 487 else:
488 488 vparts, extra = parts
489 489
490 490 vints = []
491 491 for i in vparts.split('.'):
492 492 try:
493 493 vints.append(int(i))
494 494 except ValueError:
495 495 break
496 496 # (3, 6) -> (3, 6, None)
497 497 while len(vints) < 3:
498 498 vints.append(None)
499 499
500 500 if n == 2:
501 501 return (vints[0], vints[1])
502 502 if n == 3:
503 503 return (vints[0], vints[1], vints[2])
504 504 if n == 4:
505 505 return (vints[0], vints[1], vints[2], extra)
506 506
507 507 # used by parsedate
508 508 defaultdateformats = (
509 509 '%Y-%m-%dT%H:%M:%S', # the 'real' ISO8601
510 510 '%Y-%m-%dT%H:%M', # without seconds
511 511 '%Y-%m-%dT%H%M%S', # another awful but legal variant without :
512 512 '%Y-%m-%dT%H%M', # without seconds
513 513 '%Y-%m-%d %H:%M:%S', # our common legal variant
514 514 '%Y-%m-%d %H:%M', # without seconds
515 515 '%Y-%m-%d %H%M%S', # without :
516 516 '%Y-%m-%d %H%M', # without seconds
517 517 '%Y-%m-%d %I:%M:%S%p',
518 518 '%Y-%m-%d %H:%M',
519 519 '%Y-%m-%d %I:%M%p',
520 520 '%Y-%m-%d',
521 521 '%m-%d',
522 522 '%m/%d',
523 523 '%m/%d/%y',
524 524 '%m/%d/%Y',
525 525 '%a %b %d %H:%M:%S %Y',
526 526 '%a %b %d %I:%M:%S%p %Y',
527 527 '%a, %d %b %Y %H:%M:%S', # GNU coreutils "/bin/date --rfc-2822"
528 528 '%b %d %H:%M:%S %Y',
529 529 '%b %d %I:%M:%S%p %Y',
530 530 '%b %d %H:%M:%S',
531 531 '%b %d %I:%M:%S%p',
532 532 '%b %d %H:%M',
533 533 '%b %d %I:%M%p',
534 534 '%b %d %Y',
535 535 '%b %d',
536 536 '%H:%M:%S',
537 537 '%I:%M:%S%p',
538 538 '%H:%M',
539 539 '%I:%M%p',
540 540 )
541 541
542 542 extendeddateformats = defaultdateformats + (
543 543 "%Y",
544 544 "%Y-%m",
545 545 "%b",
546 546 "%b %Y",
547 547 )
548 548
549 549 def cachefunc(func):
550 550 '''cache the result of function calls'''
551 551 # XXX doesn't handle keywords args
552 552 if func.__code__.co_argcount == 0:
553 553 cache = []
554 554 def f():
555 555 if len(cache) == 0:
556 556 cache.append(func())
557 557 return cache[0]
558 558 return f
559 559 cache = {}
560 560 if func.__code__.co_argcount == 1:
561 561 # we gain a small amount of time because
562 562 # we don't need to pack/unpack the list
563 563 def f(arg):
564 564 if arg not in cache:
565 565 cache[arg] = func(arg)
566 566 return cache[arg]
567 567 else:
568 568 def f(*args):
569 569 if args not in cache:
570 570 cache[args] = func(*args)
571 571 return cache[args]
572 572
573 573 return f
574 574
575 575 class sortdict(collections.OrderedDict):
576 576 '''a simple sorted dictionary
577 577
578 578 >>> d1 = sortdict([('a', 0), ('b', 1)])
579 579 >>> d2 = d1.copy()
580 580 >>> d2
581 581 sortdict([('a', 0), ('b', 1)])
582 582 >>> d2.update([('a', 2)])
583 583 >>> d2.keys() # should still be in last-set order
584 584 ['b', 'a']
585 585 '''
586 586
587 587 def __setitem__(self, key, value):
588 588 if key in self:
589 589 del self[key]
590 590 super(sortdict, self).__setitem__(key, value)
591 591
592 592 class _lrucachenode(object):
593 593 """A node in a doubly linked list.
594 594
595 595 Holds a reference to nodes on either side as well as a key-value
596 596 pair for the dictionary entry.
597 597 """
598 598 __slots__ = (u'next', u'prev', u'key', u'value')
599 599
600 600 def __init__(self):
601 601 self.next = None
602 602 self.prev = None
603 603
604 604 self.key = _notset
605 605 self.value = None
606 606
607 607 def markempty(self):
608 608 """Mark the node as emptied."""
609 609 self.key = _notset
610 610
611 611 class lrucachedict(object):
612 612 """Dict that caches most recent accesses and sets.
613 613
614 614 The dict consists of an actual backing dict - indexed by original
615 615 key - and a doubly linked circular list defining the order of entries in
616 616 the cache.
617 617
618 618 The head node is the newest entry in the cache. If the cache is full,
619 619 we recycle head.prev and make it the new head. Cache accesses result in
620 620 the node being moved to before the existing head and being marked as the
621 621 new head node.
622 622 """
623 623 def __init__(self, max):
624 624 self._cache = {}
625 625
626 626 self._head = head = _lrucachenode()
627 627 head.prev = head
628 628 head.next = head
629 629 self._size = 1
630 630 self._capacity = max
631 631
632 632 def __len__(self):
633 633 return len(self._cache)
634 634
635 635 def __contains__(self, k):
636 636 return k in self._cache
637 637
638 638 def __iter__(self):
639 639 # We don't have to iterate in cache order, but why not.
640 640 n = self._head
641 641 for i in range(len(self._cache)):
642 642 yield n.key
643 643 n = n.next
644 644
645 645 def __getitem__(self, k):
646 646 node = self._cache[k]
647 647 self._movetohead(node)
648 648 return node.value
649 649
650 650 def __setitem__(self, k, v):
651 651 node = self._cache.get(k)
652 652 # Replace existing value and mark as newest.
653 653 if node is not None:
654 654 node.value = v
655 655 self._movetohead(node)
656 656 return
657 657
658 658 if self._size < self._capacity:
659 659 node = self._addcapacity()
660 660 else:
661 661 # Grab the last/oldest item.
662 662 node = self._head.prev
663 663
664 664 # At capacity. Kill the old entry.
665 665 if node.key is not _notset:
666 666 del self._cache[node.key]
667 667
668 668 node.key = k
669 669 node.value = v
670 670 self._cache[k] = node
671 671 # And mark it as newest entry. No need to adjust order since it
672 672 # is already self._head.prev.
673 673 self._head = node
674 674
675 675 def __delitem__(self, k):
676 676 node = self._cache.pop(k)
677 677 node.markempty()
678 678
679 679 # Temporarily mark as newest item before re-adjusting head to make
680 680 # this node the oldest item.
681 681 self._movetohead(node)
682 682 self._head = node.next
683 683
684 684 # Additional dict methods.
685 685
686 686 def get(self, k, default=None):
687 687 try:
688 688 return self._cache[k].value
689 689 except KeyError:
690 690 return default
691 691
692 692 def clear(self):
693 693 n = self._head
694 694 while n.key is not _notset:
695 695 n.markempty()
696 696 n = n.next
697 697
698 698 self._cache.clear()
699 699
700 700 def copy(self):
701 701 result = lrucachedict(self._capacity)
702 702 n = self._head.prev
703 703 # Iterate in oldest-to-newest order, so the copy has the right ordering
704 704 for i in range(len(self._cache)):
705 705 result[n.key] = n.value
706 706 n = n.prev
707 707 return result
708 708
709 709 def _movetohead(self, node):
710 710 """Mark a node as the newest, making it the new head.
711 711
712 712 When a node is accessed, it becomes the freshest entry in the LRU
713 713 list, which is denoted by self._head.
714 714
715 715 Visually, let's make ``N`` the new head node (* denotes head):
716 716
717 717 previous/oldest <-> head <-> next/next newest
718 718
719 719 ----<->--- A* ---<->-----
720 720 | |
721 721 E <-> D <-> N <-> C <-> B
722 722
723 723 To:
724 724
725 725 ----<->--- N* ---<->-----
726 726 | |
727 727 E <-> D <-> C <-> B <-> A
728 728
729 729 This requires the following moves:
730 730
731 731 C.next = D (node.prev.next = node.next)
732 732 D.prev = C (node.next.prev = node.prev)
733 733 E.next = N (head.prev.next = node)
734 734 N.prev = E (node.prev = head.prev)
735 735 N.next = A (node.next = head)
736 736 A.prev = N (head.prev = node)
737 737 """
738 738 head = self._head
739 739 # C.next = D
740 740 node.prev.next = node.next
741 741 # D.prev = C
742 742 node.next.prev = node.prev
743 743 # N.prev = E
744 744 node.prev = head.prev
745 745 # N.next = A
746 746 # It is tempting to do just "head" here, however if node is
747 747 # adjacent to head, this will do bad things.
748 748 node.next = head.prev.next
749 749 # E.next = N
750 750 node.next.prev = node
751 751 # A.prev = N
752 752 node.prev.next = node
753 753
754 754 self._head = node
755 755
756 756 def _addcapacity(self):
757 757 """Add a node to the circular linked list.
758 758
759 759 The new node is inserted before the head node.
760 760 """
761 761 head = self._head
762 762 node = _lrucachenode()
763 763 head.prev.next = node
764 764 node.prev = head.prev
765 765 node.next = head
766 766 head.prev = node
767 767 self._size += 1
768 768 return node
769 769
770 770 def lrucachefunc(func):
771 771 '''cache most recent results of function calls'''
772 772 cache = {}
773 773 order = collections.deque()
774 774 if func.__code__.co_argcount == 1:
775 775 def f(arg):
776 776 if arg not in cache:
777 777 if len(cache) > 20:
778 778 del cache[order.popleft()]
779 779 cache[arg] = func(arg)
780 780 else:
781 781 order.remove(arg)
782 782 order.append(arg)
783 783 return cache[arg]
784 784 else:
785 785 def f(*args):
786 786 if args not in cache:
787 787 if len(cache) > 20:
788 788 del cache[order.popleft()]
789 789 cache[args] = func(*args)
790 790 else:
791 791 order.remove(args)
792 792 order.append(args)
793 793 return cache[args]
794 794
795 795 return f
796 796
797 797 class propertycache(object):
798 798 def __init__(self, func):
799 799 self.func = func
800 800 self.name = func.__name__
801 801 def __get__(self, obj, type=None):
802 802 result = self.func(obj)
803 803 self.cachevalue(obj, result)
804 804 return result
805 805
806 806 def cachevalue(self, obj, value):
807 807 # __dict__ assignment required to bypass __setattr__ (eg: repoview)
808 808 obj.__dict__[self.name] = value
809 809
810 810 def pipefilter(s, cmd):
811 811 '''filter string S through command CMD, returning its output'''
812 812 p = subprocess.Popen(cmd, shell=True, close_fds=closefds,
813 813 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
814 814 pout, perr = p.communicate(s)
815 815 return pout
816 816
817 817 def tempfilter(s, cmd):
818 818 '''filter string S through a pair of temporary files with CMD.
819 819 CMD is used as a template to create the real command to be run,
820 820 with the strings INFILE and OUTFILE replaced by the real names of
821 821 the temporary files generated.'''
822 822 inname, outname = None, None
823 823 try:
824 824 infd, inname = tempfile.mkstemp(prefix='hg-filter-in-')
825 825 fp = os.fdopen(infd, pycompat.sysstr('wb'))
826 826 fp.write(s)
827 827 fp.close()
828 828 outfd, outname = tempfile.mkstemp(prefix='hg-filter-out-')
829 829 os.close(outfd)
830 830 cmd = cmd.replace('INFILE', inname)
831 831 cmd = cmd.replace('OUTFILE', outname)
832 832 code = os.system(cmd)
833 833 if pycompat.sysplatform == 'OpenVMS' and code & 1:
834 834 code = 0
835 835 if code:
836 836 raise Abort(_("command '%s' failed: %s") %
837 837 (cmd, explainexit(code)))
838 838 return readfile(outname)
839 839 finally:
840 840 try:
841 841 if inname:
842 842 os.unlink(inname)
843 843 except OSError:
844 844 pass
845 845 try:
846 846 if outname:
847 847 os.unlink(outname)
848 848 except OSError:
849 849 pass
850 850
851 851 filtertable = {
852 852 'tempfile:': tempfilter,
853 853 'pipe:': pipefilter,
854 854 }
855 855
856 856 def filter(s, cmd):
857 857 "filter a string through a command that transforms its input to its output"
858 858 for name, fn in filtertable.iteritems():
859 859 if cmd.startswith(name):
860 860 return fn(s, cmd[len(name):].lstrip())
861 861 return pipefilter(s, cmd)
862 862
863 863 def binary(s):
864 864 """return true if a string is binary data"""
865 865 return bool(s and '\0' in s)
866 866
867 867 def increasingchunks(source, min=1024, max=65536):
868 868 '''return no less than min bytes per chunk while data remains,
869 869 doubling min after each chunk until it reaches max'''
870 870 def log2(x):
871 871 if not x:
872 872 return 0
873 873 i = 0
874 874 while x:
875 875 x >>= 1
876 876 i += 1
877 877 return i - 1
878 878
879 879 buf = []
880 880 blen = 0
881 881 for chunk in source:
882 882 buf.append(chunk)
883 883 blen += len(chunk)
884 884 if blen >= min:
885 885 if min < max:
886 886 min = min << 1
887 887 nmin = 1 << log2(blen)
888 888 if nmin > min:
889 889 min = nmin
890 890 if min > max:
891 891 min = max
892 892 yield ''.join(buf)
893 893 blen = 0
894 894 buf = []
895 895 if buf:
896 896 yield ''.join(buf)
897 897
898 898 Abort = error.Abort
899 899
900 900 def always(fn):
901 901 return True
902 902
903 903 def never(fn):
904 904 return False
905 905
906 906 def nogc(func):
907 907 """disable garbage collector
908 908
909 909 Python's garbage collector triggers a GC each time a certain number of
910 910 container objects (the number being defined by gc.get_threshold()) are
911 911 allocated even when marked not to be tracked by the collector. Tracking has
912 912 no effect on when GCs are triggered, only on what objects the GC looks
913 913 into. As a workaround, disable GC while building complex (huge)
914 914 containers.
915 915
916 916 This garbage collector issue have been fixed in 2.7.
917 917 """
918 918 if sys.version_info >= (2, 7):
919 919 return func
920 920 def wrapper(*args, **kwargs):
921 921 gcenabled = gc.isenabled()
922 922 gc.disable()
923 923 try:
924 924 return func(*args, **kwargs)
925 925 finally:
926 926 if gcenabled:
927 927 gc.enable()
928 928 return wrapper
929 929
930 930 def pathto(root, n1, n2):
931 931 '''return the relative path from one place to another.
932 932 root should use os.sep to separate directories
933 933 n1 should use os.sep to separate directories
934 934 n2 should use "/" to separate directories
935 935 returns an os.sep-separated path.
936 936
937 937 If n1 is a relative path, it's assumed it's
938 938 relative to root.
939 939 n2 should always be relative to root.
940 940 '''
941 941 if not n1:
942 942 return localpath(n2)
943 943 if os.path.isabs(n1):
944 944 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
945 945 return os.path.join(root, localpath(n2))
946 946 n2 = '/'.join((pconvert(root), n2))
947 947 a, b = splitpath(n1), n2.split('/')
948 948 a.reverse()
949 949 b.reverse()
950 950 while a and b and a[-1] == b[-1]:
951 951 a.pop()
952 952 b.pop()
953 953 b.reverse()
954 954 return pycompat.ossep.join((['..'] * len(a)) + b) or '.'
955 955
956 956 def mainfrozen():
957 957 """return True if we are a frozen executable.
958 958
959 959 The code supports py2exe (most common, Windows only) and tools/freeze
960 960 (portable, not much used).
961 961 """
962 962 return (safehasattr(sys, "frozen") or # new py2exe
963 963 safehasattr(sys, "importers") or # old py2exe
964 964 imp.is_frozen(u"__main__")) # tools/freeze
965 965
966 966 # the location of data files matching the source code
967 967 if mainfrozen() and getattr(sys, 'frozen', None) != 'macosx_app':
968 968 # executable version (py2exe) doesn't support __file__
969 969 datapath = os.path.dirname(pycompat.sysexecutable)
970 970 else:
971 971 datapath = os.path.dirname(pycompat.fsencode(__file__))
972 972
973 973 i18n.setdatapath(datapath)
974 974
975 975 _hgexecutable = None
976 976
977 977 def hgexecutable():
978 978 """return location of the 'hg' executable.
979 979
980 980 Defaults to $HG or 'hg' in the search path.
981 981 """
982 982 if _hgexecutable is None:
983 983 hg = encoding.environ.get('HG')
984 984 mainmod = sys.modules[pycompat.sysstr('__main__')]
985 985 if hg:
986 986 _sethgexecutable(hg)
987 987 elif mainfrozen():
988 988 if getattr(sys, 'frozen', None) == 'macosx_app':
989 989 # Env variable set by py2app
990 990 _sethgexecutable(encoding.environ['EXECUTABLEPATH'])
991 991 else:
992 992 _sethgexecutable(pycompat.sysexecutable)
993 993 elif (os.path.basename(
994 994 pycompat.fsencode(getattr(mainmod, '__file__', ''))) == 'hg'):
995 995 _sethgexecutable(pycompat.fsencode(mainmod.__file__))
996 996 else:
997 997 exe = findexe('hg') or os.path.basename(sys.argv[0])
998 998 _sethgexecutable(exe)
999 999 return _hgexecutable
1000 1000
1001 1001 def _sethgexecutable(path):
1002 1002 """set location of the 'hg' executable"""
1003 1003 global _hgexecutable
1004 1004 _hgexecutable = path
1005 1005
1006 1006 def _isstdout(f):
1007 1007 fileno = getattr(f, 'fileno', None)
1008 1008 return fileno and fileno() == sys.__stdout__.fileno()
1009 1009
1010 1010 def shellenviron(environ=None):
1011 1011 """return environ with optional override, useful for shelling out"""
1012 1012 def py2shell(val):
1013 1013 'convert python object into string that is useful to shell'
1014 1014 if val is None or val is False:
1015 1015 return '0'
1016 1016 if val is True:
1017 1017 return '1'
1018 1018 return str(val)
1019 1019 env = dict(encoding.environ)
1020 1020 if environ:
1021 1021 env.update((k, py2shell(v)) for k, v in environ.iteritems())
1022 1022 env['HG'] = hgexecutable()
1023 1023 return env
1024 1024
1025 1025 def system(cmd, environ=None, cwd=None, out=None):
1026 1026 '''enhanced shell command execution.
1027 1027 run with environment maybe modified, maybe in different dir.
1028 1028
1029 1029 if out is specified, it is assumed to be a file-like object that has a
1030 1030 write() method. stdout and stderr will be redirected to out.'''
1031 1031 try:
1032 1032 stdout.flush()
1033 1033 except Exception:
1034 1034 pass
1035 1035 cmd = quotecommand(cmd)
1036 1036 env = shellenviron(environ)
1037 1037 if out is None or _isstdout(out):
1038 1038 rc = subprocess.call(cmd, shell=True, close_fds=closefds,
1039 1039 env=env, cwd=cwd)
1040 1040 else:
1041 1041 proc = subprocess.Popen(cmd, shell=True, close_fds=closefds,
1042 1042 env=env, cwd=cwd, stdout=subprocess.PIPE,
1043 1043 stderr=subprocess.STDOUT)
1044 1044 for line in iter(proc.stdout.readline, ''):
1045 1045 out.write(line)
1046 1046 proc.wait()
1047 1047 rc = proc.returncode
1048 1048 if pycompat.sysplatform == 'OpenVMS' and rc & 1:
1049 1049 rc = 0
1050 1050 return rc
1051 1051
1052 1052 def checksignature(func):
1053 1053 '''wrap a function with code to check for calling errors'''
1054 1054 def check(*args, **kwargs):
1055 1055 try:
1056 1056 return func(*args, **kwargs)
1057 1057 except TypeError:
1058 1058 if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
1059 1059 raise error.SignatureError
1060 1060 raise
1061 1061
1062 1062 return check
1063 1063
1064 1064 # a whilelist of known filesystems where hardlink works reliably
1065 1065 _hardlinkfswhitelist = {
1066 1066 'btrfs',
1067 1067 'ext2',
1068 1068 'ext3',
1069 1069 'ext4',
1070 1070 'hfs',
1071 1071 'jfs',
1072 1072 'reiserfs',
1073 1073 'tmpfs',
1074 1074 'ufs',
1075 1075 'xfs',
1076 1076 'zfs',
1077 1077 }
1078 1078
1079 1079 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
1080 1080 '''copy a file, preserving mode and optionally other stat info like
1081 1081 atime/mtime
1082 1082
1083 1083 checkambig argument is used with filestat, and is useful only if
1084 1084 destination file is guarded by any lock (e.g. repo.lock or
1085 1085 repo.wlock).
1086 1086
1087 1087 copystat and checkambig should be exclusive.
1088 1088 '''
1089 1089 assert not (copystat and checkambig)
1090 1090 oldstat = None
1091 1091 if os.path.lexists(dest):
1092 1092 if checkambig:
1093 1093 oldstat = checkambig and filestat.frompath(dest)
1094 1094 unlink(dest)
1095 1095 if hardlink:
1096 1096 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
1097 1097 # unless we are confident that dest is on a whitelisted filesystem.
1098 1098 try:
1099 1099 fstype = getfstype(os.path.dirname(dest))
1100 1100 except OSError:
1101 1101 fstype = None
1102 1102 if fstype not in _hardlinkfswhitelist:
1103 1103 hardlink = False
1104 1104 if hardlink:
1105 1105 try:
1106 1106 oslink(src, dest)
1107 1107 return
1108 1108 except (IOError, OSError):
1109 1109 pass # fall back to normal copy
1110 1110 if os.path.islink(src):
1111 1111 os.symlink(os.readlink(src), dest)
1112 1112 # copytime is ignored for symlinks, but in general copytime isn't needed
1113 1113 # for them anyway
1114 1114 else:
1115 1115 try:
1116 1116 shutil.copyfile(src, dest)
1117 1117 if copystat:
1118 1118 # copystat also copies mode
1119 1119 shutil.copystat(src, dest)
1120 1120 else:
1121 1121 shutil.copymode(src, dest)
1122 1122 if oldstat and oldstat.stat:
1123 1123 newstat = filestat.frompath(dest)
1124 1124 if newstat.isambig(oldstat):
1125 1125 # stat of copied file is ambiguous to original one
1126 1126 advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
1127 1127 os.utime(dest, (advanced, advanced))
1128 1128 except shutil.Error as inst:
1129 1129 raise Abort(str(inst))
1130 1130
1131 1131 def copyfiles(src, dst, hardlink=None, progress=lambda t, pos: None):
1132 1132 """Copy a directory tree using hardlinks if possible."""
1133 1133 num = 0
1134 1134
1135 1135 gettopic = lambda: hardlink and _('linking') or _('copying')
1136 1136
1137 1137 if os.path.isdir(src):
1138 1138 if hardlink is None:
1139 1139 hardlink = (os.stat(src).st_dev ==
1140 1140 os.stat(os.path.dirname(dst)).st_dev)
1141 1141 topic = gettopic()
1142 1142 os.mkdir(dst)
1143 1143 for name, kind in listdir(src):
1144 1144 srcname = os.path.join(src, name)
1145 1145 dstname = os.path.join(dst, name)
1146 1146 def nprog(t, pos):
1147 1147 if pos is not None:
1148 1148 return progress(t, pos + num)
1149 1149 hardlink, n = copyfiles(srcname, dstname, hardlink, progress=nprog)
1150 1150 num += n
1151 1151 else:
1152 1152 if hardlink is None:
1153 1153 hardlink = (os.stat(os.path.dirname(src)).st_dev ==
1154 1154 os.stat(os.path.dirname(dst)).st_dev)
1155 1155 topic = gettopic()
1156 1156
1157 1157 if hardlink:
1158 1158 try:
1159 1159 oslink(src, dst)
1160 1160 except (IOError, OSError):
1161 1161 hardlink = False
1162 1162 shutil.copy(src, dst)
1163 1163 else:
1164 1164 shutil.copy(src, dst)
1165 1165 num += 1
1166 1166 progress(topic, num)
1167 1167 progress(topic, None)
1168 1168
1169 1169 return hardlink, num
1170 1170
1171 _winreservednames = '''con prn aux nul
1171 _winreservednames = b'''con prn aux nul
1172 1172 com1 com2 com3 com4 com5 com6 com7 com8 com9
1173 1173 lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9'''.split()
1174 1174 _winreservedchars = ':*?"<>|'
1175 1175 def checkwinfilename(path):
1176 1176 r'''Check that the base-relative path is a valid filename on Windows.
1177 1177 Returns None if the path is ok, or a UI string describing the problem.
1178 1178
1179 1179 >>> checkwinfilename("just/a/normal/path")
1180 1180 >>> checkwinfilename("foo/bar/con.xml")
1181 1181 "filename contains 'con', which is reserved on Windows"
1182 1182 >>> checkwinfilename("foo/con.xml/bar")
1183 1183 "filename contains 'con', which is reserved on Windows"
1184 1184 >>> checkwinfilename("foo/bar/xml.con")
1185 1185 >>> checkwinfilename("foo/bar/AUX/bla.txt")
1186 1186 "filename contains 'AUX', which is reserved on Windows"
1187 1187 >>> checkwinfilename("foo/bar/bla:.txt")
1188 1188 "filename contains ':', which is reserved on Windows"
1189 1189 >>> checkwinfilename("foo/bar/b\07la.txt")
1190 1190 "filename contains '\\x07', which is invalid on Windows"
1191 1191 >>> checkwinfilename("foo/bar/bla ")
1192 1192 "filename ends with ' ', which is not allowed on Windows"
1193 1193 >>> checkwinfilename("../bar")
1194 1194 >>> checkwinfilename("foo\\")
1195 1195 "filename ends with '\\', which is invalid on Windows"
1196 1196 >>> checkwinfilename("foo\\/bar")
1197 1197 "directory name ends with '\\', which is invalid on Windows"
1198 1198 '''
1199 1199 if path.endswith('\\'):
1200 1200 return _("filename ends with '\\', which is invalid on Windows")
1201 1201 if '\\/' in path:
1202 1202 return _("directory name ends with '\\', which is invalid on Windows")
1203 1203 for n in path.replace('\\', '/').split('/'):
1204 1204 if not n:
1205 1205 continue
1206 1206 for c in _filenamebytestr(n):
1207 1207 if c in _winreservedchars:
1208 1208 return _("filename contains '%s', which is reserved "
1209 1209 "on Windows") % c
1210 1210 if ord(c) <= 31:
1211 1211 return _("filename contains %r, which is invalid "
1212 1212 "on Windows") % c
1213 1213 base = n.split('.')[0]
1214 1214 if base and base.lower() in _winreservednames:
1215 1215 return _("filename contains '%s', which is reserved "
1216 1216 "on Windows") % base
1217 1217 t = n[-1]
1218 1218 if t in '. ' and n not in '..':
1219 1219 return _("filename ends with '%s', which is not allowed "
1220 1220 "on Windows") % t
1221 1221
1222 1222 if pycompat.osname == 'nt':
1223 1223 checkosfilename = checkwinfilename
1224 1224 timer = time.clock
1225 1225 else:
1226 1226 checkosfilename = platform.checkosfilename
1227 1227 timer = time.time
1228 1228
1229 1229 if safehasattr(time, "perf_counter"):
1230 1230 timer = time.perf_counter
1231 1231
1232 1232 def makelock(info, pathname):
1233 1233 try:
1234 1234 return os.symlink(info, pathname)
1235 1235 except OSError as why:
1236 1236 if why.errno == errno.EEXIST:
1237 1237 raise
1238 1238 except AttributeError: # no symlink in os
1239 1239 pass
1240 1240
1241 1241 ld = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL)
1242 1242 os.write(ld, info)
1243 1243 os.close(ld)
1244 1244
1245 1245 def readlock(pathname):
1246 1246 try:
1247 1247 return os.readlink(pathname)
1248 1248 except OSError as why:
1249 1249 if why.errno not in (errno.EINVAL, errno.ENOSYS):
1250 1250 raise
1251 1251 except AttributeError: # no symlink in os
1252 1252 pass
1253 1253 fp = posixfile(pathname)
1254 1254 r = fp.read()
1255 1255 fp.close()
1256 1256 return r
1257 1257
1258 1258 def fstat(fp):
1259 1259 '''stat file object that may not have fileno method.'''
1260 1260 try:
1261 1261 return os.fstat(fp.fileno())
1262 1262 except AttributeError:
1263 1263 return os.stat(fp.name)
1264 1264
1265 1265 # File system features
1266 1266
1267 1267 def fscasesensitive(path):
1268 1268 """
1269 1269 Return true if the given path is on a case-sensitive filesystem
1270 1270
1271 1271 Requires a path (like /foo/.hg) ending with a foldable final
1272 1272 directory component.
1273 1273 """
1274 1274 s1 = os.lstat(path)
1275 1275 d, b = os.path.split(path)
1276 1276 b2 = b.upper()
1277 1277 if b == b2:
1278 1278 b2 = b.lower()
1279 1279 if b == b2:
1280 1280 return True # no evidence against case sensitivity
1281 1281 p2 = os.path.join(d, b2)
1282 1282 try:
1283 1283 s2 = os.lstat(p2)
1284 1284 if s2 == s1:
1285 1285 return False
1286 1286 return True
1287 1287 except OSError:
1288 1288 return True
1289 1289
1290 1290 try:
1291 1291 import re2
1292 1292 _re2 = None
1293 1293 except ImportError:
1294 1294 _re2 = False
1295 1295
1296 1296 class _re(object):
1297 1297 def _checkre2(self):
1298 1298 global _re2
1299 1299 try:
1300 1300 # check if match works, see issue3964
1301 1301 _re2 = bool(re2.match(r'\[([^\[]+)\]', '[ui]'))
1302 1302 except ImportError:
1303 1303 _re2 = False
1304 1304
1305 1305 def compile(self, pat, flags=0):
1306 1306 '''Compile a regular expression, using re2 if possible
1307 1307
1308 1308 For best performance, use only re2-compatible regexp features. The
1309 1309 only flags from the re module that are re2-compatible are
1310 1310 IGNORECASE and MULTILINE.'''
1311 1311 if _re2 is None:
1312 1312 self._checkre2()
1313 1313 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
1314 1314 if flags & remod.IGNORECASE:
1315 1315 pat = '(?i)' + pat
1316 1316 if flags & remod.MULTILINE:
1317 1317 pat = '(?m)' + pat
1318 1318 try:
1319 1319 return re2.compile(pat)
1320 1320 except re2.error:
1321 1321 pass
1322 1322 return remod.compile(pat, flags)
1323 1323
1324 1324 @propertycache
1325 1325 def escape(self):
1326 1326 '''Return the version of escape corresponding to self.compile.
1327 1327
1328 1328 This is imperfect because whether re2 or re is used for a particular
1329 1329 function depends on the flags, etc, but it's the best we can do.
1330 1330 '''
1331 1331 global _re2
1332 1332 if _re2 is None:
1333 1333 self._checkre2()
1334 1334 if _re2:
1335 1335 return re2.escape
1336 1336 else:
1337 1337 return remod.escape
1338 1338
1339 1339 re = _re()
1340 1340
1341 1341 _fspathcache = {}
1342 1342 def fspath(name, root):
1343 1343 '''Get name in the case stored in the filesystem
1344 1344
1345 1345 The name should be relative to root, and be normcase-ed for efficiency.
1346 1346
1347 1347 Note that this function is unnecessary, and should not be
1348 1348 called, for case-sensitive filesystems (simply because it's expensive).
1349 1349
1350 1350 The root should be normcase-ed, too.
1351 1351 '''
1352 1352 def _makefspathcacheentry(dir):
1353 1353 return dict((normcase(n), n) for n in os.listdir(dir))
1354 1354
1355 1355 seps = pycompat.ossep
1356 1356 if pycompat.osaltsep:
1357 1357 seps = seps + pycompat.osaltsep
1358 1358 # Protect backslashes. This gets silly very quickly.
1359 1359 seps.replace('\\','\\\\')
1360 1360 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
1361 1361 dir = os.path.normpath(root)
1362 1362 result = []
1363 1363 for part, sep in pattern.findall(name):
1364 1364 if sep:
1365 1365 result.append(sep)
1366 1366 continue
1367 1367
1368 1368 if dir not in _fspathcache:
1369 1369 _fspathcache[dir] = _makefspathcacheentry(dir)
1370 1370 contents = _fspathcache[dir]
1371 1371
1372 1372 found = contents.get(part)
1373 1373 if not found:
1374 1374 # retry "once per directory" per "dirstate.walk" which
1375 1375 # may take place for each patches of "hg qpush", for example
1376 1376 _fspathcache[dir] = contents = _makefspathcacheentry(dir)
1377 1377 found = contents.get(part)
1378 1378
1379 1379 result.append(found or part)
1380 1380 dir = os.path.join(dir, part)
1381 1381
1382 1382 return ''.join(result)
1383 1383
1384 1384 def getfstype(dirpath):
1385 1385 '''Get the filesystem type name from a directory (best-effort)
1386 1386
1387 1387 Returns None if we are unsure. Raises OSError on ENOENT, EPERM, etc.
1388 1388 '''
1389 1389 return getattr(osutil, 'getfstype', lambda x: None)(dirpath)
1390 1390
1391 1391 def checknlink(testfile):
1392 1392 '''check whether hardlink count reporting works properly'''
1393 1393
1394 1394 # testfile may be open, so we need a separate file for checking to
1395 1395 # work around issue2543 (or testfile may get lost on Samba shares)
1396 1396 f1 = testfile + ".hgtmp1"
1397 1397 if os.path.lexists(f1):
1398 1398 return False
1399 1399 try:
1400 1400 posixfile(f1, 'w').close()
1401 1401 except IOError:
1402 1402 try:
1403 1403 os.unlink(f1)
1404 1404 except OSError:
1405 1405 pass
1406 1406 return False
1407 1407
1408 1408 f2 = testfile + ".hgtmp2"
1409 1409 fd = None
1410 1410 try:
1411 1411 oslink(f1, f2)
1412 1412 # nlinks() may behave differently for files on Windows shares if
1413 1413 # the file is open.
1414 1414 fd = posixfile(f2)
1415 1415 return nlinks(f2) > 1
1416 1416 except OSError:
1417 1417 return False
1418 1418 finally:
1419 1419 if fd is not None:
1420 1420 fd.close()
1421 1421 for f in (f1, f2):
1422 1422 try:
1423 1423 os.unlink(f)
1424 1424 except OSError:
1425 1425 pass
1426 1426
1427 1427 def endswithsep(path):
1428 1428 '''Check path ends with os.sep or os.altsep.'''
1429 1429 return (path.endswith(pycompat.ossep)
1430 1430 or pycompat.osaltsep and path.endswith(pycompat.osaltsep))
1431 1431
1432 1432 def splitpath(path):
1433 1433 '''Split path by os.sep.
1434 1434 Note that this function does not use os.altsep because this is
1435 1435 an alternative of simple "xxx.split(os.sep)".
1436 1436 It is recommended to use os.path.normpath() before using this
1437 1437 function if need.'''
1438 1438 return path.split(pycompat.ossep)
1439 1439
1440 1440 def gui():
1441 1441 '''Are we running in a GUI?'''
1442 1442 if pycompat.sysplatform == 'darwin':
1443 1443 if 'SSH_CONNECTION' in encoding.environ:
1444 1444 # handle SSH access to a box where the user is logged in
1445 1445 return False
1446 1446 elif getattr(osutil, 'isgui', None):
1447 1447 # check if a CoreGraphics session is available
1448 1448 return osutil.isgui()
1449 1449 else:
1450 1450 # pure build; use a safe default
1451 1451 return True
1452 1452 else:
1453 1453 return pycompat.osname == "nt" or encoding.environ.get("DISPLAY")
1454 1454
1455 1455 def mktempcopy(name, emptyok=False, createmode=None):
1456 1456 """Create a temporary file with the same contents from name
1457 1457
1458 1458 The permission bits are copied from the original file.
1459 1459
1460 1460 If the temporary file is going to be truncated immediately, you
1461 1461 can use emptyok=True as an optimization.
1462 1462
1463 1463 Returns the name of the temporary file.
1464 1464 """
1465 1465 d, fn = os.path.split(name)
1466 1466 fd, temp = tempfile.mkstemp(prefix='.%s-' % fn, dir=d)
1467 1467 os.close(fd)
1468 1468 # Temporary files are created with mode 0600, which is usually not
1469 1469 # what we want. If the original file already exists, just copy
1470 1470 # its mode. Otherwise, manually obey umask.
1471 1471 copymode(name, temp, createmode)
1472 1472 if emptyok:
1473 1473 return temp
1474 1474 try:
1475 1475 try:
1476 1476 ifp = posixfile(name, "rb")
1477 1477 except IOError as inst:
1478 1478 if inst.errno == errno.ENOENT:
1479 1479 return temp
1480 1480 if not getattr(inst, 'filename', None):
1481 1481 inst.filename = name
1482 1482 raise
1483 1483 ofp = posixfile(temp, "wb")
1484 1484 for chunk in filechunkiter(ifp):
1485 1485 ofp.write(chunk)
1486 1486 ifp.close()
1487 1487 ofp.close()
1488 1488 except: # re-raises
1489 1489 try: os.unlink(temp)
1490 1490 except OSError: pass
1491 1491 raise
1492 1492 return temp
1493 1493
1494 1494 class filestat(object):
1495 1495 """help to exactly detect change of a file
1496 1496
1497 1497 'stat' attribute is result of 'os.stat()' if specified 'path'
1498 1498 exists. Otherwise, it is None. This can avoid preparative
1499 1499 'exists()' examination on client side of this class.
1500 1500 """
1501 1501 def __init__(self, stat):
1502 1502 self.stat = stat
1503 1503
1504 1504 @classmethod
1505 1505 def frompath(cls, path):
1506 1506 try:
1507 1507 stat = os.stat(path)
1508 1508 except OSError as err:
1509 1509 if err.errno != errno.ENOENT:
1510 1510 raise
1511 1511 stat = None
1512 1512 return cls(stat)
1513 1513
1514 1514 @classmethod
1515 1515 def fromfp(cls, fp):
1516 1516 stat = os.fstat(fp.fileno())
1517 1517 return cls(stat)
1518 1518
1519 1519 __hash__ = object.__hash__
1520 1520
1521 1521 def __eq__(self, old):
1522 1522 try:
1523 1523 # if ambiguity between stat of new and old file is
1524 1524 # avoided, comparison of size, ctime and mtime is enough
1525 1525 # to exactly detect change of a file regardless of platform
1526 1526 return (self.stat.st_size == old.stat.st_size and
1527 1527 self.stat.st_ctime == old.stat.st_ctime and
1528 1528 self.stat.st_mtime == old.stat.st_mtime)
1529 1529 except AttributeError:
1530 1530 pass
1531 1531 try:
1532 1532 return self.stat is None and old.stat is None
1533 1533 except AttributeError:
1534 1534 return False
1535 1535
1536 1536 def isambig(self, old):
1537 1537 """Examine whether new (= self) stat is ambiguous against old one
1538 1538
1539 1539 "S[N]" below means stat of a file at N-th change:
1540 1540
1541 1541 - S[n-1].ctime < S[n].ctime: can detect change of a file
1542 1542 - S[n-1].ctime == S[n].ctime
1543 1543 - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
1544 1544 - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
1545 1545 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
1546 1546 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
1547 1547
1548 1548 Case (*2) above means that a file was changed twice or more at
1549 1549 same time in sec (= S[n-1].ctime), and comparison of timestamp
1550 1550 is ambiguous.
1551 1551
1552 1552 Base idea to avoid such ambiguity is "advance mtime 1 sec, if
1553 1553 timestamp is ambiguous".
1554 1554
1555 1555 But advancing mtime only in case (*2) doesn't work as
1556 1556 expected, because naturally advanced S[n].mtime in case (*1)
1557 1557 might be equal to manually advanced S[n-1 or earlier].mtime.
1558 1558
1559 1559 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
1560 1560 treated as ambiguous regardless of mtime, to avoid overlooking
1561 1561 by confliction between such mtime.
1562 1562
1563 1563 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
1564 1564 S[n].mtime", even if size of a file isn't changed.
1565 1565 """
1566 1566 try:
1567 1567 return (self.stat.st_ctime == old.stat.st_ctime)
1568 1568 except AttributeError:
1569 1569 return False
1570 1570
1571 1571 def avoidambig(self, path, old):
1572 1572 """Change file stat of specified path to avoid ambiguity
1573 1573
1574 1574 'old' should be previous filestat of 'path'.
1575 1575
1576 1576 This skips avoiding ambiguity, if a process doesn't have
1577 1577 appropriate privileges for 'path'. This returns False in this
1578 1578 case.
1579 1579
1580 1580 Otherwise, this returns True, as "ambiguity is avoided".
1581 1581 """
1582 1582 advanced = (old.stat.st_mtime + 1) & 0x7fffffff
1583 1583 try:
1584 1584 os.utime(path, (advanced, advanced))
1585 1585 except OSError as inst:
1586 1586 if inst.errno == errno.EPERM:
1587 1587 # utime() on the file created by another user causes EPERM,
1588 1588 # if a process doesn't have appropriate privileges
1589 1589 return False
1590 1590 raise
1591 1591 return True
1592 1592
1593 1593 def __ne__(self, other):
1594 1594 return not self == other
1595 1595
1596 1596 class atomictempfile(object):
1597 1597 '''writable file object that atomically updates a file
1598 1598
1599 1599 All writes will go to a temporary copy of the original file. Call
1600 1600 close() when you are done writing, and atomictempfile will rename
1601 1601 the temporary copy to the original name, making the changes
1602 1602 visible. If the object is destroyed without being closed, all your
1603 1603 writes are discarded.
1604 1604
1605 1605 checkambig argument of constructor is used with filestat, and is
1606 1606 useful only if target file is guarded by any lock (e.g. repo.lock
1607 1607 or repo.wlock).
1608 1608 '''
1609 1609 def __init__(self, name, mode='w+b', createmode=None, checkambig=False):
1610 1610 self.__name = name # permanent name
1611 1611 self._tempname = mktempcopy(name, emptyok=('w' in mode),
1612 1612 createmode=createmode)
1613 1613 self._fp = posixfile(self._tempname, mode)
1614 1614 self._checkambig = checkambig
1615 1615
1616 1616 # delegated methods
1617 1617 self.read = self._fp.read
1618 1618 self.write = self._fp.write
1619 1619 self.seek = self._fp.seek
1620 1620 self.tell = self._fp.tell
1621 1621 self.fileno = self._fp.fileno
1622 1622
1623 1623 def close(self):
1624 1624 if not self._fp.closed:
1625 1625 self._fp.close()
1626 1626 filename = localpath(self.__name)
1627 1627 oldstat = self._checkambig and filestat.frompath(filename)
1628 1628 if oldstat and oldstat.stat:
1629 1629 rename(self._tempname, filename)
1630 1630 newstat = filestat.frompath(filename)
1631 1631 if newstat.isambig(oldstat):
1632 1632 # stat of changed file is ambiguous to original one
1633 1633 advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
1634 1634 os.utime(filename, (advanced, advanced))
1635 1635 else:
1636 1636 rename(self._tempname, filename)
1637 1637
1638 1638 def discard(self):
1639 1639 if not self._fp.closed:
1640 1640 try:
1641 1641 os.unlink(self._tempname)
1642 1642 except OSError:
1643 1643 pass
1644 1644 self._fp.close()
1645 1645
1646 1646 def __del__(self):
1647 1647 if safehasattr(self, '_fp'): # constructor actually did something
1648 1648 self.discard()
1649 1649
1650 1650 def __enter__(self):
1651 1651 return self
1652 1652
1653 1653 def __exit__(self, exctype, excvalue, traceback):
1654 1654 if exctype is not None:
1655 1655 self.discard()
1656 1656 else:
1657 1657 self.close()
1658 1658
1659 1659 def unlinkpath(f, ignoremissing=False):
1660 1660 """unlink and remove the directory if it is empty"""
1661 1661 if ignoremissing:
1662 1662 tryunlink(f)
1663 1663 else:
1664 1664 unlink(f)
1665 1665 # try removing directories that might now be empty
1666 1666 try:
1667 1667 removedirs(os.path.dirname(f))
1668 1668 except OSError:
1669 1669 pass
1670 1670
1671 1671 def tryunlink(f):
1672 1672 """Attempt to remove a file, ignoring ENOENT errors."""
1673 1673 try:
1674 1674 unlink(f)
1675 1675 except OSError as e:
1676 1676 if e.errno != errno.ENOENT:
1677 1677 raise
1678 1678
1679 1679 def makedirs(name, mode=None, notindexed=False):
1680 1680 """recursive directory creation with parent mode inheritance
1681 1681
1682 1682 Newly created directories are marked as "not to be indexed by
1683 1683 the content indexing service", if ``notindexed`` is specified
1684 1684 for "write" mode access.
1685 1685 """
1686 1686 try:
1687 1687 makedir(name, notindexed)
1688 1688 except OSError as err:
1689 1689 if err.errno == errno.EEXIST:
1690 1690 return
1691 1691 if err.errno != errno.ENOENT or not name:
1692 1692 raise
1693 1693 parent = os.path.dirname(os.path.abspath(name))
1694 1694 if parent == name:
1695 1695 raise
1696 1696 makedirs(parent, mode, notindexed)
1697 1697 try:
1698 1698 makedir(name, notindexed)
1699 1699 except OSError as err:
1700 1700 # Catch EEXIST to handle races
1701 1701 if err.errno == errno.EEXIST:
1702 1702 return
1703 1703 raise
1704 1704 if mode is not None:
1705 1705 os.chmod(name, mode)
1706 1706
1707 1707 def readfile(path):
1708 1708 with open(path, 'rb') as fp:
1709 1709 return fp.read()
1710 1710
1711 1711 def writefile(path, text):
1712 1712 with open(path, 'wb') as fp:
1713 1713 fp.write(text)
1714 1714
1715 1715 def appendfile(path, text):
1716 1716 with open(path, 'ab') as fp:
1717 1717 fp.write(text)
1718 1718
1719 1719 class chunkbuffer(object):
1720 1720 """Allow arbitrary sized chunks of data to be efficiently read from an
1721 1721 iterator over chunks of arbitrary size."""
1722 1722
1723 1723 def __init__(self, in_iter):
1724 1724 """in_iter is the iterator that's iterating over the input chunks."""
1725 1725 def splitbig(chunks):
1726 1726 for chunk in chunks:
1727 1727 if len(chunk) > 2**20:
1728 1728 pos = 0
1729 1729 while pos < len(chunk):
1730 1730 end = pos + 2 ** 18
1731 1731 yield chunk[pos:end]
1732 1732 pos = end
1733 1733 else:
1734 1734 yield chunk
1735 1735 self.iter = splitbig(in_iter)
1736 1736 self._queue = collections.deque()
1737 1737 self._chunkoffset = 0
1738 1738
1739 1739 def read(self, l=None):
1740 1740 """Read L bytes of data from the iterator of chunks of data.
1741 1741 Returns less than L bytes if the iterator runs dry.
1742 1742
1743 1743 If size parameter is omitted, read everything"""
1744 1744 if l is None:
1745 1745 return ''.join(self.iter)
1746 1746
1747 1747 left = l
1748 1748 buf = []
1749 1749 queue = self._queue
1750 1750 while left > 0:
1751 1751 # refill the queue
1752 1752 if not queue:
1753 1753 target = 2**18
1754 1754 for chunk in self.iter:
1755 1755 queue.append(chunk)
1756 1756 target -= len(chunk)
1757 1757 if target <= 0:
1758 1758 break
1759 1759 if not queue:
1760 1760 break
1761 1761
1762 1762 # The easy way to do this would be to queue.popleft(), modify the
1763 1763 # chunk (if necessary), then queue.appendleft(). However, for cases
1764 1764 # where we read partial chunk content, this incurs 2 dequeue
1765 1765 # mutations and creates a new str for the remaining chunk in the
1766 1766 # queue. Our code below avoids this overhead.
1767 1767
1768 1768 chunk = queue[0]
1769 1769 chunkl = len(chunk)
1770 1770 offset = self._chunkoffset
1771 1771
1772 1772 # Use full chunk.
1773 1773 if offset == 0 and left >= chunkl:
1774 1774 left -= chunkl
1775 1775 queue.popleft()
1776 1776 buf.append(chunk)
1777 1777 # self._chunkoffset remains at 0.
1778 1778 continue
1779 1779
1780 1780 chunkremaining = chunkl - offset
1781 1781
1782 1782 # Use all of unconsumed part of chunk.
1783 1783 if left >= chunkremaining:
1784 1784 left -= chunkremaining
1785 1785 queue.popleft()
1786 1786 # offset == 0 is enabled by block above, so this won't merely
1787 1787 # copy via ``chunk[0:]``.
1788 1788 buf.append(chunk[offset:])
1789 1789 self._chunkoffset = 0
1790 1790
1791 1791 # Partial chunk needed.
1792 1792 else:
1793 1793 buf.append(chunk[offset:offset + left])
1794 1794 self._chunkoffset += left
1795 1795 left -= chunkremaining
1796 1796
1797 1797 return ''.join(buf)
1798 1798
1799 1799 def filechunkiter(f, size=131072, limit=None):
1800 1800 """Create a generator that produces the data in the file size
1801 1801 (default 131072) bytes at a time, up to optional limit (default is
1802 1802 to read all data). Chunks may be less than size bytes if the
1803 1803 chunk is the last chunk in the file, or the file is a socket or
1804 1804 some other type of file that sometimes reads less data than is
1805 1805 requested."""
1806 1806 assert size >= 0
1807 1807 assert limit is None or limit >= 0
1808 1808 while True:
1809 1809 if limit is None:
1810 1810 nbytes = size
1811 1811 else:
1812 1812 nbytes = min(limit, size)
1813 1813 s = nbytes and f.read(nbytes)
1814 1814 if not s:
1815 1815 break
1816 1816 if limit:
1817 1817 limit -= len(s)
1818 1818 yield s
1819 1819
1820 1820 def makedate(timestamp=None):
1821 1821 '''Return a unix timestamp (or the current time) as a (unixtime,
1822 1822 offset) tuple based off the local timezone.'''
1823 1823 if timestamp is None:
1824 1824 timestamp = time.time()
1825 1825 if timestamp < 0:
1826 1826 hint = _("check your clock")
1827 1827 raise Abort(_("negative timestamp: %d") % timestamp, hint=hint)
1828 1828 delta = (datetime.datetime.utcfromtimestamp(timestamp) -
1829 1829 datetime.datetime.fromtimestamp(timestamp))
1830 1830 tz = delta.days * 86400 + delta.seconds
1831 1831 return timestamp, tz
1832 1832
1833 1833 def datestr(date=None, format='%a %b %d %H:%M:%S %Y %1%2'):
1834 1834 """represent a (unixtime, offset) tuple as a localized time.
1835 1835 unixtime is seconds since the epoch, and offset is the time zone's
1836 1836 number of seconds away from UTC.
1837 1837
1838 1838 >>> datestr((0, 0))
1839 1839 'Thu Jan 01 00:00:00 1970 +0000'
1840 1840 >>> datestr((42, 0))
1841 1841 'Thu Jan 01 00:00:42 1970 +0000'
1842 1842 >>> datestr((-42, 0))
1843 1843 'Wed Dec 31 23:59:18 1969 +0000'
1844 1844 >>> datestr((0x7fffffff, 0))
1845 1845 'Tue Jan 19 03:14:07 2038 +0000'
1846 1846 >>> datestr((-0x80000000, 0))
1847 1847 'Fri Dec 13 20:45:52 1901 +0000'
1848 1848 """
1849 1849 t, tz = date or makedate()
1850 1850 if "%1" in format or "%2" in format or "%z" in format:
1851 1851 sign = (tz > 0) and "-" or "+"
1852 1852 minutes = abs(tz) // 60
1853 1853 q, r = divmod(minutes, 60)
1854 1854 format = format.replace("%z", "%1%2")
1855 1855 format = format.replace("%1", "%c%02d" % (sign, q))
1856 1856 format = format.replace("%2", "%02d" % r)
1857 1857 d = t - tz
1858 1858 if d > 0x7fffffff:
1859 1859 d = 0x7fffffff
1860 1860 elif d < -0x80000000:
1861 1861 d = -0x80000000
1862 1862 # Never use time.gmtime() and datetime.datetime.fromtimestamp()
1863 1863 # because they use the gmtime() system call which is buggy on Windows
1864 1864 # for negative values.
1865 1865 t = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=d)
1866 1866 s = encoding.strtolocal(t.strftime(encoding.strfromlocal(format)))
1867 1867 return s
1868 1868
1869 1869 def shortdate(date=None):
1870 1870 """turn (timestamp, tzoff) tuple into iso 8631 date."""
1871 1871 return datestr(date, format='%Y-%m-%d')
1872 1872
1873 1873 def parsetimezone(s):
1874 1874 """find a trailing timezone, if any, in string, and return a
1875 1875 (offset, remainder) pair"""
1876 1876
1877 1877 if s.endswith("GMT") or s.endswith("UTC"):
1878 1878 return 0, s[:-3].rstrip()
1879 1879
1880 1880 # Unix-style timezones [+-]hhmm
1881 1881 if len(s) >= 5 and s[-5] in "+-" and s[-4:].isdigit():
1882 1882 sign = (s[-5] == "+") and 1 or -1
1883 1883 hours = int(s[-4:-2])
1884 1884 minutes = int(s[-2:])
1885 1885 return -sign * (hours * 60 + minutes) * 60, s[:-5].rstrip()
1886 1886
1887 1887 # ISO8601 trailing Z
1888 1888 if s.endswith("Z") and s[-2:-1].isdigit():
1889 1889 return 0, s[:-1]
1890 1890
1891 1891 # ISO8601-style [+-]hh:mm
1892 1892 if (len(s) >= 6 and s[-6] in "+-" and s[-3] == ":" and
1893 1893 s[-5:-3].isdigit() and s[-2:].isdigit()):
1894 1894 sign = (s[-6] == "+") and 1 or -1
1895 1895 hours = int(s[-5:-3])
1896 1896 minutes = int(s[-2:])
1897 1897 return -sign * (hours * 60 + minutes) * 60, s[:-6]
1898 1898
1899 1899 return None, s
1900 1900
1901 1901 def strdate(string, format, defaults=None):
1902 1902 """parse a localized time string and return a (unixtime, offset) tuple.
1903 1903 if the string cannot be parsed, ValueError is raised."""
1904 1904 if defaults is None:
1905 1905 defaults = {}
1906 1906
1907 1907 # NOTE: unixtime = localunixtime + offset
1908 1908 offset, date = parsetimezone(string)
1909 1909
1910 1910 # add missing elements from defaults
1911 1911 usenow = False # default to using biased defaults
1912 1912 for part in ("S", "M", "HI", "d", "mb", "yY"): # decreasing specificity
1913 1913 part = pycompat.bytestr(part)
1914 1914 found = [True for p in part if ("%"+p) in format]
1915 1915 if not found:
1916 1916 date += "@" + defaults[part][usenow]
1917 1917 format += "@%" + part[0]
1918 1918 else:
1919 1919 # We've found a specific time element, less specific time
1920 1920 # elements are relative to today
1921 1921 usenow = True
1922 1922
1923 1923 timetuple = time.strptime(encoding.strfromlocal(date),
1924 1924 encoding.strfromlocal(format))
1925 1925 localunixtime = int(calendar.timegm(timetuple))
1926 1926 if offset is None:
1927 1927 # local timezone
1928 1928 unixtime = int(time.mktime(timetuple))
1929 1929 offset = unixtime - localunixtime
1930 1930 else:
1931 1931 unixtime = localunixtime + offset
1932 1932 return unixtime, offset
1933 1933
1934 1934 def parsedate(date, formats=None, bias=None):
1935 1935 """parse a localized date/time and return a (unixtime, offset) tuple.
1936 1936
1937 1937 The date may be a "unixtime offset" string or in one of the specified
1938 1938 formats. If the date already is a (unixtime, offset) tuple, it is returned.
1939 1939
1940 1940 >>> parsedate(' today ') == parsedate(\
1941 1941 datetime.date.today().strftime('%b %d'))
1942 1942 True
1943 1943 >>> parsedate( 'yesterday ') == parsedate((datetime.date.today() -\
1944 1944 datetime.timedelta(days=1)\
1945 1945 ).strftime('%b %d'))
1946 1946 True
1947 1947 >>> now, tz = makedate()
1948 1948 >>> strnow, strtz = parsedate('now')
1949 1949 >>> (strnow - now) < 1
1950 1950 True
1951 1951 >>> tz == strtz
1952 1952 True
1953 1953 """
1954 1954 if bias is None:
1955 1955 bias = {}
1956 1956 if not date:
1957 1957 return 0, 0
1958 1958 if isinstance(date, tuple) and len(date) == 2:
1959 1959 return date
1960 1960 if not formats:
1961 1961 formats = defaultdateformats
1962 1962 date = date.strip()
1963 1963
1964 1964 if date == 'now' or date == _('now'):
1965 1965 return makedate()
1966 1966 if date == 'today' or date == _('today'):
1967 1967 date = datetime.date.today().strftime('%b %d')
1968 1968 elif date == 'yesterday' or date == _('yesterday'):
1969 1969 date = (datetime.date.today() -
1970 1970 datetime.timedelta(days=1)).strftime('%b %d')
1971 1971
1972 1972 try:
1973 1973 when, offset = map(int, date.split(' '))
1974 1974 except ValueError:
1975 1975 # fill out defaults
1976 1976 now = makedate()
1977 1977 defaults = {}
1978 1978 for part in ("d", "mb", "yY", "HI", "M", "S"):
1979 1979 # this piece is for rounding the specific end of unknowns
1980 1980 b = bias.get(part)
1981 1981 if b is None:
1982 1982 if part[0:1] in "HMS":
1983 1983 b = "00"
1984 1984 else:
1985 1985 b = "0"
1986 1986
1987 1987 # this piece is for matching the generic end to today's date
1988 1988 n = datestr(now, "%" + part[0:1])
1989 1989
1990 1990 defaults[part] = (b, n)
1991 1991
1992 1992 for format in formats:
1993 1993 try:
1994 1994 when, offset = strdate(date, format, defaults)
1995 1995 except (ValueError, OverflowError):
1996 1996 pass
1997 1997 else:
1998 1998 break
1999 1999 else:
2000 2000 raise error.ParseError(_('invalid date: %r') % date)
2001 2001 # validate explicit (probably user-specified) date and
2002 2002 # time zone offset. values must fit in signed 32 bits for
2003 2003 # current 32-bit linux runtimes. timezones go from UTC-12
2004 2004 # to UTC+14
2005 2005 if when < -0x80000000 or when > 0x7fffffff:
2006 2006 raise error.ParseError(_('date exceeds 32 bits: %d') % when)
2007 2007 if offset < -50400 or offset > 43200:
2008 2008 raise error.ParseError(_('impossible time zone offset: %d') % offset)
2009 2009 return when, offset
2010 2010
2011 2011 def matchdate(date):
2012 2012 """Return a function that matches a given date match specifier
2013 2013
2014 2014 Formats include:
2015 2015
2016 2016 '{date}' match a given date to the accuracy provided
2017 2017
2018 2018 '<{date}' on or before a given date
2019 2019
2020 2020 '>{date}' on or after a given date
2021 2021
2022 2022 >>> p1 = parsedate("10:29:59")
2023 2023 >>> p2 = parsedate("10:30:00")
2024 2024 >>> p3 = parsedate("10:30:59")
2025 2025 >>> p4 = parsedate("10:31:00")
2026 2026 >>> p5 = parsedate("Sep 15 10:30:00 1999")
2027 2027 >>> f = matchdate("10:30")
2028 2028 >>> f(p1[0])
2029 2029 False
2030 2030 >>> f(p2[0])
2031 2031 True
2032 2032 >>> f(p3[0])
2033 2033 True
2034 2034 >>> f(p4[0])
2035 2035 False
2036 2036 >>> f(p5[0])
2037 2037 False
2038 2038 """
2039 2039
2040 2040 def lower(date):
2041 2041 d = {'mb': "1", 'd': "1"}
2042 2042 return parsedate(date, extendeddateformats, d)[0]
2043 2043
2044 2044 def upper(date):
2045 2045 d = {'mb': "12", 'HI': "23", 'M': "59", 'S': "59"}
2046 2046 for days in ("31", "30", "29"):
2047 2047 try:
2048 2048 d["d"] = days
2049 2049 return parsedate(date, extendeddateformats, d)[0]
2050 2050 except Abort:
2051 2051 pass
2052 2052 d["d"] = "28"
2053 2053 return parsedate(date, extendeddateformats, d)[0]
2054 2054
2055 2055 date = date.strip()
2056 2056
2057 2057 if not date:
2058 2058 raise Abort(_("dates cannot consist entirely of whitespace"))
2059 2059 elif date[0] == "<":
2060 2060 if not date[1:]:
2061 2061 raise Abort(_("invalid day spec, use '<DATE'"))
2062 2062 when = upper(date[1:])
2063 2063 return lambda x: x <= when
2064 2064 elif date[0] == ">":
2065 2065 if not date[1:]:
2066 2066 raise Abort(_("invalid day spec, use '>DATE'"))
2067 2067 when = lower(date[1:])
2068 2068 return lambda x: x >= when
2069 2069 elif date[0] == "-":
2070 2070 try:
2071 2071 days = int(date[1:])
2072 2072 except ValueError:
2073 2073 raise Abort(_("invalid day spec: %s") % date[1:])
2074 2074 if days < 0:
2075 2075 raise Abort(_("%s must be nonnegative (see 'hg help dates')")
2076 2076 % date[1:])
2077 2077 when = makedate()[0] - days * 3600 * 24
2078 2078 return lambda x: x >= when
2079 2079 elif " to " in date:
2080 2080 a, b = date.split(" to ")
2081 2081 start, stop = lower(a), upper(b)
2082 2082 return lambda x: x >= start and x <= stop
2083 2083 else:
2084 2084 start, stop = lower(date), upper(date)
2085 2085 return lambda x: x >= start and x <= stop
2086 2086
2087 2087 def stringmatcher(pattern, casesensitive=True):
2088 2088 """
2089 2089 accepts a string, possibly starting with 're:' or 'literal:' prefix.
2090 2090 returns the matcher name, pattern, and matcher function.
2091 2091 missing or unknown prefixes are treated as literal matches.
2092 2092
2093 2093 helper for tests:
2094 2094 >>> def test(pattern, *tests):
2095 2095 ... kind, pattern, matcher = stringmatcher(pattern)
2096 2096 ... return (kind, pattern, [bool(matcher(t)) for t in tests])
2097 2097 >>> def itest(pattern, *tests):
2098 2098 ... kind, pattern, matcher = stringmatcher(pattern, casesensitive=False)
2099 2099 ... return (kind, pattern, [bool(matcher(t)) for t in tests])
2100 2100
2101 2101 exact matching (no prefix):
2102 2102 >>> test('abcdefg', 'abc', 'def', 'abcdefg')
2103 2103 ('literal', 'abcdefg', [False, False, True])
2104 2104
2105 2105 regex matching ('re:' prefix)
2106 2106 >>> test('re:a.+b', 'nomatch', 'fooadef', 'fooadefbar')
2107 2107 ('re', 'a.+b', [False, False, True])
2108 2108
2109 2109 force exact matches ('literal:' prefix)
2110 2110 >>> test('literal:re:foobar', 'foobar', 're:foobar')
2111 2111 ('literal', 're:foobar', [False, True])
2112 2112
2113 2113 unknown prefixes are ignored and treated as literals
2114 2114 >>> test('foo:bar', 'foo', 'bar', 'foo:bar')
2115 2115 ('literal', 'foo:bar', [False, False, True])
2116 2116
2117 2117 case insensitive regex matches
2118 2118 >>> itest('re:A.+b', 'nomatch', 'fooadef', 'fooadefBar')
2119 2119 ('re', 'A.+b', [False, False, True])
2120 2120
2121 2121 case insensitive literal matches
2122 2122 >>> itest('ABCDEFG', 'abc', 'def', 'abcdefg')
2123 2123 ('literal', 'ABCDEFG', [False, False, True])
2124 2124 """
2125 2125 if pattern.startswith('re:'):
2126 2126 pattern = pattern[3:]
2127 2127 try:
2128 2128 flags = 0
2129 2129 if not casesensitive:
2130 2130 flags = remod.I
2131 2131 regex = remod.compile(pattern, flags)
2132 2132 except remod.error as e:
2133 2133 raise error.ParseError(_('invalid regular expression: %s')
2134 2134 % e)
2135 2135 return 're', pattern, regex.search
2136 2136 elif pattern.startswith('literal:'):
2137 2137 pattern = pattern[8:]
2138 2138
2139 2139 match = pattern.__eq__
2140 2140
2141 2141 if not casesensitive:
2142 2142 ipat = encoding.lower(pattern)
2143 2143 match = lambda s: ipat == encoding.lower(s)
2144 2144 return 'literal', pattern, match
2145 2145
2146 2146 def shortuser(user):
2147 2147 """Return a short representation of a user name or email address."""
2148 2148 f = user.find('@')
2149 2149 if f >= 0:
2150 2150 user = user[:f]
2151 2151 f = user.find('<')
2152 2152 if f >= 0:
2153 2153 user = user[f + 1:]
2154 2154 f = user.find(' ')
2155 2155 if f >= 0:
2156 2156 user = user[:f]
2157 2157 f = user.find('.')
2158 2158 if f >= 0:
2159 2159 user = user[:f]
2160 2160 return user
2161 2161
2162 2162 def emailuser(user):
2163 2163 """Return the user portion of an email address."""
2164 2164 f = user.find('@')
2165 2165 if f >= 0:
2166 2166 user = user[:f]
2167 2167 f = user.find('<')
2168 2168 if f >= 0:
2169 2169 user = user[f + 1:]
2170 2170 return user
2171 2171
2172 2172 def email(author):
2173 2173 '''get email of author.'''
2174 2174 r = author.find('>')
2175 2175 if r == -1:
2176 2176 r = None
2177 2177 return author[author.find('<') + 1:r]
2178 2178
2179 2179 def ellipsis(text, maxlength=400):
2180 2180 """Trim string to at most maxlength (default: 400) columns in display."""
2181 2181 return encoding.trim(text, maxlength, ellipsis='...')
2182 2182
2183 2183 def unitcountfn(*unittable):
2184 2184 '''return a function that renders a readable count of some quantity'''
2185 2185
2186 2186 def go(count):
2187 2187 for multiplier, divisor, format in unittable:
2188 2188 if abs(count) >= divisor * multiplier:
2189 2189 return format % (count / float(divisor))
2190 2190 return unittable[-1][2] % count
2191 2191
2192 2192 return go
2193 2193
2194 2194 def processlinerange(fromline, toline):
2195 2195 """Check that linerange <fromline>:<toline> makes sense and return a
2196 2196 0-based range.
2197 2197
2198 2198 >>> processlinerange(10, 20)
2199 2199 (9, 20)
2200 2200 >>> processlinerange(2, 1)
2201 2201 Traceback (most recent call last):
2202 2202 ...
2203 2203 ParseError: line range must be positive
2204 2204 >>> processlinerange(0, 5)
2205 2205 Traceback (most recent call last):
2206 2206 ...
2207 2207 ParseError: fromline must be strictly positive
2208 2208 """
2209 2209 if toline - fromline < 0:
2210 2210 raise error.ParseError(_("line range must be positive"))
2211 2211 if fromline < 1:
2212 2212 raise error.ParseError(_("fromline must be strictly positive"))
2213 2213 return fromline - 1, toline
2214 2214
2215 2215 bytecount = unitcountfn(
2216 2216 (100, 1 << 30, _('%.0f GB')),
2217 2217 (10, 1 << 30, _('%.1f GB')),
2218 2218 (1, 1 << 30, _('%.2f GB')),
2219 2219 (100, 1 << 20, _('%.0f MB')),
2220 2220 (10, 1 << 20, _('%.1f MB')),
2221 2221 (1, 1 << 20, _('%.2f MB')),
2222 2222 (100, 1 << 10, _('%.0f KB')),
2223 2223 (10, 1 << 10, _('%.1f KB')),
2224 2224 (1, 1 << 10, _('%.2f KB')),
2225 2225 (1, 1, _('%.0f bytes')),
2226 2226 )
2227 2227
2228 2228 # Matches a single EOL which can either be a CRLF where repeated CR
2229 2229 # are removed or a LF. We do not care about old Macintosh files, so a
2230 2230 # stray CR is an error.
2231 2231 _eolre = remod.compile(br'\r*\n')
2232 2232
2233 2233 def tolf(s):
2234 2234 return _eolre.sub('\n', s)
2235 2235
2236 2236 def tocrlf(s):
2237 2237 return _eolre.sub('\r\n', s)
2238 2238
2239 2239 if pycompat.oslinesep == '\r\n':
2240 2240 tonativeeol = tocrlf
2241 2241 fromnativeeol = tolf
2242 2242 else:
2243 2243 tonativeeol = pycompat.identity
2244 2244 fromnativeeol = pycompat.identity
2245 2245
2246 2246 def escapestr(s):
2247 2247 # call underlying function of s.encode('string_escape') directly for
2248 2248 # Python 3 compatibility
2249 2249 return codecs.escape_encode(s)[0]
2250 2250
2251 2251 def unescapestr(s):
2252 2252 return codecs.escape_decode(s)[0]
2253 2253
2254 2254 def uirepr(s):
2255 2255 # Avoid double backslash in Windows path repr()
2256 2256 return repr(s).replace('\\\\', '\\')
2257 2257
2258 2258 # delay import of textwrap
2259 2259 def MBTextWrapper(**kwargs):
2260 2260 class tw(textwrap.TextWrapper):
2261 2261 """
2262 2262 Extend TextWrapper for width-awareness.
2263 2263
2264 2264 Neither number of 'bytes' in any encoding nor 'characters' is
2265 2265 appropriate to calculate terminal columns for specified string.
2266 2266
2267 2267 Original TextWrapper implementation uses built-in 'len()' directly,
2268 2268 so overriding is needed to use width information of each characters.
2269 2269
2270 2270 In addition, characters classified into 'ambiguous' width are
2271 2271 treated as wide in East Asian area, but as narrow in other.
2272 2272
2273 2273 This requires use decision to determine width of such characters.
2274 2274 """
2275 2275 def _cutdown(self, ucstr, space_left):
2276 2276 l = 0
2277 2277 colwidth = encoding.ucolwidth
2278 2278 for i in xrange(len(ucstr)):
2279 2279 l += colwidth(ucstr[i])
2280 2280 if space_left < l:
2281 2281 return (ucstr[:i], ucstr[i:])
2282 2282 return ucstr, ''
2283 2283
2284 2284 # overriding of base class
2285 2285 def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
2286 2286 space_left = max(width - cur_len, 1)
2287 2287
2288 2288 if self.break_long_words:
2289 2289 cut, res = self._cutdown(reversed_chunks[-1], space_left)
2290 2290 cur_line.append(cut)
2291 2291 reversed_chunks[-1] = res
2292 2292 elif not cur_line:
2293 2293 cur_line.append(reversed_chunks.pop())
2294 2294
2295 2295 # this overriding code is imported from TextWrapper of Python 2.6
2296 2296 # to calculate columns of string by 'encoding.ucolwidth()'
2297 2297 def _wrap_chunks(self, chunks):
2298 2298 colwidth = encoding.ucolwidth
2299 2299
2300 2300 lines = []
2301 2301 if self.width <= 0:
2302 2302 raise ValueError("invalid width %r (must be > 0)" % self.width)
2303 2303
2304 2304 # Arrange in reverse order so items can be efficiently popped
2305 2305 # from a stack of chucks.
2306 2306 chunks.reverse()
2307 2307
2308 2308 while chunks:
2309 2309
2310 2310 # Start the list of chunks that will make up the current line.
2311 2311 # cur_len is just the length of all the chunks in cur_line.
2312 2312 cur_line = []
2313 2313 cur_len = 0
2314 2314
2315 2315 # Figure out which static string will prefix this line.
2316 2316 if lines:
2317 2317 indent = self.subsequent_indent
2318 2318 else:
2319 2319 indent = self.initial_indent
2320 2320
2321 2321 # Maximum width for this line.
2322 2322 width = self.width - len(indent)
2323 2323
2324 2324 # First chunk on line is whitespace -- drop it, unless this
2325 2325 # is the very beginning of the text (i.e. no lines started yet).
2326 2326 if self.drop_whitespace and chunks[-1].strip() == r'' and lines:
2327 2327 del chunks[-1]
2328 2328
2329 2329 while chunks:
2330 2330 l = colwidth(chunks[-1])
2331 2331
2332 2332 # Can at least squeeze this chunk onto the current line.
2333 2333 if cur_len + l <= width:
2334 2334 cur_line.append(chunks.pop())
2335 2335 cur_len += l
2336 2336
2337 2337 # Nope, this line is full.
2338 2338 else:
2339 2339 break
2340 2340
2341 2341 # The current line is full, and the next chunk is too big to
2342 2342 # fit on *any* line (not just this one).
2343 2343 if chunks and colwidth(chunks[-1]) > width:
2344 2344 self._handle_long_word(chunks, cur_line, cur_len, width)
2345 2345
2346 2346 # If the last chunk on this line is all whitespace, drop it.
2347 2347 if (self.drop_whitespace and
2348 2348 cur_line and cur_line[-1].strip() == r''):
2349 2349 del cur_line[-1]
2350 2350
2351 2351 # Convert current line back to a string and store it in list
2352 2352 # of all lines (return value).
2353 2353 if cur_line:
2354 2354 lines.append(indent + r''.join(cur_line))
2355 2355
2356 2356 return lines
2357 2357
2358 2358 global MBTextWrapper
2359 2359 MBTextWrapper = tw
2360 2360 return tw(**kwargs)
2361 2361
2362 2362 def wrap(line, width, initindent='', hangindent=''):
2363 2363 maxindent = max(len(hangindent), len(initindent))
2364 2364 if width <= maxindent:
2365 2365 # adjust for weird terminal size
2366 2366 width = max(78, maxindent + 1)
2367 2367 line = line.decode(pycompat.sysstr(encoding.encoding),
2368 2368 pycompat.sysstr(encoding.encodingmode))
2369 2369 initindent = initindent.decode(pycompat.sysstr(encoding.encoding),
2370 2370 pycompat.sysstr(encoding.encodingmode))
2371 2371 hangindent = hangindent.decode(pycompat.sysstr(encoding.encoding),
2372 2372 pycompat.sysstr(encoding.encodingmode))
2373 2373 wrapper = MBTextWrapper(width=width,
2374 2374 initial_indent=initindent,
2375 2375 subsequent_indent=hangindent)
2376 2376 return wrapper.fill(line).encode(pycompat.sysstr(encoding.encoding))
2377 2377
2378 2378 if (pyplatform.python_implementation() == 'CPython' and
2379 2379 sys.version_info < (3, 0)):
2380 2380 # There is an issue in CPython that some IO methods do not handle EINTR
2381 2381 # correctly. The following table shows what CPython version (and functions)
2382 2382 # are affected (buggy: has the EINTR bug, okay: otherwise):
2383 2383 #
2384 2384 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
2385 2385 # --------------------------------------------------
2386 2386 # fp.__iter__ | buggy | buggy | okay
2387 2387 # fp.read* | buggy | okay [1] | okay
2388 2388 #
2389 2389 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
2390 2390 #
2391 2391 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
2392 2392 # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
2393 2393 #
2394 2394 # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
2395 2395 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
2396 2396 # CPython 2, because CPython 2 maintains an internal readahead buffer for
2397 2397 # fp.__iter__ but not other fp.read* methods.
2398 2398 #
2399 2399 # On modern systems like Linux, the "read" syscall cannot be interrupted
2400 2400 # when reading "fast" files like on-disk files. So the EINTR issue only
2401 2401 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
2402 2402 # files approximately as "fast" files and use the fast (unsafe) code path,
2403 2403 # to minimize the performance impact.
2404 2404 if sys.version_info >= (2, 7, 4):
2405 2405 # fp.readline deals with EINTR correctly, use it as a workaround.
2406 2406 def _safeiterfile(fp):
2407 2407 return iter(fp.readline, '')
2408 2408 else:
2409 2409 # fp.read* are broken too, manually deal with EINTR in a stupid way.
2410 2410 # note: this may block longer than necessary because of bufsize.
2411 2411 def _safeiterfile(fp, bufsize=4096):
2412 2412 fd = fp.fileno()
2413 2413 line = ''
2414 2414 while True:
2415 2415 try:
2416 2416 buf = os.read(fd, bufsize)
2417 2417 except OSError as ex:
2418 2418 # os.read only raises EINTR before any data is read
2419 2419 if ex.errno == errno.EINTR:
2420 2420 continue
2421 2421 else:
2422 2422 raise
2423 2423 line += buf
2424 2424 if '\n' in buf:
2425 2425 splitted = line.splitlines(True)
2426 2426 line = ''
2427 2427 for l in splitted:
2428 2428 if l[-1] == '\n':
2429 2429 yield l
2430 2430 else:
2431 2431 line = l
2432 2432 if not buf:
2433 2433 break
2434 2434 if line:
2435 2435 yield line
2436 2436
2437 2437 def iterfile(fp):
2438 2438 fastpath = True
2439 2439 if type(fp) is file:
2440 2440 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
2441 2441 if fastpath:
2442 2442 return fp
2443 2443 else:
2444 2444 return _safeiterfile(fp)
2445 2445 else:
2446 2446 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
2447 2447 def iterfile(fp):
2448 2448 return fp
2449 2449
2450 2450 def iterlines(iterator):
2451 2451 for chunk in iterator:
2452 2452 for line in chunk.splitlines():
2453 2453 yield line
2454 2454
2455 2455 def expandpath(path):
2456 2456 return os.path.expanduser(os.path.expandvars(path))
2457 2457
2458 2458 def hgcmd():
2459 2459 """Return the command used to execute current hg
2460 2460
2461 2461 This is different from hgexecutable() because on Windows we want
2462 2462 to avoid things opening new shell windows like batch files, so we
2463 2463 get either the python call or current executable.
2464 2464 """
2465 2465 if mainfrozen():
2466 2466 if getattr(sys, 'frozen', None) == 'macosx_app':
2467 2467 # Env variable set by py2app
2468 2468 return [encoding.environ['EXECUTABLEPATH']]
2469 2469 else:
2470 2470 return [pycompat.sysexecutable]
2471 2471 return gethgcmd()
2472 2472
2473 2473 def rundetached(args, condfn):
2474 2474 """Execute the argument list in a detached process.
2475 2475
2476 2476 condfn is a callable which is called repeatedly and should return
2477 2477 True once the child process is known to have started successfully.
2478 2478 At this point, the child process PID is returned. If the child
2479 2479 process fails to start or finishes before condfn() evaluates to
2480 2480 True, return -1.
2481 2481 """
2482 2482 # Windows case is easier because the child process is either
2483 2483 # successfully starting and validating the condition or exiting
2484 2484 # on failure. We just poll on its PID. On Unix, if the child
2485 2485 # process fails to start, it will be left in a zombie state until
2486 2486 # the parent wait on it, which we cannot do since we expect a long
2487 2487 # running process on success. Instead we listen for SIGCHLD telling
2488 2488 # us our child process terminated.
2489 2489 terminated = set()
2490 2490 def handler(signum, frame):
2491 2491 terminated.add(os.wait())
2492 2492 prevhandler = None
2493 2493 SIGCHLD = getattr(signal, 'SIGCHLD', None)
2494 2494 if SIGCHLD is not None:
2495 2495 prevhandler = signal.signal(SIGCHLD, handler)
2496 2496 try:
2497 2497 pid = spawndetached(args)
2498 2498 while not condfn():
2499 2499 if ((pid in terminated or not testpid(pid))
2500 2500 and not condfn()):
2501 2501 return -1
2502 2502 time.sleep(0.1)
2503 2503 return pid
2504 2504 finally:
2505 2505 if prevhandler is not None:
2506 2506 signal.signal(signal.SIGCHLD, prevhandler)
2507 2507
2508 2508 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
2509 2509 """Return the result of interpolating items in the mapping into string s.
2510 2510
2511 2511 prefix is a single character string, or a two character string with
2512 2512 a backslash as the first character if the prefix needs to be escaped in
2513 2513 a regular expression.
2514 2514
2515 2515 fn is an optional function that will be applied to the replacement text
2516 2516 just before replacement.
2517 2517
2518 2518 escape_prefix is an optional flag that allows using doubled prefix for
2519 2519 its escaping.
2520 2520 """
2521 2521 fn = fn or (lambda s: s)
2522 2522 patterns = '|'.join(mapping.keys())
2523 2523 if escape_prefix:
2524 2524 patterns += '|' + prefix
2525 2525 if len(prefix) > 1:
2526 2526 prefix_char = prefix[1:]
2527 2527 else:
2528 2528 prefix_char = prefix
2529 2529 mapping[prefix_char] = prefix_char
2530 2530 r = remod.compile(r'%s(%s)' % (prefix, patterns))
2531 2531 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
2532 2532
2533 2533 def getport(port):
2534 2534 """Return the port for a given network service.
2535 2535
2536 2536 If port is an integer, it's returned as is. If it's a string, it's
2537 2537 looked up using socket.getservbyname(). If there's no matching
2538 2538 service, error.Abort is raised.
2539 2539 """
2540 2540 try:
2541 2541 return int(port)
2542 2542 except ValueError:
2543 2543 pass
2544 2544
2545 2545 try:
2546 2546 return socket.getservbyname(port)
2547 2547 except socket.error:
2548 2548 raise Abort(_("no port number associated with service '%s'") % port)
2549 2549
2550 2550 _booleans = {'1': True, 'yes': True, 'true': True, 'on': True, 'always': True,
2551 2551 '0': False, 'no': False, 'false': False, 'off': False,
2552 2552 'never': False}
2553 2553
2554 2554 def parsebool(s):
2555 2555 """Parse s into a boolean.
2556 2556
2557 2557 If s is not a valid boolean, returns None.
2558 2558 """
2559 2559 return _booleans.get(s.lower(), None)
2560 2560
2561 2561 _hextochr = dict((a + b, chr(int(a + b, 16)))
2562 2562 for a in string.hexdigits for b in string.hexdigits)
2563 2563
2564 2564 class url(object):
2565 2565 r"""Reliable URL parser.
2566 2566
2567 2567 This parses URLs and provides attributes for the following
2568 2568 components:
2569 2569
2570 2570 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
2571 2571
2572 2572 Missing components are set to None. The only exception is
2573 2573 fragment, which is set to '' if present but empty.
2574 2574
2575 2575 If parsefragment is False, fragment is included in query. If
2576 2576 parsequery is False, query is included in path. If both are
2577 2577 False, both fragment and query are included in path.
2578 2578
2579 2579 See http://www.ietf.org/rfc/rfc2396.txt for more information.
2580 2580
2581 2581 Note that for backward compatibility reasons, bundle URLs do not
2582 2582 take host names. That means 'bundle://../' has a path of '../'.
2583 2583
2584 2584 Examples:
2585 2585
2586 2586 >>> url('http://www.ietf.org/rfc/rfc2396.txt')
2587 2587 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
2588 2588 >>> url('ssh://[::1]:2200//home/joe/repo')
2589 2589 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
2590 2590 >>> url('file:///home/joe/repo')
2591 2591 <url scheme: 'file', path: '/home/joe/repo'>
2592 2592 >>> url('file:///c:/temp/foo/')
2593 2593 <url scheme: 'file', path: 'c:/temp/foo/'>
2594 2594 >>> url('bundle:foo')
2595 2595 <url scheme: 'bundle', path: 'foo'>
2596 2596 >>> url('bundle://../foo')
2597 2597 <url scheme: 'bundle', path: '../foo'>
2598 2598 >>> url(r'c:\foo\bar')
2599 2599 <url path: 'c:\\foo\\bar'>
2600 2600 >>> url(r'\\blah\blah\blah')
2601 2601 <url path: '\\\\blah\\blah\\blah'>
2602 2602 >>> url(r'\\blah\blah\blah#baz')
2603 2603 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
2604 2604 >>> url(r'file:///C:\users\me')
2605 2605 <url scheme: 'file', path: 'C:\\users\\me'>
2606 2606
2607 2607 Authentication credentials:
2608 2608
2609 2609 >>> url('ssh://joe:xyz@x/repo')
2610 2610 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
2611 2611 >>> url('ssh://joe@x/repo')
2612 2612 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
2613 2613
2614 2614 Query strings and fragments:
2615 2615
2616 2616 >>> url('http://host/a?b#c')
2617 2617 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
2618 2618 >>> url('http://host/a?b#c', parsequery=False, parsefragment=False)
2619 2619 <url scheme: 'http', host: 'host', path: 'a?b#c'>
2620 2620
2621 2621 Empty path:
2622 2622
2623 2623 >>> url('')
2624 2624 <url path: ''>
2625 2625 >>> url('#a')
2626 2626 <url path: '', fragment: 'a'>
2627 2627 >>> url('http://host/')
2628 2628 <url scheme: 'http', host: 'host', path: ''>
2629 2629 >>> url('http://host/#a')
2630 2630 <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
2631 2631
2632 2632 Only scheme:
2633 2633
2634 2634 >>> url('http:')
2635 2635 <url scheme: 'http'>
2636 2636 """
2637 2637
2638 2638 _safechars = "!~*'()+"
2639 2639 _safepchars = "/!~*'()+:\\"
2640 2640 _matchscheme = remod.compile('^[a-zA-Z0-9+.\\-]+:').match
2641 2641
2642 2642 def __init__(self, path, parsequery=True, parsefragment=True):
2643 2643 # We slowly chomp away at path until we have only the path left
2644 2644 self.scheme = self.user = self.passwd = self.host = None
2645 2645 self.port = self.path = self.query = self.fragment = None
2646 2646 self._localpath = True
2647 2647 self._hostport = ''
2648 2648 self._origpath = path
2649 2649
2650 2650 if parsefragment and '#' in path:
2651 2651 path, self.fragment = path.split('#', 1)
2652 2652
2653 2653 # special case for Windows drive letters and UNC paths
2654 2654 if hasdriveletter(path) or path.startswith('\\\\'):
2655 2655 self.path = path
2656 2656 return
2657 2657
2658 2658 # For compatibility reasons, we can't handle bundle paths as
2659 2659 # normal URLS
2660 2660 if path.startswith('bundle:'):
2661 2661 self.scheme = 'bundle'
2662 2662 path = path[7:]
2663 2663 if path.startswith('//'):
2664 2664 path = path[2:]
2665 2665 self.path = path
2666 2666 return
2667 2667
2668 2668 if self._matchscheme(path):
2669 2669 parts = path.split(':', 1)
2670 2670 if parts[0]:
2671 2671 self.scheme, path = parts
2672 2672 self._localpath = False
2673 2673
2674 2674 if not path:
2675 2675 path = None
2676 2676 if self._localpath:
2677 2677 self.path = ''
2678 2678 return
2679 2679 else:
2680 2680 if self._localpath:
2681 2681 self.path = path
2682 2682 return
2683 2683
2684 2684 if parsequery and '?' in path:
2685 2685 path, self.query = path.split('?', 1)
2686 2686 if not path:
2687 2687 path = None
2688 2688 if not self.query:
2689 2689 self.query = None
2690 2690
2691 2691 # // is required to specify a host/authority
2692 2692 if path and path.startswith('//'):
2693 2693 parts = path[2:].split('/', 1)
2694 2694 if len(parts) > 1:
2695 2695 self.host, path = parts
2696 2696 else:
2697 2697 self.host = parts[0]
2698 2698 path = None
2699 2699 if not self.host:
2700 2700 self.host = None
2701 2701 # path of file:///d is /d
2702 2702 # path of file:///d:/ is d:/, not /d:/
2703 2703 if path and not hasdriveletter(path):
2704 2704 path = '/' + path
2705 2705
2706 2706 if self.host and '@' in self.host:
2707 2707 self.user, self.host = self.host.rsplit('@', 1)
2708 2708 if ':' in self.user:
2709 2709 self.user, self.passwd = self.user.split(':', 1)
2710 2710 if not self.host:
2711 2711 self.host = None
2712 2712
2713 2713 # Don't split on colons in IPv6 addresses without ports
2714 2714 if (self.host and ':' in self.host and
2715 2715 not (self.host.startswith('[') and self.host.endswith(']'))):
2716 2716 self._hostport = self.host
2717 2717 self.host, self.port = self.host.rsplit(':', 1)
2718 2718 if not self.host:
2719 2719 self.host = None
2720 2720
2721 2721 if (self.host and self.scheme == 'file' and
2722 2722 self.host not in ('localhost', '127.0.0.1', '[::1]')):
2723 2723 raise Abort(_('file:// URLs can only refer to localhost'))
2724 2724
2725 2725 self.path = path
2726 2726
2727 2727 # leave the query string escaped
2728 2728 for a in ('user', 'passwd', 'host', 'port',
2729 2729 'path', 'fragment'):
2730 2730 v = getattr(self, a)
2731 2731 if v is not None:
2732 2732 setattr(self, a, urlreq.unquote(v))
2733 2733
2734 2734 def __repr__(self):
2735 2735 attrs = []
2736 2736 for a in ('scheme', 'user', 'passwd', 'host', 'port', 'path',
2737 2737 'query', 'fragment'):
2738 2738 v = getattr(self, a)
2739 2739 if v is not None:
2740 2740 attrs.append('%s: %r' % (a, v))
2741 2741 return '<url %s>' % ', '.join(attrs)
2742 2742
2743 2743 def __bytes__(self):
2744 2744 r"""Join the URL's components back into a URL string.
2745 2745
2746 2746 Examples:
2747 2747
2748 2748 >>> str(url('http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
2749 2749 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
2750 2750 >>> str(url('http://user:pw@host:80/?foo=bar&baz=42'))
2751 2751 'http://user:pw@host:80/?foo=bar&baz=42'
2752 2752 >>> str(url('http://user:pw@host:80/?foo=bar%3dbaz'))
2753 2753 'http://user:pw@host:80/?foo=bar%3dbaz'
2754 2754 >>> str(url('ssh://user:pw@[::1]:2200//home/joe#'))
2755 2755 'ssh://user:pw@[::1]:2200//home/joe#'
2756 2756 >>> str(url('http://localhost:80//'))
2757 2757 'http://localhost:80//'
2758 2758 >>> str(url('http://localhost:80/'))
2759 2759 'http://localhost:80/'
2760 2760 >>> str(url('http://localhost:80'))
2761 2761 'http://localhost:80/'
2762 2762 >>> str(url('bundle:foo'))
2763 2763 'bundle:foo'
2764 2764 >>> str(url('bundle://../foo'))
2765 2765 'bundle:../foo'
2766 2766 >>> str(url('path'))
2767 2767 'path'
2768 2768 >>> str(url('file:///tmp/foo/bar'))
2769 2769 'file:///tmp/foo/bar'
2770 2770 >>> str(url('file:///c:/tmp/foo/bar'))
2771 2771 'file:///c:/tmp/foo/bar'
2772 2772 >>> print url(r'bundle:foo\bar')
2773 2773 bundle:foo\bar
2774 2774 >>> print url(r'file:///D:\data\hg')
2775 2775 file:///D:\data\hg
2776 2776 """
2777 2777 if self._localpath:
2778 2778 s = self.path
2779 2779 if self.scheme == 'bundle':
2780 2780 s = 'bundle:' + s
2781 2781 if self.fragment:
2782 2782 s += '#' + self.fragment
2783 2783 return s
2784 2784
2785 2785 s = self.scheme + ':'
2786 2786 if self.user or self.passwd or self.host:
2787 2787 s += '//'
2788 2788 elif self.scheme and (not self.path or self.path.startswith('/')
2789 2789 or hasdriveletter(self.path)):
2790 2790 s += '//'
2791 2791 if hasdriveletter(self.path):
2792 2792 s += '/'
2793 2793 if self.user:
2794 2794 s += urlreq.quote(self.user, safe=self._safechars)
2795 2795 if self.passwd:
2796 2796 s += ':' + urlreq.quote(self.passwd, safe=self._safechars)
2797 2797 if self.user or self.passwd:
2798 2798 s += '@'
2799 2799 if self.host:
2800 2800 if not (self.host.startswith('[') and self.host.endswith(']')):
2801 2801 s += urlreq.quote(self.host)
2802 2802 else:
2803 2803 s += self.host
2804 2804 if self.port:
2805 2805 s += ':' + urlreq.quote(self.port)
2806 2806 if self.host:
2807 2807 s += '/'
2808 2808 if self.path:
2809 2809 # TODO: similar to the query string, we should not unescape the
2810 2810 # path when we store it, the path might contain '%2f' = '/',
2811 2811 # which we should *not* escape.
2812 2812 s += urlreq.quote(self.path, safe=self._safepchars)
2813 2813 if self.query:
2814 2814 # we store the query in escaped form.
2815 2815 s += '?' + self.query
2816 2816 if self.fragment is not None:
2817 2817 s += '#' + urlreq.quote(self.fragment, safe=self._safepchars)
2818 2818 return s
2819 2819
2820 2820 __str__ = encoding.strmethod(__bytes__)
2821 2821
2822 2822 def authinfo(self):
2823 2823 user, passwd = self.user, self.passwd
2824 2824 try:
2825 2825 self.user, self.passwd = None, None
2826 2826 s = bytes(self)
2827 2827 finally:
2828 2828 self.user, self.passwd = user, passwd
2829 2829 if not self.user:
2830 2830 return (s, None)
2831 2831 # authinfo[1] is passed to urllib2 password manager, and its
2832 2832 # URIs must not contain credentials. The host is passed in the
2833 2833 # URIs list because Python < 2.4.3 uses only that to search for
2834 2834 # a password.
2835 2835 return (s, (None, (s, self.host),
2836 2836 self.user, self.passwd or ''))
2837 2837
2838 2838 def isabs(self):
2839 2839 if self.scheme and self.scheme != 'file':
2840 2840 return True # remote URL
2841 2841 if hasdriveletter(self.path):
2842 2842 return True # absolute for our purposes - can't be joined()
2843 2843 if self.path.startswith(r'\\'):
2844 2844 return True # Windows UNC path
2845 2845 if self.path.startswith('/'):
2846 2846 return True # POSIX-style
2847 2847 return False
2848 2848
2849 2849 def localpath(self):
2850 2850 if self.scheme == 'file' or self.scheme == 'bundle':
2851 2851 path = self.path or '/'
2852 2852 # For Windows, we need to promote hosts containing drive
2853 2853 # letters to paths with drive letters.
2854 2854 if hasdriveletter(self._hostport):
2855 2855 path = self._hostport + '/' + self.path
2856 2856 elif (self.host is not None and self.path
2857 2857 and not hasdriveletter(path)):
2858 2858 path = '/' + path
2859 2859 return path
2860 2860 return self._origpath
2861 2861
2862 2862 def islocal(self):
2863 2863 '''whether localpath will return something that posixfile can open'''
2864 2864 return (not self.scheme or self.scheme == 'file'
2865 2865 or self.scheme == 'bundle')
2866 2866
2867 2867 def hasscheme(path):
2868 2868 return bool(url(path).scheme)
2869 2869
2870 2870 def hasdriveletter(path):
2871 2871 return path and path[1:2] == ':' and path[0:1].isalpha()
2872 2872
2873 2873 def urllocalpath(path):
2874 2874 return url(path, parsequery=False, parsefragment=False).localpath()
2875 2875
2876 2876 def hidepassword(u):
2877 2877 '''hide user credential in a url string'''
2878 2878 u = url(u)
2879 2879 if u.passwd:
2880 2880 u.passwd = '***'
2881 2881 return bytes(u)
2882 2882
2883 2883 def removeauth(u):
2884 2884 '''remove all authentication information from a url string'''
2885 2885 u = url(u)
2886 2886 u.user = u.passwd = None
2887 2887 return str(u)
2888 2888
2889 2889 timecount = unitcountfn(
2890 2890 (1, 1e3, _('%.0f s')),
2891 2891 (100, 1, _('%.1f s')),
2892 2892 (10, 1, _('%.2f s')),
2893 2893 (1, 1, _('%.3f s')),
2894 2894 (100, 0.001, _('%.1f ms')),
2895 2895 (10, 0.001, _('%.2f ms')),
2896 2896 (1, 0.001, _('%.3f ms')),
2897 2897 (100, 0.000001, _('%.1f us')),
2898 2898 (10, 0.000001, _('%.2f us')),
2899 2899 (1, 0.000001, _('%.3f us')),
2900 2900 (100, 0.000000001, _('%.1f ns')),
2901 2901 (10, 0.000000001, _('%.2f ns')),
2902 2902 (1, 0.000000001, _('%.3f ns')),
2903 2903 )
2904 2904
2905 2905 _timenesting = [0]
2906 2906
2907 2907 def timed(func):
2908 2908 '''Report the execution time of a function call to stderr.
2909 2909
2910 2910 During development, use as a decorator when you need to measure
2911 2911 the cost of a function, e.g. as follows:
2912 2912
2913 2913 @util.timed
2914 2914 def foo(a, b, c):
2915 2915 pass
2916 2916 '''
2917 2917
2918 2918 def wrapper(*args, **kwargs):
2919 2919 start = timer()
2920 2920 indent = 2
2921 2921 _timenesting[0] += indent
2922 2922 try:
2923 2923 return func(*args, **kwargs)
2924 2924 finally:
2925 2925 elapsed = timer() - start
2926 2926 _timenesting[0] -= indent
2927 2927 stderr.write('%s%s: %s\n' %
2928 2928 (' ' * _timenesting[0], func.__name__,
2929 2929 timecount(elapsed)))
2930 2930 return wrapper
2931 2931
2932 2932 _sizeunits = (('m', 2**20), ('k', 2**10), ('g', 2**30),
2933 2933 ('kb', 2**10), ('mb', 2**20), ('gb', 2**30), ('b', 1))
2934 2934
2935 2935 def sizetoint(s):
2936 2936 '''Convert a space specifier to a byte count.
2937 2937
2938 2938 >>> sizetoint('30')
2939 2939 30
2940 2940 >>> sizetoint('2.2kb')
2941 2941 2252
2942 2942 >>> sizetoint('6M')
2943 2943 6291456
2944 2944 '''
2945 2945 t = s.strip().lower()
2946 2946 try:
2947 2947 for k, u in _sizeunits:
2948 2948 if t.endswith(k):
2949 2949 return int(float(t[:-len(k)]) * u)
2950 2950 return int(t)
2951 2951 except ValueError:
2952 2952 raise error.ParseError(_("couldn't parse size: %s") % s)
2953 2953
2954 2954 class hooks(object):
2955 2955 '''A collection of hook functions that can be used to extend a
2956 2956 function's behavior. Hooks are called in lexicographic order,
2957 2957 based on the names of their sources.'''
2958 2958
2959 2959 def __init__(self):
2960 2960 self._hooks = []
2961 2961
2962 2962 def add(self, source, hook):
2963 2963 self._hooks.append((source, hook))
2964 2964
2965 2965 def __call__(self, *args):
2966 2966 self._hooks.sort(key=lambda x: x[0])
2967 2967 results = []
2968 2968 for source, hook in self._hooks:
2969 2969 results.append(hook(*args))
2970 2970 return results
2971 2971
2972 2972 def getstackframes(skip=0, line=' %-*s in %s\n', fileline='%s:%s', depth=0):
2973 2973 '''Yields lines for a nicely formatted stacktrace.
2974 2974 Skips the 'skip' last entries, then return the last 'depth' entries.
2975 2975 Each file+linenumber is formatted according to fileline.
2976 2976 Each line is formatted according to line.
2977 2977 If line is None, it yields:
2978 2978 length of longest filepath+line number,
2979 2979 filepath+linenumber,
2980 2980 function
2981 2981
2982 2982 Not be used in production code but very convenient while developing.
2983 2983 '''
2984 2984 entries = [(fileline % (fn, ln), func)
2985 2985 for fn, ln, func, _text in traceback.extract_stack()[:-skip - 1]
2986 2986 ][-depth:]
2987 2987 if entries:
2988 2988 fnmax = max(len(entry[0]) for entry in entries)
2989 2989 for fnln, func in entries:
2990 2990 if line is None:
2991 2991 yield (fnmax, fnln, func)
2992 2992 else:
2993 2993 yield line % (fnmax, fnln, func)
2994 2994
2995 2995 def debugstacktrace(msg='stacktrace', skip=0,
2996 2996 f=stderr, otherf=stdout, depth=0):
2997 2997 '''Writes a message to f (stderr) with a nicely formatted stacktrace.
2998 2998 Skips the 'skip' entries closest to the call, then show 'depth' entries.
2999 2999 By default it will flush stdout first.
3000 3000 It can be used everywhere and intentionally does not require an ui object.
3001 3001 Not be used in production code but very convenient while developing.
3002 3002 '''
3003 3003 if otherf:
3004 3004 otherf.flush()
3005 3005 f.write('%s at:\n' % msg.rstrip())
3006 3006 for line in getstackframes(skip + 1, depth=depth):
3007 3007 f.write(line)
3008 3008 f.flush()
3009 3009
3010 3010 class dirs(object):
3011 3011 '''a multiset of directory names from a dirstate or manifest'''
3012 3012
3013 3013 def __init__(self, map, skip=None):
3014 3014 self._dirs = {}
3015 3015 addpath = self.addpath
3016 3016 if safehasattr(map, 'iteritems') and skip is not None:
3017 3017 for f, s in map.iteritems():
3018 3018 if s[0] != skip:
3019 3019 addpath(f)
3020 3020 else:
3021 3021 for f in map:
3022 3022 addpath(f)
3023 3023
3024 3024 def addpath(self, path):
3025 3025 dirs = self._dirs
3026 3026 for base in finddirs(path):
3027 3027 if base in dirs:
3028 3028 dirs[base] += 1
3029 3029 return
3030 3030 dirs[base] = 1
3031 3031
3032 3032 def delpath(self, path):
3033 3033 dirs = self._dirs
3034 3034 for base in finddirs(path):
3035 3035 if dirs[base] > 1:
3036 3036 dirs[base] -= 1
3037 3037 return
3038 3038 del dirs[base]
3039 3039
3040 3040 def __iter__(self):
3041 3041 return iter(self._dirs)
3042 3042
3043 3043 def __contains__(self, d):
3044 3044 return d in self._dirs
3045 3045
3046 3046 if safehasattr(parsers, 'dirs'):
3047 3047 dirs = parsers.dirs
3048 3048
3049 3049 def finddirs(path):
3050 3050 pos = path.rfind('/')
3051 3051 while pos != -1:
3052 3052 yield path[:pos]
3053 3053 pos = path.rfind('/', 0, pos)
3054 3054
3055 3055 class ctxmanager(object):
3056 3056 '''A context manager for use in 'with' blocks to allow multiple
3057 3057 contexts to be entered at once. This is both safer and more
3058 3058 flexible than contextlib.nested.
3059 3059
3060 3060 Once Mercurial supports Python 2.7+, this will become mostly
3061 3061 unnecessary.
3062 3062 '''
3063 3063
3064 3064 def __init__(self, *args):
3065 3065 '''Accepts a list of no-argument functions that return context
3066 3066 managers. These will be invoked at __call__ time.'''
3067 3067 self._pending = args
3068 3068 self._atexit = []
3069 3069
3070 3070 def __enter__(self):
3071 3071 return self
3072 3072
3073 3073 def enter(self):
3074 3074 '''Create and enter context managers in the order in which they were
3075 3075 passed to the constructor.'''
3076 3076 values = []
3077 3077 for func in self._pending:
3078 3078 obj = func()
3079 3079 values.append(obj.__enter__())
3080 3080 self._atexit.append(obj.__exit__)
3081 3081 del self._pending
3082 3082 return values
3083 3083
3084 3084 def atexit(self, func, *args, **kwargs):
3085 3085 '''Add a function to call when this context manager exits. The
3086 3086 ordering of multiple atexit calls is unspecified, save that
3087 3087 they will happen before any __exit__ functions.'''
3088 3088 def wrapper(exc_type, exc_val, exc_tb):
3089 3089 func(*args, **kwargs)
3090 3090 self._atexit.append(wrapper)
3091 3091 return func
3092 3092
3093 3093 def __exit__(self, exc_type, exc_val, exc_tb):
3094 3094 '''Context managers are exited in the reverse order from which
3095 3095 they were created.'''
3096 3096 received = exc_type is not None
3097 3097 suppressed = False
3098 3098 pending = None
3099 3099 self._atexit.reverse()
3100 3100 for exitfunc in self._atexit:
3101 3101 try:
3102 3102 if exitfunc(exc_type, exc_val, exc_tb):
3103 3103 suppressed = True
3104 3104 exc_type = None
3105 3105 exc_val = None
3106 3106 exc_tb = None
3107 3107 except BaseException:
3108 3108 pending = sys.exc_info()
3109 3109 exc_type, exc_val, exc_tb = pending = sys.exc_info()
3110 3110 del self._atexit
3111 3111 if pending:
3112 3112 raise exc_val
3113 3113 return received and suppressed
3114 3114
3115 3115 # compression code
3116 3116
3117 3117 SERVERROLE = 'server'
3118 3118 CLIENTROLE = 'client'
3119 3119
3120 3120 compewireprotosupport = collections.namedtuple(u'compenginewireprotosupport',
3121 3121 (u'name', u'serverpriority',
3122 3122 u'clientpriority'))
3123 3123
3124 3124 class compressormanager(object):
3125 3125 """Holds registrations of various compression engines.
3126 3126
3127 3127 This class essentially abstracts the differences between compression
3128 3128 engines to allow new compression formats to be added easily, possibly from
3129 3129 extensions.
3130 3130
3131 3131 Compressors are registered against the global instance by calling its
3132 3132 ``register()`` method.
3133 3133 """
3134 3134 def __init__(self):
3135 3135 self._engines = {}
3136 3136 # Bundle spec human name to engine name.
3137 3137 self._bundlenames = {}
3138 3138 # Internal bundle identifier to engine name.
3139 3139 self._bundletypes = {}
3140 3140 # Revlog header to engine name.
3141 3141 self._revlogheaders = {}
3142 3142 # Wire proto identifier to engine name.
3143 3143 self._wiretypes = {}
3144 3144
3145 3145 def __getitem__(self, key):
3146 3146 return self._engines[key]
3147 3147
3148 3148 def __contains__(self, key):
3149 3149 return key in self._engines
3150 3150
3151 3151 def __iter__(self):
3152 3152 return iter(self._engines.keys())
3153 3153
3154 3154 def register(self, engine):
3155 3155 """Register a compression engine with the manager.
3156 3156
3157 3157 The argument must be a ``compressionengine`` instance.
3158 3158 """
3159 3159 if not isinstance(engine, compressionengine):
3160 3160 raise ValueError(_('argument must be a compressionengine'))
3161 3161
3162 3162 name = engine.name()
3163 3163
3164 3164 if name in self._engines:
3165 3165 raise error.Abort(_('compression engine %s already registered') %
3166 3166 name)
3167 3167
3168 3168 bundleinfo = engine.bundletype()
3169 3169 if bundleinfo:
3170 3170 bundlename, bundletype = bundleinfo
3171 3171
3172 3172 if bundlename in self._bundlenames:
3173 3173 raise error.Abort(_('bundle name %s already registered') %
3174 3174 bundlename)
3175 3175 if bundletype in self._bundletypes:
3176 3176 raise error.Abort(_('bundle type %s already registered by %s') %
3177 3177 (bundletype, self._bundletypes[bundletype]))
3178 3178
3179 3179 # No external facing name declared.
3180 3180 if bundlename:
3181 3181 self._bundlenames[bundlename] = name
3182 3182
3183 3183 self._bundletypes[bundletype] = name
3184 3184
3185 3185 wiresupport = engine.wireprotosupport()
3186 3186 if wiresupport:
3187 3187 wiretype = wiresupport.name
3188 3188 if wiretype in self._wiretypes:
3189 3189 raise error.Abort(_('wire protocol compression %s already '
3190 3190 'registered by %s') %
3191 3191 (wiretype, self._wiretypes[wiretype]))
3192 3192
3193 3193 self._wiretypes[wiretype] = name
3194 3194
3195 3195 revlogheader = engine.revlogheader()
3196 3196 if revlogheader and revlogheader in self._revlogheaders:
3197 3197 raise error.Abort(_('revlog header %s already registered by %s') %
3198 3198 (revlogheader, self._revlogheaders[revlogheader]))
3199 3199
3200 3200 if revlogheader:
3201 3201 self._revlogheaders[revlogheader] = name
3202 3202
3203 3203 self._engines[name] = engine
3204 3204
3205 3205 @property
3206 3206 def supportedbundlenames(self):
3207 3207 return set(self._bundlenames.keys())
3208 3208
3209 3209 @property
3210 3210 def supportedbundletypes(self):
3211 3211 return set(self._bundletypes.keys())
3212 3212
3213 3213 def forbundlename(self, bundlename):
3214 3214 """Obtain a compression engine registered to a bundle name.
3215 3215
3216 3216 Will raise KeyError if the bundle type isn't registered.
3217 3217
3218 3218 Will abort if the engine is known but not available.
3219 3219 """
3220 3220 engine = self._engines[self._bundlenames[bundlename]]
3221 3221 if not engine.available():
3222 3222 raise error.Abort(_('compression engine %s could not be loaded') %
3223 3223 engine.name())
3224 3224 return engine
3225 3225
3226 3226 def forbundletype(self, bundletype):
3227 3227 """Obtain a compression engine registered to a bundle type.
3228 3228
3229 3229 Will raise KeyError if the bundle type isn't registered.
3230 3230
3231 3231 Will abort if the engine is known but not available.
3232 3232 """
3233 3233 engine = self._engines[self._bundletypes[bundletype]]
3234 3234 if not engine.available():
3235 3235 raise error.Abort(_('compression engine %s could not be loaded') %
3236 3236 engine.name())
3237 3237 return engine
3238 3238
3239 3239 def supportedwireengines(self, role, onlyavailable=True):
3240 3240 """Obtain compression engines that support the wire protocol.
3241 3241
3242 3242 Returns a list of engines in prioritized order, most desired first.
3243 3243
3244 3244 If ``onlyavailable`` is set, filter out engines that can't be
3245 3245 loaded.
3246 3246 """
3247 3247 assert role in (SERVERROLE, CLIENTROLE)
3248 3248
3249 3249 attr = 'serverpriority' if role == SERVERROLE else 'clientpriority'
3250 3250
3251 3251 engines = [self._engines[e] for e in self._wiretypes.values()]
3252 3252 if onlyavailable:
3253 3253 engines = [e for e in engines if e.available()]
3254 3254
3255 3255 def getkey(e):
3256 3256 # Sort first by priority, highest first. In case of tie, sort
3257 3257 # alphabetically. This is arbitrary, but ensures output is
3258 3258 # stable.
3259 3259 w = e.wireprotosupport()
3260 3260 return -1 * getattr(w, attr), w.name
3261 3261
3262 3262 return list(sorted(engines, key=getkey))
3263 3263
3264 3264 def forwiretype(self, wiretype):
3265 3265 engine = self._engines[self._wiretypes[wiretype]]
3266 3266 if not engine.available():
3267 3267 raise error.Abort(_('compression engine %s could not be loaded') %
3268 3268 engine.name())
3269 3269 return engine
3270 3270
3271 3271 def forrevlogheader(self, header):
3272 3272 """Obtain a compression engine registered to a revlog header.
3273 3273
3274 3274 Will raise KeyError if the revlog header value isn't registered.
3275 3275 """
3276 3276 return self._engines[self._revlogheaders[header]]
3277 3277
3278 3278 compengines = compressormanager()
3279 3279
3280 3280 class compressionengine(object):
3281 3281 """Base class for compression engines.
3282 3282
3283 3283 Compression engines must implement the interface defined by this class.
3284 3284 """
3285 3285 def name(self):
3286 3286 """Returns the name of the compression engine.
3287 3287
3288 3288 This is the key the engine is registered under.
3289 3289
3290 3290 This method must be implemented.
3291 3291 """
3292 3292 raise NotImplementedError()
3293 3293
3294 3294 def available(self):
3295 3295 """Whether the compression engine is available.
3296 3296
3297 3297 The intent of this method is to allow optional compression engines
3298 3298 that may not be available in all installations (such as engines relying
3299 3299 on C extensions that may not be present).
3300 3300 """
3301 3301 return True
3302 3302
3303 3303 def bundletype(self):
3304 3304 """Describes bundle identifiers for this engine.
3305 3305
3306 3306 If this compression engine isn't supported for bundles, returns None.
3307 3307
3308 3308 If this engine can be used for bundles, returns a 2-tuple of strings of
3309 3309 the user-facing "bundle spec" compression name and an internal
3310 3310 identifier used to denote the compression format within bundles. To
3311 3311 exclude the name from external usage, set the first element to ``None``.
3312 3312
3313 3313 If bundle compression is supported, the class must also implement
3314 3314 ``compressstream`` and `decompressorreader``.
3315 3315
3316 3316 The docstring of this method is used in the help system to tell users
3317 3317 about this engine.
3318 3318 """
3319 3319 return None
3320 3320
3321 3321 def wireprotosupport(self):
3322 3322 """Declare support for this compression format on the wire protocol.
3323 3323
3324 3324 If this compression engine isn't supported for compressing wire
3325 3325 protocol payloads, returns None.
3326 3326
3327 3327 Otherwise, returns ``compenginewireprotosupport`` with the following
3328 3328 fields:
3329 3329
3330 3330 * String format identifier
3331 3331 * Integer priority for the server
3332 3332 * Integer priority for the client
3333 3333
3334 3334 The integer priorities are used to order the advertisement of format
3335 3335 support by server and client. The highest integer is advertised
3336 3336 first. Integers with non-positive values aren't advertised.
3337 3337
3338 3338 The priority values are somewhat arbitrary and only used for default
3339 3339 ordering. The relative order can be changed via config options.
3340 3340
3341 3341 If wire protocol compression is supported, the class must also implement
3342 3342 ``compressstream`` and ``decompressorreader``.
3343 3343 """
3344 3344 return None
3345 3345
3346 3346 def revlogheader(self):
3347 3347 """Header added to revlog chunks that identifies this engine.
3348 3348
3349 3349 If this engine can be used to compress revlogs, this method should
3350 3350 return the bytes used to identify chunks compressed with this engine.
3351 3351 Else, the method should return ``None`` to indicate it does not
3352 3352 participate in revlog compression.
3353 3353 """
3354 3354 return None
3355 3355
3356 3356 def compressstream(self, it, opts=None):
3357 3357 """Compress an iterator of chunks.
3358 3358
3359 3359 The method receives an iterator (ideally a generator) of chunks of
3360 3360 bytes to be compressed. It returns an iterator (ideally a generator)
3361 3361 of bytes of chunks representing the compressed output.
3362 3362
3363 3363 Optionally accepts an argument defining how to perform compression.
3364 3364 Each engine treats this argument differently.
3365 3365 """
3366 3366 raise NotImplementedError()
3367 3367
3368 3368 def decompressorreader(self, fh):
3369 3369 """Perform decompression on a file object.
3370 3370
3371 3371 Argument is an object with a ``read(size)`` method that returns
3372 3372 compressed data. Return value is an object with a ``read(size)`` that
3373 3373 returns uncompressed data.
3374 3374 """
3375 3375 raise NotImplementedError()
3376 3376
3377 3377 def revlogcompressor(self, opts=None):
3378 3378 """Obtain an object that can be used to compress revlog entries.
3379 3379
3380 3380 The object has a ``compress(data)`` method that compresses binary
3381 3381 data. This method returns compressed binary data or ``None`` if
3382 3382 the data could not be compressed (too small, not compressible, etc).
3383 3383 The returned data should have a header uniquely identifying this
3384 3384 compression format so decompression can be routed to this engine.
3385 3385 This header should be identified by the ``revlogheader()`` return
3386 3386 value.
3387 3387
3388 3388 The object has a ``decompress(data)`` method that decompresses
3389 3389 data. The method will only be called if ``data`` begins with
3390 3390 ``revlogheader()``. The method should return the raw, uncompressed
3391 3391 data or raise a ``RevlogError``.
3392 3392
3393 3393 The object is reusable but is not thread safe.
3394 3394 """
3395 3395 raise NotImplementedError()
3396 3396
3397 3397 class _zlibengine(compressionengine):
3398 3398 def name(self):
3399 3399 return 'zlib'
3400 3400
3401 3401 def bundletype(self):
3402 3402 """zlib compression using the DEFLATE algorithm.
3403 3403
3404 3404 All Mercurial clients should support this format. The compression
3405 3405 algorithm strikes a reasonable balance between compression ratio
3406 3406 and size.
3407 3407 """
3408 3408 return 'gzip', 'GZ'
3409 3409
3410 3410 def wireprotosupport(self):
3411 3411 return compewireprotosupport('zlib', 20, 20)
3412 3412
3413 3413 def revlogheader(self):
3414 3414 return 'x'
3415 3415
3416 3416 def compressstream(self, it, opts=None):
3417 3417 opts = opts or {}
3418 3418
3419 3419 z = zlib.compressobj(opts.get('level', -1))
3420 3420 for chunk in it:
3421 3421 data = z.compress(chunk)
3422 3422 # Not all calls to compress emit data. It is cheaper to inspect
3423 3423 # here than to feed empty chunks through generator.
3424 3424 if data:
3425 3425 yield data
3426 3426
3427 3427 yield z.flush()
3428 3428
3429 3429 def decompressorreader(self, fh):
3430 3430 def gen():
3431 3431 d = zlib.decompressobj()
3432 3432 for chunk in filechunkiter(fh):
3433 3433 while chunk:
3434 3434 # Limit output size to limit memory.
3435 3435 yield d.decompress(chunk, 2 ** 18)
3436 3436 chunk = d.unconsumed_tail
3437 3437
3438 3438 return chunkbuffer(gen())
3439 3439
3440 3440 class zlibrevlogcompressor(object):
3441 3441 def compress(self, data):
3442 3442 insize = len(data)
3443 3443 # Caller handles empty input case.
3444 3444 assert insize > 0
3445 3445
3446 3446 if insize < 44:
3447 3447 return None
3448 3448
3449 3449 elif insize <= 1000000:
3450 3450 compressed = zlib.compress(data)
3451 3451 if len(compressed) < insize:
3452 3452 return compressed
3453 3453 return None
3454 3454
3455 3455 # zlib makes an internal copy of the input buffer, doubling
3456 3456 # memory usage for large inputs. So do streaming compression
3457 3457 # on large inputs.
3458 3458 else:
3459 3459 z = zlib.compressobj()
3460 3460 parts = []
3461 3461 pos = 0
3462 3462 while pos < insize:
3463 3463 pos2 = pos + 2**20
3464 3464 parts.append(z.compress(data[pos:pos2]))
3465 3465 pos = pos2
3466 3466 parts.append(z.flush())
3467 3467
3468 3468 if sum(map(len, parts)) < insize:
3469 3469 return ''.join(parts)
3470 3470 return None
3471 3471
3472 3472 def decompress(self, data):
3473 3473 try:
3474 3474 return zlib.decompress(data)
3475 3475 except zlib.error as e:
3476 3476 raise error.RevlogError(_('revlog decompress error: %s') %
3477 3477 str(e))
3478 3478
3479 3479 def revlogcompressor(self, opts=None):
3480 3480 return self.zlibrevlogcompressor()
3481 3481
3482 3482 compengines.register(_zlibengine())
3483 3483
3484 3484 class _bz2engine(compressionengine):
3485 3485 def name(self):
3486 3486 return 'bz2'
3487 3487
3488 3488 def bundletype(self):
3489 3489 """An algorithm that produces smaller bundles than ``gzip``.
3490 3490
3491 3491 All Mercurial clients should support this format.
3492 3492
3493 3493 This engine will likely produce smaller bundles than ``gzip`` but
3494 3494 will be significantly slower, both during compression and
3495 3495 decompression.
3496 3496
3497 3497 If available, the ``zstd`` engine can yield similar or better
3498 3498 compression at much higher speeds.
3499 3499 """
3500 3500 return 'bzip2', 'BZ'
3501 3501
3502 3502 # We declare a protocol name but don't advertise by default because
3503 3503 # it is slow.
3504 3504 def wireprotosupport(self):
3505 3505 return compewireprotosupport('bzip2', 0, 0)
3506 3506
3507 3507 def compressstream(self, it, opts=None):
3508 3508 opts = opts or {}
3509 3509 z = bz2.BZ2Compressor(opts.get('level', 9))
3510 3510 for chunk in it:
3511 3511 data = z.compress(chunk)
3512 3512 if data:
3513 3513 yield data
3514 3514
3515 3515 yield z.flush()
3516 3516
3517 3517 def decompressorreader(self, fh):
3518 3518 def gen():
3519 3519 d = bz2.BZ2Decompressor()
3520 3520 for chunk in filechunkiter(fh):
3521 3521 yield d.decompress(chunk)
3522 3522
3523 3523 return chunkbuffer(gen())
3524 3524
3525 3525 compengines.register(_bz2engine())
3526 3526
3527 3527 class _truncatedbz2engine(compressionengine):
3528 3528 def name(self):
3529 3529 return 'bz2truncated'
3530 3530
3531 3531 def bundletype(self):
3532 3532 return None, '_truncatedBZ'
3533 3533
3534 3534 # We don't implement compressstream because it is hackily handled elsewhere.
3535 3535
3536 3536 def decompressorreader(self, fh):
3537 3537 def gen():
3538 3538 # The input stream doesn't have the 'BZ' header. So add it back.
3539 3539 d = bz2.BZ2Decompressor()
3540 3540 d.decompress('BZ')
3541 3541 for chunk in filechunkiter(fh):
3542 3542 yield d.decompress(chunk)
3543 3543
3544 3544 return chunkbuffer(gen())
3545 3545
3546 3546 compengines.register(_truncatedbz2engine())
3547 3547
3548 3548 class _noopengine(compressionengine):
3549 3549 def name(self):
3550 3550 return 'none'
3551 3551
3552 3552 def bundletype(self):
3553 3553 """No compression is performed.
3554 3554
3555 3555 Use this compression engine to explicitly disable compression.
3556 3556 """
3557 3557 return 'none', 'UN'
3558 3558
3559 3559 # Clients always support uncompressed payloads. Servers don't because
3560 3560 # unless you are on a fast network, uncompressed payloads can easily
3561 3561 # saturate your network pipe.
3562 3562 def wireprotosupport(self):
3563 3563 return compewireprotosupport('none', 0, 10)
3564 3564
3565 3565 # We don't implement revlogheader because it is handled specially
3566 3566 # in the revlog class.
3567 3567
3568 3568 def compressstream(self, it, opts=None):
3569 3569 return it
3570 3570
3571 3571 def decompressorreader(self, fh):
3572 3572 return fh
3573 3573
3574 3574 class nooprevlogcompressor(object):
3575 3575 def compress(self, data):
3576 3576 return None
3577 3577
3578 3578 def revlogcompressor(self, opts=None):
3579 3579 return self.nooprevlogcompressor()
3580 3580
3581 3581 compengines.register(_noopengine())
3582 3582
3583 3583 class _zstdengine(compressionengine):
3584 3584 def name(self):
3585 3585 return 'zstd'
3586 3586
3587 3587 @propertycache
3588 3588 def _module(self):
3589 3589 # Not all installs have the zstd module available. So defer importing
3590 3590 # until first access.
3591 3591 try:
3592 3592 from . import zstd
3593 3593 # Force delayed import.
3594 3594 zstd.__version__
3595 3595 return zstd
3596 3596 except ImportError:
3597 3597 return None
3598 3598
3599 3599 def available(self):
3600 3600 return bool(self._module)
3601 3601
3602 3602 def bundletype(self):
3603 3603 """A modern compression algorithm that is fast and highly flexible.
3604 3604
3605 3605 Only supported by Mercurial 4.1 and newer clients.
3606 3606
3607 3607 With the default settings, zstd compression is both faster and yields
3608 3608 better compression than ``gzip``. It also frequently yields better
3609 3609 compression than ``bzip2`` while operating at much higher speeds.
3610 3610
3611 3611 If this engine is available and backwards compatibility is not a
3612 3612 concern, it is likely the best available engine.
3613 3613 """
3614 3614 return 'zstd', 'ZS'
3615 3615
3616 3616 def wireprotosupport(self):
3617 3617 return compewireprotosupport('zstd', 50, 50)
3618 3618
3619 3619 def revlogheader(self):
3620 3620 return '\x28'
3621 3621
3622 3622 def compressstream(self, it, opts=None):
3623 3623 opts = opts or {}
3624 3624 # zstd level 3 is almost always significantly faster than zlib
3625 3625 # while providing no worse compression. It strikes a good balance
3626 3626 # between speed and compression.
3627 3627 level = opts.get('level', 3)
3628 3628
3629 3629 zstd = self._module
3630 3630 z = zstd.ZstdCompressor(level=level).compressobj()
3631 3631 for chunk in it:
3632 3632 data = z.compress(chunk)
3633 3633 if data:
3634 3634 yield data
3635 3635
3636 3636 yield z.flush()
3637 3637
3638 3638 def decompressorreader(self, fh):
3639 3639 zstd = self._module
3640 3640 dctx = zstd.ZstdDecompressor()
3641 3641 return chunkbuffer(dctx.read_from(fh))
3642 3642
3643 3643 class zstdrevlogcompressor(object):
3644 3644 def __init__(self, zstd, level=3):
3645 3645 # Writing the content size adds a few bytes to the output. However,
3646 3646 # it allows decompression to be more optimal since we can
3647 3647 # pre-allocate a buffer to hold the result.
3648 3648 self._cctx = zstd.ZstdCompressor(level=level,
3649 3649 write_content_size=True)
3650 3650 self._dctx = zstd.ZstdDecompressor()
3651 3651 self._compinsize = zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE
3652 3652 self._decompinsize = zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
3653 3653
3654 3654 def compress(self, data):
3655 3655 insize = len(data)
3656 3656 # Caller handles empty input case.
3657 3657 assert insize > 0
3658 3658
3659 3659 if insize < 50:
3660 3660 return None
3661 3661
3662 3662 elif insize <= 1000000:
3663 3663 compressed = self._cctx.compress(data)
3664 3664 if len(compressed) < insize:
3665 3665 return compressed
3666 3666 return None
3667 3667 else:
3668 3668 z = self._cctx.compressobj()
3669 3669 chunks = []
3670 3670 pos = 0
3671 3671 while pos < insize:
3672 3672 pos2 = pos + self._compinsize
3673 3673 chunk = z.compress(data[pos:pos2])
3674 3674 if chunk:
3675 3675 chunks.append(chunk)
3676 3676 pos = pos2
3677 3677 chunks.append(z.flush())
3678 3678
3679 3679 if sum(map(len, chunks)) < insize:
3680 3680 return ''.join(chunks)
3681 3681 return None
3682 3682
3683 3683 def decompress(self, data):
3684 3684 insize = len(data)
3685 3685
3686 3686 try:
3687 3687 # This was measured to be faster than other streaming
3688 3688 # decompressors.
3689 3689 dobj = self._dctx.decompressobj()
3690 3690 chunks = []
3691 3691 pos = 0
3692 3692 while pos < insize:
3693 3693 pos2 = pos + self._decompinsize
3694 3694 chunk = dobj.decompress(data[pos:pos2])
3695 3695 if chunk:
3696 3696 chunks.append(chunk)
3697 3697 pos = pos2
3698 3698 # Frame should be exhausted, so no finish() API.
3699 3699
3700 3700 return ''.join(chunks)
3701 3701 except Exception as e:
3702 3702 raise error.RevlogError(_('revlog decompress error: %s') %
3703 3703 str(e))
3704 3704
3705 3705 def revlogcompressor(self, opts=None):
3706 3706 opts = opts or {}
3707 3707 return self.zstdrevlogcompressor(self._module,
3708 3708 level=opts.get('level', 3))
3709 3709
3710 3710 compengines.register(_zstdengine())
3711 3711
3712 3712 def bundlecompressiontopics():
3713 3713 """Obtains a list of available bundle compressions for use in help."""
3714 3714 # help.makeitemsdocs() expects a dict of names to items with a .__doc__.
3715 3715 items = {}
3716 3716
3717 3717 # We need to format the docstring. So use a dummy object/type to hold it
3718 3718 # rather than mutating the original.
3719 3719 class docobject(object):
3720 3720 pass
3721 3721
3722 3722 for name in compengines:
3723 3723 engine = compengines[name]
3724 3724
3725 3725 if not engine.available():
3726 3726 continue
3727 3727
3728 3728 bt = engine.bundletype()
3729 3729 if not bt or not bt[0]:
3730 3730 continue
3731 3731
3732 3732 doc = pycompat.sysstr('``%s``\n %s') % (
3733 3733 bt[0], engine.bundletype.__doc__)
3734 3734
3735 3735 value = docobject()
3736 3736 value.__doc__ = doc
3737 3737
3738 3738 items[bt[0]] = value
3739 3739
3740 3740 return items
3741 3741
3742 3742 # convenient shortcut
3743 3743 dst = debugstacktrace
General Comments 0
You need to be logged in to leave comments. Login now