##// END OF EJS Templates
profiling: add an assertion to help pytype...
Matt Harbison -
r53297:5ff6fba7 default
parent child Browse files
Show More
@@ -1,358 +1,361
1 # profiling.py - profiling functions
1 # profiling.py - profiling functions
2 #
2 #
3 # Copyright 2016 Gregory Szorc <gregory.szorc@gmail.com>
3 # Copyright 2016 Gregory Szorc <gregory.szorc@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import annotations
8 from __future__ import annotations
9
9
10 import contextlib
10 import contextlib
11 import os
11 import os
12 import signal
12 import signal
13 import subprocess
13 import subprocess
14 import sys
14 import sys
15
15
16 from .i18n import _
16 from .i18n import _
17 from . import (
17 from . import (
18 encoding,
18 encoding,
19 error,
19 error,
20 extensions,
20 extensions,
21 pycompat,
21 pycompat,
22 util,
22 util,
23 )
23 )
24
24
25
25
26 def _loadprofiler(ui, profiler):
26 def _loadprofiler(ui, profiler):
27 """load profiler extension. return profile method, or None on failure"""
27 """load profiler extension. return profile method, or None on failure"""
28 extname = profiler
28 extname = profiler
29 extensions.loadall(ui, whitelist=[extname])
29 extensions.loadall(ui, whitelist=[extname])
30 try:
30 try:
31 mod = extensions.find(extname)
31 mod = extensions.find(extname)
32 except KeyError:
32 except KeyError:
33 return None
33 return None
34 else:
34 else:
35 return getattr(mod, 'profile', None)
35 return getattr(mod, 'profile', None)
36
36
37
37
38 @contextlib.contextmanager
38 @contextlib.contextmanager
39 def lsprofile(ui, fp):
39 def lsprofile(ui, fp):
40 format = ui.config(b'profiling', b'format')
40 format = ui.config(b'profiling', b'format')
41 field = ui.config(b'profiling', b'sort')
41 field = ui.config(b'profiling', b'sort')
42 limit = ui.configint(b'profiling', b'limit')
42 limit = ui.configint(b'profiling', b'limit')
43 climit = ui.configint(b'profiling', b'nested')
43 climit = ui.configint(b'profiling', b'nested')
44
44
45 if format not in [b'text', b'kcachegrind']:
45 if format not in [b'text', b'kcachegrind']:
46 ui.warn(_(b"unrecognized profiling format '%s' - Ignored\n") % format)
46 ui.warn(_(b"unrecognized profiling format '%s' - Ignored\n") % format)
47 format = b'text'
47 format = b'text'
48
48
49 try:
49 try:
50 from . import lsprof
50 from . import lsprof
51 except ImportError:
51 except ImportError:
52 raise error.Abort(
52 raise error.Abort(
53 _(
53 _(
54 b'lsprof not available - install from '
54 b'lsprof not available - install from '
55 b'http://codespeak.net/svn/user/arigo/hack/misc/lsprof/'
55 b'http://codespeak.net/svn/user/arigo/hack/misc/lsprof/'
56 )
56 )
57 )
57 )
58 p = lsprof.Profiler()
58 p = lsprof.Profiler()
59 try:
59 try:
60 p.enable(subcalls=True)
60 p.enable(subcalls=True)
61 except ValueError as exc:
61 except ValueError as exc:
62 if str(exc) != "Another profiling tool is already active":
62 if str(exc) != "Another profiling tool is already active":
63 raise
63 raise
64 if not hasattr(sys, "monitoring"):
64 if not hasattr(sys, "monitoring"):
65 raise
65 raise
66 # python >=3.12 prevent more than one profiler to run at the same
66 # python >=3.12 prevent more than one profiler to run at the same
67 # time, tries to improve the report to help the user understand
67 # time, tries to improve the report to help the user understand
68 # what is going on.
68 # what is going on.
69 other_tool_name = sys.monitoring.get_tool(sys.monitoring.PROFILER_ID)
69 other_tool_name = sys.monitoring.get_tool(sys.monitoring.PROFILER_ID)
70 if other_tool_name == "cProfile":
70 if other_tool_name == "cProfile":
71 msg = b'cannot recursively call `lsprof`'
71 msg = b'cannot recursively call `lsprof`'
72 raise error.Abort(msg) from None
72 raise error.Abort(msg) from None
73 else:
73 else:
74 tool = b'<unknown>'
74 tool = b'<unknown>'
75 if other_tool_name:
75 if other_tool_name:
76 tool = encoding.strtolocal(other_tool_name)
76 tool = encoding.strtolocal(other_tool_name)
77 m = b'failed to start "lsprofile"; another profiler already running: %s'
77 m = b'failed to start "lsprofile"; another profiler already running: %s'
78 raise error.Abort(_(m) % tool) from None
78 raise error.Abort(_(m) % tool) from None
79 try:
79 try:
80 yield
80 yield
81 finally:
81 finally:
82 p.disable()
82 p.disable()
83
83
84 if format == b'kcachegrind':
84 if format == b'kcachegrind':
85 from . import lsprofcalltree
85 from . import lsprofcalltree
86
86
87 calltree = lsprofcalltree.KCacheGrind(p)
87 calltree = lsprofcalltree.KCacheGrind(p)
88 calltree.output(fp)
88 calltree.output(fp)
89 else:
89 else:
90 # format == 'text'
90 # format == 'text'
91 stats = lsprof.Stats(p.getstats())
91 stats = lsprof.Stats(p.getstats())
92 stats.sort(pycompat.sysstr(field))
92 stats.sort(pycompat.sysstr(field))
93 stats.pprint(limit=limit, file=fp, climit=climit)
93 stats.pprint(limit=limit, file=fp, climit=climit)
94 fp.flush()
94 fp.flush()
95
95
96
96
97 @contextlib.contextmanager
97 @contextlib.contextmanager
98 def flameprofile(ui, fp):
98 def flameprofile(ui, fp):
99 try:
99 try:
100 from flamegraph import flamegraph # pytype: disable=import-error
100 from flamegraph import flamegraph # pytype: disable=import-error
101 except ImportError:
101 except ImportError:
102 raise error.Abort(
102 raise error.Abort(
103 _(
103 _(
104 b'flamegraph not available - install from '
104 b'flamegraph not available - install from '
105 b'https://github.com/evanhempel/python-flamegraph'
105 b'https://github.com/evanhempel/python-flamegraph'
106 )
106 )
107 )
107 )
108 # developer config: profiling.freq
108 # developer config: profiling.freq
109 freq = ui.configint(b'profiling', b'freq')
109 freq = ui.configint(b'profiling', b'freq')
110 filter_ = None
110 filter_ = None
111 collapse_recursion = True
111 collapse_recursion = True
112 thread = flamegraph.ProfileThread(
112 thread = flamegraph.ProfileThread(
113 fp, 1.0 / freq, filter_, collapse_recursion
113 fp, 1.0 / freq, filter_, collapse_recursion
114 )
114 )
115 start_time = util.timer()
115 start_time = util.timer()
116 try:
116 try:
117 thread.start()
117 thread.start()
118 yield
118 yield
119 finally:
119 finally:
120 thread.stop()
120 thread.stop()
121 thread.join()
121 thread.join()
122 m = b'Collected %d stack frames (%d unique) in %2.2f seconds.'
122 m = b'Collected %d stack frames (%d unique) in %2.2f seconds.'
123 m %= (
123 m %= (
124 (
124 (
125 util.timer() - start_time,
125 util.timer() - start_time,
126 thread.num_frames(),
126 thread.num_frames(),
127 thread.num_frames(unique=True),
127 thread.num_frames(unique=True),
128 ),
128 ),
129 )
129 )
130 print(m, flush=True)
130 print(m, flush=True)
131
131
132
132
133 @contextlib.contextmanager
133 @contextlib.contextmanager
134 def statprofile(ui, fp):
134 def statprofile(ui, fp):
135 from . import statprof
135 from . import statprof
136
136
137 freq = ui.configint(b'profiling', b'freq')
137 freq = ui.configint(b'profiling', b'freq')
138 if freq > 0:
138 if freq > 0:
139 # Cannot reset when profiler is already active. So silently no-op.
139 # Cannot reset when profiler is already active. So silently no-op.
140 if statprof.state.profile_level == 0:
140 if statprof.state.profile_level == 0:
141 statprof.reset(freq)
141 statprof.reset(freq)
142 else:
142 else:
143 ui.warn(_(b"invalid sampling frequency '%s' - ignoring\n") % freq)
143 ui.warn(_(b"invalid sampling frequency '%s' - ignoring\n") % freq)
144
144
145 track = ui.config(
145 track = ui.config(
146 b'profiling', b'time-track', pycompat.iswindows and b'cpu' or b'real'
146 b'profiling', b'time-track', pycompat.iswindows and b'cpu' or b'real'
147 )
147 )
148 statprof.start(mechanism=b'thread', track=track)
148 statprof.start(mechanism=b'thread', track=track)
149
149
150 try:
150 try:
151 yield
151 yield
152 finally:
152 finally:
153 data = statprof.stop()
153 data = statprof.stop()
154
154
155 profformat = ui.config(b'profiling', b'statformat')
155 profformat = ui.config(b'profiling', b'statformat')
156
156
157 formats = {
157 formats = {
158 b'byline': statprof.DisplayFormats.ByLine,
158 b'byline': statprof.DisplayFormats.ByLine,
159 b'bymethod': statprof.DisplayFormats.ByMethod,
159 b'bymethod': statprof.DisplayFormats.ByMethod,
160 b'hotpath': statprof.DisplayFormats.Hotpath,
160 b'hotpath': statprof.DisplayFormats.Hotpath,
161 b'json': statprof.DisplayFormats.Json,
161 b'json': statprof.DisplayFormats.Json,
162 b'chrome': statprof.DisplayFormats.Chrome,
162 b'chrome': statprof.DisplayFormats.Chrome,
163 }
163 }
164
164
165 if profformat in formats:
165 if profformat in formats:
166 displayformat = formats[profformat]
166 displayformat = formats[profformat]
167 else:
167 else:
168 ui.warn(_(b'unknown profiler output format: %s\n') % profformat)
168 ui.warn(_(b'unknown profiler output format: %s\n') % profformat)
169 displayformat = statprof.DisplayFormats.Hotpath
169 displayformat = statprof.DisplayFormats.Hotpath
170
170
171 kwargs = {}
171 kwargs = {}
172
172
173 def fraction(s):
173 def fraction(s):
174 if isinstance(s, (float, int)):
174 if isinstance(s, (float, int)):
175 return float(s)
175 return float(s)
176 if s.endswith(b'%'):
176 if s.endswith(b'%'):
177 v = float(s[:-1]) / 100
177 v = float(s[:-1]) / 100
178 else:
178 else:
179 v = float(s)
179 v = float(s)
180 if 0 <= v <= 1:
180 if 0 <= v <= 1:
181 return v
181 return v
182 raise ValueError(s)
182 raise ValueError(s)
183
183
184 if profformat == b'chrome':
184 if profformat == b'chrome':
185 showmin = ui.configwith(fraction, b'profiling', b'showmin', 0.005)
185 showmin = ui.configwith(fraction, b'profiling', b'showmin', 0.005)
186 showmax = ui.configwith(fraction, b'profiling', b'showmax')
186 showmax = ui.configwith(fraction, b'profiling', b'showmax')
187 kwargs.update(minthreshold=showmin, maxthreshold=showmax)
187 kwargs.update(minthreshold=showmin, maxthreshold=showmax)
188 elif profformat == b'hotpath':
188 elif profformat == b'hotpath':
189 # inconsistent config: profiling.showmin
189 # inconsistent config: profiling.showmin
190 limit = ui.configwith(fraction, b'profiling', b'showmin', 0.05)
190 limit = ui.configwith(fraction, b'profiling', b'showmin', 0.05)
191 kwargs['limit'] = limit
191 kwargs['limit'] = limit
192 showtime = ui.configbool(b'profiling', b'showtime')
192 showtime = ui.configbool(b'profiling', b'showtime')
193 kwargs['showtime'] = showtime
193 kwargs['showtime'] = showtime
194
194
195 statprof.display(fp, data=data, format=displayformat, **kwargs)
195 statprof.display(fp, data=data, format=displayformat, **kwargs)
196 fp.flush()
196 fp.flush()
197
197
198
198
199 @contextlib.contextmanager
199 @contextlib.contextmanager
200 def pyspy_profile(ui, fp):
200 def pyspy_profile(ui, fp):
201 exe = ui.config(b'profiling', b'py-spy.exe')
201 exe = ui.config(b'profiling', b'py-spy.exe')
202
202
203 freq = ui.configint(b'profiling', b'py-spy.freq')
203 freq = ui.configint(b'profiling', b'py-spy.freq')
204
204
205 format = ui.config(b'profiling', b'py-spy.format')
205 format = ui.config(b'profiling', b'py-spy.format')
206
206
207 fd = fp.fileno()
207 fd = fp.fileno()
208
208
209 output_path = "/dev/fd/%d" % (fd)
209 output_path = "/dev/fd/%d" % (fd)
210
210
211 my_pid = os.getpid()
211 my_pid = os.getpid()
212
212
213 cmd = [
213 cmd = [
214 exe,
214 exe,
215 "record",
215 "record",
216 "--pid",
216 "--pid",
217 str(my_pid),
217 str(my_pid),
218 "--native",
218 "--native",
219 "--rate",
219 "--rate",
220 str(freq),
220 str(freq),
221 "--output",
221 "--output",
222 output_path,
222 output_path,
223 ]
223 ]
224
224
225 if format:
225 if format:
226 cmd.extend(["--format", format])
226 cmd.extend(["--format", format])
227
227
228 proc = subprocess.Popen(
228 proc = subprocess.Popen(
229 cmd,
229 cmd,
230 pass_fds={fd},
230 pass_fds={fd},
231 stdout=subprocess.PIPE,
231 stdout=subprocess.PIPE,
232 )
232 )
233
233
234 _ = proc.stdout.readline()
234 _ = proc.stdout.readline()
235
235
236 try:
236 try:
237 yield
237 yield
238 finally:
238 finally:
239 os.kill(proc.pid, signal.SIGINT)
239 os.kill(proc.pid, signal.SIGINT)
240 proc.communicate()
240 proc.communicate()
241
241
242
242
243 class profile:
243 class profile:
244 """Start profiling.
244 """Start profiling.
245
245
246 Profiling is active when the context manager is active. When the context
246 Profiling is active when the context manager is active. When the context
247 manager exits, profiling results will be written to the configured output.
247 manager exits, profiling results will be written to the configured output.
248 """
248 """
249
249
250 def __init__(self, ui, enabled=True):
250 def __init__(self, ui, enabled=True):
251 self._ui = ui
251 self._ui = ui
252 self._output = None
252 self._output = None
253 self._fp = None
253 self._fp = None
254 self._fpdoclose = True
254 self._fpdoclose = True
255 self._flushfp = None
255 self._flushfp = None
256 self._profiler = None
256 self._profiler = None
257 self._enabled = enabled
257 self._enabled = enabled
258 self._entered = False
258 self._entered = False
259 self._started = False
259 self._started = False
260
260
261 def __enter__(self):
261 def __enter__(self):
262 self._entered = True
262 self._entered = True
263 if self._enabled:
263 if self._enabled:
264 self.start()
264 self.start()
265 return self
265 return self
266
266
267 def start(self):
267 def start(self):
268 """Start profiling.
268 """Start profiling.
269
269
270 The profiling will stop at the context exit.
270 The profiling will stop at the context exit.
271
271
272 If the profiler was already started, this has no effect."""
272 If the profiler was already started, this has no effect."""
273 if not self._entered:
273 if not self._entered:
274 raise error.ProgrammingError(b'use a context manager to start')
274 raise error.ProgrammingError(b'use a context manager to start')
275 if self._started:
275 if self._started:
276 return
276 return
277 self._started = True
277 self._started = True
278 profiler = encoding.environ.get(b'HGPROF')
278 profiler = encoding.environ.get(b'HGPROF')
279 proffn = None
279 proffn = None
280 if profiler is None:
280 if profiler is None:
281 profiler = self._ui.config(b'profiling', b'type')
281 profiler = self._ui.config(b'profiling', b'type')
282 if profiler not in (b'ls', b'stat', b'flame', b'py-spy'):
282 if profiler not in (b'ls', b'stat', b'flame', b'py-spy'):
283 # try load profiler from extension with the same name
283 # try load profiler from extension with the same name
284 proffn = _loadprofiler(self._ui, profiler)
284 proffn = _loadprofiler(self._ui, profiler)
285 if proffn is None:
285 if proffn is None:
286 self._ui.warn(
286 self._ui.warn(
287 _(b"unrecognized profiler '%s' - ignored\n") % profiler
287 _(b"unrecognized profiler '%s' - ignored\n") % profiler
288 )
288 )
289 profiler = b'stat'
289 profiler = b'stat'
290
290
291 self._output = self._ui.config(b'profiling', b'output')
291 self._output = self._ui.config(b'profiling', b'output')
292
292
293 try:
293 try:
294 if self._output == b'blackbox':
294 if self._output == b'blackbox':
295 self._fp = util.stringio()
295 self._fp = util.stringio()
296 elif self._output:
296 elif self._output:
297 path = util.expandpath(self._output)
297 path = util.expandpath(self._output)
298 self._fp = open(path, 'wb')
298 self._fp = open(path, 'wb')
299 elif pycompat.iswindows:
299 elif pycompat.iswindows:
300 # parse escape sequence by win32print()
300 # parse escape sequence by win32print()
301 class uifp:
301 class uifp:
302 def __init__(self, ui):
302 def __init__(self, ui):
303 self._ui = ui
303 self._ui = ui
304
304
305 def write(self, data):
305 def write(self, data):
306 self._ui.write_err(data)
306 self._ui.write_err(data)
307
307
308 def flush(self):
308 def flush(self):
309 self._ui.flush()
309 self._ui.flush()
310
310
311 self._fpdoclose = False
311 self._fpdoclose = False
312 self._fp = uifp(self._ui)
312 self._fp = uifp(self._ui)
313 else:
313 else:
314 self._fpdoclose = False
314 self._fpdoclose = False
315 self._fp = self._ui.ferr
315 self._fp = self._ui.ferr
316 # Ensure we've flushed fout before writing to ferr.
316 # Ensure we've flushed fout before writing to ferr.
317 self._flushfp = self._ui.fout
317 self._flushfp = self._ui.fout
318
318
319 if proffn is not None:
319 if proffn is not None:
320 pass
320 pass
321 elif profiler == b'ls':
321 elif profiler == b'ls':
322 proffn = lsprofile
322 proffn = lsprofile
323 elif profiler == b'flame':
323 elif profiler == b'flame':
324 proffn = flameprofile
324 proffn = flameprofile
325 elif profiler == b'py-spy':
325 elif profiler == b'py-spy':
326 proffn = pyspy_profile
326 proffn = pyspy_profile
327 else:
327 else:
328 proffn = statprofile
328 proffn = statprofile
329
329
330 self._profiler = proffn(self._ui, self._fp)
330 self._profiler = proffn(self._ui, self._fp)
331 self._profiler.__enter__()
331 self._profiler.__enter__()
332 except: # re-raises
332 except: # re-raises
333 self._closefp()
333 self._closefp()
334 raise
334 raise
335
335
336 def __exit__(self, exception_type, exception_value, traceback):
336 def __exit__(self, exception_type, exception_value, traceback):
337 propagate = None
337 propagate = None
338 if self._profiler is not None:
338 if self._profiler is not None:
339 self._uiflush()
339 self._uiflush()
340 propagate = self._profiler.__exit__(
340 propagate = self._profiler.__exit__(
341 exception_type, exception_value, traceback
341 exception_type, exception_value, traceback
342 )
342 )
343 if self._output == b'blackbox':
343 if self._output == b'blackbox':
344 val = b'Profile:\n%s' % self._fp.getvalue()
344 fp = self._fp
345 # Help pytype: blackbox output uses io.BytesIO instead of a file
346 assert isinstance(fp, util.stringio)
347 val = b'Profile:\n%s' % fp.getvalue()
345 # ui.log treats the input as a format string,
348 # ui.log treats the input as a format string,
346 # so we need to escape any % signs.
349 # so we need to escape any % signs.
347 val = val.replace(b'%', b'%%')
350 val = val.replace(b'%', b'%%')
348 self._ui.log(b'profile', val)
351 self._ui.log(b'profile', val)
349 self._closefp()
352 self._closefp()
350 return propagate
353 return propagate
351
354
352 def _closefp(self):
355 def _closefp(self):
353 if self._fpdoclose and self._fp is not None:
356 if self._fpdoclose and self._fp is not None:
354 self._fp.close()
357 self._fp.close()
355
358
356 def _uiflush(self):
359 def _uiflush(self):
357 if self._flushfp:
360 if self._flushfp:
358 self._flushfp.flush()
361 self._flushfp.flush()
General Comments 0
You need to be logged in to leave comments. Login now