##// END OF EJS Templates
Move strip_ansi function to utils
Thomas Kluyver -
Show More
@@ -1,178 +1,166 b''
1 1 """Filters for processing ANSI colors within Jinja templates.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (c) 2013, the IPython Development Team.
5 5 #
6 6 # Distributed under the terms of the Modified BSD License.
7 7 #
8 8 # The full license is in the file COPYING.txt, distributed with this software.
9 9 #-----------------------------------------------------------------------------
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Imports
13 13 #-----------------------------------------------------------------------------
14 14
15 15 import re
16 16 from IPython.utils import coloransi
17 from IPython.utils.text import strip_ansi
17 18
18 19 #-----------------------------------------------------------------------------
19 20 # Classes and functions
20 21 #-----------------------------------------------------------------------------
21 22
22 23 __all__ = [
23 24 'strip_ansi',
24 25 'ansi2html',
25 26 'single_ansi2latex',
26 27 'ansi2latex'
27 28 ]
28 29
29 def strip_ansi(source):
30 """
31 Remove ansi from text
32
33 Parameters
34 ----------
35 source : str
36 Source to remove the ansi from
37 """
38
39 return re.sub(r'\033\[(\d|;)+?m', '', source)
40
41
42 30 ansi_colormap = {
43 31 '30': 'ansiblack',
44 32 '31': 'ansired',
45 33 '32': 'ansigreen',
46 34 '33': 'ansiyellow',
47 35 '34': 'ansiblue',
48 36 '35': 'ansipurple',
49 37 '36': 'ansicyan',
50 38 '37': 'ansigrey',
51 39 '01': 'ansibold',
52 40 }
53 41
54 42 html_escapes = {
55 43 '<': '&lt;',
56 44 '>': '&gt;',
57 45 "'": '&apos;',
58 46 '"': '&quot;',
59 47 '`': '&#96;',
60 48 }
61 49 ansi_re = re.compile('\x1b' + r'\[([\dA-Fa-f;]*?)m')
62 50
63 51 def ansi2html(text):
64 52 """
65 53 Convert ansi colors to html colors.
66 54
67 55 Parameters
68 56 ----------
69 57 text : str
70 58 Text containing ansi colors to convert to html
71 59 """
72 60
73 61 # do ampersand first
74 62 text = text.replace('&', '&amp;')
75 63
76 64 for c, escape in html_escapes.items():
77 65 text = text.replace(c, escape)
78 66
79 67 m = ansi_re.search(text)
80 68 opened = False
81 69 cmds = []
82 70 opener = ''
83 71 closer = ''
84 72 while m:
85 73 cmds = m.groups()[0].split(';')
86 74 closer = '</span>' if opened else ''
87 75
88 76 # True if there is there more than one element in cmds, *or*
89 77 # if there is only one but it is not equal to a string of zeroes.
90 78 opened = len(cmds) > 1 or cmds[0] != '0' * len(cmds[0])
91 79 classes = []
92 80 for cmd in cmds:
93 81 if cmd in ansi_colormap:
94 82 classes.append(ansi_colormap[cmd])
95 83
96 84 if classes:
97 85 opener = '<span class="%s">' % (' '.join(classes))
98 86 else:
99 87 opener = ''
100 88 text = re.sub(ansi_re, closer + opener, text, 1)
101 89
102 90 m = ansi_re.search(text)
103 91
104 92 if opened:
105 93 text += '</span>'
106 94 return text
107 95
108 96
109 97 def single_ansi2latex(code):
110 98 """Converts single ansi markup to latex format.
111 99
112 100 Return latex code and number of open brackets.
113 101
114 102 Accepts codes like '\x1b[1;32m' (bold, red) and the short form '\x1b[32m' (red)
115 103
116 104 Colors are matched to those defined in coloransi, which defines colors
117 105 using the 0, 1 (bold) and 5 (blinking) styles. Styles 1 and 5 are
118 106 interpreted as bold. All other styles are mapped to 0. Note that in
119 107 coloransi, a style of 1 does not just mean bold; for example, Brown is
120 108 "0;33", but Yellow is "1;33". An empty string is returned for unrecognised
121 109 codes and the "reset" code '\x1b[m'.
122 110 """
123 111 components = code.split(';')
124 112 if len(components) > 1:
125 113 # Style is digits after '['
126 114 style = int(components[0].split('[')[-1])
127 115 color = components[1][:-1]
128 116 else:
129 117 style = 0
130 118 color = components[0][-3:-1]
131 119
132 120 # If the style is not normal (0), bold (1) or blinking (5) then treat it as normal
133 121 if style not in [0, 1, 5]:
134 122 style = 0
135 123
136 124 for name, tcode in coloransi.color_templates:
137 125 tstyle, tcolor = tcode.split(';')
138 126 tstyle = int(tstyle)
139 127 if tstyle == style and tcolor == color:
140 128 break
141 129 else:
142 130 return '', 0
143 131
144 132 if style == 5:
145 133 name = name[5:] # BlinkRed -> Red, etc
146 134 name = name.lower()
147 135
148 136 if style in [1, 5]:
149 137 return r'\textbf{\color{'+name+'}', 1
150 138 else:
151 139 return r'{\color{'+name+'}', 1
152 140
153 141 def ansi2latex(text):
154 142 """Converts ansi formated text to latex version
155 143
156 144 based on https://bitbucket.org/birkenfeld/sphinx-contrib/ansi.py
157 145 """
158 146 color_pattern = re.compile('\x1b\\[([^m]*)m')
159 147 last_end = 0
160 148 openbrack = 0
161 149 outstring = ''
162 150 for match in color_pattern.finditer(text):
163 151 head = text[last_end:match.start()]
164 152 outstring += head
165 153 if openbrack:
166 154 outstring += '}'*openbrack
167 155 openbrack = 0
168 156 code = match.group()
169 157 if not (code == coloransi.TermColors.Normal or openbrack):
170 158 texform, openbrack = single_ansi2latex(code)
171 159 outstring += texform
172 160 last_end = match.end()
173 161
174 162 # Add the remainer of the string and THEN close any remaining color brackets.
175 163 outstring += text[last_end:]
176 164 if openbrack:
177 165 outstring += '}'*openbrack
178 166 return outstring.strip()
@@ -1,672 +1,672 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Process Controller
3 3
4 4 This module runs one or more subprocesses which will actually run the IPython
5 5 test suite.
6 6
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 from __future__ import print_function
13 13
14 14 import argparse
15 15 import json
16 16 import multiprocessing.pool
17 17 import os
18 18 import shutil
19 19 import signal
20 20 import sys
21 21 import subprocess
22 22 import time
23 23 import re
24 24
25 25 from .iptest import have, test_group_names as py_test_group_names, test_sections, StreamCapturer
26 26 from IPython.utils.path import compress_user
27 27 from IPython.utils.py3compat import bytes_to_str
28 28 from IPython.utils.sysinfo import get_sys_info
29 29 from IPython.utils.tempdir import TemporaryDirectory
30 from IPython.nbconvert.filters.ansi import strip_ansi
30 from IPython.utils.text import strip_ansi
31 31
32 32 try:
33 33 # Python >= 3.3
34 34 from subprocess import TimeoutExpired
35 35 def popen_wait(p, timeout):
36 36 return p.wait(timeout)
37 37 except ImportError:
38 38 class TimeoutExpired(Exception):
39 39 pass
40 40 def popen_wait(p, timeout):
41 41 """backport of Popen.wait from Python 3"""
42 42 for i in range(int(10 * timeout)):
43 43 if p.poll() is not None:
44 44 return
45 45 time.sleep(0.1)
46 46 if p.poll() is None:
47 47 raise TimeoutExpired
48 48
49 49 NOTEBOOK_SHUTDOWN_TIMEOUT = 10
50 50
51 51 class TestController(object):
52 52 """Run tests in a subprocess
53 53 """
54 54 #: str, IPython test suite to be executed.
55 55 section = None
56 56 #: list, command line arguments to be executed
57 57 cmd = None
58 58 #: dict, extra environment variables to set for the subprocess
59 59 env = None
60 60 #: list, TemporaryDirectory instances to clear up when the process finishes
61 61 dirs = None
62 62 #: subprocess.Popen instance
63 63 process = None
64 64 #: str, process stdout+stderr
65 65 stdout = None
66 66
67 67 def __init__(self):
68 68 self.cmd = []
69 69 self.env = {}
70 70 self.dirs = []
71 71
72 72 def setup(self):
73 73 """Create temporary directories etc.
74 74
75 75 This is only called when we know the test group will be run. Things
76 76 created here may be cleaned up by self.cleanup().
77 77 """
78 78 pass
79 79
80 80 def launch(self, buffer_output=False):
81 81 # print('*** ENV:', self.env) # dbg
82 82 # print('*** CMD:', self.cmd) # dbg
83 83 env = os.environ.copy()
84 84 env.update(self.env)
85 85 output = subprocess.PIPE if buffer_output else None
86 86 stdout = subprocess.STDOUT if buffer_output else None
87 87 self.process = subprocess.Popen(self.cmd, stdout=output,
88 88 stderr=stdout, env=env)
89 89
90 90 def wait(self):
91 91 self.stdout, _ = self.process.communicate()
92 92 return self.process.returncode
93 93
94 94 def print_extra_info(self):
95 95 """Print extra information about this test run.
96 96
97 97 If we're running in parallel and showing the concise view, this is only
98 98 called if the test group fails. Otherwise, it's called before the test
99 99 group is started.
100 100
101 101 The base implementation does nothing, but it can be overridden by
102 102 subclasses.
103 103 """
104 104 return
105 105
106 106 def cleanup_process(self):
107 107 """Cleanup on exit by killing any leftover processes."""
108 108 subp = self.process
109 109 if subp is None or (subp.poll() is not None):
110 110 return # Process doesn't exist, or is already dead.
111 111
112 112 try:
113 113 print('Cleaning up stale PID: %d' % subp.pid)
114 114 subp.kill()
115 115 except: # (OSError, WindowsError) ?
116 116 # This is just a best effort, if we fail or the process was
117 117 # really gone, ignore it.
118 118 pass
119 119 else:
120 120 for i in range(10):
121 121 if subp.poll() is None:
122 122 time.sleep(0.1)
123 123 else:
124 124 break
125 125
126 126 if subp.poll() is None:
127 127 # The process did not die...
128 128 print('... failed. Manual cleanup may be required.')
129 129
130 130 def cleanup(self):
131 131 "Kill process if it's still alive, and clean up temporary directories"
132 132 self.cleanup_process()
133 133 for td in self.dirs:
134 134 td.cleanup()
135 135
136 136 __del__ = cleanup
137 137
138 138
139 139 class PyTestController(TestController):
140 140 """Run Python tests using IPython.testing.iptest"""
141 141 #: str, Python command to execute in subprocess
142 142 pycmd = None
143 143
144 144 def __init__(self, section, options):
145 145 """Create new test runner."""
146 146 TestController.__init__(self)
147 147 self.section = section
148 148 # pycmd is put into cmd[2] in PyTestController.launch()
149 149 self.cmd = [sys.executable, '-c', None, section]
150 150 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
151 151 self.options = options
152 152
153 153 def setup(self):
154 154 ipydir = TemporaryDirectory()
155 155 self.dirs.append(ipydir)
156 156 self.env['IPYTHONDIR'] = ipydir.name
157 157 self.workingdir = workingdir = TemporaryDirectory()
158 158 self.dirs.append(workingdir)
159 159 self.env['IPTEST_WORKING_DIR'] = workingdir.name
160 160 # This means we won't get odd effects from our own matplotlib config
161 161 self.env['MPLCONFIGDIR'] = workingdir.name
162 162
163 163 # From options:
164 164 if self.options.xunit:
165 165 self.add_xunit()
166 166 if self.options.coverage:
167 167 self.add_coverage()
168 168 self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams
169 169 self.cmd.extend(self.options.extra_args)
170 170
171 171 @property
172 172 def will_run(self):
173 173 try:
174 174 return test_sections[self.section].will_run
175 175 except KeyError:
176 176 return True
177 177
178 178 def add_xunit(self):
179 179 xunit_file = os.path.abspath(self.section + '.xunit.xml')
180 180 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
181 181
182 182 def add_coverage(self):
183 183 try:
184 184 sources = test_sections[self.section].includes
185 185 except KeyError:
186 186 sources = ['IPython']
187 187
188 188 coverage_rc = ("[run]\n"
189 189 "data_file = {data_file}\n"
190 190 "source =\n"
191 191 " {source}\n"
192 192 ).format(data_file=os.path.abspath('.coverage.'+self.section),
193 193 source="\n ".join(sources))
194 194 config_file = os.path.join(self.workingdir.name, '.coveragerc')
195 195 with open(config_file, 'w') as f:
196 196 f.write(coverage_rc)
197 197
198 198 self.env['COVERAGE_PROCESS_START'] = config_file
199 199 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
200 200
201 201 def launch(self, buffer_output=False):
202 202 self.cmd[2] = self.pycmd
203 203 super(PyTestController, self).launch(buffer_output=buffer_output)
204 204
205 205
206 206 js_prefix = 'js/'
207 207
208 208 def get_js_test_dir():
209 209 import IPython.html.tests as t
210 210 return os.path.join(os.path.dirname(t.__file__), '')
211 211
212 212 def all_js_groups():
213 213 import glob
214 214 test_dir = get_js_test_dir()
215 215 all_subdirs = glob.glob(test_dir + '[!_]*/')
216 216 return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs]
217 217
218 218 class JSController(TestController):
219 219 """Run CasperJS tests """
220 220 requirements = ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3']
221 221 display_slimer_output = False
222 222
223 223 def __init__(self, section, enabled=True, engine='phantomjs'):
224 224 """Create new test runner."""
225 225 TestController.__init__(self)
226 226 self.engine = engine
227 227 self.section = section
228 228 self.enabled = enabled
229 229 self.slimer_failure = re.compile('^FAIL.*', flags=re.MULTILINE)
230 230 js_test_dir = get_js_test_dir()
231 231 includes = '--includes=' + os.path.join(js_test_dir,'util.js')
232 232 test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
233 233 self.cmd = ['casperjs', 'test', includes, test_cases, '--engine=%s' % self.engine]
234 234
235 235 def setup(self):
236 236 self.ipydir = TemporaryDirectory()
237 237 self.nbdir = TemporaryDirectory()
238 238 self.dirs.append(self.ipydir)
239 239 self.dirs.append(self.nbdir)
240 240 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir1', u'sub βˆ‚ir 1a')))
241 241 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir2', u'sub βˆ‚ir 1b')))
242 242
243 243 # start the ipython notebook, so we get the port number
244 244 self.server_port = 0
245 245 self._init_server()
246 246 if self.server_port:
247 247 self.cmd.append("--port=%i" % self.server_port)
248 248 else:
249 249 # don't launch tests if the server didn't start
250 250 self.cmd = [sys.executable, '-c', 'raise SystemExit(1)']
251 251
252 252 def launch(self, buffer_output):
253 253 # If the engine is SlimerJS, we need to buffer the output because
254 254 # SlimerJS does not support exit codes, so CasperJS always returns 0.
255 255 if self.engine == 'slimerjs' and not buffer_output:
256 256 self.display_slimer_output = True
257 257 return super(JSController, self).launch(buffer_output=True)
258 258
259 259 else:
260 260 return super(JSController, self).launch(buffer_output=buffer_output)
261 261
262 262 def wait(self, *pargs, **kwargs):
263 263 """Wait for the JSController to finish"""
264 264 ret = super(JSController, self).wait(*pargs, **kwargs)
265 265 # If this is a SlimerJS controller, check the captured stdout for
266 266 # errors. Otherwise, just return the return code.
267 267 if self.engine == 'slimerjs':
268 268 stdout = bytes_to_str(self.stdout)
269 269 if self.display_slimer_output:
270 270 print(stdout)
271 271 if ret != 0:
272 272 # This could still happen e.g. if it's stopped by SIGINT
273 273 return ret
274 274 return bool(self.slimer_failure.search(strip_ansi(stdout)))
275 275 else:
276 276 return ret
277 277
278 278 def print_extra_info(self):
279 279 print("Running tests with notebook directory %r" % self.nbdir.name)
280 280
281 281 @property
282 282 def will_run(self):
283 283 return self.enabled and all(have[a] for a in self.requirements + [self.engine])
284 284
285 285 def _init_server(self):
286 286 "Start the notebook server in a separate process"
287 287 self.server_command = command = [sys.executable,
288 288 '-m', 'IPython.html',
289 289 '--no-browser',
290 290 '--ipython-dir', self.ipydir.name,
291 291 '--notebook-dir', self.nbdir.name,
292 292 ]
293 293 # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
294 294 # which run afoul of ipc's maximum path length.
295 295 if sys.platform.startswith('linux'):
296 296 command.append('--KernelManager.transport=ipc')
297 297 self.stream_capturer = c = StreamCapturer()
298 298 c.start()
299 299 self.server = subprocess.Popen(command, stdout=c.writefd, stderr=subprocess.STDOUT)
300 300 self.server_info_file = os.path.join(self.ipydir.name,
301 301 'profile_default', 'security', 'nbserver-%i.json' % self.server.pid
302 302 )
303 303 self._wait_for_server()
304 304
305 305 def _wait_for_server(self):
306 306 """Wait 30 seconds for the notebook server to start"""
307 307 for i in range(300):
308 308 if self.server.poll() is not None:
309 309 return self._failed_to_start()
310 310 if os.path.exists(self.server_info_file):
311 311 try:
312 312 self._load_server_info()
313 313 except ValueError:
314 314 # If the server is halfway through writing the file, we may
315 315 # get invalid JSON; it should be ready next iteration.
316 316 pass
317 317 else:
318 318 return
319 319 time.sleep(0.1)
320 320 print("Notebook server-info file never arrived: %s" % self.server_info_file,
321 321 file=sys.stderr
322 322 )
323 323
324 324 def _failed_to_start(self):
325 325 """Notebook server exited prematurely"""
326 326 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
327 327 print("Notebook failed to start: ", file=sys.stderr)
328 328 print(self.server_command)
329 329 print(captured, file=sys.stderr)
330 330
331 331 def _load_server_info(self):
332 332 """Notebook server started, load connection info from JSON"""
333 333 with open(self.server_info_file) as f:
334 334 info = json.load(f)
335 335 self.server_port = info['port']
336 336
337 337 def cleanup(self):
338 338 try:
339 339 self.server.terminate()
340 340 except OSError:
341 341 # already dead
342 342 pass
343 343 # wait 10s for the server to shutdown
344 344 try:
345 345 popen_wait(self.server, NOTEBOOK_SHUTDOWN_TIMEOUT)
346 346 except TimeoutExpired:
347 347 # server didn't terminate, kill it
348 348 try:
349 349 print("Failed to terminate notebook server, killing it.",
350 350 file=sys.stderr
351 351 )
352 352 self.server.kill()
353 353 except OSError:
354 354 # already dead
355 355 pass
356 356 # wait another 10s
357 357 try:
358 358 popen_wait(self.server, NOTEBOOK_SHUTDOWN_TIMEOUT)
359 359 except TimeoutExpired:
360 360 print("Notebook server still running (%s)" % self.server_info_file,
361 361 file=sys.stderr
362 362 )
363 363
364 364 self.stream_capturer.halt()
365 365 TestController.cleanup(self)
366 366
367 367
368 368 def prepare_controllers(options):
369 369 """Returns two lists of TestController instances, those to run, and those
370 370 not to run."""
371 371 testgroups = options.testgroups
372 372 if testgroups:
373 373 if 'js' in testgroups:
374 374 js_testgroups = all_js_groups()
375 375 else:
376 376 js_testgroups = [g for g in testgroups if g.startswith(js_prefix)]
377 377
378 378 py_testgroups = [g for g in testgroups if g not in ['js'] + js_testgroups]
379 379 else:
380 380 py_testgroups = py_test_group_names
381 381 if not options.all:
382 382 js_testgroups = []
383 383 test_sections['parallel'].enabled = False
384 384 else:
385 385 js_testgroups = all_js_groups()
386 386
387 387 engine = 'slimerjs' if options.slimerjs else 'phantomjs'
388 388 c_js = [JSController(name, engine=engine) for name in js_testgroups]
389 389 c_py = [PyTestController(name, options) for name in py_testgroups]
390 390
391 391 controllers = c_py + c_js
392 392 to_run = [c for c in controllers if c.will_run]
393 393 not_run = [c for c in controllers if not c.will_run]
394 394 return to_run, not_run
395 395
396 396 def do_run(controller, buffer_output=True):
397 397 """Setup and run a test controller.
398 398
399 399 If buffer_output is True, no output is displayed, to avoid it appearing
400 400 interleaved. In this case, the caller is responsible for displaying test
401 401 output on failure.
402 402
403 403 Returns
404 404 -------
405 405 controller : TestController
406 406 The same controller as passed in, as a convenience for using map() type
407 407 APIs.
408 408 exitcode : int
409 409 The exit code of the test subprocess. Non-zero indicates failure.
410 410 """
411 411 try:
412 412 try:
413 413 controller.setup()
414 414 if not buffer_output:
415 415 controller.print_extra_info()
416 416 controller.launch(buffer_output=buffer_output)
417 417 except Exception:
418 418 import traceback
419 419 traceback.print_exc()
420 420 return controller, 1 # signal failure
421 421
422 422 exitcode = controller.wait()
423 423 return controller, exitcode
424 424
425 425 except KeyboardInterrupt:
426 426 return controller, -signal.SIGINT
427 427 finally:
428 428 controller.cleanup()
429 429
430 430 def report():
431 431 """Return a string with a summary report of test-related variables."""
432 432 inf = get_sys_info()
433 433 out = []
434 434 def _add(name, value):
435 435 out.append((name, value))
436 436
437 437 _add('IPython version', inf['ipython_version'])
438 438 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
439 439 _add('IPython package', compress_user(inf['ipython_path']))
440 440 _add('Python version', inf['sys_version'].replace('\n',''))
441 441 _add('sys.executable', compress_user(inf['sys_executable']))
442 442 _add('Platform', inf['platform'])
443 443
444 444 width = max(len(n) for (n,v) in out)
445 445 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
446 446
447 447 avail = []
448 448 not_avail = []
449 449
450 450 for k, is_avail in have.items():
451 451 if is_avail:
452 452 avail.append(k)
453 453 else:
454 454 not_avail.append(k)
455 455
456 456 if avail:
457 457 out.append('\nTools and libraries available at test time:\n')
458 458 avail.sort()
459 459 out.append(' ' + ' '.join(avail)+'\n')
460 460
461 461 if not_avail:
462 462 out.append('\nTools and libraries NOT available at test time:\n')
463 463 not_avail.sort()
464 464 out.append(' ' + ' '.join(not_avail)+'\n')
465 465
466 466 return ''.join(out)
467 467
468 468 def run_iptestall(options):
469 469 """Run the entire IPython test suite by calling nose and trial.
470 470
471 471 This function constructs :class:`IPTester` instances for all IPython
472 472 modules and package and then runs each of them. This causes the modules
473 473 and packages of IPython to be tested each in their own subprocess using
474 474 nose.
475 475
476 476 Parameters
477 477 ----------
478 478
479 479 All parameters are passed as attributes of the options object.
480 480
481 481 testgroups : list of str
482 482 Run only these sections of the test suite. If empty, run all the available
483 483 sections.
484 484
485 485 fast : int or None
486 486 Run the test suite in parallel, using n simultaneous processes. If None
487 487 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
488 488
489 489 inc_slow : bool
490 490 Include slow tests, like IPython.parallel. By default, these tests aren't
491 491 run.
492 492
493 493 slimerjs : bool
494 494 Use slimerjs if it's installed instead of phantomjs for casperjs tests.
495 495
496 496 xunit : bool
497 497 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
498 498
499 499 coverage : bool or str
500 500 Measure code coverage from tests. True will store the raw coverage data,
501 501 or pass 'html' or 'xml' to get reports.
502 502
503 503 extra_args : list
504 504 Extra arguments to pass to the test subprocesses, e.g. '-v'
505 505 """
506 506 to_run, not_run = prepare_controllers(options)
507 507
508 508 def justify(ltext, rtext, width=70, fill='-'):
509 509 ltext += ' '
510 510 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
511 511 return ltext + rtext
512 512
513 513 # Run all test runners, tracking execution time
514 514 failed = []
515 515 t_start = time.time()
516 516
517 517 print()
518 518 if options.fast == 1:
519 519 # This actually means sequential, i.e. with 1 job
520 520 for controller in to_run:
521 521 print('Test group:', controller.section)
522 522 sys.stdout.flush() # Show in correct order when output is piped
523 523 controller, res = do_run(controller, buffer_output=False)
524 524 if res:
525 525 failed.append(controller)
526 526 if res == -signal.SIGINT:
527 527 print("Interrupted")
528 528 break
529 529 print()
530 530
531 531 else:
532 532 # Run tests concurrently
533 533 try:
534 534 pool = multiprocessing.pool.ThreadPool(options.fast)
535 535 for (controller, res) in pool.imap_unordered(do_run, to_run):
536 536 res_string = 'OK' if res == 0 else 'FAILED'
537 537 print(justify('Test group: ' + controller.section, res_string))
538 538 if res:
539 539 controller.print_extra_info()
540 540 print(bytes_to_str(controller.stdout))
541 541 failed.append(controller)
542 542 if res == -signal.SIGINT:
543 543 print("Interrupted")
544 544 break
545 545 except KeyboardInterrupt:
546 546 return
547 547
548 548 for controller in not_run:
549 549 print(justify('Test group: ' + controller.section, 'NOT RUN'))
550 550
551 551 t_end = time.time()
552 552 t_tests = t_end - t_start
553 553 nrunners = len(to_run)
554 554 nfail = len(failed)
555 555 # summarize results
556 556 print('_'*70)
557 557 print('Test suite completed for system with the following information:')
558 558 print(report())
559 559 took = "Took %.3fs." % t_tests
560 560 print('Status: ', end='')
561 561 if not failed:
562 562 print('OK (%d test groups).' % nrunners, took)
563 563 else:
564 564 # If anything went wrong, point out what command to rerun manually to
565 565 # see the actual errors and individual summary
566 566 failed_sections = [c.section for c in failed]
567 567 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
568 568 nrunners, ', '.join(failed_sections)), took)
569 569 print()
570 570 print('You may wish to rerun these, with:')
571 571 print(' iptest', *failed_sections)
572 572 print()
573 573
574 574 if options.coverage:
575 575 from coverage import coverage
576 576 cov = coverage(data_file='.coverage')
577 577 cov.combine()
578 578 cov.save()
579 579
580 580 # Coverage HTML report
581 581 if options.coverage == 'html':
582 582 html_dir = 'ipy_htmlcov'
583 583 shutil.rmtree(html_dir, ignore_errors=True)
584 584 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
585 585 sys.stdout.flush()
586 586
587 587 # Custom HTML reporter to clean up module names.
588 588 from coverage.html import HtmlReporter
589 589 class CustomHtmlReporter(HtmlReporter):
590 590 def find_code_units(self, morfs):
591 591 super(CustomHtmlReporter, self).find_code_units(morfs)
592 592 for cu in self.code_units:
593 593 nameparts = cu.name.split(os.sep)
594 594 if 'IPython' not in nameparts:
595 595 continue
596 596 ix = nameparts.index('IPython')
597 597 cu.name = '.'.join(nameparts[ix:])
598 598
599 599 # Reimplement the html_report method with our custom reporter
600 600 cov._harvest_data()
601 601 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
602 602 html_title='IPython test coverage',
603 603 )
604 604 reporter = CustomHtmlReporter(cov, cov.config)
605 605 reporter.report(None)
606 606 print('done.')
607 607
608 608 # Coverage XML report
609 609 elif options.coverage == 'xml':
610 610 cov.xml_report(outfile='ipy_coverage.xml')
611 611
612 612 if failed:
613 613 # Ensure that our exit code indicates failure
614 614 sys.exit(1)
615 615
616 616 argparser = argparse.ArgumentParser(description='Run IPython test suite')
617 617 argparser.add_argument('testgroups', nargs='*',
618 618 help='Run specified groups of tests. If omitted, run '
619 619 'all tests.')
620 620 argparser.add_argument('--all', action='store_true',
621 621 help='Include slow tests not run by default.')
622 622 argparser.add_argument('--slimerjs', action='store_true',
623 623 help="Use slimerjs if it's installed instead of phantomjs for casperjs tests.")
624 624 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
625 625 help='Run test sections in parallel. This starts as many '
626 626 'processes as you have cores, or you can specify a number.')
627 627 argparser.add_argument('--xunit', action='store_true',
628 628 help='Produce Xunit XML results')
629 629 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
630 630 help="Measure test coverage. Specify 'html' or "
631 631 "'xml' to get reports.")
632 632 argparser.add_argument('--subproc-streams', default='capture',
633 633 help="What to do with stdout/stderr from subprocesses. "
634 634 "'capture' (default), 'show' and 'discard' are the options.")
635 635
636 636 def default_options():
637 637 """Get an argparse Namespace object with the default arguments, to pass to
638 638 :func:`run_iptestall`.
639 639 """
640 640 options = argparser.parse_args([])
641 641 options.extra_args = []
642 642 return options
643 643
644 644 def main():
645 645 # iptest doesn't work correctly if the working directory is the
646 646 # root of the IPython source tree. Tell the user to avoid
647 647 # frustration.
648 648 if os.path.exists(os.path.join(os.getcwd(),
649 649 'IPython', 'testing', '__main__.py')):
650 650 print("Don't run iptest from the IPython source directory",
651 651 file=sys.stderr)
652 652 sys.exit(1)
653 653 # Arguments after -- should be passed through to nose. Argparse treats
654 654 # everything after -- as regular positional arguments, so we separate them
655 655 # first.
656 656 try:
657 657 ix = sys.argv.index('--')
658 658 except ValueError:
659 659 to_parse = sys.argv[1:]
660 660 extra_args = []
661 661 else:
662 662 to_parse = sys.argv[1:ix]
663 663 extra_args = sys.argv[ix+1:]
664 664
665 665 options = argparser.parse_args(to_parse)
666 666 options.extra_args = extra_args
667 667
668 668 run_iptestall(options)
669 669
670 670
671 671 if __name__ == '__main__':
672 672 main()
@@ -1,773 +1,784 b''
1 1 # encoding: utf-8
2 2 """
3 3 Utilities for working with strings and text.
4 4
5 5 Inheritance diagram:
6 6
7 7 .. inheritance-diagram:: IPython.utils.text
8 8 :parts: 3
9 9 """
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Copyright (C) 2008-2011 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Imports
20 20 #-----------------------------------------------------------------------------
21 21
22 22 import os
23 23 import re
24 24 import sys
25 25 import textwrap
26 26 from string import Formatter
27 27
28 28 from IPython.external.path import path
29 29 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
30 30 from IPython.utils import py3compat
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Declarations
34 34 #-----------------------------------------------------------------------------
35 35
36 36 # datetime.strftime date format for ipython
37 37 if sys.platform == 'win32':
38 38 date_format = "%B %d, %Y"
39 39 else:
40 40 date_format = "%B %-d, %Y"
41 41
42 42
43 43 #-----------------------------------------------------------------------------
44 44 # Code
45 45 #-----------------------------------------------------------------------------
46 46
47 47 class LSString(str):
48 48 """String derivative with a special access attributes.
49 49
50 50 These are normal strings, but with the special attributes:
51 51
52 52 .l (or .list) : value as list (split on newlines).
53 53 .n (or .nlstr): original value (the string itself).
54 54 .s (or .spstr): value as whitespace-separated string.
55 55 .p (or .paths): list of path objects
56 56
57 57 Any values which require transformations are computed only once and
58 58 cached.
59 59
60 60 Such strings are very useful to efficiently interact with the shell, which
61 61 typically only understands whitespace-separated options for commands."""
62 62
63 63 def get_list(self):
64 64 try:
65 65 return self.__list
66 66 except AttributeError:
67 67 self.__list = self.split('\n')
68 68 return self.__list
69 69
70 70 l = list = property(get_list)
71 71
72 72 def get_spstr(self):
73 73 try:
74 74 return self.__spstr
75 75 except AttributeError:
76 76 self.__spstr = self.replace('\n',' ')
77 77 return self.__spstr
78 78
79 79 s = spstr = property(get_spstr)
80 80
81 81 def get_nlstr(self):
82 82 return self
83 83
84 84 n = nlstr = property(get_nlstr)
85 85
86 86 def get_paths(self):
87 87 try:
88 88 return self.__paths
89 89 except AttributeError:
90 90 self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
91 91 return self.__paths
92 92
93 93 p = paths = property(get_paths)
94 94
95 95 # FIXME: We need to reimplement type specific displayhook and then add this
96 96 # back as a custom printer. This should also be moved outside utils into the
97 97 # core.
98 98
99 99 # def print_lsstring(arg):
100 100 # """ Prettier (non-repr-like) and more informative printer for LSString """
101 101 # print "LSString (.p, .n, .l, .s available). Value:"
102 102 # print arg
103 103 #
104 104 #
105 105 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
106 106
107 107
108 108 class SList(list):
109 109 """List derivative with a special access attributes.
110 110
111 111 These are normal lists, but with the special attributes:
112 112
113 113 * .l (or .list) : value as list (the list itself).
114 114 * .n (or .nlstr): value as a string, joined on newlines.
115 115 * .s (or .spstr): value as a string, joined on spaces.
116 116 * .p (or .paths): list of path objects
117 117
118 118 Any values which require transformations are computed only once and
119 119 cached."""
120 120
121 121 def get_list(self):
122 122 return self
123 123
124 124 l = list = property(get_list)
125 125
126 126 def get_spstr(self):
127 127 try:
128 128 return self.__spstr
129 129 except AttributeError:
130 130 self.__spstr = ' '.join(self)
131 131 return self.__spstr
132 132
133 133 s = spstr = property(get_spstr)
134 134
135 135 def get_nlstr(self):
136 136 try:
137 137 return self.__nlstr
138 138 except AttributeError:
139 139 self.__nlstr = '\n'.join(self)
140 140 return self.__nlstr
141 141
142 142 n = nlstr = property(get_nlstr)
143 143
144 144 def get_paths(self):
145 145 try:
146 146 return self.__paths
147 147 except AttributeError:
148 148 self.__paths = [path(p) for p in self if os.path.exists(p)]
149 149 return self.__paths
150 150
151 151 p = paths = property(get_paths)
152 152
153 153 def grep(self, pattern, prune = False, field = None):
154 154 """ Return all strings matching 'pattern' (a regex or callable)
155 155
156 156 This is case-insensitive. If prune is true, return all items
157 157 NOT matching the pattern.
158 158
159 159 If field is specified, the match must occur in the specified
160 160 whitespace-separated field.
161 161
162 162 Examples::
163 163
164 164 a.grep( lambda x: x.startswith('C') )
165 165 a.grep('Cha.*log', prune=1)
166 166 a.grep('chm', field=-1)
167 167 """
168 168
169 169 def match_target(s):
170 170 if field is None:
171 171 return s
172 172 parts = s.split()
173 173 try:
174 174 tgt = parts[field]
175 175 return tgt
176 176 except IndexError:
177 177 return ""
178 178
179 179 if isinstance(pattern, py3compat.string_types):
180 180 pred = lambda x : re.search(pattern, x, re.IGNORECASE)
181 181 else:
182 182 pred = pattern
183 183 if not prune:
184 184 return SList([el for el in self if pred(match_target(el))])
185 185 else:
186 186 return SList([el for el in self if not pred(match_target(el))])
187 187
188 188 def fields(self, *fields):
189 189 """ Collect whitespace-separated fields from string list
190 190
191 191 Allows quick awk-like usage of string lists.
192 192
193 193 Example data (in var a, created by 'a = !ls -l')::
194 194
195 195 -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog
196 196 drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython
197 197
198 198 * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
199 199 * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
200 200 (note the joining by space).
201 201 * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
202 202
203 203 IndexErrors are ignored.
204 204
205 205 Without args, fields() just split()'s the strings.
206 206 """
207 207 if len(fields) == 0:
208 208 return [el.split() for el in self]
209 209
210 210 res = SList()
211 211 for el in [f.split() for f in self]:
212 212 lineparts = []
213 213
214 214 for fd in fields:
215 215 try:
216 216 lineparts.append(el[fd])
217 217 except IndexError:
218 218 pass
219 219 if lineparts:
220 220 res.append(" ".join(lineparts))
221 221
222 222 return res
223 223
224 224 def sort(self,field= None, nums = False):
225 225 """ sort by specified fields (see fields())
226 226
227 227 Example::
228 228
229 229 a.sort(1, nums = True)
230 230
231 231 Sorts a by second field, in numerical order (so that 21 > 3)
232 232
233 233 """
234 234
235 235 #decorate, sort, undecorate
236 236 if field is not None:
237 237 dsu = [[SList([line]).fields(field), line] for line in self]
238 238 else:
239 239 dsu = [[line, line] for line in self]
240 240 if nums:
241 241 for i in range(len(dsu)):
242 242 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
243 243 try:
244 244 n = int(numstr)
245 245 except ValueError:
246 246 n = 0;
247 247 dsu[i][0] = n
248 248
249 249
250 250 dsu.sort()
251 251 return SList([t[1] for t in dsu])
252 252
253 253
254 254 # FIXME: We need to reimplement type specific displayhook and then add this
255 255 # back as a custom printer. This should also be moved outside utils into the
256 256 # core.
257 257
258 258 # def print_slist(arg):
259 259 # """ Prettier (non-repr-like) and more informative printer for SList """
260 260 # print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
261 261 # if hasattr(arg, 'hideonce') and arg.hideonce:
262 262 # arg.hideonce = False
263 263 # return
264 264 #
265 265 # nlprint(arg) # This was a nested list printer, now removed.
266 266 #
267 267 # print_slist = result_display.when_type(SList)(print_slist)
268 268
269 269
270 270 def indent(instr,nspaces=4, ntabs=0, flatten=False):
271 271 """Indent a string a given number of spaces or tabstops.
272 272
273 273 indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
274 274
275 275 Parameters
276 276 ----------
277 277
278 278 instr : basestring
279 279 The string to be indented.
280 280 nspaces : int (default: 4)
281 281 The number of spaces to be indented.
282 282 ntabs : int (default: 0)
283 283 The number of tabs to be indented.
284 284 flatten : bool (default: False)
285 285 Whether to scrub existing indentation. If True, all lines will be
286 286 aligned to the same indentation. If False, existing indentation will
287 287 be strictly increased.
288 288
289 289 Returns
290 290 -------
291 291
292 292 str|unicode : string indented by ntabs and nspaces.
293 293
294 294 """
295 295 if instr is None:
296 296 return
297 297 ind = '\t'*ntabs+' '*nspaces
298 298 if flatten:
299 299 pat = re.compile(r'^\s*', re.MULTILINE)
300 300 else:
301 301 pat = re.compile(r'^', re.MULTILINE)
302 302 outstr = re.sub(pat, ind, instr)
303 303 if outstr.endswith(os.linesep+ind):
304 304 return outstr[:-len(ind)]
305 305 else:
306 306 return outstr
307 307
308 308
309 309 def list_strings(arg):
310 310 """Always return a list of strings, given a string or list of strings
311 311 as input.
312 312
313 313 Examples
314 314 --------
315 315 ::
316 316
317 317 In [7]: list_strings('A single string')
318 318 Out[7]: ['A single string']
319 319
320 320 In [8]: list_strings(['A single string in a list'])
321 321 Out[8]: ['A single string in a list']
322 322
323 323 In [9]: list_strings(['A','list','of','strings'])
324 324 Out[9]: ['A', 'list', 'of', 'strings']
325 325 """
326 326
327 327 if isinstance(arg, py3compat.string_types): return [arg]
328 328 else: return arg
329 329
330 330
331 331 def marquee(txt='',width=78,mark='*'):
332 332 """Return the input string centered in a 'marquee'.
333 333
334 334 Examples
335 335 --------
336 336 ::
337 337
338 338 In [16]: marquee('A test',40)
339 339 Out[16]: '**************** A test ****************'
340 340
341 341 In [17]: marquee('A test',40,'-')
342 342 Out[17]: '---------------- A test ----------------'
343 343
344 344 In [18]: marquee('A test',40,' ')
345 345 Out[18]: ' A test '
346 346
347 347 """
348 348 if not txt:
349 349 return (mark*width)[:width]
350 350 nmark = (width-len(txt)-2)//len(mark)//2
351 351 if nmark < 0: nmark =0
352 352 marks = mark*nmark
353 353 return '%s %s %s' % (marks,txt,marks)
354 354
355 355
356 356 ini_spaces_re = re.compile(r'^(\s+)')
357 357
358 358 def num_ini_spaces(strng):
359 359 """Return the number of initial spaces in a string"""
360 360
361 361 ini_spaces = ini_spaces_re.match(strng)
362 362 if ini_spaces:
363 363 return ini_spaces.end()
364 364 else:
365 365 return 0
366 366
367 367
368 368 def format_screen(strng):
369 369 """Format a string for screen printing.
370 370
371 371 This removes some latex-type format codes."""
372 372 # Paragraph continue
373 373 par_re = re.compile(r'\\$',re.MULTILINE)
374 374 strng = par_re.sub('',strng)
375 375 return strng
376 376
377 377
378 378 def dedent(text):
379 379 """Equivalent of textwrap.dedent that ignores unindented first line.
380 380
381 381 This means it will still dedent strings like:
382 382 '''foo
383 383 is a bar
384 384 '''
385 385
386 386 For use in wrap_paragraphs.
387 387 """
388 388
389 389 if text.startswith('\n'):
390 390 # text starts with blank line, don't ignore the first line
391 391 return textwrap.dedent(text)
392 392
393 393 # split first line
394 394 splits = text.split('\n',1)
395 395 if len(splits) == 1:
396 396 # only one line
397 397 return textwrap.dedent(text)
398 398
399 399 first, rest = splits
400 400 # dedent everything but the first line
401 401 rest = textwrap.dedent(rest)
402 402 return '\n'.join([first, rest])
403 403
404 404
405 405 def wrap_paragraphs(text, ncols=80):
406 406 """Wrap multiple paragraphs to fit a specified width.
407 407
408 408 This is equivalent to textwrap.wrap, but with support for multiple
409 409 paragraphs, as separated by empty lines.
410 410
411 411 Returns
412 412 -------
413 413
414 414 list of complete paragraphs, wrapped to fill `ncols` columns.
415 415 """
416 416 paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
417 417 text = dedent(text).strip()
418 418 paragraphs = paragraph_re.split(text)[::2] # every other entry is space
419 419 out_ps = []
420 420 indent_re = re.compile(r'\n\s+', re.MULTILINE)
421 421 for p in paragraphs:
422 422 # presume indentation that survives dedent is meaningful formatting,
423 423 # so don't fill unless text is flush.
424 424 if indent_re.search(p) is None:
425 425 # wrap paragraph
426 426 p = textwrap.fill(p, ncols)
427 427 out_ps.append(p)
428 428 return out_ps
429 429
430 430
431 431 def long_substr(data):
432 432 """Return the longest common substring in a list of strings.
433 433
434 434 Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
435 435 """
436 436 substr = ''
437 437 if len(data) > 1 and len(data[0]) > 0:
438 438 for i in range(len(data[0])):
439 439 for j in range(len(data[0])-i+1):
440 440 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
441 441 substr = data[0][i:i+j]
442 442 elif len(data) == 1:
443 443 substr = data[0]
444 444 return substr
445 445
446 446
447 447 def strip_email_quotes(text):
448 448 """Strip leading email quotation characters ('>').
449 449
450 450 Removes any combination of leading '>' interspersed with whitespace that
451 451 appears *identically* in all lines of the input text.
452 452
453 453 Parameters
454 454 ----------
455 455 text : str
456 456
457 457 Examples
458 458 --------
459 459
460 460 Simple uses::
461 461
462 462 In [2]: strip_email_quotes('> > text')
463 463 Out[2]: 'text'
464 464
465 465 In [3]: strip_email_quotes('> > text\\n> > more')
466 466 Out[3]: 'text\\nmore'
467 467
468 468 Note how only the common prefix that appears in all lines is stripped::
469 469
470 470 In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
471 471 Out[4]: '> text\\n> more\\nmore...'
472 472
473 473 So if any line has no quote marks ('>') , then none are stripped from any
474 474 of them ::
475 475
476 476 In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
477 477 Out[5]: '> > text\\n> > more\\nlast different'
478 478 """
479 479 lines = text.splitlines()
480 480 matches = set()
481 481 for line in lines:
482 482 prefix = re.match(r'^(\s*>[ >]*)', line)
483 483 if prefix:
484 484 matches.add(prefix.group(1))
485 485 else:
486 486 break
487 487 else:
488 488 prefix = long_substr(list(matches))
489 489 if prefix:
490 490 strip = len(prefix)
491 491 text = '\n'.join([ ln[strip:] for ln in lines])
492 492 return text
493 493
494 def strip_ansi(source):
495 """
496 Remove ansi escape codes from text.
497
498 Parameters
499 ----------
500 source : str
501 Source to remove the ansi from
502 """
503 return re.sub(r'\033\[(\d|;)+?m', '', source)
504
494 505
495 506 class EvalFormatter(Formatter):
496 507 """A String Formatter that allows evaluation of simple expressions.
497 508
498 509 Note that this version interprets a : as specifying a format string (as per
499 510 standard string formatting), so if slicing is required, you must explicitly
500 511 create a slice.
501 512
502 513 This is to be used in templating cases, such as the parallel batch
503 514 script templates, where simple arithmetic on arguments is useful.
504 515
505 516 Examples
506 517 --------
507 518 ::
508 519
509 520 In [1]: f = EvalFormatter()
510 521 In [2]: f.format('{n//4}', n=8)
511 522 Out[2]: '2'
512 523
513 524 In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
514 525 Out[3]: 'll'
515 526 """
516 527 def get_field(self, name, args, kwargs):
517 528 v = eval(name, kwargs)
518 529 return v, name
519 530
520 531 #XXX: As of Python 3.4, the format string parsing no longer splits on a colon
521 532 # inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
522 533 # above, it should be possible to remove FullEvalFormatter.
523 534
524 535 @skip_doctest_py3
525 536 class FullEvalFormatter(Formatter):
526 537 """A String Formatter that allows evaluation of simple expressions.
527 538
528 539 Any time a format key is not found in the kwargs,
529 540 it will be tried as an expression in the kwargs namespace.
530 541
531 542 Note that this version allows slicing using [1:2], so you cannot specify
532 543 a format string. Use :class:`EvalFormatter` to permit format strings.
533 544
534 545 Examples
535 546 --------
536 547 ::
537 548
538 549 In [1]: f = FullEvalFormatter()
539 550 In [2]: f.format('{n//4}', n=8)
540 551 Out[2]: u'2'
541 552
542 553 In [3]: f.format('{list(range(5))[2:4]}')
543 554 Out[3]: u'[2, 3]'
544 555
545 556 In [4]: f.format('{3*2}')
546 557 Out[4]: u'6'
547 558 """
548 559 # copied from Formatter._vformat with minor changes to allow eval
549 560 # and replace the format_spec code with slicing
550 561 def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
551 562 if recursion_depth < 0:
552 563 raise ValueError('Max string recursion exceeded')
553 564 result = []
554 565 for literal_text, field_name, format_spec, conversion in \
555 566 self.parse(format_string):
556 567
557 568 # output the literal text
558 569 if literal_text:
559 570 result.append(literal_text)
560 571
561 572 # if there's a field, output it
562 573 if field_name is not None:
563 574 # this is some markup, find the object and do
564 575 # the formatting
565 576
566 577 if format_spec:
567 578 # override format spec, to allow slicing:
568 579 field_name = ':'.join([field_name, format_spec])
569 580
570 581 # eval the contents of the field for the object
571 582 # to be formatted
572 583 obj = eval(field_name, kwargs)
573 584
574 585 # do any conversion on the resulting object
575 586 obj = self.convert_field(obj, conversion)
576 587
577 588 # format the object and append to the result
578 589 result.append(self.format_field(obj, ''))
579 590
580 591 return u''.join(py3compat.cast_unicode(s) for s in result)
581 592
582 593
583 594 @skip_doctest_py3
584 595 class DollarFormatter(FullEvalFormatter):
585 596 """Formatter allowing Itpl style $foo replacement, for names and attribute
586 597 access only. Standard {foo} replacement also works, and allows full
587 598 evaluation of its arguments.
588 599
589 600 Examples
590 601 --------
591 602 ::
592 603
593 604 In [1]: f = DollarFormatter()
594 605 In [2]: f.format('{n//4}', n=8)
595 606 Out[2]: u'2'
596 607
597 608 In [3]: f.format('23 * 76 is $result', result=23*76)
598 609 Out[3]: u'23 * 76 is 1748'
599 610
600 611 In [4]: f.format('$a or {b}', a=1, b=2)
601 612 Out[4]: u'1 or 2'
602 613 """
603 614 _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
604 615 def parse(self, fmt_string):
605 616 for literal_txt, field_name, format_spec, conversion \
606 617 in Formatter.parse(self, fmt_string):
607 618
608 619 # Find $foo patterns in the literal text.
609 620 continue_from = 0
610 621 txt = ""
611 622 for m in self._dollar_pattern.finditer(literal_txt):
612 623 new_txt, new_field = m.group(1,2)
613 624 # $$foo --> $foo
614 625 if new_field.startswith("$"):
615 626 txt += new_txt + new_field
616 627 else:
617 628 yield (txt + new_txt, new_field, "", None)
618 629 txt = ""
619 630 continue_from = m.end()
620 631
621 632 # Re-yield the {foo} style pattern
622 633 yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
623 634
624 635 #-----------------------------------------------------------------------------
625 636 # Utils to columnize a list of string
626 637 #-----------------------------------------------------------------------------
627 638
628 639 def _chunks(l, n):
629 640 """Yield successive n-sized chunks from l."""
630 641 for i in py3compat.xrange(0, len(l), n):
631 642 yield l[i:i+n]
632 643
633 644
634 645 def _find_optimal(rlist , separator_size=2 , displaywidth=80):
635 646 """Calculate optimal info to columnize a list of string"""
636 647 for nrow in range(1, len(rlist)+1) :
637 648 chk = list(map(max,_chunks(rlist, nrow)))
638 649 sumlength = sum(chk)
639 650 ncols = len(chk)
640 651 if sumlength+separator_size*(ncols-1) <= displaywidth :
641 652 break;
642 653 return {'columns_numbers' : ncols,
643 654 'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
644 655 'rows_numbers' : nrow,
645 656 'columns_width' : chk
646 657 }
647 658
648 659
649 660 def _get_or_default(mylist, i, default=None):
650 661 """return list item number, or default if don't exist"""
651 662 if i >= len(mylist):
652 663 return default
653 664 else :
654 665 return mylist[i]
655 666
656 667
657 668 @skip_doctest
658 669 def compute_item_matrix(items, empty=None, *args, **kwargs) :
659 670 """Returns a nested list, and info to columnize items
660 671
661 672 Parameters
662 673 ----------
663 674
664 675 items
665 676 list of strings to columize
666 677 empty : (default None)
667 678 default value to fill list if needed
668 679 separator_size : int (default=2)
669 680 How much caracters will be used as a separation between each columns.
670 681 displaywidth : int (default=80)
671 682 The width of the area onto wich the columns should enter
672 683
673 684 Returns
674 685 -------
675 686
676 687 strings_matrix
677 688
678 689 nested list of string, the outer most list contains as many list as
679 690 rows, the innermost lists have each as many element as colums. If the
680 691 total number of elements in `items` does not equal the product of
681 692 rows*columns, the last element of some lists are filled with `None`.
682 693
683 694 dict_info
684 695 some info to make columnize easier:
685 696
686 697 columns_numbers
687 698 number of columns
688 699 rows_numbers
689 700 number of rows
690 701 columns_width
691 702 list of with of each columns
692 703 optimal_separator_width
693 704 best separator width between columns
694 705
695 706 Examples
696 707 --------
697 708 ::
698 709
699 710 In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
700 711 ...: compute_item_matrix(l,displaywidth=12)
701 712 Out[1]:
702 713 ([['aaa', 'f', 'k'],
703 714 ['b', 'g', 'l'],
704 715 ['cc', 'h', None],
705 716 ['d', 'i', None],
706 717 ['eeeee', 'j', None]],
707 718 {'columns_numbers': 3,
708 719 'columns_width': [5, 1, 1],
709 720 'optimal_separator_width': 2,
710 721 'rows_numbers': 5})
711 722 """
712 723 info = _find_optimal(list(map(len, items)), *args, **kwargs)
713 724 nrow, ncol = info['rows_numbers'], info['columns_numbers']
714 725 return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)
715 726
716 727
717 728 def columnize(items, separator=' ', displaywidth=80):
718 729 """ Transform a list of strings into a single string with columns.
719 730
720 731 Parameters
721 732 ----------
722 733 items : sequence of strings
723 734 The strings to process.
724 735
725 736 separator : str, optional [default is two spaces]
726 737 The string that separates columns.
727 738
728 739 displaywidth : int, optional [default is 80]
729 740 Width of the display in number of characters.
730 741
731 742 Returns
732 743 -------
733 744 The formatted string.
734 745 """
735 746 if not items :
736 747 return '\n'
737 748 matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
738 749 fmatrix = [filter(None, x) for x in matrix]
739 750 sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
740 751 return '\n'.join(map(sjoin, fmatrix))+'\n'
741 752
742 753
743 754 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
744 755 """
745 756 Return a string with a natural enumeration of items
746 757
747 758 >>> get_text_list(['a', 'b', 'c', 'd'])
748 759 'a, b, c and d'
749 760 >>> get_text_list(['a', 'b', 'c'], ' or ')
750 761 'a, b or c'
751 762 >>> get_text_list(['a', 'b', 'c'], ', ')
752 763 'a, b, c'
753 764 >>> get_text_list(['a', 'b'], ' or ')
754 765 'a or b'
755 766 >>> get_text_list(['a'])
756 767 'a'
757 768 >>> get_text_list([])
758 769 ''
759 770 >>> get_text_list(['a', 'b'], wrap_item_with="`")
760 771 '`a` and `b`'
761 772 >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
762 773 'a + b + c = d'
763 774 """
764 775 if len(list_) == 0:
765 776 return ''
766 777 if wrap_item_with:
767 778 list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
768 779 item in list_]
769 780 if len(list_) == 1:
770 781 return list_[0]
771 782 return '%s%s%s' % (
772 783 sep.join(i for i in list_[:-1]),
773 784 last_sep, list_[-1]) No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now