##// END OF EJS Templates
add text.getdefaultencoding() for central default encoding guess...
MinRK -
Show More
@@ -1,660 +1,661
1 1 """A simple configuration system.
2 2
3 3 Authors
4 4 -------
5 5 * Brian Granger
6 6 * Fernando Perez
7 7 * Min RK
8 8 """
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Copyright (C) 2008-2011 The IPython Development Team
12 12 #
13 13 # Distributed under the terms of the BSD License. The full license is in
14 14 # the file COPYING, distributed as part of this software.
15 15 #-----------------------------------------------------------------------------
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Imports
19 19 #-----------------------------------------------------------------------------
20 20
21 21 import __builtin__ as builtin_mod
22 22 import re
23 23 import sys
24 24
25 25 from IPython.external import argparse
26 26 from IPython.utils.path import filefind, get_ipython_dir
27 from IPython.utils import py3compat, warn
27 from IPython.utils import py3compat, text, warn
28 28
29 29 #-----------------------------------------------------------------------------
30 30 # Exceptions
31 31 #-----------------------------------------------------------------------------
32 32
33 33
34 34 class ConfigError(Exception):
35 35 pass
36 36
37 37
38 38 class ConfigLoaderError(ConfigError):
39 39 pass
40 40
41 41 class ArgumentError(ConfigLoaderError):
42 42 pass
43 43
44 44 #-----------------------------------------------------------------------------
45 45 # Argparse fix
46 46 #-----------------------------------------------------------------------------
47 47
48 48 # Unfortunately argparse by default prints help messages to stderr instead of
49 49 # stdout. This makes it annoying to capture long help screens at the command
50 50 # line, since one must know how to pipe stderr, which many users don't know how
51 51 # to do. So we override the print_help method with one that defaults to
52 52 # stdout and use our class instead.
53 53
54 54 class ArgumentParser(argparse.ArgumentParser):
55 55 """Simple argparse subclass that prints help to stdout by default."""
56 56
57 57 def print_help(self, file=None):
58 58 if file is None:
59 59 file = sys.stdout
60 60 return super(ArgumentParser, self).print_help(file)
61 61
62 62 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
63 63
64 64 #-----------------------------------------------------------------------------
65 65 # Config class for holding config information
66 66 #-----------------------------------------------------------------------------
67 67
68 68
69 69 class Config(dict):
70 70 """An attribute based dict that can do smart merges."""
71 71
72 72 def __init__(self, *args, **kwds):
73 73 dict.__init__(self, *args, **kwds)
74 74 # This sets self.__dict__ = self, but it has to be done this way
75 75 # because we are also overriding __setattr__.
76 76 dict.__setattr__(self, '__dict__', self)
77 77
78 78 def _merge(self, other):
79 79 to_update = {}
80 80 for k, v in other.iteritems():
81 81 if not self.has_key(k):
82 82 to_update[k] = v
83 83 else: # I have this key
84 84 if isinstance(v, Config):
85 85 # Recursively merge common sub Configs
86 86 self[k]._merge(v)
87 87 else:
88 88 # Plain updates for non-Configs
89 89 to_update[k] = v
90 90
91 91 self.update(to_update)
92 92
93 93 def _is_section_key(self, key):
94 94 if key[0].upper()==key[0] and not key.startswith('_'):
95 95 return True
96 96 else:
97 97 return False
98 98
99 99 def __contains__(self, key):
100 100 if self._is_section_key(key):
101 101 return True
102 102 else:
103 103 return super(Config, self).__contains__(key)
104 104 # .has_key is deprecated for dictionaries.
105 105 has_key = __contains__
106 106
107 107 def _has_section(self, key):
108 108 if self._is_section_key(key):
109 109 if super(Config, self).__contains__(key):
110 110 return True
111 111 return False
112 112
113 113 def copy(self):
114 114 return type(self)(dict.copy(self))
115 115
116 116 def __copy__(self):
117 117 return self.copy()
118 118
119 119 def __deepcopy__(self, memo):
120 120 import copy
121 121 return type(self)(copy.deepcopy(self.items()))
122 122
123 123 def __getitem__(self, key):
124 124 # We cannot use directly self._is_section_key, because it triggers
125 125 # infinite recursion on top of PyPy. Instead, we manually fish the
126 126 # bound method.
127 127 is_section_key = self.__class__._is_section_key.__get__(self)
128 128
129 129 # Because we use this for an exec namespace, we need to delegate
130 130 # the lookup of names in __builtin__ to itself. This means
131 131 # that you can't have section or attribute names that are
132 132 # builtins.
133 133 try:
134 134 return getattr(builtin_mod, key)
135 135 except AttributeError:
136 136 pass
137 137 if is_section_key(key):
138 138 try:
139 139 return dict.__getitem__(self, key)
140 140 except KeyError:
141 141 c = Config()
142 142 dict.__setitem__(self, key, c)
143 143 return c
144 144 else:
145 145 return dict.__getitem__(self, key)
146 146
147 147 def __setitem__(self, key, value):
148 148 # Don't allow names in __builtin__ to be modified.
149 149 if hasattr(builtin_mod, key):
150 150 raise ConfigError('Config variable names cannot have the same name '
151 151 'as a Python builtin: %s' % key)
152 152 if self._is_section_key(key):
153 153 if not isinstance(value, Config):
154 154 raise ValueError('values whose keys begin with an uppercase '
155 155 'char must be Config instances: %r, %r' % (key, value))
156 156 else:
157 157 dict.__setitem__(self, key, value)
158 158
159 159 def __getattr__(self, key):
160 160 try:
161 161 return self.__getitem__(key)
162 162 except KeyError, e:
163 163 raise AttributeError(e)
164 164
165 165 def __setattr__(self, key, value):
166 166 try:
167 167 self.__setitem__(key, value)
168 168 except KeyError, e:
169 169 raise AttributeError(e)
170 170
171 171 def __delattr__(self, key):
172 172 try:
173 173 dict.__delitem__(self, key)
174 174 except KeyError, e:
175 175 raise AttributeError(e)
176 176
177 177
178 178 #-----------------------------------------------------------------------------
179 179 # Config loading classes
180 180 #-----------------------------------------------------------------------------
181 181
182 182
183 183 class ConfigLoader(object):
184 184 """A object for loading configurations from just about anywhere.
185 185
186 186 The resulting configuration is packaged as a :class:`Struct`.
187 187
188 188 Notes
189 189 -----
190 190 A :class:`ConfigLoader` does one thing: load a config from a source
191 191 (file, command line arguments) and returns the data as a :class:`Struct`.
192 192 There are lots of things that :class:`ConfigLoader` does not do. It does
193 193 not implement complex logic for finding config files. It does not handle
194 194 default values or merge multiple configs. These things need to be
195 195 handled elsewhere.
196 196 """
197 197
198 198 def __init__(self):
199 199 """A base class for config loaders.
200 200
201 201 Examples
202 202 --------
203 203
204 204 >>> cl = ConfigLoader()
205 205 >>> config = cl.load_config()
206 206 >>> config
207 207 {}
208 208 """
209 209 self.clear()
210 210
211 211 def clear(self):
212 212 self.config = Config()
213 213
214 214 def load_config(self):
215 215 """Load a config from somewhere, return a :class:`Config` instance.
216 216
217 217 Usually, this will cause self.config to be set and then returned.
218 218 However, in most cases, :meth:`ConfigLoader.clear` should be called
219 219 to erase any previous state.
220 220 """
221 221 self.clear()
222 222 return self.config
223 223
224 224
225 225 class FileConfigLoader(ConfigLoader):
226 226 """A base class for file based configurations.
227 227
228 228 As we add more file based config loaders, the common logic should go
229 229 here.
230 230 """
231 231 pass
232 232
233 233
234 234 class PyFileConfigLoader(FileConfigLoader):
235 235 """A config loader for pure python files.
236 236
237 237 This calls execfile on a plain python file and looks for attributes
238 238 that are all caps. These attribute are added to the config Struct.
239 239 """
240 240
241 241 def __init__(self, filename, path=None):
242 242 """Build a config loader for a filename and path.
243 243
244 244 Parameters
245 245 ----------
246 246 filename : str
247 247 The file name of the config file.
248 248 path : str, list, tuple
249 249 The path to search for the config file on, or a sequence of
250 250 paths to try in order.
251 251 """
252 252 super(PyFileConfigLoader, self).__init__()
253 253 self.filename = filename
254 254 self.path = path
255 255 self.full_filename = ''
256 256 self.data = None
257 257
258 258 def load_config(self):
259 259 """Load the config from a file and return it as a Struct."""
260 260 self.clear()
261 261 self._find_file()
262 262 self._read_file_as_dict()
263 263 self._convert_to_config()
264 264 return self.config
265 265
266 266 def _find_file(self):
267 267 """Try to find the file by searching the paths."""
268 268 self.full_filename = filefind(self.filename, self.path)
269 269
270 270 def _read_file_as_dict(self):
271 271 """Load the config file into self.config, with recursive loading."""
272 272 # This closure is made available in the namespace that is used
273 273 # to exec the config file. It allows users to call
274 274 # load_subconfig('myconfig.py') to load config files recursively.
275 275 # It needs to be a closure because it has references to self.path
276 276 # and self.config. The sub-config is loaded with the same path
277 277 # as the parent, but it uses an empty config which is then merged
278 278 # with the parents.
279 279
280 280 # If a profile is specified, the config file will be loaded
281 281 # from that profile
282 282
283 283 def load_subconfig(fname, profile=None):
284 284 # import here to prevent circular imports
285 285 from IPython.core.profiledir import ProfileDir, ProfileDirError
286 286 if profile is not None:
287 287 try:
288 288 profile_dir = ProfileDir.find_profile_dir_by_name(
289 289 get_ipython_dir(),
290 290 profile,
291 291 )
292 292 except ProfileDirError:
293 293 return
294 294 path = profile_dir.location
295 295 else:
296 296 path = self.path
297 297 loader = PyFileConfigLoader(fname, path)
298 298 try:
299 299 sub_config = loader.load_config()
300 300 except IOError:
301 301 # Pass silently if the sub config is not there. This happens
302 302 # when a user s using a profile, but not the default config.
303 303 pass
304 304 else:
305 305 self.config._merge(sub_config)
306 306
307 307 # Again, this needs to be a closure and should be used in config
308 308 # files to get the config being loaded.
309 309 def get_config():
310 310 return self.config
311 311
312 312 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
313 313 fs_encoding = sys.getfilesystemencoding() or 'ascii'
314 314 conf_filename = self.full_filename.encode(fs_encoding)
315 315 py3compat.execfile(conf_filename, namespace)
316 316
317 317 def _convert_to_config(self):
318 318 if self.data is None:
319 319 ConfigLoaderError('self.data does not exist')
320 320
321 321
322 322 class CommandLineConfigLoader(ConfigLoader):
323 323 """A config loader for command line arguments.
324 324
325 325 As we add more command line based loaders, the common logic should go
326 326 here.
327 327 """
328 328
329 329 def _exec_config_str(self, lhs, rhs):
330 330 exec_str = 'self.config.' + lhs + '=' + rhs
331 331 try:
332 332 # Try to see if regular Python syntax will work. This
333 333 # won't handle strings as the quote marks are removed
334 334 # by the system shell.
335 335 exec exec_str in locals(), globals()
336 336 except (NameError, SyntaxError):
337 337 # This case happens if the rhs is a string but without
338 338 # the quote marks. Use repr, to get quote marks, and
339 339 # 'u' prefix and see if
340 340 # it succeeds. If it still fails, we let it raise.
341 341 exec_str = u'self.config.' + lhs + '=' + repr(rhs)
342 342 exec exec_str in locals(), globals()
343 343
344 344 def _load_flag(self, cfg):
345 345 """update self.config from a flag, which can be a dict or Config"""
346 346 if isinstance(cfg, (dict, Config)):
347 347 # don't clobber whole config sections, update
348 348 # each section from config:
349 349 for sec,c in cfg.iteritems():
350 350 self.config[sec].update(c)
351 351 else:
352 352 raise ValueError("Invalid flag: '%s'"%raw)
353 353
354 354 # raw --identifier=value pattern
355 355 # but *also* accept '-' as wordsep, for aliases
356 356 # accepts: --foo=a
357 357 # --Class.trait=value
358 358 # --alias-name=value
359 359 # rejects: -foo=value
360 360 # --foo
361 361 # --Class.trait
362 362 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
363 363
364 364 # just flags, no assignments, with two *or one* leading '-'
365 365 # accepts: --foo
366 366 # -foo-bar-again
367 367 # rejects: --anything=anything
368 368 # --two.word
369 369
370 370 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
371 371
372 372 class KeyValueConfigLoader(CommandLineConfigLoader):
373 373 """A config loader that loads key value pairs from the command line.
374 374
375 375 This allows command line options to be gives in the following form::
376 376
377 377 ipython --profile="foo" --InteractiveShell.autocall=False
378 378 """
379 379
380 380 def __init__(self, argv=None, aliases=None, flags=None):
381 381 """Create a key value pair config loader.
382 382
383 383 Parameters
384 384 ----------
385 385 argv : list
386 386 A list that has the form of sys.argv[1:] which has unicode
387 387 elements of the form u"key=value". If this is None (default),
388 388 then sys.argv[1:] will be used.
389 389 aliases : dict
390 390 A dict of aliases for configurable traits.
391 391 Keys are the short aliases, Values are the resolved trait.
392 392 Of the form: `{'alias' : 'Configurable.trait'}`
393 393 flags : dict
394 394 A dict of flags, keyed by str name. Vaues can be Config objects,
395 395 dicts, or "key=value" strings. If Config or dict, when the flag
396 396 is triggered, The flag is loaded as `self.config.update(m)`.
397 397
398 398 Returns
399 399 -------
400 400 config : Config
401 401 The resulting Config object.
402 402
403 403 Examples
404 404 --------
405 405
406 406 >>> from IPython.config.loader import KeyValueConfigLoader
407 407 >>> cl = KeyValueConfigLoader()
408 408 >>> cl.load_config(["--A.name='brian'","--B.number=0"])
409 409 {'A': {'name': 'brian'}, 'B': {'number': 0}}
410 410 """
411 411 self.clear()
412 412 if argv is None:
413 413 argv = sys.argv[1:]
414 414 self.argv = argv
415 415 self.aliases = aliases or {}
416 416 self.flags = flags or {}
417 417
418 418
419 419 def clear(self):
420 420 super(KeyValueConfigLoader, self).clear()
421 421 self.extra_args = []
422 422
423 423
424 424 def _decode_argv(self, argv, enc=None):
425 425 """decode argv if bytes, using stin.encoding, falling back on default enc"""
426 426 uargv = []
427 427 if enc is None:
428 enc = sys.stdin.encoding or sys.getdefaultencoding()
428 enc = text.getdefaultencoding()
429 429 for arg in argv:
430 430 if not isinstance(arg, unicode):
431 431 # only decode if not already decoded
432 432 arg = arg.decode(enc)
433 433 uargv.append(arg)
434 434 return uargv
435 435
436 436
437 437 def load_config(self, argv=None, aliases=None, flags=None):
438 438 """Parse the configuration and generate the Config object.
439 439
440 440 After loading, any arguments that are not key-value or
441 441 flags will be stored in self.extra_args - a list of
442 442 unparsed command-line arguments. This is used for
443 443 arguments such as input files or subcommands.
444 444
445 445 Parameters
446 446 ----------
447 447 argv : list, optional
448 448 A list that has the form of sys.argv[1:] which has unicode
449 449 elements of the form u"key=value". If this is None (default),
450 450 then self.argv will be used.
451 451 aliases : dict
452 452 A dict of aliases for configurable traits.
453 453 Keys are the short aliases, Values are the resolved trait.
454 454 Of the form: `{'alias' : 'Configurable.trait'}`
455 455 flags : dict
456 456 A dict of flags, keyed by str name. Values can be Config objects
457 457 or dicts. When the flag is triggered, The config is loaded as
458 458 `self.config.update(cfg)`.
459 459 """
460 460 from IPython.config.configurable import Configurable
461 461
462 462 self.clear()
463 463 if argv is None:
464 464 argv = self.argv
465 465 if aliases is None:
466 466 aliases = self.aliases
467 467 if flags is None:
468 468 flags = self.flags
469 469
470 470 # ensure argv is a list of unicode strings:
471 471 uargv = self._decode_argv(argv)
472 472 for idx,raw in enumerate(uargv):
473 473 # strip leading '-'
474 474 item = raw.lstrip('-')
475 475
476 476 if raw == '--':
477 477 # don't parse arguments after '--'
478 478 # this is useful for relaying arguments to scripts, e.g.
479 479 # ipython -i foo.py --pylab=qt -- args after '--' go-to-foo.py
480 480 self.extra_args.extend(uargv[idx+1:])
481 481 break
482 482
483 483 if kv_pattern.match(raw):
484 484 lhs,rhs = item.split('=',1)
485 485 # Substitute longnames for aliases.
486 486 if lhs in aliases:
487 487 lhs = aliases[lhs]
488 488 if '.' not in lhs:
489 489 # probably a mistyped alias, but not technically illegal
490 490 warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
491 491 self._exec_config_str(lhs, rhs)
492 492
493 493 elif flag_pattern.match(raw):
494 494 if item in flags:
495 495 cfg,help = flags[item]
496 496 self._load_flag(cfg)
497 497 else:
498 498 raise ArgumentError("Unrecognized flag: '%s'"%raw)
499 499 elif raw.startswith('-'):
500 500 kv = '--'+item
501 501 if kv_pattern.match(kv):
502 502 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
503 503 else:
504 504 raise ArgumentError("Invalid argument: '%s'"%raw)
505 505 else:
506 506 # keep all args that aren't valid in a list,
507 507 # in case our parent knows what to do with them.
508 508 self.extra_args.append(item)
509 509 return self.config
510 510
511 511 class ArgParseConfigLoader(CommandLineConfigLoader):
512 512 """A loader that uses the argparse module to load from the command line."""
513 513
514 514 def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw):
515 515 """Create a config loader for use with argparse.
516 516
517 517 Parameters
518 518 ----------
519 519
520 520 argv : optional, list
521 521 If given, used to read command-line arguments from, otherwise
522 522 sys.argv[1:] is used.
523 523
524 524 parser_args : tuple
525 525 A tuple of positional arguments that will be passed to the
526 526 constructor of :class:`argparse.ArgumentParser`.
527 527
528 528 parser_kw : dict
529 529 A tuple of keyword arguments that will be passed to the
530 530 constructor of :class:`argparse.ArgumentParser`.
531 531
532 532 Returns
533 533 -------
534 534 config : Config
535 535 The resulting Config object.
536 536 """
537 537 super(CommandLineConfigLoader, self).__init__()
538 538 self.clear()
539 539 if argv is None:
540 540 argv = sys.argv[1:]
541 541 self.argv = argv
542 542 self.aliases = aliases or {}
543 543 self.flags = flags or {}
544 544
545 545 self.parser_args = parser_args
546 546 self.version = parser_kw.pop("version", None)
547 547 kwargs = dict(argument_default=argparse.SUPPRESS)
548 548 kwargs.update(parser_kw)
549 549 self.parser_kw = kwargs
550 550
551 551 def load_config(self, argv=None, aliases=None, flags=None):
552 552 """Parse command line arguments and return as a Config object.
553 553
554 554 Parameters
555 555 ----------
556 556
557 557 args : optional, list
558 558 If given, a list with the structure of sys.argv[1:] to parse
559 559 arguments from. If not given, the instance's self.argv attribute
560 560 (given at construction time) is used."""
561 561 self.clear()
562 562 if argv is None:
563 563 argv = self.argv
564 564 if aliases is None:
565 565 aliases = self.aliases
566 566 if flags is None:
567 567 flags = self.flags
568 568 self._create_parser(aliases, flags)
569 569 self._parse_args(argv)
570 570 self._convert_to_config()
571 571 return self.config
572 572
573 573 def get_extra_args(self):
574 574 if hasattr(self, 'extra_args'):
575 575 return self.extra_args
576 576 else:
577 577 return []
578 578
579 579 def _create_parser(self, aliases=None, flags=None):
580 580 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
581 581 self._add_arguments(aliases, flags)
582 582
583 583 def _add_arguments(self, aliases=None, flags=None):
584 584 raise NotImplementedError("subclasses must implement _add_arguments")
585 585
586 586 def _parse_args(self, args):
587 587 """self.parser->self.parsed_data"""
588 588 # decode sys.argv to support unicode command-line options
589 uargs = [py3compat.cast_unicode(a) for a in args]
589 enc = text.getdefaultencoding()
590 uargs = [py3compat.cast_unicode(a, enc) for a in args]
590 591 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
591 592
592 593 def _convert_to_config(self):
593 594 """self.parsed_data->self.config"""
594 595 for k, v in vars(self.parsed_data).iteritems():
595 596 exec "self.config.%s = v"%k in locals(), globals()
596 597
597 598 class KVArgParseConfigLoader(ArgParseConfigLoader):
598 599 """A config loader that loads aliases and flags with argparse,
599 600 but will use KVLoader for the rest. This allows better parsing
600 601 of common args, such as `ipython -c 'print 5'`, but still gets
601 602 arbitrary config with `ipython --InteractiveShell.use_readline=False`"""
602 603
603 604 def _convert_to_config(self):
604 605 """self.parsed_data->self.config"""
605 606 for k, v in vars(self.parsed_data).iteritems():
606 607 self._exec_config_str(k, v)
607 608
608 609 def _add_arguments(self, aliases=None, flags=None):
609 610 self.alias_flags = {}
610 611 # print aliases, flags
611 612 if aliases is None:
612 613 aliases = self.aliases
613 614 if flags is None:
614 615 flags = self.flags
615 616 paa = self.parser.add_argument
616 617 for key,value in aliases.iteritems():
617 618 if key in flags:
618 619 # flags
619 620 nargs = '?'
620 621 else:
621 622 nargs = None
622 623 if len(key) is 1:
623 624 paa('-'+key, '--'+key, type=str, dest=value, nargs=nargs)
624 625 else:
625 626 paa('--'+key, type=str, dest=value, nargs=nargs)
626 627 for key, (value, help) in flags.iteritems():
627 628 if key in self.aliases:
628 629 #
629 630 self.alias_flags[self.aliases[key]] = value
630 631 continue
631 632 if len(key) is 1:
632 633 paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
633 634 else:
634 635 paa('--'+key, action='append_const', dest='_flags', const=value)
635 636
636 637 def _convert_to_config(self):
637 638 """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
638 639 # remove subconfigs list from namespace before transforming the Namespace
639 640 if '_flags' in self.parsed_data:
640 641 subcs = self.parsed_data._flags
641 642 del self.parsed_data._flags
642 643 else:
643 644 subcs = []
644 645
645 646 for k, v in vars(self.parsed_data).iteritems():
646 647 if v is None:
647 648 # it was a flag that shares the name of an alias
648 649 subcs.append(self.alias_flags[k])
649 650 else:
650 651 # eval the KV assignment
651 652 self._exec_config_str(k, v)
652 653
653 654 for subc in subcs:
654 655 self._load_flag(subc)
655 656
656 657 if self.extra_args:
657 658 sub_parser = KeyValueConfigLoader()
658 659 sub_parser.load_config(self.extra_args)
659 660 self.config._merge(sub_parser.config)
660 661 self.extra_args = sub_parser.extra_args
@@ -1,147 +1,148
1 1 """Windows-specific implementation of process utilities.
2 2
3 3 This file is only meant to be imported by process.py, not by end-users.
4 4 """
5 5
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (C) 2010 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16 from __future__ import print_function
17 17
18 18 # stdlib
19 19 import os
20 20 import sys
21 21
22 22 from subprocess import STDOUT
23 23
24 24 # our own imports
25 25 from ._process_common import read_no_interrupt, process_handler
26 from . import text
26 27
27 28 #-----------------------------------------------------------------------------
28 29 # Function definitions
29 30 #-----------------------------------------------------------------------------
30 31
31 32 class AvoidUNCPath(object):
32 33 """A context manager to protect command execution from UNC paths.
33 34
34 35 In the Win32 API, commands can't be invoked with the cwd being a UNC path.
35 36 This context manager temporarily changes directory to the 'C:' drive on
36 37 entering, and restores the original working directory on exit.
37 38
38 39 The context manager returns the starting working directory *if* it made a
39 40 change and None otherwise, so that users can apply the necessary adjustment
40 41 to their system calls in the event of a change.
41 42
42 43 Example
43 44 -------
44 45 ::
45 46 cmd = 'dir'
46 47 with AvoidUNCPath() as path:
47 48 if path is not None:
48 49 cmd = '"pushd %s &&"%s' % (path, cmd)
49 50 os.system(cmd)
50 51 """
51 52 def __enter__(self):
52 53 self.path = os.getcwdu()
53 54 self.is_unc_path = self.path.startswith(r"\\")
54 55 if self.is_unc_path:
55 56 # change to c drive (as cmd.exe cannot handle UNC addresses)
56 57 os.chdir("C:")
57 58 return self.path
58 59 else:
59 60 # We return None to signal that there was no change in the working
60 61 # directory
61 62 return None
62 63
63 64 def __exit__(self, exc_type, exc_value, traceback):
64 65 if self.is_unc_path:
65 66 os.chdir(self.path)
66 67
67 68
68 69 def _find_cmd(cmd):
69 70 """Find the full path to a .bat or .exe using the win32api module."""
70 71 try:
71 72 from win32api import SearchPath
72 73 except ImportError:
73 74 raise ImportError('you need to have pywin32 installed for this to work')
74 75 else:
75 76 PATH = os.environ['PATH']
76 77 extensions = ['.exe', '.com', '.bat', '.py']
77 78 path = None
78 79 for ext in extensions:
79 80 try:
80 81 path = SearchPath(PATH, cmd + ext)[0]
81 82 except:
82 83 pass
83 84 if path is None:
84 85 raise OSError("command %r not found" % cmd)
85 86 else:
86 87 return path
87 88
88 89
89 90 def _system_body(p):
90 91 """Callback for _system."""
91 enc = sys.stdin.encoding or sys.getdefaultencoding()
92 enc = text.getdefaultencoding()
92 93 for line in read_no_interrupt(p.stdout).splitlines():
93 94 line = line.decode(enc, 'replace')
94 95 print(line, file=sys.stdout)
95 96 for line in read_no_interrupt(p.stderr).splitlines():
96 97 line = line.decode(enc, 'replace')
97 98 print(line, file=sys.stderr)
98 99
99 100 # Wait to finish for returncode
100 101 return p.wait()
101 102
102 103
103 104 def system(cmd):
104 105 """Win32 version of os.system() that works with network shares.
105 106
106 107 Note that this implementation returns None, as meant for use in IPython.
107 108
108 109 Parameters
109 110 ----------
110 111 cmd : str
111 112 A command to be executed in the system shell.
112 113
113 114 Returns
114 115 -------
115 116 None : we explicitly do NOT return the subprocess status code, as this
116 117 utility is meant to be used extensively in IPython, where any return value
117 118 would trigger :func:`sys.displayhook` calls.
118 119 """
119 120 with AvoidUNCPath() as path:
120 121 if path is not None:
121 122 cmd = '"pushd %s &&"%s' % (path, cmd)
122 123 return process_handler(cmd, _system_body)
123 124
124 125
125 126 def getoutput(cmd):
126 127 """Return standard output of executing cmd in a shell.
127 128
128 129 Accepts the same arguments as os.system().
129 130
130 131 Parameters
131 132 ----------
132 133 cmd : str
133 134 A command to be executed in the system shell.
134 135
135 136 Returns
136 137 -------
137 138 stdout : str
138 139 """
139 140
140 141 with AvoidUNCPath() as path:
141 142 if path is not None:
142 143 cmd = '"pushd %s &&"%s' % (path, cmd)
143 144 out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
144 145
145 146 if out is None:
146 147 out = ''
147 148 return out
@@ -1,164 +1,165
1 1 """Utilities to manipulate JSON objects.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (C) 2010 The IPython Development Team
5 5 #
6 6 # Distributed under the terms of the BSD License. The full license is in
7 7 # the file COPYING.txt, distributed as part of this software.
8 8 #-----------------------------------------------------------------------------
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Imports
12 12 #-----------------------------------------------------------------------------
13 13 # stdlib
14 14 import re
15 15 import sys
16 16 import types
17 17 from datetime import datetime
18 18
19 19 from IPython.utils import py3compat
20 from IPython.utils import text
20 21 next_attr_name = '__next__' if py3compat.PY3 else 'next'
21 22
22 23 #-----------------------------------------------------------------------------
23 24 # Globals and constants
24 25 #-----------------------------------------------------------------------------
25 26
26 27 # timestamp formats
27 28 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
28 29 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
29 30
30 31 #-----------------------------------------------------------------------------
31 32 # Classes and functions
32 33 #-----------------------------------------------------------------------------
33 34
34 35 def rekey(dikt):
35 36 """Rekey a dict that has been forced to use str keys where there should be
36 37 ints by json."""
37 38 for k in dikt.iterkeys():
38 39 if isinstance(k, basestring):
39 40 ik=fk=None
40 41 try:
41 42 ik = int(k)
42 43 except ValueError:
43 44 try:
44 45 fk = float(k)
45 46 except ValueError:
46 47 continue
47 48 if ik is not None:
48 49 nk = ik
49 50 else:
50 51 nk = fk
51 52 if nk in dikt:
52 53 raise KeyError("already have key %r"%nk)
53 54 dikt[nk] = dikt.pop(k)
54 55 return dikt
55 56
56 57
57 58 def extract_dates(obj):
58 59 """extract ISO8601 dates from unpacked JSON"""
59 60 if isinstance(obj, dict):
60 61 obj = dict(obj) # don't clobber
61 62 for k,v in obj.iteritems():
62 63 obj[k] = extract_dates(v)
63 64 elif isinstance(obj, (list, tuple)):
64 65 obj = [ extract_dates(o) for o in obj ]
65 66 elif isinstance(obj, basestring):
66 67 if ISO8601_PAT.match(obj):
67 68 obj = datetime.strptime(obj, ISO8601)
68 69 return obj
69 70
70 71 def squash_dates(obj):
71 72 """squash datetime objects into ISO8601 strings"""
72 73 if isinstance(obj, dict):
73 74 obj = dict(obj) # don't clobber
74 75 for k,v in obj.iteritems():
75 76 obj[k] = squash_dates(v)
76 77 elif isinstance(obj, (list, tuple)):
77 78 obj = [ squash_dates(o) for o in obj ]
78 79 elif isinstance(obj, datetime):
79 80 obj = obj.strftime(ISO8601)
80 81 return obj
81 82
82 83 def date_default(obj):
83 84 """default function for packing datetime objects in JSON."""
84 85 if isinstance(obj, datetime):
85 86 return obj.strftime(ISO8601)
86 87 else:
87 88 raise TypeError("%r is not JSON serializable"%obj)
88 89
89 90
90 91
91 92 def json_clean(obj):
92 93 """Clean an object to ensure it's safe to encode in JSON.
93 94
94 95 Atomic, immutable objects are returned unmodified. Sets and tuples are
95 96 converted to lists, lists are copied and dicts are also copied.
96 97
97 98 Note: dicts whose keys could cause collisions upon encoding (such as a dict
98 99 with both the number 1 and the string '1' as keys) will cause a ValueError
99 100 to be raised.
100 101
101 102 Parameters
102 103 ----------
103 104 obj : any python object
104 105
105 106 Returns
106 107 -------
107 108 out : object
108 109
109 110 A version of the input which will not cause an encoding error when
110 111 encoded as JSON. Note that this function does not *encode* its inputs,
111 112 it simply sanitizes it so that there will be no encoding errors later.
112 113
113 114 Examples
114 115 --------
115 116 >>> json_clean(4)
116 117 4
117 118 >>> json_clean(range(10))
118 119 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
119 120 >>> json_clean(dict(x=1, y=2))
120 121 {'y': 2, 'x': 1}
121 122 >>> json_clean(dict(x=1, y=2, z=[1,2,3]))
122 123 {'y': 2, 'x': 1, 'z': [1, 2, 3]}
123 124 >>> json_clean(True)
124 125 True
125 126 """
126 127 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
127 128 # listed explicitly because bools pass as int instances
128 129 atomic_ok = (unicode, int, float, types.NoneType)
129 130
130 131 # containers that we need to convert into lists
131 132 container_to_list = (tuple, set, types.GeneratorType)
132 133
133 134 if isinstance(obj, atomic_ok):
134 135 return obj
135 136
136 137 if isinstance(obj, bytes):
137 return obj.decode(sys.getdefaultencoding(), 'replace')
138 return obj.decode(text.getdefaultencoding(), 'replace')
138 139
139 140 if isinstance(obj, container_to_list) or (
140 141 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
141 142 obj = list(obj)
142 143
143 144 if isinstance(obj, list):
144 145 return [json_clean(x) for x in obj]
145 146
146 147 if isinstance(obj, dict):
147 148 # First, validate that the dict won't lose data in conversion due to
148 149 # key collisions after stringification. This can happen with keys like
149 150 # True and 'true' or 1 and '1', which collide in JSON.
150 151 nkeys = len(obj)
151 152 nkeys_collapsed = len(set(map(str, obj)))
152 153 if nkeys != nkeys_collapsed:
153 154 raise ValueError('dict can not be safely converted to JSON: '
154 155 'key collision would lead to dropped values')
155 156 # If all OK, proceed by making the new dict that will be json-safe
156 157 out = {}
157 158 for k,v in obj.iteritems():
158 159 out[str(k)] = json_clean(v)
159 160 return out
160 161
161 162 # If we get here, we don't know how to handle the object, so we just get
162 163 # its repr and return that. This will catch lambdas, open sockets, class
163 164 # objects, and any other complicated contraption that json can't encode
164 165 return repr(obj)
@@ -1,715 +1,739
1 1 # encoding: utf-8
2 2 """
3 3 Utilities for working with strings and text.
4 4 """
5 5
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (C) 2008-2009 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 import __main__
18 18
19 import locale
19 20 import os
20 21 import re
21 22 import shutil
23 import sys
22 24 import textwrap
23 25 from string import Formatter
24 26
25 27 from IPython.external.path import path
26 28 from IPython.utils import py3compat
27 29 from IPython.utils.io import nlprint
28 30 from IPython.utils.data import flatten
29 31
30 32 #-----------------------------------------------------------------------------
31 33 # Code
32 34 #-----------------------------------------------------------------------------
33 35
36 # Less conservative replacement for sys.getdefaultencoding, that will try
37 # to match the environment.
38 # Defined here as central function, so if we find better choices, we
39 # won't need to make changes all over IPython.
40 def getdefaultencoding():
41 """Return IPython's guess for the default encoding for bytes as text.
42
43 Asks for stdin.encoding first, to match the calling Terminal, but that
44 is often None for subprocesses. Fall back on locale.getpreferredencoding()
45 which should be a sensible platform default (that respects LANG environment),
46 and finally to sys.getdefaultencoding() which is the most conservative option,
47 and usually ASCII.
48 """
49 enc = sys.stdin.encoding
50 if not enc:
51 try:
52 # There are reports of getpreferredencoding raising errors
53 # in some cases, which may well be fixed, but let's be conservative here.
54 enc = locale.getpreferredencoding(False)
55 except Exception:
56 pass
57 return enc or sys.getdefaultencoding()
34 58
35 59 def unquote_ends(istr):
36 60 """Remove a single pair of quotes from the endpoints of a string."""
37 61
38 62 if not istr:
39 63 return istr
40 64 if (istr[0]=="'" and istr[-1]=="'") or \
41 65 (istr[0]=='"' and istr[-1]=='"'):
42 66 return istr[1:-1]
43 67 else:
44 68 return istr
45 69
46 70
47 71 class LSString(str):
48 72 """String derivative with a special access attributes.
49 73
50 74 These are normal strings, but with the special attributes:
51 75
52 76 .l (or .list) : value as list (split on newlines).
53 77 .n (or .nlstr): original value (the string itself).
54 78 .s (or .spstr): value as whitespace-separated string.
55 79 .p (or .paths): list of path objects
56 80
57 81 Any values which require transformations are computed only once and
58 82 cached.
59 83
60 84 Such strings are very useful to efficiently interact with the shell, which
61 85 typically only understands whitespace-separated options for commands."""
62 86
63 87 def get_list(self):
64 88 try:
65 89 return self.__list
66 90 except AttributeError:
67 91 self.__list = self.split('\n')
68 92 return self.__list
69 93
70 94 l = list = property(get_list)
71 95
72 96 def get_spstr(self):
73 97 try:
74 98 return self.__spstr
75 99 except AttributeError:
76 100 self.__spstr = self.replace('\n',' ')
77 101 return self.__spstr
78 102
79 103 s = spstr = property(get_spstr)
80 104
81 105 def get_nlstr(self):
82 106 return self
83 107
84 108 n = nlstr = property(get_nlstr)
85 109
86 110 def get_paths(self):
87 111 try:
88 112 return self.__paths
89 113 except AttributeError:
90 114 self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
91 115 return self.__paths
92 116
93 117 p = paths = property(get_paths)
94 118
95 119 # FIXME: We need to reimplement type specific displayhook and then add this
96 120 # back as a custom printer. This should also be moved outside utils into the
97 121 # core.
98 122
99 123 # def print_lsstring(arg):
100 124 # """ Prettier (non-repr-like) and more informative printer for LSString """
101 125 # print "LSString (.p, .n, .l, .s available). Value:"
102 126 # print arg
103 127 #
104 128 #
105 129 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
106 130
107 131
108 132 class SList(list):
109 133 """List derivative with a special access attributes.
110 134
111 135 These are normal lists, but with the special attributes:
112 136
113 137 .l (or .list) : value as list (the list itself).
114 138 .n (or .nlstr): value as a string, joined on newlines.
115 139 .s (or .spstr): value as a string, joined on spaces.
116 140 .p (or .paths): list of path objects
117 141
118 142 Any values which require transformations are computed only once and
119 143 cached."""
120 144
121 145 def get_list(self):
122 146 return self
123 147
124 148 l = list = property(get_list)
125 149
126 150 def get_spstr(self):
127 151 try:
128 152 return self.__spstr
129 153 except AttributeError:
130 154 self.__spstr = ' '.join(self)
131 155 return self.__spstr
132 156
133 157 s = spstr = property(get_spstr)
134 158
135 159 def get_nlstr(self):
136 160 try:
137 161 return self.__nlstr
138 162 except AttributeError:
139 163 self.__nlstr = '\n'.join(self)
140 164 return self.__nlstr
141 165
142 166 n = nlstr = property(get_nlstr)
143 167
144 168 def get_paths(self):
145 169 try:
146 170 return self.__paths
147 171 except AttributeError:
148 172 self.__paths = [path(p) for p in self if os.path.exists(p)]
149 173 return self.__paths
150 174
151 175 p = paths = property(get_paths)
152 176
153 177 def grep(self, pattern, prune = False, field = None):
154 178 """ Return all strings matching 'pattern' (a regex or callable)
155 179
156 180 This is case-insensitive. If prune is true, return all items
157 181 NOT matching the pattern.
158 182
159 183 If field is specified, the match must occur in the specified
160 184 whitespace-separated field.
161 185
162 186 Examples::
163 187
164 188 a.grep( lambda x: x.startswith('C') )
165 189 a.grep('Cha.*log', prune=1)
166 190 a.grep('chm', field=-1)
167 191 """
168 192
169 193 def match_target(s):
170 194 if field is None:
171 195 return s
172 196 parts = s.split()
173 197 try:
174 198 tgt = parts[field]
175 199 return tgt
176 200 except IndexError:
177 201 return ""
178 202
179 203 if isinstance(pattern, basestring):
180 204 pred = lambda x : re.search(pattern, x, re.IGNORECASE)
181 205 else:
182 206 pred = pattern
183 207 if not prune:
184 208 return SList([el for el in self if pred(match_target(el))])
185 209 else:
186 210 return SList([el for el in self if not pred(match_target(el))])
187 211
188 212 def fields(self, *fields):
189 213 """ Collect whitespace-separated fields from string list
190 214
191 215 Allows quick awk-like usage of string lists.
192 216
193 217 Example data (in var a, created by 'a = !ls -l')::
194 218 -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog
195 219 drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython
196 220
197 221 a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
198 222 a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
199 223 (note the joining by space).
200 224 a.fields(-1) is ['ChangeLog', 'IPython']
201 225
202 226 IndexErrors are ignored.
203 227
204 228 Without args, fields() just split()'s the strings.
205 229 """
206 230 if len(fields) == 0:
207 231 return [el.split() for el in self]
208 232
209 233 res = SList()
210 234 for el in [f.split() for f in self]:
211 235 lineparts = []
212 236
213 237 for fd in fields:
214 238 try:
215 239 lineparts.append(el[fd])
216 240 except IndexError:
217 241 pass
218 242 if lineparts:
219 243 res.append(" ".join(lineparts))
220 244
221 245 return res
222 246
223 247 def sort(self,field= None, nums = False):
224 248 """ sort by specified fields (see fields())
225 249
226 250 Example::
227 251 a.sort(1, nums = True)
228 252
229 253 Sorts a by second field, in numerical order (so that 21 > 3)
230 254
231 255 """
232 256
233 257 #decorate, sort, undecorate
234 258 if field is not None:
235 259 dsu = [[SList([line]).fields(field), line] for line in self]
236 260 else:
237 261 dsu = [[line, line] for line in self]
238 262 if nums:
239 263 for i in range(len(dsu)):
240 264 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
241 265 try:
242 266 n = int(numstr)
243 267 except ValueError:
244 268 n = 0;
245 269 dsu[i][0] = n
246 270
247 271
248 272 dsu.sort()
249 273 return SList([t[1] for t in dsu])
250 274
251 275
252 276 # FIXME: We need to reimplement type specific displayhook and then add this
253 277 # back as a custom printer. This should also be moved outside utils into the
254 278 # core.
255 279
256 280 # def print_slist(arg):
257 281 # """ Prettier (non-repr-like) and more informative printer for SList """
258 282 # print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
259 283 # if hasattr(arg, 'hideonce') and arg.hideonce:
260 284 # arg.hideonce = False
261 285 # return
262 286 #
263 287 # nlprint(arg)
264 288 #
265 289 # print_slist = result_display.when_type(SList)(print_slist)
266 290
267 291
268 292 def esc_quotes(strng):
269 293 """Return the input string with single and double quotes escaped out"""
270 294
271 295 return strng.replace('"','\\"').replace("'","\\'")
272 296
273 297
274 298 def make_quoted_expr(s):
275 299 """Return string s in appropriate quotes, using raw string if possible.
276 300
277 301 XXX - example removed because it caused encoding errors in documentation
278 302 generation. We need a new example that doesn't contain invalid chars.
279 303
280 304 Note the use of raw string and padding at the end to allow trailing
281 305 backslash.
282 306 """
283 307
284 308 tail = ''
285 309 tailpadding = ''
286 310 raw = ''
287 311 ucode = '' if py3compat.PY3 else 'u'
288 312 if "\\" in s:
289 313 raw = 'r'
290 314 if s.endswith('\\'):
291 315 tail = '[:-1]'
292 316 tailpadding = '_'
293 317 if '"' not in s:
294 318 quote = '"'
295 319 elif "'" not in s:
296 320 quote = "'"
297 321 elif '"""' not in s and not s.endswith('"'):
298 322 quote = '"""'
299 323 elif "'''" not in s and not s.endswith("'"):
300 324 quote = "'''"
301 325 else:
302 326 # give up, backslash-escaped string will do
303 327 return '"%s"' % esc_quotes(s)
304 328 res = ucode + raw + quote + s + tailpadding + quote + tail
305 329 return res
306 330
307 331
308 332 def qw(words,flat=0,sep=None,maxsplit=-1):
309 333 """Similar to Perl's qw() operator, but with some more options.
310 334
311 335 qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
312 336
313 337 words can also be a list itself, and with flat=1, the output will be
314 338 recursively flattened.
315 339
316 340 Examples:
317 341
318 342 >>> qw('1 2')
319 343 ['1', '2']
320 344
321 345 >>> qw(['a b','1 2',['m n','p q']])
322 346 [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
323 347
324 348 >>> qw(['a b','1 2',['m n','p q']],flat=1)
325 349 ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
326 350 """
327 351
328 352 if isinstance(words, basestring):
329 353 return [word.strip() for word in words.split(sep,maxsplit)
330 354 if word and not word.isspace() ]
331 355 if flat:
332 356 return flatten(map(qw,words,[1]*len(words)))
333 357 return map(qw,words)
334 358
335 359
336 360 def qwflat(words,sep=None,maxsplit=-1):
337 361 """Calls qw(words) in flat mode. It's just a convenient shorthand."""
338 362 return qw(words,1,sep,maxsplit)
339 363
340 364
341 365 def qw_lol(indata):
342 366 """qw_lol('a b') -> [['a','b']],
343 367 otherwise it's just a call to qw().
344 368
345 369 We need this to make sure the modules_some keys *always* end up as a
346 370 list of lists."""
347 371
348 372 if isinstance(indata, basestring):
349 373 return [qw(indata)]
350 374 else:
351 375 return qw(indata)
352 376
353 377
354 378 def grep(pat,list,case=1):
355 379 """Simple minded grep-like function.
356 380 grep(pat,list) returns occurrences of pat in list, None on failure.
357 381
358 382 It only does simple string matching, with no support for regexps. Use the
359 383 option case=0 for case-insensitive matching."""
360 384
361 385 # This is pretty crude. At least it should implement copying only references
362 386 # to the original data in case it's big. Now it copies the data for output.
363 387 out=[]
364 388 if case:
365 389 for term in list:
366 390 if term.find(pat)>-1: out.append(term)
367 391 else:
368 392 lpat=pat.lower()
369 393 for term in list:
370 394 if term.lower().find(lpat)>-1: out.append(term)
371 395
372 396 if len(out): return out
373 397 else: return None
374 398
375 399
376 400 def dgrep(pat,*opts):
377 401 """Return grep() on dir()+dir(__builtins__).
378 402
379 403 A very common use of grep() when working interactively."""
380 404
381 405 return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
382 406
383 407
384 408 def idgrep(pat):
385 409 """Case-insensitive dgrep()"""
386 410
387 411 return dgrep(pat,0)
388 412
389 413
390 414 def igrep(pat,list):
391 415 """Synonym for case-insensitive grep."""
392 416
393 417 return grep(pat,list,case=0)
394 418
395 419
396 420 def indent(instr,nspaces=4, ntabs=0, flatten=False):
397 421 """Indent a string a given number of spaces or tabstops.
398 422
399 423 indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
400 424
401 425 Parameters
402 426 ----------
403 427
404 428 instr : basestring
405 429 The string to be indented.
406 430 nspaces : int (default: 4)
407 431 The number of spaces to be indented.
408 432 ntabs : int (default: 0)
409 433 The number of tabs to be indented.
410 434 flatten : bool (default: False)
411 435 Whether to scrub existing indentation. If True, all lines will be
412 436 aligned to the same indentation. If False, existing indentation will
413 437 be strictly increased.
414 438
415 439 Returns
416 440 -------
417 441
418 442 str|unicode : string indented by ntabs and nspaces.
419 443
420 444 """
421 445 if instr is None:
422 446 return
423 447 ind = '\t'*ntabs+' '*nspaces
424 448 if flatten:
425 449 pat = re.compile(r'^\s*', re.MULTILINE)
426 450 else:
427 451 pat = re.compile(r'^', re.MULTILINE)
428 452 outstr = re.sub(pat, ind, instr)
429 453 if outstr.endswith(os.linesep+ind):
430 454 return outstr[:-len(ind)]
431 455 else:
432 456 return outstr
433 457
434 458 def native_line_ends(filename,backup=1):
435 459 """Convert (in-place) a file to line-ends native to the current OS.
436 460
437 461 If the optional backup argument is given as false, no backup of the
438 462 original file is left. """
439 463
440 464 backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
441 465
442 466 bak_filename = filename + backup_suffixes[os.name]
443 467
444 468 original = open(filename).read()
445 469 shutil.copy2(filename,bak_filename)
446 470 try:
447 471 new = open(filename,'wb')
448 472 new.write(os.linesep.join(original.splitlines()))
449 473 new.write(os.linesep) # ALWAYS put an eol at the end of the file
450 474 new.close()
451 475 except:
452 476 os.rename(bak_filename,filename)
453 477 if not backup:
454 478 try:
455 479 os.remove(bak_filename)
456 480 except:
457 481 pass
458 482
459 483
460 484 def list_strings(arg):
461 485 """Always return a list of strings, given a string or list of strings
462 486 as input.
463 487
464 488 :Examples:
465 489
466 490 In [7]: list_strings('A single string')
467 491 Out[7]: ['A single string']
468 492
469 493 In [8]: list_strings(['A single string in a list'])
470 494 Out[8]: ['A single string in a list']
471 495
472 496 In [9]: list_strings(['A','list','of','strings'])
473 497 Out[9]: ['A', 'list', 'of', 'strings']
474 498 """
475 499
476 500 if isinstance(arg,basestring): return [arg]
477 501 else: return arg
478 502
479 503
480 504 def marquee(txt='',width=78,mark='*'):
481 505 """Return the input string centered in a 'marquee'.
482 506
483 507 :Examples:
484 508
485 509 In [16]: marquee('A test',40)
486 510 Out[16]: '**************** A test ****************'
487 511
488 512 In [17]: marquee('A test',40,'-')
489 513 Out[17]: '---------------- A test ----------------'
490 514
491 515 In [18]: marquee('A test',40,' ')
492 516 Out[18]: ' A test '
493 517
494 518 """
495 519 if not txt:
496 520 return (mark*width)[:width]
497 521 nmark = (width-len(txt)-2)//len(mark)//2
498 522 if nmark < 0: nmark =0
499 523 marks = mark*nmark
500 524 return '%s %s %s' % (marks,txt,marks)
501 525
502 526
503 527 ini_spaces_re = re.compile(r'^(\s+)')
504 528
505 529 def num_ini_spaces(strng):
506 530 """Return the number of initial spaces in a string"""
507 531
508 532 ini_spaces = ini_spaces_re.match(strng)
509 533 if ini_spaces:
510 534 return ini_spaces.end()
511 535 else:
512 536 return 0
513 537
514 538
515 539 def format_screen(strng):
516 540 """Format a string for screen printing.
517 541
518 542 This removes some latex-type format codes."""
519 543 # Paragraph continue
520 544 par_re = re.compile(r'\\$',re.MULTILINE)
521 545 strng = par_re.sub('',strng)
522 546 return strng
523 547
524 548 def dedent(text):
525 549 """Equivalent of textwrap.dedent that ignores unindented first line.
526 550
527 551 This means it will still dedent strings like:
528 552 '''foo
529 553 is a bar
530 554 '''
531 555
532 556 For use in wrap_paragraphs.
533 557 """
534 558
535 559 if text.startswith('\n'):
536 560 # text starts with blank line, don't ignore the first line
537 561 return textwrap.dedent(text)
538 562
539 563 # split first line
540 564 splits = text.split('\n',1)
541 565 if len(splits) == 1:
542 566 # only one line
543 567 return textwrap.dedent(text)
544 568
545 569 first, rest = splits
546 570 # dedent everything but the first line
547 571 rest = textwrap.dedent(rest)
548 572 return '\n'.join([first, rest])
549 573
550 574 def wrap_paragraphs(text, ncols=80):
551 575 """Wrap multiple paragraphs to fit a specified width.
552 576
553 577 This is equivalent to textwrap.wrap, but with support for multiple
554 578 paragraphs, as separated by empty lines.
555 579
556 580 Returns
557 581 -------
558 582
559 583 list of complete paragraphs, wrapped to fill `ncols` columns.
560 584 """
561 585 paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
562 586 text = dedent(text).strip()
563 587 paragraphs = paragraph_re.split(text)[::2] # every other entry is space
564 588 out_ps = []
565 589 indent_re = re.compile(r'\n\s+', re.MULTILINE)
566 590 for p in paragraphs:
567 591 # presume indentation that survives dedent is meaningful formatting,
568 592 # so don't fill unless text is flush.
569 593 if indent_re.search(p) is None:
570 594 # wrap paragraph
571 595 p = textwrap.fill(p, ncols)
572 596 out_ps.append(p)
573 597 return out_ps
574 598
575 599
576 600
577 601 class EvalFormatter(Formatter):
578 602 """A String Formatter that allows evaluation of simple expressions.
579 603
580 604 Any time a format key is not found in the kwargs,
581 605 it will be tried as an expression in the kwargs namespace.
582 606
583 607 This is to be used in templating cases, such as the parallel batch
584 608 script templates, where simple arithmetic on arguments is useful.
585 609
586 610 Examples
587 611 --------
588 612
589 613 In [1]: f = EvalFormatter()
590 614 In [2]: f.format('{n//4}', n=8)
591 615 Out[2]: '2'
592 616
593 617 In [3]: f.format('{list(range(3))}')
594 618 Out[3]: '[0, 1, 2]'
595 619
596 620 In [4]: f.format('{3*2}')
597 621 Out[4]: '6'
598 622 """
599 623
600 624 # should we allow slicing by disabling the format_spec feature?
601 625 allow_slicing = True
602 626
603 627 # copied from Formatter._vformat with minor changes to allow eval
604 628 # and replace the format_spec code with slicing
605 629 def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
606 630 if recursion_depth < 0:
607 631 raise ValueError('Max string recursion exceeded')
608 632 result = []
609 633 for literal_text, field_name, format_spec, conversion in \
610 634 self.parse(format_string):
611 635
612 636 # output the literal text
613 637 if literal_text:
614 638 result.append(literal_text)
615 639
616 640 # if there's a field, output it
617 641 if field_name is not None:
618 642 # this is some markup, find the object and do
619 643 # the formatting
620 644
621 645 if self.allow_slicing and format_spec:
622 646 # override format spec, to allow slicing:
623 647 field_name = ':'.join([field_name, format_spec])
624 648 format_spec = ''
625 649
626 650 # eval the contents of the field for the object
627 651 # to be formatted
628 652 obj = eval(field_name, kwargs)
629 653
630 654 # do any conversion on the resulting object
631 655 obj = self.convert_field(obj, conversion)
632 656
633 657 # expand the format spec, if needed
634 658 format_spec = self._vformat(format_spec, args, kwargs,
635 659 used_args, recursion_depth-1)
636 660
637 661 # format the object and append to the result
638 662 result.append(self.format_field(obj, format_spec))
639 663
640 664 return ''.join(result)
641 665
642 666
643 667 def columnize(items, separator=' ', displaywidth=80):
644 668 """ Transform a list of strings into a single string with columns.
645 669
646 670 Parameters
647 671 ----------
648 672 items : sequence of strings
649 673 The strings to process.
650 674
651 675 separator : str, optional [default is two spaces]
652 676 The string that separates columns.
653 677
654 678 displaywidth : int, optional [default is 80]
655 679 Width of the display in number of characters.
656 680
657 681 Returns
658 682 -------
659 683 The formatted string.
660 684 """
661 685 # Note: this code is adapted from columnize 0.3.2.
662 686 # See http://code.google.com/p/pycolumnize/
663 687
664 688 # Some degenerate cases.
665 689 size = len(items)
666 690 if size == 0:
667 691 return '\n'
668 692 elif size == 1:
669 693 return '%s\n' % items[0]
670 694
671 695 # Special case: if any item is longer than the maximum width, there's no
672 696 # point in triggering the logic below...
673 697 item_len = map(len, items) # save these, we can reuse them below
674 698 longest = max(item_len)
675 699 if longest >= displaywidth:
676 700 return '\n'.join(items+[''])
677 701
678 702 # Try every row count from 1 upwards
679 703 array_index = lambda nrows, row, col: nrows*col + row
680 704 for nrows in range(1, size):
681 705 ncols = (size + nrows - 1) // nrows
682 706 colwidths = []
683 707 totwidth = -len(separator)
684 708 for col in range(ncols):
685 709 # Get max column width for this column
686 710 colwidth = 0
687 711 for row in range(nrows):
688 712 i = array_index(nrows, row, col)
689 713 if i >= size: break
690 714 x, len_x = items[i], item_len[i]
691 715 colwidth = max(colwidth, len_x)
692 716 colwidths.append(colwidth)
693 717 totwidth += colwidth + len(separator)
694 718 if totwidth > displaywidth:
695 719 break
696 720 if totwidth <= displaywidth:
697 721 break
698 722
699 723 # The smallest number of rows computed and the max widths for each
700 724 # column has been obtained. Now we just have to format each of the rows.
701 725 string = ''
702 726 for row in range(nrows):
703 727 texts = []
704 728 for col in range(ncols):
705 729 i = row + nrows*col
706 730 if i >= size:
707 731 texts.append('')
708 732 else:
709 733 texts.append(items[i])
710 734 while texts and not texts[-1]:
711 735 del texts[-1]
712 736 for col in range(len(texts)):
713 737 texts[col] = texts[col].ljust(colwidths[col])
714 738 string += '%s\n' % separator.join(texts)
715 739 return string
@@ -1,91 +1,91
1 1 import sys
2 2 import time
3 3 from io import StringIO
4 4
5 5 from session import extract_header, Message
6 6
7 from IPython.utils import io
7 from IPython.utils import io, text
8 8
9 9 #-----------------------------------------------------------------------------
10 10 # Globals
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Stream classes
15 15 #-----------------------------------------------------------------------------
16 16
17 17 class OutStream(object):
18 18 """A file like object that publishes the stream to a 0MQ PUB socket."""
19 19
20 20 # The time interval between automatic flushes, in seconds.
21 21 flush_interval = 0.05
22 22 topic=None
23 23
24 24 def __init__(self, session, pub_socket, name):
25 25 self.session = session
26 26 self.pub_socket = pub_socket
27 27 self.name = name
28 28 self.parent_header = {}
29 29 self._new_buffer()
30 30
31 31 def set_parent(self, parent):
32 32 self.parent_header = extract_header(parent)
33 33
34 34 def close(self):
35 35 self.pub_socket = None
36 36
37 37 def flush(self):
38 38 #io.rprint('>>>flushing output buffer: %s<<<' % self.name) # dbg
39 39 if self.pub_socket is None:
40 40 raise ValueError(u'I/O operation on closed file')
41 41 else:
42 42 data = self._buffer.getvalue()
43 43 if data:
44 44 content = {u'name':self.name, u'data':data}
45 45 msg = self.session.send(self.pub_socket, u'stream', content=content,
46 46 parent=self.parent_header, ident=self.topic)
47 47
48 48 if hasattr(self.pub_socket, 'flush'):
49 49 # socket itself has flush (presumably ZMQStream)
50 50 self.pub_socket.flush()
51 51 self._buffer.close()
52 52 self._new_buffer()
53 53
54 54 def isatty(self):
55 55 return False
56 56
57 57 def next(self):
58 58 raise IOError('Read not supported on a write only stream.')
59 59
60 60 def read(self, size=-1):
61 61 raise IOError('Read not supported on a write only stream.')
62 62
63 63 def readline(self, size=-1):
64 64 raise IOError('Read not supported on a write only stream.')
65 65
66 66 def write(self, string):
67 67 if self.pub_socket is None:
68 68 raise ValueError('I/O operation on closed file')
69 69 else:
70 70 # Make sure that we're handling unicode
71 71 if not isinstance(string, unicode):
72 enc = sys.stdin.encoding or sys.getdefaultencoding()
72 enc = text.getdefaultencoding()
73 73 string = string.decode(enc, 'replace')
74 74
75 75 self._buffer.write(string)
76 76 current_time = time.time()
77 77 if self._start <= 0:
78 78 self._start = current_time
79 79 elif current_time - self._start > self.flush_interval:
80 80 self.flush()
81 81
82 82 def writelines(self, sequence):
83 83 if self.pub_socket is None:
84 84 raise ValueError('I/O operation on closed file')
85 85 else:
86 86 for string in sequence:
87 87 self.write(string)
88 88
89 89 def _new_buffer(self):
90 90 self._buffer = StringIO()
91 91 self._start = -1
General Comments 0
You need to be logged in to leave comments. Login now