##// END OF EJS Templates
Remove old workaround for a bug fixed in Python 3.4...
Nikita Kniazev -
Show More
@@ -1,2257 +1,2260 b''
1 1 """Completion for IPython.
2 2
3 3 This module started as fork of the rlcompleter module in the Python standard
4 4 library. The original enhancements made to rlcompleter have been sent
5 5 upstream and were accepted as of Python 2.3,
6 6
7 7 This module now support a wide variety of completion mechanism both available
8 8 for normal classic Python code, as well as completer for IPython specific
9 9 Syntax like magics.
10 10
11 11 Latex and Unicode completion
12 12 ============================
13 13
14 14 IPython and compatible frontends not only can complete your code, but can help
15 15 you to input a wide range of characters. In particular we allow you to insert
16 16 a unicode character using the tab completion mechanism.
17 17
18 18 Forward latex/unicode completion
19 19 --------------------------------
20 20
21 21 Forward completion allows you to easily type a unicode character using its latex
22 22 name, or unicode long description. To do so type a backslash follow by the
23 23 relevant name and press tab:
24 24
25 25
26 26 Using latex completion:
27 27
28 28 .. code::
29 29
30 30 \\alpha<tab>
31 31 Ξ±
32 32
33 33 or using unicode completion:
34 34
35 35
36 36 .. code::
37 37
38 38 \\GREEK SMALL LETTER ALPHA<tab>
39 39 Ξ±
40 40
41 41
42 42 Only valid Python identifiers will complete. Combining characters (like arrow or
43 43 dots) are also available, unlike latex they need to be put after the their
44 44 counterpart that is to say, `F\\\\vec<tab>` is correct, not `\\\\vec<tab>F`.
45 45
46 46 Some browsers are known to display combining characters incorrectly.
47 47
48 48 Backward latex completion
49 49 -------------------------
50 50
51 51 It is sometime challenging to know how to type a character, if you are using
52 52 IPython, or any compatible frontend you can prepend backslash to the character
53 53 and press `<tab>` to expand it to its latex form.
54 54
55 55 .. code::
56 56
57 57 \\Ξ±<tab>
58 58 \\alpha
59 59
60 60
61 61 Both forward and backward completions can be deactivated by setting the
62 62 ``Completer.backslash_combining_completions`` option to ``False``.
63 63
64 64
65 65 Experimental
66 66 ============
67 67
68 68 Starting with IPython 6.0, this module can make use of the Jedi library to
69 69 generate completions both using static analysis of the code, and dynamically
70 70 inspecting multiple namespaces. Jedi is an autocompletion and static analysis
71 71 for Python. The APIs attached to this new mechanism is unstable and will
72 72 raise unless use in an :any:`provisionalcompleter` context manager.
73 73
74 74 You will find that the following are experimental:
75 75
76 76 - :any:`provisionalcompleter`
77 77 - :any:`IPCompleter.completions`
78 78 - :any:`Completion`
79 79 - :any:`rectify_completions`
80 80
81 81 .. note::
82 82
83 83 better name for :any:`rectify_completions` ?
84 84
85 85 We welcome any feedback on these new API, and we also encourage you to try this
86 86 module in debug mode (start IPython with ``--Completer.debug=True``) in order
87 87 to have extra logging information if :any:`jedi` is crashing, or if current
88 88 IPython completer pending deprecations are returning results not yet handled
89 89 by :any:`jedi`
90 90
91 91 Using Jedi for tab completion allow snippets like the following to work without
92 92 having to execute any code:
93 93
94 94 >>> myvar = ['hello', 42]
95 95 ... myvar[1].bi<tab>
96 96
97 97 Tab completion will be able to infer that ``myvar[1]`` is a real number without
98 98 executing any code unlike the previously available ``IPCompleter.greedy``
99 99 option.
100 100
101 101 Be sure to update :any:`jedi` to the latest stable version or to try the
102 102 current development version to get better completions.
103 103 """
104 104
105 105
106 106 # Copyright (c) IPython Development Team.
107 107 # Distributed under the terms of the Modified BSD License.
108 108 #
109 109 # Some of this code originated from rlcompleter in the Python standard library
110 110 # Copyright (C) 2001 Python Software Foundation, www.python.org
111 111
112 112
113 113 import builtins as builtin_mod
114 114 import glob
115 115 import inspect
116 116 import itertools
117 117 import keyword
118 118 import os
119 119 import re
120 120 import string
121 121 import sys
122 122 import time
123 123 import unicodedata
124 124 import uuid
125 125 import warnings
126 126 from contextlib import contextmanager
127 127 from importlib import import_module
128 128 from types import SimpleNamespace
129 129 from typing import Iterable, Iterator, List, Tuple, Union, Any, Sequence, Dict, NamedTuple, Pattern, Optional
130 130
131 131 from IPython.core.error import TryNext
132 132 from IPython.core.inputtransformer2 import ESC_MAGIC
133 133 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
134 134 from IPython.core.oinspect import InspectColors
135 from IPython.testing.skipdoctest import skip_doctest
135 136 from IPython.utils import generics
136 137 from IPython.utils.dir2 import dir2, get_real_method
137 138 from IPython.utils.path import ensure_dir_exists
138 139 from IPython.utils.process import arg_split
139 140 from traitlets import Bool, Enum, Int, List as ListTrait, Unicode, default, observe
140 141 from traitlets.config.configurable import Configurable
141 142
142 143 import __main__
143 144
144 145 # skip module docstests
145 146 __skip_doctest__ = True
146 147
147 148 try:
148 149 import jedi
149 150 jedi.settings.case_insensitive_completion = False
150 151 import jedi.api.helpers
151 152 import jedi.api.classes
152 153 JEDI_INSTALLED = True
153 154 except ImportError:
154 155 JEDI_INSTALLED = False
155 156 #-----------------------------------------------------------------------------
156 157 # Globals
157 158 #-----------------------------------------------------------------------------
158 159
159 160 # ranges where we have most of the valid unicode names. We could be more finer
160 161 # grained but is it worth it for performance While unicode have character in the
161 162 # range 0, 0x110000, we seem to have name for about 10% of those. (131808 as I
162 163 # write this). With below range we cover them all, with a density of ~67%
163 164 # biggest next gap we consider only adds up about 1% density and there are 600
164 165 # gaps that would need hard coding.
165 166 _UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)]
166 167
167 168 # Public API
168 169 __all__ = ['Completer','IPCompleter']
169 170
170 171 if sys.platform == 'win32':
171 172 PROTECTABLES = ' '
172 173 else:
173 174 PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
174 175
175 176 # Protect against returning an enormous number of completions which the frontend
176 177 # may have trouble processing.
177 178 MATCHES_LIMIT = 500
178 179
179 180 _deprecation_readline_sentinel = object()
180 181
181 182
182 183 class ProvisionalCompleterWarning(FutureWarning):
183 184 """
184 185 Exception raise by an experimental feature in this module.
185 186
186 187 Wrap code in :any:`provisionalcompleter` context manager if you
187 188 are certain you want to use an unstable feature.
188 189 """
189 190 pass
190 191
191 192 warnings.filterwarnings('error', category=ProvisionalCompleterWarning)
192 193
194
195 @skip_doctest
193 196 @contextmanager
194 197 def provisionalcompleter(action='ignore'):
195 198 """
196 199 This context manager has to be used in any place where unstable completer
197 200 behavior and API may be called.
198 201
199 202 >>> with provisionalcompleter():
200 203 ... completer.do_experimental_things() # works
201 204
202 205 >>> completer.do_experimental_things() # raises.
203 206
204 207 .. note::
205 208
206 209 Unstable
207 210
208 211 By using this context manager you agree that the API in use may change
209 212 without warning, and that you won't complain if they do so.
210 213
211 214 You also understand that, if the API is not to your liking, you should report
212 215 a bug to explain your use case upstream.
213 216
214 217 We'll be happy to get your feedback, feature requests, and improvements on
215 218 any of the unstable APIs!
216 219 """
217 220 with warnings.catch_warnings():
218 221 warnings.filterwarnings(action, category=ProvisionalCompleterWarning)
219 222 yield
220 223
221 224
222 225 def has_open_quotes(s):
223 226 """Return whether a string has open quotes.
224 227
225 228 This simply counts whether the number of quote characters of either type in
226 229 the string is odd.
227 230
228 231 Returns
229 232 -------
230 233 If there is an open quote, the quote character is returned. Else, return
231 234 False.
232 235 """
233 236 # We check " first, then ', so complex cases with nested quotes will get
234 237 # the " to take precedence.
235 238 if s.count('"') % 2:
236 239 return '"'
237 240 elif s.count("'") % 2:
238 241 return "'"
239 242 else:
240 243 return False
241 244
242 245
243 246 def protect_filename(s, protectables=PROTECTABLES):
244 247 """Escape a string to protect certain characters."""
245 248 if set(s) & set(protectables):
246 249 if sys.platform == "win32":
247 250 return '"' + s + '"'
248 251 else:
249 252 return "".join(("\\" + c if c in protectables else c) for c in s)
250 253 else:
251 254 return s
252 255
253 256
254 257 def expand_user(path:str) -> Tuple[str, bool, str]:
255 258 """Expand ``~``-style usernames in strings.
256 259
257 260 This is similar to :func:`os.path.expanduser`, but it computes and returns
258 261 extra information that will be useful if the input was being used in
259 262 computing completions, and you wish to return the completions with the
260 263 original '~' instead of its expanded value.
261 264
262 265 Parameters
263 266 ----------
264 267 path : str
265 268 String to be expanded. If no ~ is present, the output is the same as the
266 269 input.
267 270
268 271 Returns
269 272 -------
270 273 newpath : str
271 274 Result of ~ expansion in the input path.
272 275 tilde_expand : bool
273 276 Whether any expansion was performed or not.
274 277 tilde_val : str
275 278 The value that ~ was replaced with.
276 279 """
277 280 # Default values
278 281 tilde_expand = False
279 282 tilde_val = ''
280 283 newpath = path
281 284
282 285 if path.startswith('~'):
283 286 tilde_expand = True
284 287 rest = len(path)-1
285 288 newpath = os.path.expanduser(path)
286 289 if rest:
287 290 tilde_val = newpath[:-rest]
288 291 else:
289 292 tilde_val = newpath
290 293
291 294 return newpath, tilde_expand, tilde_val
292 295
293 296
294 297 def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str:
295 298 """Does the opposite of expand_user, with its outputs.
296 299 """
297 300 if tilde_expand:
298 301 return path.replace(tilde_val, '~')
299 302 else:
300 303 return path
301 304
302 305
303 306 def completions_sorting_key(word):
304 307 """key for sorting completions
305 308
306 309 This does several things:
307 310
308 311 - Demote any completions starting with underscores to the end
309 312 - Insert any %magic and %%cellmagic completions in the alphabetical order
310 313 by their name
311 314 """
312 315 prio1, prio2 = 0, 0
313 316
314 317 if word.startswith('__'):
315 318 prio1 = 2
316 319 elif word.startswith('_'):
317 320 prio1 = 1
318 321
319 322 if word.endswith('='):
320 323 prio1 = -1
321 324
322 325 if word.startswith('%%'):
323 326 # If there's another % in there, this is something else, so leave it alone
324 327 if not "%" in word[2:]:
325 328 word = word[2:]
326 329 prio2 = 2
327 330 elif word.startswith('%'):
328 331 if not "%" in word[1:]:
329 332 word = word[1:]
330 333 prio2 = 1
331 334
332 335 return prio1, word, prio2
333 336
334 337
335 338 class _FakeJediCompletion:
336 339 """
337 340 This is a workaround to communicate to the UI that Jedi has crashed and to
338 341 report a bug. Will be used only id :any:`IPCompleter.debug` is set to true.
339 342
340 343 Added in IPython 6.0 so should likely be removed for 7.0
341 344
342 345 """
343 346
344 347 def __init__(self, name):
345 348
346 349 self.name = name
347 350 self.complete = name
348 351 self.type = 'crashed'
349 352 self.name_with_symbols = name
350 353 self.signature = ''
351 354 self._origin = 'fake'
352 355
353 356 def __repr__(self):
354 357 return '<Fake completion object jedi has crashed>'
355 358
356 359
357 360 class Completion:
358 361 """
359 362 Completion object used and return by IPython completers.
360 363
361 364 .. warning::
362 365
363 366 Unstable
364 367
365 368 This function is unstable, API may change without warning.
366 369 It will also raise unless use in proper context manager.
367 370
368 371 This act as a middle ground :any:`Completion` object between the
369 372 :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion
370 373 object. While Jedi need a lot of information about evaluator and how the
371 374 code should be ran/inspected, PromptToolkit (and other frontend) mostly
372 375 need user facing information.
373 376
374 377 - Which range should be replaced replaced by what.
375 378 - Some metadata (like completion type), or meta information to displayed to
376 379 the use user.
377 380
378 381 For debugging purpose we can also store the origin of the completion (``jedi``,
379 382 ``IPython.python_matches``, ``IPython.magics_matches``...).
380 383 """
381 384
382 385 __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
383 386
384 387 def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None:
385 388 warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
386 389 "It may change without warnings. "
387 390 "Use in corresponding context manager.",
388 391 category=ProvisionalCompleterWarning, stacklevel=2)
389 392
390 393 self.start = start
391 394 self.end = end
392 395 self.text = text
393 396 self.type = type
394 397 self.signature = signature
395 398 self._origin = _origin
396 399
397 400 def __repr__(self):
398 401 return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
399 402 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
400 403
401 404 def __eq__(self, other)->Bool:
402 405 """
403 406 Equality and hash do not hash the type (as some completer may not be
404 407 able to infer the type), but are use to (partially) de-duplicate
405 408 completion.
406 409
407 410 Completely de-duplicating completion is a bit tricker that just
408 411 comparing as it depends on surrounding text, which Completions are not
409 412 aware of.
410 413 """
411 414 return self.start == other.start and \
412 415 self.end == other.end and \
413 416 self.text == other.text
414 417
415 418 def __hash__(self):
416 419 return hash((self.start, self.end, self.text))
417 420
418 421
419 422 _IC = Iterable[Completion]
420 423
421 424
422 425 def _deduplicate_completions(text: str, completions: _IC)-> _IC:
423 426 """
424 427 Deduplicate a set of completions.
425 428
426 429 .. warning::
427 430
428 431 Unstable
429 432
430 433 This function is unstable, API may change without warning.
431 434
432 435 Parameters
433 436 ----------
434 437 text : str
435 438 text that should be completed.
436 439 completions : Iterator[Completion]
437 440 iterator over the completions to deduplicate
438 441
439 442 Yields
440 443 ------
441 444 `Completions` objects
442 445 Completions coming from multiple sources, may be different but end up having
443 446 the same effect when applied to ``text``. If this is the case, this will
444 447 consider completions as equal and only emit the first encountered.
445 448 Not folded in `completions()` yet for debugging purpose, and to detect when
446 449 the IPython completer does return things that Jedi does not, but should be
447 450 at some point.
448 451 """
449 452 completions = list(completions)
450 453 if not completions:
451 454 return
452 455
453 456 new_start = min(c.start for c in completions)
454 457 new_end = max(c.end for c in completions)
455 458
456 459 seen = set()
457 460 for c in completions:
458 461 new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
459 462 if new_text not in seen:
460 463 yield c
461 464 seen.add(new_text)
462 465
463 466
464 467 def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC:
465 468 """
466 469 Rectify a set of completions to all have the same ``start`` and ``end``
467 470
468 471 .. warning::
469 472
470 473 Unstable
471 474
472 475 This function is unstable, API may change without warning.
473 476 It will also raise unless use in proper context manager.
474 477
475 478 Parameters
476 479 ----------
477 480 text : str
478 481 text that should be completed.
479 482 completions : Iterator[Completion]
480 483 iterator over the completions to rectify
481 484
482 485 Notes
483 486 -----
484 487 :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though
485 488 the Jupyter Protocol requires them to behave like so. This will readjust
486 489 the completion to have the same ``start`` and ``end`` by padding both
487 490 extremities with surrounding text.
488 491
489 492 During stabilisation should support a ``_debug`` option to log which
490 493 completion are return by the IPython completer and not found in Jedi in
491 494 order to make upstream bug report.
492 495 """
493 496 warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). "
494 497 "It may change without warnings. "
495 498 "Use in corresponding context manager.",
496 499 category=ProvisionalCompleterWarning, stacklevel=2)
497 500
498 501 completions = list(completions)
499 502 if not completions:
500 503 return
501 504 starts = (c.start for c in completions)
502 505 ends = (c.end for c in completions)
503 506
504 507 new_start = min(starts)
505 508 new_end = max(ends)
506 509
507 510 seen_jedi = set()
508 511 seen_python_matches = set()
509 512 for c in completions:
510 513 new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
511 514 if c._origin == 'jedi':
512 515 seen_jedi.add(new_text)
513 516 elif c._origin == 'IPCompleter.python_matches':
514 517 seen_python_matches.add(new_text)
515 518 yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature)
516 519 diff = seen_python_matches.difference(seen_jedi)
517 520 if diff and _debug:
518 521 print('IPython.python matches have extras:', diff)
519 522
520 523
521 524 if sys.platform == 'win32':
522 525 DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?'
523 526 else:
524 527 DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
525 528
526 529 GREEDY_DELIMS = ' =\r\n'
527 530
528 531
529 532 class CompletionSplitter(object):
530 533 """An object to split an input line in a manner similar to readline.
531 534
532 535 By having our own implementation, we can expose readline-like completion in
533 536 a uniform manner to all frontends. This object only needs to be given the
534 537 line of text to be split and the cursor position on said line, and it
535 538 returns the 'word' to be completed on at the cursor after splitting the
536 539 entire line.
537 540
538 541 What characters are used as splitting delimiters can be controlled by
539 542 setting the ``delims`` attribute (this is a property that internally
540 543 automatically builds the necessary regular expression)"""
541 544
542 545 # Private interface
543 546
544 547 # A string of delimiter characters. The default value makes sense for
545 548 # IPython's most typical usage patterns.
546 549 _delims = DELIMS
547 550
548 551 # The expression (a normal string) to be compiled into a regular expression
549 552 # for actual splitting. We store it as an attribute mostly for ease of
550 553 # debugging, since this type of code can be so tricky to debug.
551 554 _delim_expr = None
552 555
553 556 # The regular expression that does the actual splitting
554 557 _delim_re = None
555 558
556 559 def __init__(self, delims=None):
557 560 delims = CompletionSplitter._delims if delims is None else delims
558 561 self.delims = delims
559 562
560 563 @property
561 564 def delims(self):
562 565 """Return the string of delimiter characters."""
563 566 return self._delims
564 567
565 568 @delims.setter
566 569 def delims(self, delims):
567 570 """Set the delimiters for line splitting."""
568 571 expr = '[' + ''.join('\\'+ c for c in delims) + ']'
569 572 self._delim_re = re.compile(expr)
570 573 self._delims = delims
571 574 self._delim_expr = expr
572 575
573 576 def split_line(self, line, cursor_pos=None):
574 577 """Split a line of text with a cursor at the given position.
575 578 """
576 579 l = line if cursor_pos is None else line[:cursor_pos]
577 580 return self._delim_re.split(l)[-1]
578 581
579 582
580 583
581 584 class Completer(Configurable):
582 585
583 586 greedy = Bool(False,
584 587 help="""Activate greedy completion
585 588 PENDING DEPRECTION. this is now mostly taken care of with Jedi.
586 589
587 590 This will enable completion on elements of lists, results of function calls, etc.,
588 591 but can be unsafe because the code is actually evaluated on TAB.
589 592 """
590 593 ).tag(config=True)
591 594
592 595 use_jedi = Bool(default_value=JEDI_INSTALLED,
593 596 help="Experimental: Use Jedi to generate autocompletions. "
594 597 "Default to True if jedi is installed.").tag(config=True)
595 598
596 599 jedi_compute_type_timeout = Int(default_value=400,
597 600 help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types.
598 601 Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
599 602 performance by preventing jedi to build its cache.
600 603 """).tag(config=True)
601 604
602 605 debug = Bool(default_value=False,
603 606 help='Enable debug for the Completer. Mostly print extra '
604 607 'information for experimental jedi integration.')\
605 608 .tag(config=True)
606 609
607 610 backslash_combining_completions = Bool(True,
608 611 help="Enable unicode completions, e.g. \\alpha<tab> . "
609 612 "Includes completion of latex commands, unicode names, and expanding "
610 613 "unicode characters back to latex commands.").tag(config=True)
611 614
612 615
613 616
614 617 def __init__(self, namespace=None, global_namespace=None, **kwargs):
615 618 """Create a new completer for the command line.
616 619
617 620 Completer(namespace=ns, global_namespace=ns2) -> completer instance.
618 621
619 622 If unspecified, the default namespace where completions are performed
620 623 is __main__ (technically, __main__.__dict__). Namespaces should be
621 624 given as dictionaries.
622 625
623 626 An optional second namespace can be given. This allows the completer
624 627 to handle cases where both the local and global scopes need to be
625 628 distinguished.
626 629 """
627 630
628 631 # Don't bind to namespace quite yet, but flag whether the user wants a
629 632 # specific namespace or to use __main__.__dict__. This will allow us
630 633 # to bind to __main__.__dict__ at completion time, not now.
631 634 if namespace is None:
632 635 self.use_main_ns = True
633 636 else:
634 637 self.use_main_ns = False
635 638 self.namespace = namespace
636 639
637 640 # The global namespace, if given, can be bound directly
638 641 if global_namespace is None:
639 642 self.global_namespace = {}
640 643 else:
641 644 self.global_namespace = global_namespace
642 645
643 646 self.custom_matchers = []
644 647
645 648 super(Completer, self).__init__(**kwargs)
646 649
647 650 def complete(self, text, state):
648 651 """Return the next possible completion for 'text'.
649 652
650 653 This is called successively with state == 0, 1, 2, ... until it
651 654 returns None. The completion should begin with 'text'.
652 655
653 656 """
654 657 if self.use_main_ns:
655 658 self.namespace = __main__.__dict__
656 659
657 660 if state == 0:
658 661 if "." in text:
659 662 self.matches = self.attr_matches(text)
660 663 else:
661 664 self.matches = self.global_matches(text)
662 665 try:
663 666 return self.matches[state]
664 667 except IndexError:
665 668 return None
666 669
667 670 def global_matches(self, text):
668 671 """Compute matches when text is a simple name.
669 672
670 673 Return a list of all keywords, built-in functions and names currently
671 674 defined in self.namespace or self.global_namespace that match.
672 675
673 676 """
674 677 matches = []
675 678 match_append = matches.append
676 679 n = len(text)
677 680 for lst in [keyword.kwlist,
678 681 builtin_mod.__dict__.keys(),
679 682 self.namespace.keys(),
680 683 self.global_namespace.keys()]:
681 684 for word in lst:
682 685 if word[:n] == text and word != "__builtins__":
683 686 match_append(word)
684 687
685 688 snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z")
686 689 for lst in [self.namespace.keys(),
687 690 self.global_namespace.keys()]:
688 691 shortened = {"_".join([sub[0] for sub in word.split('_')]) : word
689 692 for word in lst if snake_case_re.match(word)}
690 693 for word in shortened.keys():
691 694 if word[:n] == text and word != "__builtins__":
692 695 match_append(shortened[word])
693 696 return matches
694 697
695 698 def attr_matches(self, text):
696 699 """Compute matches when text contains a dot.
697 700
698 701 Assuming the text is of the form NAME.NAME....[NAME], and is
699 702 evaluatable in self.namespace or self.global_namespace, it will be
700 703 evaluated and its attributes (as revealed by dir()) are used as
701 704 possible completions. (For class instances, class members are
702 705 also considered.)
703 706
704 707 WARNING: this can still invoke arbitrary C code, if an object
705 708 with a __getattr__ hook is evaluated.
706 709
707 710 """
708 711
709 712 # Another option, seems to work great. Catches things like ''.<tab>
710 713 m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
711 714
712 715 if m:
713 716 expr, attr = m.group(1, 3)
714 717 elif self.greedy:
715 718 m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
716 719 if not m2:
717 720 return []
718 721 expr, attr = m2.group(1,2)
719 722 else:
720 723 return []
721 724
722 725 try:
723 726 obj = eval(expr, self.namespace)
724 727 except:
725 728 try:
726 729 obj = eval(expr, self.global_namespace)
727 730 except:
728 731 return []
729 732
730 733 if self.limit_to__all__ and hasattr(obj, '__all__'):
731 734 words = get__all__entries(obj)
732 735 else:
733 736 words = dir2(obj)
734 737
735 738 try:
736 739 words = generics.complete_object(obj, words)
737 740 except TryNext:
738 741 pass
739 742 except AssertionError:
740 743 raise
741 744 except Exception:
742 745 # Silence errors from completion function
743 746 #raise # dbg
744 747 pass
745 748 # Build match list to return
746 749 n = len(attr)
747 750 return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ]
748 751
749 752
750 753 def get__all__entries(obj):
751 754 """returns the strings in the __all__ attribute"""
752 755 try:
753 756 words = getattr(obj, '__all__')
754 757 except:
755 758 return []
756 759
757 760 return [w for w in words if isinstance(w, str)]
758 761
759 762
760 763 def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str,
761 764 extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]:
762 765 """Used by dict_key_matches, matching the prefix to a list of keys
763 766
764 767 Parameters
765 768 ----------
766 769 keys
767 770 list of keys in dictionary currently being completed.
768 771 prefix
769 772 Part of the text already typed by the user. E.g. `mydict[b'fo`
770 773 delims
771 774 String of delimiters to consider when finding the current key.
772 775 extra_prefix : optional
773 776 Part of the text already typed in multi-key index cases. E.g. for
774 777 `mydict['foo', "bar", 'b`, this would be `('foo', 'bar')`.
775 778
776 779 Returns
777 780 -------
778 781 A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
779 782 ``quote`` being the quote that need to be used to close current string.
780 783 ``token_start`` the position where the replacement should start occurring,
781 784 ``matches`` a list of replacement/completion
782 785
783 786 """
784 787 prefix_tuple = extra_prefix if extra_prefix else ()
785 788 Nprefix = len(prefix_tuple)
786 789 def filter_prefix_tuple(key):
787 790 # Reject too short keys
788 791 if len(key) <= Nprefix:
789 792 return False
790 793 # Reject keys with non str/bytes in it
791 794 for k in key:
792 795 if not isinstance(k, (str, bytes)):
793 796 return False
794 797 # Reject keys that do not match the prefix
795 798 for k, pt in zip(key, prefix_tuple):
796 799 if k != pt:
797 800 return False
798 801 # All checks passed!
799 802 return True
800 803
801 804 filtered_keys:List[Union[str,bytes]] = []
802 805 def _add_to_filtered_keys(key):
803 806 if isinstance(key, (str, bytes)):
804 807 filtered_keys.append(key)
805 808
806 809 for k in keys:
807 810 if isinstance(k, tuple):
808 811 if filter_prefix_tuple(k):
809 812 _add_to_filtered_keys(k[Nprefix])
810 813 else:
811 814 _add_to_filtered_keys(k)
812 815
813 816 if not prefix:
814 817 return '', 0, [repr(k) for k in filtered_keys]
815 818 quote_match = re.search('["\']', prefix)
816 819 assert quote_match is not None # silence mypy
817 820 quote = quote_match.group()
818 821 try:
819 822 prefix_str = eval(prefix + quote, {})
820 823 except Exception:
821 824 return '', 0, []
822 825
823 826 pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
824 827 token_match = re.search(pattern, prefix, re.UNICODE)
825 828 assert token_match is not None # silence mypy
826 829 token_start = token_match.start()
827 830 token_prefix = token_match.group()
828 831
829 832 matched:List[str] = []
830 833 for key in filtered_keys:
831 834 try:
832 835 if not key.startswith(prefix_str):
833 836 continue
834 837 except (AttributeError, TypeError, UnicodeError):
835 838 # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
836 839 continue
837 840
838 841 # reformat remainder of key to begin with prefix
839 842 rem = key[len(prefix_str):]
840 843 # force repr wrapped in '
841 844 rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
842 845 rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
843 846 if quote == '"':
844 847 # The entered prefix is quoted with ",
845 848 # but the match is quoted with '.
846 849 # A contained " hence needs escaping for comparison:
847 850 rem_repr = rem_repr.replace('"', '\\"')
848 851
849 852 # then reinsert prefix from start of token
850 853 matched.append('%s%s' % (token_prefix, rem_repr))
851 854 return quote, token_start, matched
852 855
853 856
854 857 def cursor_to_position(text:str, line:int, column:int)->int:
855 858 """
856 859 Convert the (line,column) position of the cursor in text to an offset in a
857 860 string.
858 861
859 862 Parameters
860 863 ----------
861 864 text : str
862 865 The text in which to calculate the cursor offset
863 866 line : int
864 867 Line of the cursor; 0-indexed
865 868 column : int
866 869 Column of the cursor 0-indexed
867 870
868 871 Returns
869 872 -------
870 873 Position of the cursor in ``text``, 0-indexed.
871 874
872 875 See Also
873 876 --------
874 877 position_to_cursor : reciprocal of this function
875 878
876 879 """
877 880 lines = text.split('\n')
878 881 assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
879 882
880 883 return sum(len(l) + 1 for l in lines[:line]) + column
881 884
882 885 def position_to_cursor(text:str, offset:int)->Tuple[int, int]:
883 886 """
884 887 Convert the position of the cursor in text (0 indexed) to a line
885 888 number(0-indexed) and a column number (0-indexed) pair
886 889
887 890 Position should be a valid position in ``text``.
888 891
889 892 Parameters
890 893 ----------
891 894 text : str
892 895 The text in which to calculate the cursor offset
893 896 offset : int
894 897 Position of the cursor in ``text``, 0-indexed.
895 898
896 899 Returns
897 900 -------
898 901 (line, column) : (int, int)
899 902 Line of the cursor; 0-indexed, column of the cursor 0-indexed
900 903
901 904 See Also
902 905 --------
903 906 cursor_to_position : reciprocal of this function
904 907
905 908 """
906 909
907 910 assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text))
908 911
909 912 before = text[:offset]
910 913 blines = before.split('\n') # ! splitnes trim trailing \n
911 914 line = before.count('\n')
912 915 col = len(blines[-1])
913 916 return line, col
914 917
915 918
916 919 def _safe_isinstance(obj, module, class_name):
917 920 """Checks if obj is an instance of module.class_name if loaded
918 921 """
919 922 return (module in sys.modules and
920 923 isinstance(obj, getattr(import_module(module), class_name)))
921 924
922 925 def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:
923 926 """Match Unicode characters back to Unicode name
924 927
925 928 This does ``β˜ƒ`` -> ``\\snowman``
926 929
927 930 Note that snowman is not a valid python3 combining character but will be expanded.
928 931 Though it will not recombine back to the snowman character by the completion machinery.
929 932
930 933 This will not either back-complete standard sequences like \\n, \\b ...
931 934
932 935 Returns
933 936 =======
934 937
935 938 Return a tuple with two elements:
936 939
937 940 - The Unicode character that was matched (preceded with a backslash), or
938 941 empty string,
939 942 - a sequence (of 1), name for the match Unicode character, preceded by
940 943 backslash, or empty if no match.
941 944
942 945 """
943 946 if len(text)<2:
944 947 return '', ()
945 948 maybe_slash = text[-2]
946 949 if maybe_slash != '\\':
947 950 return '', ()
948 951
949 952 char = text[-1]
950 953 # no expand on quote for completion in strings.
951 954 # nor backcomplete standard ascii keys
952 955 if char in string.ascii_letters or char in ('"',"'"):
953 956 return '', ()
954 957 try :
955 958 unic = unicodedata.name(char)
956 959 return '\\'+char,('\\'+unic,)
957 960 except KeyError:
958 961 pass
959 962 return '', ()
960 963
961 964 def back_latex_name_matches(text:str) -> Tuple[str, Sequence[str]] :
962 965 """Match latex characters back to unicode name
963 966
964 967 This does ``\\β„΅`` -> ``\\aleph``
965 968
966 969 """
967 970 if len(text)<2:
968 971 return '', ()
969 972 maybe_slash = text[-2]
970 973 if maybe_slash != '\\':
971 974 return '', ()
972 975
973 976
974 977 char = text[-1]
975 978 # no expand on quote for completion in strings.
976 979 # nor backcomplete standard ascii keys
977 980 if char in string.ascii_letters or char in ('"',"'"):
978 981 return '', ()
979 982 try :
980 983 latex = reverse_latex_symbol[char]
981 984 # '\\' replace the \ as well
982 985 return '\\'+char,[latex]
983 986 except KeyError:
984 987 pass
985 988 return '', ()
986 989
987 990
988 991 def _formatparamchildren(parameter) -> str:
989 992 """
990 993 Get parameter name and value from Jedi Private API
991 994
992 995 Jedi does not expose a simple way to get `param=value` from its API.
993 996
994 997 Parameters
995 998 ----------
996 999 parameter
997 1000 Jedi's function `Param`
998 1001
999 1002 Returns
1000 1003 -------
1001 1004 A string like 'a', 'b=1', '*args', '**kwargs'
1002 1005
1003 1006 """
1004 1007 description = parameter.description
1005 1008 if not description.startswith('param '):
1006 1009 raise ValueError('Jedi function parameter description have change format.'
1007 1010 'Expected "param ...", found %r".' % description)
1008 1011 return description[6:]
1009 1012
1010 1013 def _make_signature(completion)-> str:
1011 1014 """
1012 1015 Make the signature from a jedi completion
1013 1016
1014 1017 Parameters
1015 1018 ----------
1016 1019 completion : jedi.Completion
1017 1020 object does not complete a function type
1018 1021
1019 1022 Returns
1020 1023 -------
1021 1024 a string consisting of the function signature, with the parenthesis but
1022 1025 without the function name. example:
1023 1026 `(a, *args, b=1, **kwargs)`
1024 1027
1025 1028 """
1026 1029
1027 1030 # it looks like this might work on jedi 0.17
1028 1031 if hasattr(completion, 'get_signatures'):
1029 1032 signatures = completion.get_signatures()
1030 1033 if not signatures:
1031 1034 return '(?)'
1032 1035
1033 1036 c0 = completion.get_signatures()[0]
1034 1037 return '('+c0.to_string().split('(', maxsplit=1)[1]
1035 1038
1036 1039 return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures()
1037 1040 for p in signature.defined_names()) if f])
1038 1041
1039 1042
1040 1043 class _CompleteResult(NamedTuple):
1041 1044 matched_text : str
1042 1045 matches: Sequence[str]
1043 1046 matches_origin: Sequence[str]
1044 1047 jedi_matches: Any
1045 1048
1046 1049
1047 1050 class IPCompleter(Completer):
1048 1051 """Extension of the completer class with IPython-specific features"""
1049 1052
1050 1053 __dict_key_regexps: Optional[Dict[bool,Pattern]] = None
1051 1054
1052 1055 @observe('greedy')
1053 1056 def _greedy_changed(self, change):
1054 1057 """update the splitter and readline delims when greedy is changed"""
1055 1058 if change['new']:
1056 1059 self.splitter.delims = GREEDY_DELIMS
1057 1060 else:
1058 1061 self.splitter.delims = DELIMS
1059 1062
1060 1063 dict_keys_only = Bool(False,
1061 1064 help="""Whether to show dict key matches only""")
1062 1065
1063 1066 merge_completions = Bool(True,
1064 1067 help="""Whether to merge completion results into a single list
1065 1068
1066 1069 If False, only the completion results from the first non-empty
1067 1070 completer will be returned.
1068 1071 """
1069 1072 ).tag(config=True)
1070 1073 omit__names = Enum((0,1,2), default_value=2,
1071 1074 help="""Instruct the completer to omit private method names
1072 1075
1073 1076 Specifically, when completing on ``object.<tab>``.
1074 1077
1075 1078 When 2 [default]: all names that start with '_' will be excluded.
1076 1079
1077 1080 When 1: all 'magic' names (``__foo__``) will be excluded.
1078 1081
1079 1082 When 0: nothing will be excluded.
1080 1083 """
1081 1084 ).tag(config=True)
1082 1085 limit_to__all__ = Bool(False,
1083 1086 help="""
1084 1087 DEPRECATED as of version 5.0.
1085 1088
1086 1089 Instruct the completer to use __all__ for the completion
1087 1090
1088 1091 Specifically, when completing on ``object.<tab>``.
1089 1092
1090 1093 When True: only those names in obj.__all__ will be included.
1091 1094
1092 1095 When False [default]: the __all__ attribute is ignored
1093 1096 """,
1094 1097 ).tag(config=True)
1095 1098
1096 1099 profile_completions = Bool(
1097 1100 default_value=False,
1098 1101 help="If True, emit profiling data for completion subsystem using cProfile."
1099 1102 ).tag(config=True)
1100 1103
1101 1104 profiler_output_dir = Unicode(
1102 1105 default_value=".completion_profiles",
1103 1106 help="Template for path at which to output profile data for completions."
1104 1107 ).tag(config=True)
1105 1108
1106 1109 @observe('limit_to__all__')
1107 1110 def _limit_to_all_changed(self, change):
1108 1111 warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration '
1109 1112 'value has been deprecated since IPython 5.0, will be made to have '
1110 1113 'no effects and then removed in future version of IPython.',
1111 1114 UserWarning)
1112 1115
1113 1116 def __init__(self, shell=None, namespace=None, global_namespace=None,
1114 1117 use_readline=_deprecation_readline_sentinel, config=None, **kwargs):
1115 1118 """IPCompleter() -> completer
1116 1119
1117 1120 Return a completer object.
1118 1121
1119 1122 Parameters
1120 1123 ----------
1121 1124 shell
1122 1125 a pointer to the ipython shell itself. This is needed
1123 1126 because this completer knows about magic functions, and those can
1124 1127 only be accessed via the ipython instance.
1125 1128 namespace : dict, optional
1126 1129 an optional dict where completions are performed.
1127 1130 global_namespace : dict, optional
1128 1131 secondary optional dict for completions, to
1129 1132 handle cases (such as IPython embedded inside functions) where
1130 1133 both Python scopes are visible.
1131 1134 use_readline : bool, optional
1132 1135 DEPRECATED, ignored since IPython 6.0, will have no effects
1133 1136 """
1134 1137
1135 1138 self.magic_escape = ESC_MAGIC
1136 1139 self.splitter = CompletionSplitter()
1137 1140
1138 1141 if use_readline is not _deprecation_readline_sentinel:
1139 1142 warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.',
1140 1143 DeprecationWarning, stacklevel=2)
1141 1144
1142 1145 # _greedy_changed() depends on splitter and readline being defined:
1143 1146 Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
1144 1147 config=config, **kwargs)
1145 1148
1146 1149 # List where completion matches will be stored
1147 1150 self.matches = []
1148 1151 self.shell = shell
1149 1152 # Regexp to split filenames with spaces in them
1150 1153 self.space_name_re = re.compile(r'([^\\] )')
1151 1154 # Hold a local ref. to glob.glob for speed
1152 1155 self.glob = glob.glob
1153 1156
1154 1157 # Determine if we are running on 'dumb' terminals, like (X)Emacs
1155 1158 # buffers, to avoid completion problems.
1156 1159 term = os.environ.get('TERM','xterm')
1157 1160 self.dumb_terminal = term in ['dumb','emacs']
1158 1161
1159 1162 # Special handling of backslashes needed in win32 platforms
1160 1163 if sys.platform == "win32":
1161 1164 self.clean_glob = self._clean_glob_win32
1162 1165 else:
1163 1166 self.clean_glob = self._clean_glob
1164 1167
1165 1168 #regexp to parse docstring for function signature
1166 1169 self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
1167 1170 self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
1168 1171 #use this if positional argument name is also needed
1169 1172 #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
1170 1173
1171 1174 self.magic_arg_matchers = [
1172 1175 self.magic_config_matches,
1173 1176 self.magic_color_matches,
1174 1177 ]
1175 1178
1176 1179 # This is set externally by InteractiveShell
1177 1180 self.custom_completers = None
1178 1181
1179 1182 # This is a list of names of unicode characters that can be completed
1180 1183 # into their corresponding unicode value. The list is large, so we
1181 1184 # laziliy initialize it on first use. Consuming code should access this
1182 1185 # attribute through the `@unicode_names` property.
1183 1186 self._unicode_names = None
1184 1187
1185 1188 @property
1186 1189 def matchers(self) -> List[Any]:
1187 1190 """All active matcher routines for completion"""
1188 1191 if self.dict_keys_only:
1189 1192 return [self.dict_key_matches]
1190 1193
1191 1194 if self.use_jedi:
1192 1195 return [
1193 1196 *self.custom_matchers,
1194 1197 self.dict_key_matches,
1195 1198 self.file_matches,
1196 1199 self.magic_matches,
1197 1200 ]
1198 1201 else:
1199 1202 return [
1200 1203 *self.custom_matchers,
1201 1204 self.dict_key_matches,
1202 1205 self.python_matches,
1203 1206 self.file_matches,
1204 1207 self.magic_matches,
1205 1208 self.python_func_kw_matches,
1206 1209 ]
1207 1210
1208 1211 def all_completions(self, text:str) -> List[str]:
1209 1212 """
1210 1213 Wrapper around the completion methods for the benefit of emacs.
1211 1214 """
1212 1215 prefix = text.rpartition('.')[0]
1213 1216 with provisionalcompleter():
1214 1217 return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text
1215 1218 for c in self.completions(text, len(text))]
1216 1219
1217 1220 return self.complete(text)[1]
1218 1221
1219 1222 def _clean_glob(self, text:str):
1220 1223 return self.glob("%s*" % text)
1221 1224
1222 1225 def _clean_glob_win32(self, text:str):
1223 1226 return [f.replace("\\","/")
1224 1227 for f in self.glob("%s*" % text)]
1225 1228
1226 1229 def file_matches(self, text:str)->List[str]:
1227 1230 """Match filenames, expanding ~USER type strings.
1228 1231
1229 1232 Most of the seemingly convoluted logic in this completer is an
1230 1233 attempt to handle filenames with spaces in them. And yet it's not
1231 1234 quite perfect, because Python's readline doesn't expose all of the
1232 1235 GNU readline details needed for this to be done correctly.
1233 1236
1234 1237 For a filename with a space in it, the printed completions will be
1235 1238 only the parts after what's already been typed (instead of the
1236 1239 full completions, as is normally done). I don't think with the
1237 1240 current (as of Python 2.3) Python readline it's possible to do
1238 1241 better."""
1239 1242
1240 1243 # chars that require escaping with backslash - i.e. chars
1241 1244 # that readline treats incorrectly as delimiters, but we
1242 1245 # don't want to treat as delimiters in filename matching
1243 1246 # when escaped with backslash
1244 1247 if text.startswith('!'):
1245 1248 text = text[1:]
1246 1249 text_prefix = u'!'
1247 1250 else:
1248 1251 text_prefix = u''
1249 1252
1250 1253 text_until_cursor = self.text_until_cursor
1251 1254 # track strings with open quotes
1252 1255 open_quotes = has_open_quotes(text_until_cursor)
1253 1256
1254 1257 if '(' in text_until_cursor or '[' in text_until_cursor:
1255 1258 lsplit = text
1256 1259 else:
1257 1260 try:
1258 1261 # arg_split ~ shlex.split, but with unicode bugs fixed by us
1259 1262 lsplit = arg_split(text_until_cursor)[-1]
1260 1263 except ValueError:
1261 1264 # typically an unmatched ", or backslash without escaped char.
1262 1265 if open_quotes:
1263 1266 lsplit = text_until_cursor.split(open_quotes)[-1]
1264 1267 else:
1265 1268 return []
1266 1269 except IndexError:
1267 1270 # tab pressed on empty line
1268 1271 lsplit = ""
1269 1272
1270 1273 if not open_quotes and lsplit != protect_filename(lsplit):
1271 1274 # if protectables are found, do matching on the whole escaped name
1272 1275 has_protectables = True
1273 1276 text0,text = text,lsplit
1274 1277 else:
1275 1278 has_protectables = False
1276 1279 text = os.path.expanduser(text)
1277 1280
1278 1281 if text == "":
1279 1282 return [text_prefix + protect_filename(f) for f in self.glob("*")]
1280 1283
1281 1284 # Compute the matches from the filesystem
1282 1285 if sys.platform == 'win32':
1283 1286 m0 = self.clean_glob(text)
1284 1287 else:
1285 1288 m0 = self.clean_glob(text.replace('\\', ''))
1286 1289
1287 1290 if has_protectables:
1288 1291 # If we had protectables, we need to revert our changes to the
1289 1292 # beginning of filename so that we don't double-write the part
1290 1293 # of the filename we have so far
1291 1294 len_lsplit = len(lsplit)
1292 1295 matches = [text_prefix + text0 +
1293 1296 protect_filename(f[len_lsplit:]) for f in m0]
1294 1297 else:
1295 1298 if open_quotes:
1296 1299 # if we have a string with an open quote, we don't need to
1297 1300 # protect the names beyond the quote (and we _shouldn't_, as
1298 1301 # it would cause bugs when the filesystem call is made).
1299 1302 matches = m0 if sys.platform == "win32" else\
1300 1303 [protect_filename(f, open_quotes) for f in m0]
1301 1304 else:
1302 1305 matches = [text_prefix +
1303 1306 protect_filename(f) for f in m0]
1304 1307
1305 1308 # Mark directories in input list by appending '/' to their names.
1306 1309 return [x+'/' if os.path.isdir(x) else x for x in matches]
1307 1310
1308 1311 def magic_matches(self, text:str):
1309 1312 """Match magics"""
1310 1313 # Get all shell magics now rather than statically, so magics loaded at
1311 1314 # runtime show up too.
1312 1315 lsm = self.shell.magics_manager.lsmagic()
1313 1316 line_magics = lsm['line']
1314 1317 cell_magics = lsm['cell']
1315 1318 pre = self.magic_escape
1316 1319 pre2 = pre+pre
1317 1320
1318 1321 explicit_magic = text.startswith(pre)
1319 1322
1320 1323 # Completion logic:
1321 1324 # - user gives %%: only do cell magics
1322 1325 # - user gives %: do both line and cell magics
1323 1326 # - no prefix: do both
1324 1327 # In other words, line magics are skipped if the user gives %% explicitly
1325 1328 #
1326 1329 # We also exclude magics that match any currently visible names:
1327 1330 # https://github.com/ipython/ipython/issues/4877, unless the user has
1328 1331 # typed a %:
1329 1332 # https://github.com/ipython/ipython/issues/10754
1330 1333 bare_text = text.lstrip(pre)
1331 1334 global_matches = self.global_matches(bare_text)
1332 1335 if not explicit_magic:
1333 1336 def matches(magic):
1334 1337 """
1335 1338 Filter magics, in particular remove magics that match
1336 1339 a name present in global namespace.
1337 1340 """
1338 1341 return ( magic.startswith(bare_text) and
1339 1342 magic not in global_matches )
1340 1343 else:
1341 1344 def matches(magic):
1342 1345 return magic.startswith(bare_text)
1343 1346
1344 1347 comp = [ pre2+m for m in cell_magics if matches(m)]
1345 1348 if not text.startswith(pre2):
1346 1349 comp += [ pre+m for m in line_magics if matches(m)]
1347 1350
1348 1351 return comp
1349 1352
1350 1353 def magic_config_matches(self, text:str) -> List[str]:
1351 1354 """ Match class names and attributes for %config magic """
1352 1355 texts = text.strip().split()
1353 1356
1354 1357 if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'):
1355 1358 # get all configuration classes
1356 1359 classes = sorted(set([ c for c in self.shell.configurables
1357 1360 if c.__class__.class_traits(config=True)
1358 1361 ]), key=lambda x: x.__class__.__name__)
1359 1362 classnames = [ c.__class__.__name__ for c in classes ]
1360 1363
1361 1364 # return all classnames if config or %config is given
1362 1365 if len(texts) == 1:
1363 1366 return classnames
1364 1367
1365 1368 # match classname
1366 1369 classname_texts = texts[1].split('.')
1367 1370 classname = classname_texts[0]
1368 1371 classname_matches = [ c for c in classnames
1369 1372 if c.startswith(classname) ]
1370 1373
1371 1374 # return matched classes or the matched class with attributes
1372 1375 if texts[1].find('.') < 0:
1373 1376 return classname_matches
1374 1377 elif len(classname_matches) == 1 and \
1375 1378 classname_matches[0] == classname:
1376 1379 cls = classes[classnames.index(classname)].__class__
1377 1380 help = cls.class_get_help()
1378 1381 # strip leading '--' from cl-args:
1379 1382 help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
1380 1383 return [ attr.split('=')[0]
1381 1384 for attr in help.strip().splitlines()
1382 1385 if attr.startswith(texts[1]) ]
1383 1386 return []
1384 1387
1385 1388 def magic_color_matches(self, text:str) -> List[str] :
1386 1389 """ Match color schemes for %colors magic"""
1387 1390 texts = text.split()
1388 1391 if text.endswith(' '):
1389 1392 # .split() strips off the trailing whitespace. Add '' back
1390 1393 # so that: '%colors ' -> ['%colors', '']
1391 1394 texts.append('')
1392 1395
1393 1396 if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'):
1394 1397 prefix = texts[1]
1395 1398 return [ color for color in InspectColors.keys()
1396 1399 if color.startswith(prefix) ]
1397 1400 return []
1398 1401
1399 1402 def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str) -> Iterable[Any]:
1400 1403 """
1401 1404 Return a list of :any:`jedi.api.Completions` object from a ``text`` and
1402 1405 cursor position.
1403 1406
1404 1407 Parameters
1405 1408 ----------
1406 1409 cursor_column : int
1407 1410 column position of the cursor in ``text``, 0-indexed.
1408 1411 cursor_line : int
1409 1412 line position of the cursor in ``text``, 0-indexed
1410 1413 text : str
1411 1414 text to complete
1412 1415
1413 1416 Notes
1414 1417 -----
1415 1418 If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
1416 1419 object containing a string with the Jedi debug information attached.
1417 1420 """
1418 1421 namespaces = [self.namespace]
1419 1422 if self.global_namespace is not None:
1420 1423 namespaces.append(self.global_namespace)
1421 1424
1422 1425 completion_filter = lambda x:x
1423 1426 offset = cursor_to_position(text, cursor_line, cursor_column)
1424 1427 # filter output if we are completing for object members
1425 1428 if offset:
1426 1429 pre = text[offset-1]
1427 1430 if pre == '.':
1428 1431 if self.omit__names == 2:
1429 1432 completion_filter = lambda c:not c.name.startswith('_')
1430 1433 elif self.omit__names == 1:
1431 1434 completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__'))
1432 1435 elif self.omit__names == 0:
1433 1436 completion_filter = lambda x:x
1434 1437 else:
1435 1438 raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
1436 1439
1437 1440 interpreter = jedi.Interpreter(text[:offset], namespaces)
1438 1441 try_jedi = True
1439 1442
1440 1443 try:
1441 1444 # find the first token in the current tree -- if it is a ' or " then we are in a string
1442 1445 completing_string = False
1443 1446 try:
1444 1447 first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value'))
1445 1448 except StopIteration:
1446 1449 pass
1447 1450 else:
1448 1451 # note the value may be ', ", or it may also be ''' or """, or
1449 1452 # in some cases, """what/you/typed..., but all of these are
1450 1453 # strings.
1451 1454 completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'}
1452 1455
1453 1456 # if we are in a string jedi is likely not the right candidate for
1454 1457 # now. Skip it.
1455 1458 try_jedi = not completing_string
1456 1459 except Exception as e:
1457 1460 # many of things can go wrong, we are using private API just don't crash.
1458 1461 if self.debug:
1459 1462 print("Error detecting if completing a non-finished string :", e, '|')
1460 1463
1461 1464 if not try_jedi:
1462 1465 return []
1463 1466 try:
1464 1467 return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1))
1465 1468 except Exception as e:
1466 1469 if self.debug:
1467 1470 return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))]
1468 1471 else:
1469 1472 return []
1470 1473
1471 1474 def python_matches(self, text:str)->List[str]:
1472 1475 """Match attributes or global python names"""
1473 1476 if "." in text:
1474 1477 try:
1475 1478 matches = self.attr_matches(text)
1476 1479 if text.endswith('.') and self.omit__names:
1477 1480 if self.omit__names == 1:
1478 1481 # true if txt is _not_ a __ name, false otherwise:
1479 1482 no__name = (lambda txt:
1480 1483 re.match(r'.*\.__.*?__',txt) is None)
1481 1484 else:
1482 1485 # true if txt is _not_ a _ name, false otherwise:
1483 1486 no__name = (lambda txt:
1484 1487 re.match(r'\._.*?',txt[txt.rindex('.'):]) is None)
1485 1488 matches = filter(no__name, matches)
1486 1489 except NameError:
1487 1490 # catches <undefined attributes>.<tab>
1488 1491 matches = []
1489 1492 else:
1490 1493 matches = self.global_matches(text)
1491 1494 return matches
1492 1495
1493 1496 def _default_arguments_from_docstring(self, doc):
1494 1497 """Parse the first line of docstring for call signature.
1495 1498
1496 1499 Docstring should be of the form 'min(iterable[, key=func])\n'.
1497 1500 It can also parse cython docstring of the form
1498 1501 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
1499 1502 """
1500 1503 if doc is None:
1501 1504 return []
1502 1505
1503 1506 #care only the firstline
1504 1507 line = doc.lstrip().splitlines()[0]
1505 1508
1506 1509 #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
1507 1510 #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
1508 1511 sig = self.docstring_sig_re.search(line)
1509 1512 if sig is None:
1510 1513 return []
1511 1514 # iterable[, key=func]' -> ['iterable[' ,' key=func]']
1512 1515 sig = sig.groups()[0].split(',')
1513 1516 ret = []
1514 1517 for s in sig:
1515 1518 #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
1516 1519 ret += self.docstring_kwd_re.findall(s)
1517 1520 return ret
1518 1521
1519 1522 def _default_arguments(self, obj):
1520 1523 """Return the list of default arguments of obj if it is callable,
1521 1524 or empty list otherwise."""
1522 1525 call_obj = obj
1523 1526 ret = []
1524 1527 if inspect.isbuiltin(obj):
1525 1528 pass
1526 1529 elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
1527 1530 if inspect.isclass(obj):
1528 1531 #for cython embedsignature=True the constructor docstring
1529 1532 #belongs to the object itself not __init__
1530 1533 ret += self._default_arguments_from_docstring(
1531 1534 getattr(obj, '__doc__', ''))
1532 1535 # for classes, check for __init__,__new__
1533 1536 call_obj = (getattr(obj, '__init__', None) or
1534 1537 getattr(obj, '__new__', None))
1535 1538 # for all others, check if they are __call__able
1536 1539 elif hasattr(obj, '__call__'):
1537 1540 call_obj = obj.__call__
1538 1541 ret += self._default_arguments_from_docstring(
1539 1542 getattr(call_obj, '__doc__', ''))
1540 1543
1541 1544 _keeps = (inspect.Parameter.KEYWORD_ONLY,
1542 1545 inspect.Parameter.POSITIONAL_OR_KEYWORD)
1543 1546
1544 1547 try:
1545 1548 sig = inspect.signature(obj)
1546 1549 ret.extend(k for k, v in sig.parameters.items() if
1547 1550 v.kind in _keeps)
1548 1551 except ValueError:
1549 1552 pass
1550 1553
1551 1554 return list(set(ret))
1552 1555
1553 1556 def python_func_kw_matches(self, text):
1554 1557 """Match named parameters (kwargs) of the last open function"""
1555 1558
1556 1559 if "." in text: # a parameter cannot be dotted
1557 1560 return []
1558 1561 try: regexp = self.__funcParamsRegex
1559 1562 except AttributeError:
1560 1563 regexp = self.__funcParamsRegex = re.compile(r'''
1561 1564 '.*?(?<!\\)' | # single quoted strings or
1562 1565 ".*?(?<!\\)" | # double quoted strings or
1563 1566 \w+ | # identifier
1564 1567 \S # other characters
1565 1568 ''', re.VERBOSE | re.DOTALL)
1566 1569 # 1. find the nearest identifier that comes before an unclosed
1567 1570 # parenthesis before the cursor
1568 1571 # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
1569 1572 tokens = regexp.findall(self.text_until_cursor)
1570 1573 iterTokens = reversed(tokens); openPar = 0
1571 1574
1572 1575 for token in iterTokens:
1573 1576 if token == ')':
1574 1577 openPar -= 1
1575 1578 elif token == '(':
1576 1579 openPar += 1
1577 1580 if openPar > 0:
1578 1581 # found the last unclosed parenthesis
1579 1582 break
1580 1583 else:
1581 1584 return []
1582 1585 # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
1583 1586 ids = []
1584 1587 isId = re.compile(r'\w+$').match
1585 1588
1586 1589 while True:
1587 1590 try:
1588 1591 ids.append(next(iterTokens))
1589 1592 if not isId(ids[-1]):
1590 1593 ids.pop(); break
1591 1594 if not next(iterTokens) == '.':
1592 1595 break
1593 1596 except StopIteration:
1594 1597 break
1595 1598
1596 1599 # Find all named arguments already assigned to, as to avoid suggesting
1597 1600 # them again
1598 1601 usedNamedArgs = set()
1599 1602 par_level = -1
1600 1603 for token, next_token in zip(tokens, tokens[1:]):
1601 1604 if token == '(':
1602 1605 par_level += 1
1603 1606 elif token == ')':
1604 1607 par_level -= 1
1605 1608
1606 1609 if par_level != 0:
1607 1610 continue
1608 1611
1609 1612 if next_token != '=':
1610 1613 continue
1611 1614
1612 1615 usedNamedArgs.add(token)
1613 1616
1614 1617 argMatches = []
1615 1618 try:
1616 1619 callableObj = '.'.join(ids[::-1])
1617 1620 namedArgs = self._default_arguments(eval(callableObj,
1618 1621 self.namespace))
1619 1622
1620 1623 # Remove used named arguments from the list, no need to show twice
1621 1624 for namedArg in set(namedArgs) - usedNamedArgs:
1622 1625 if namedArg.startswith(text):
1623 1626 argMatches.append("%s=" %namedArg)
1624 1627 except:
1625 1628 pass
1626 1629
1627 1630 return argMatches
1628 1631
1629 1632 @staticmethod
1630 1633 def _get_keys(obj: Any) -> List[Any]:
1631 1634 # Objects can define their own completions by defining an
1632 1635 # _ipy_key_completions_() method.
1633 1636 method = get_real_method(obj, '_ipython_key_completions_')
1634 1637 if method is not None:
1635 1638 return method()
1636 1639
1637 1640 # Special case some common in-memory dict-like types
1638 1641 if isinstance(obj, dict) or\
1639 1642 _safe_isinstance(obj, 'pandas', 'DataFrame'):
1640 1643 try:
1641 1644 return list(obj.keys())
1642 1645 except Exception:
1643 1646 return []
1644 1647 elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
1645 1648 _safe_isinstance(obj, 'numpy', 'void'):
1646 1649 return obj.dtype.names or []
1647 1650 return []
1648 1651
1649 1652 def dict_key_matches(self, text:str) -> List[str]:
1650 1653 "Match string keys in a dictionary, after e.g. 'foo[' "
1651 1654
1652 1655
1653 1656 if self.__dict_key_regexps is not None:
1654 1657 regexps = self.__dict_key_regexps
1655 1658 else:
1656 1659 dict_key_re_fmt = r'''(?x)
1657 1660 ( # match dict-referring expression wrt greedy setting
1658 1661 %s
1659 1662 )
1660 1663 \[ # open bracket
1661 1664 \s* # and optional whitespace
1662 1665 # Capture any number of str-like objects (e.g. "a", "b", 'c')
1663 1666 ((?:[uUbB]? # string prefix (r not handled)
1664 1667 (?:
1665 1668 '(?:[^']|(?<!\\)\\')*'
1666 1669 |
1667 1670 "(?:[^"]|(?<!\\)\\")*"
1668 1671 )
1669 1672 \s*,\s*
1670 1673 )*)
1671 1674 ([uUbB]? # string prefix (r not handled)
1672 1675 (?: # unclosed string
1673 1676 '(?:[^']|(?<!\\)\\')*
1674 1677 |
1675 1678 "(?:[^"]|(?<!\\)\\")*
1676 1679 )
1677 1680 )?
1678 1681 $
1679 1682 '''
1680 1683 regexps = self.__dict_key_regexps = {
1681 1684 False: re.compile(dict_key_re_fmt % r'''
1682 1685 # identifiers separated by .
1683 1686 (?!\d)\w+
1684 1687 (?:\.(?!\d)\w+)*
1685 1688 '''),
1686 1689 True: re.compile(dict_key_re_fmt % '''
1687 1690 .+
1688 1691 ''')
1689 1692 }
1690 1693
1691 1694 match = regexps[self.greedy].search(self.text_until_cursor)
1692 1695
1693 1696 if match is None:
1694 1697 return []
1695 1698
1696 1699 expr, prefix0, prefix = match.groups()
1697 1700 try:
1698 1701 obj = eval(expr, self.namespace)
1699 1702 except Exception:
1700 1703 try:
1701 1704 obj = eval(expr, self.global_namespace)
1702 1705 except Exception:
1703 1706 return []
1704 1707
1705 1708 keys = self._get_keys(obj)
1706 1709 if not keys:
1707 1710 return keys
1708 1711
1709 1712 extra_prefix = eval(prefix0) if prefix0 != '' else None
1710 1713
1711 1714 closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims, extra_prefix=extra_prefix)
1712 1715 if not matches:
1713 1716 return matches
1714 1717
1715 1718 # get the cursor position of
1716 1719 # - the text being completed
1717 1720 # - the start of the key text
1718 1721 # - the start of the completion
1719 1722 text_start = len(self.text_until_cursor) - len(text)
1720 1723 if prefix:
1721 1724 key_start = match.start(3)
1722 1725 completion_start = key_start + token_offset
1723 1726 else:
1724 1727 key_start = completion_start = match.end()
1725 1728
1726 1729 # grab the leading prefix, to make sure all completions start with `text`
1727 1730 if text_start > key_start:
1728 1731 leading = ''
1729 1732 else:
1730 1733 leading = text[text_start:completion_start]
1731 1734
1732 1735 # the index of the `[` character
1733 1736 bracket_idx = match.end(1)
1734 1737
1735 1738 # append closing quote and bracket as appropriate
1736 1739 # this is *not* appropriate if the opening quote or bracket is outside
1737 1740 # the text given to this method
1738 1741 suf = ''
1739 1742 continuation = self.line_buffer[len(self.text_until_cursor):]
1740 1743 if key_start > text_start and closing_quote:
1741 1744 # quotes were opened inside text, maybe close them
1742 1745 if continuation.startswith(closing_quote):
1743 1746 continuation = continuation[len(closing_quote):]
1744 1747 else:
1745 1748 suf += closing_quote
1746 1749 if bracket_idx > text_start:
1747 1750 # brackets were opened inside text, maybe close them
1748 1751 if not continuation.startswith(']'):
1749 1752 suf += ']'
1750 1753
1751 1754 return [leading + k + suf for k in matches]
1752 1755
1753 1756 @staticmethod
1754 1757 def unicode_name_matches(text:str) -> Tuple[str, List[str]] :
1755 1758 """Match Latex-like syntax for unicode characters base
1756 1759 on the name of the character.
1757 1760
1758 1761 This does ``\\GREEK SMALL LETTER ETA`` -> ``Ξ·``
1759 1762
1760 1763 Works only on valid python 3 identifier, or on combining characters that
1761 1764 will combine to form a valid identifier.
1762 1765 """
1763 1766 slashpos = text.rfind('\\')
1764 1767 if slashpos > -1:
1765 1768 s = text[slashpos+1:]
1766 1769 try :
1767 1770 unic = unicodedata.lookup(s)
1768 1771 # allow combining chars
1769 1772 if ('a'+unic).isidentifier():
1770 1773 return '\\'+s,[unic]
1771 1774 except KeyError:
1772 1775 pass
1773 1776 return '', []
1774 1777
1775 1778
1776 1779 def latex_matches(self, text:str) -> Tuple[str, Sequence[str]]:
1777 1780 """Match Latex syntax for unicode characters.
1778 1781
1779 1782 This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``Ξ±``
1780 1783 """
1781 1784 slashpos = text.rfind('\\')
1782 1785 if slashpos > -1:
1783 1786 s = text[slashpos:]
1784 1787 if s in latex_symbols:
1785 1788 # Try to complete a full latex symbol to unicode
1786 1789 # \\alpha -> Ξ±
1787 1790 return s, [latex_symbols[s]]
1788 1791 else:
1789 1792 # If a user has partially typed a latex symbol, give them
1790 1793 # a full list of options \al -> [\aleph, \alpha]
1791 1794 matches = [k for k in latex_symbols if k.startswith(s)]
1792 1795 if matches:
1793 1796 return s, matches
1794 1797 return '', ()
1795 1798
1796 1799 def dispatch_custom_completer(self, text):
1797 1800 if not self.custom_completers:
1798 1801 return
1799 1802
1800 1803 line = self.line_buffer
1801 1804 if not line.strip():
1802 1805 return None
1803 1806
1804 1807 # Create a little structure to pass all the relevant information about
1805 1808 # the current completion to any custom completer.
1806 1809 event = SimpleNamespace()
1807 1810 event.line = line
1808 1811 event.symbol = text
1809 1812 cmd = line.split(None,1)[0]
1810 1813 event.command = cmd
1811 1814 event.text_until_cursor = self.text_until_cursor
1812 1815
1813 1816 # for foo etc, try also to find completer for %foo
1814 1817 if not cmd.startswith(self.magic_escape):
1815 1818 try_magic = self.custom_completers.s_matches(
1816 1819 self.magic_escape + cmd)
1817 1820 else:
1818 1821 try_magic = []
1819 1822
1820 1823 for c in itertools.chain(self.custom_completers.s_matches(cmd),
1821 1824 try_magic,
1822 1825 self.custom_completers.flat_matches(self.text_until_cursor)):
1823 1826 try:
1824 1827 res = c(event)
1825 1828 if res:
1826 1829 # first, try case sensitive match
1827 1830 withcase = [r for r in res if r.startswith(text)]
1828 1831 if withcase:
1829 1832 return withcase
1830 1833 # if none, then case insensitive ones are ok too
1831 1834 text_low = text.lower()
1832 1835 return [r for r in res if r.lower().startswith(text_low)]
1833 1836 except TryNext:
1834 1837 pass
1835 1838 except KeyboardInterrupt:
1836 1839 """
1837 1840 If custom completer take too long,
1838 1841 let keyboard interrupt abort and return nothing.
1839 1842 """
1840 1843 break
1841 1844
1842 1845 return None
1843 1846
1844 1847 def completions(self, text: str, offset: int)->Iterator[Completion]:
1845 1848 """
1846 1849 Returns an iterator over the possible completions
1847 1850
1848 1851 .. warning::
1849 1852
1850 1853 Unstable
1851 1854
1852 1855 This function is unstable, API may change without warning.
1853 1856 It will also raise unless use in proper context manager.
1854 1857
1855 1858 Parameters
1856 1859 ----------
1857 1860 text : str
1858 1861 Full text of the current input, multi line string.
1859 1862 offset : int
1860 1863 Integer representing the position of the cursor in ``text``. Offset
1861 1864 is 0-based indexed.
1862 1865
1863 1866 Yields
1864 1867 ------
1865 1868 Completion
1866 1869
1867 1870 Notes
1868 1871 -----
1869 1872 The cursor on a text can either be seen as being "in between"
1870 1873 characters or "On" a character depending on the interface visible to
1871 1874 the user. For consistency the cursor being on "in between" characters X
1872 1875 and Y is equivalent to the cursor being "on" character Y, that is to say
1873 1876 the character the cursor is on is considered as being after the cursor.
1874 1877
1875 1878 Combining characters may span more that one position in the
1876 1879 text.
1877 1880
1878 1881 .. note::
1879 1882
1880 1883 If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--``
1881 1884 fake Completion token to distinguish completion returned by Jedi
1882 1885 and usual IPython completion.
1883 1886
1884 1887 .. note::
1885 1888
1886 1889 Completions are not completely deduplicated yet. If identical
1887 1890 completions are coming from different sources this function does not
1888 1891 ensure that each completion object will only be present once.
1889 1892 """
1890 1893 warnings.warn("_complete is a provisional API (as of IPython 6.0). "
1891 1894 "It may change without warnings. "
1892 1895 "Use in corresponding context manager.",
1893 1896 category=ProvisionalCompleterWarning, stacklevel=2)
1894 1897
1895 1898 seen = set()
1896 1899 profiler:Optional[cProfile.Profile]
1897 1900 try:
1898 1901 if self.profile_completions:
1899 1902 import cProfile
1900 1903 profiler = cProfile.Profile()
1901 1904 profiler.enable()
1902 1905 else:
1903 1906 profiler = None
1904 1907
1905 1908 for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
1906 1909 if c and (c in seen):
1907 1910 continue
1908 1911 yield c
1909 1912 seen.add(c)
1910 1913 except KeyboardInterrupt:
1911 1914 """if completions take too long and users send keyboard interrupt,
1912 1915 do not crash and return ASAP. """
1913 1916 pass
1914 1917 finally:
1915 1918 if profiler is not None:
1916 1919 profiler.disable()
1917 1920 ensure_dir_exists(self.profiler_output_dir)
1918 1921 output_path = os.path.join(self.profiler_output_dir, str(uuid.uuid4()))
1919 1922 print("Writing profiler output to", output_path)
1920 1923 profiler.dump_stats(output_path)
1921 1924
1922 1925 def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Completion]:
1923 1926 """
1924 1927 Core completion module.Same signature as :any:`completions`, with the
1925 1928 extra `timeout` parameter (in seconds).
1926 1929
1927 1930 Computing jedi's completion ``.type`` can be quite expensive (it is a
1928 1931 lazy property) and can require some warm-up, more warm up than just
1929 1932 computing the ``name`` of a completion. The warm-up can be :
1930 1933
1931 1934 - Long warm-up the first time a module is encountered after
1932 1935 install/update: actually build parse/inference tree.
1933 1936
1934 1937 - first time the module is encountered in a session: load tree from
1935 1938 disk.
1936 1939
1937 1940 We don't want to block completions for tens of seconds so we give the
1938 1941 completer a "budget" of ``_timeout`` seconds per invocation to compute
1939 1942 completions types, the completions that have not yet been computed will
1940 1943 be marked as "unknown" an will have a chance to be computed next round
1941 1944 are things get cached.
1942 1945
1943 1946 Keep in mind that Jedi is not the only thing treating the completion so
1944 1947 keep the timeout short-ish as if we take more than 0.3 second we still
1945 1948 have lots of processing to do.
1946 1949
1947 1950 """
1948 1951 deadline = time.monotonic() + _timeout
1949 1952
1950 1953
1951 1954 before = full_text[:offset]
1952 1955 cursor_line, cursor_column = position_to_cursor(full_text, offset)
1953 1956
1954 1957 matched_text, matches, matches_origin, jedi_matches = self._complete(
1955 1958 full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column)
1956 1959
1957 1960 iter_jm = iter(jedi_matches)
1958 1961 if _timeout:
1959 1962 for jm in iter_jm:
1960 1963 try:
1961 1964 type_ = jm.type
1962 1965 except Exception:
1963 1966 if self.debug:
1964 1967 print("Error in Jedi getting type of ", jm)
1965 1968 type_ = None
1966 1969 delta = len(jm.name_with_symbols) - len(jm.complete)
1967 1970 if type_ == 'function':
1968 1971 signature = _make_signature(jm)
1969 1972 else:
1970 1973 signature = ''
1971 1974 yield Completion(start=offset - delta,
1972 1975 end=offset,
1973 1976 text=jm.name_with_symbols,
1974 1977 type=type_,
1975 1978 signature=signature,
1976 1979 _origin='jedi')
1977 1980
1978 1981 if time.monotonic() > deadline:
1979 1982 break
1980 1983
1981 1984 for jm in iter_jm:
1982 1985 delta = len(jm.name_with_symbols) - len(jm.complete)
1983 1986 yield Completion(start=offset - delta,
1984 1987 end=offset,
1985 1988 text=jm.name_with_symbols,
1986 1989 type='<unknown>', # don't compute type for speed
1987 1990 _origin='jedi',
1988 1991 signature='')
1989 1992
1990 1993
1991 1994 start_offset = before.rfind(matched_text)
1992 1995
1993 1996 # TODO:
1994 1997 # Suppress this, right now just for debug.
1995 1998 if jedi_matches and matches and self.debug:
1996 1999 yield Completion(start=start_offset, end=offset, text='--jedi/ipython--',
1997 2000 _origin='debug', type='none', signature='')
1998 2001
1999 2002 # I'm unsure if this is always true, so let's assert and see if it
2000 2003 # crash
2001 2004 assert before.endswith(matched_text)
2002 2005 for m, t in zip(matches, matches_origin):
2003 2006 yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='<unknown>')
2004 2007
2005 2008
2006 2009 def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]:
2007 2010 """Find completions for the given text and line context.
2008 2011
2009 2012 Note that both the text and the line_buffer are optional, but at least
2010 2013 one of them must be given.
2011 2014
2012 2015 Parameters
2013 2016 ----------
2014 2017 text : string, optional
2015 2018 Text to perform the completion on. If not given, the line buffer
2016 2019 is split using the instance's CompletionSplitter object.
2017 2020 line_buffer : string, optional
2018 2021 If not given, the completer attempts to obtain the current line
2019 2022 buffer via readline. This keyword allows clients which are
2020 2023 requesting for text completions in non-readline contexts to inform
2021 2024 the completer of the entire text.
2022 2025 cursor_pos : int, optional
2023 2026 Index of the cursor in the full line buffer. Should be provided by
2024 2027 remote frontends where kernel has no access to frontend state.
2025 2028
2026 2029 Returns
2027 2030 -------
2028 2031 Tuple of two items:
2029 2032 text : str
2030 2033 Text that was actually used in the completion.
2031 2034 matches : list
2032 2035 A list of completion matches.
2033 2036
2034 2037 Notes
2035 2038 -----
2036 2039 This API is likely to be deprecated and replaced by
2037 2040 :any:`IPCompleter.completions` in the future.
2038 2041
2039 2042 """
2040 2043 warnings.warn('`Completer.complete` is pending deprecation since '
2041 2044 'IPython 6.0 and will be replaced by `Completer.completions`.',
2042 2045 PendingDeprecationWarning)
2043 2046 # potential todo, FOLD the 3rd throw away argument of _complete
2044 2047 # into the first 2 one.
2045 2048 return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2]
2046 2049
2047 2050 def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
2048 2051 full_text=None) -> _CompleteResult:
2049 2052 """
2050 2053 Like complete but can also returns raw jedi completions as well as the
2051 2054 origin of the completion text. This could (and should) be made much
2052 2055 cleaner but that will be simpler once we drop the old (and stateful)
2053 2056 :any:`complete` API.
2054 2057
2055 2058 With current provisional API, cursor_pos act both (depending on the
2056 2059 caller) as the offset in the ``text`` or ``line_buffer``, or as the
2057 2060 ``column`` when passing multiline strings this could/should be renamed
2058 2061 but would add extra noise.
2059 2062
2060 2063 Parameters
2061 2064 ----------
2062 2065 cursor_line :
2063 2066 Index of the line the cursor is on. 0 indexed.
2064 2067 cursor_pos :
2065 2068 Position of the cursor in the current line/line_buffer/text. 0
2066 2069 indexed.
2067 2070 line_buffer : optional, str
2068 2071 The current line the cursor is in, this is mostly due to legacy
2069 2072 reason that readline coudl only give a us the single current line.
2070 2073 Prefer `full_text`.
2071 2074 text : str
2072 2075 The current "token" the cursor is in, mostly also for historical
2073 2076 reasons. as the completer would trigger only after the current line
2074 2077 was parsed.
2075 2078 full_text : str
2076 2079 Full text of the current cell.
2077 2080
2078 2081 Returns
2079 2082 -------
2080 2083 A tuple of N elements which are (likely):
2081 2084 matched_text: ? the text that the complete matched
2082 2085 matches: list of completions ?
2083 2086 matches_origin: ? list same length as matches, and where each completion came from
2084 2087 jedi_matches: list of Jedi matches, have it's own structure.
2085 2088 """
2086 2089
2087 2090
2088 2091 # if the cursor position isn't given, the only sane assumption we can
2089 2092 # make is that it's at the end of the line (the common case)
2090 2093 if cursor_pos is None:
2091 2094 cursor_pos = len(line_buffer) if text is None else len(text)
2092 2095
2093 2096 if self.use_main_ns:
2094 2097 self.namespace = __main__.__dict__
2095 2098
2096 2099 # if text is either None or an empty string, rely on the line buffer
2097 2100 if (not line_buffer) and full_text:
2098 2101 line_buffer = full_text.split('\n')[cursor_line]
2099 2102 if not text: # issue #11508: check line_buffer before calling split_line
2100 2103 text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else ''
2101 2104
2102 2105 if self.backslash_combining_completions:
2103 2106 # allow deactivation of these on windows.
2104 2107 base_text = text if not line_buffer else line_buffer[:cursor_pos]
2105 2108
2106 2109 for meth in (self.latex_matches,
2107 2110 self.unicode_name_matches,
2108 2111 back_latex_name_matches,
2109 2112 back_unicode_name_matches,
2110 2113 self.fwd_unicode_match):
2111 2114 name_text, name_matches = meth(base_text)
2112 2115 if name_text:
2113 2116 return _CompleteResult(name_text, name_matches[:MATCHES_LIMIT], \
2114 2117 [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), ())
2115 2118
2116 2119
2117 2120 # If no line buffer is given, assume the input text is all there was
2118 2121 if line_buffer is None:
2119 2122 line_buffer = text
2120 2123
2121 2124 self.line_buffer = line_buffer
2122 2125 self.text_until_cursor = self.line_buffer[:cursor_pos]
2123 2126
2124 2127 # Do magic arg matches
2125 2128 for matcher in self.magic_arg_matchers:
2126 2129 matches = list(matcher(line_buffer))[:MATCHES_LIMIT]
2127 2130 if matches:
2128 2131 origins = [matcher.__qualname__] * len(matches)
2129 2132 return _CompleteResult(text, matches, origins, ())
2130 2133
2131 2134 # Start with a clean slate of completions
2132 2135 matches = []
2133 2136
2134 2137 # FIXME: we should extend our api to return a dict with completions for
2135 2138 # different types of objects. The rlcomplete() method could then
2136 2139 # simply collapse the dict into a list for readline, but we'd have
2137 2140 # richer completion semantics in other environments.
2138 2141 completions:Iterable[Any] = []
2139 2142 if self.use_jedi:
2140 2143 if not full_text:
2141 2144 full_text = line_buffer
2142 2145 completions = self._jedi_matches(
2143 2146 cursor_pos, cursor_line, full_text)
2144 2147
2145 2148 if self.merge_completions:
2146 2149 matches = []
2147 2150 for matcher in self.matchers:
2148 2151 try:
2149 2152 matches.extend([(m, matcher.__qualname__)
2150 2153 for m in matcher(text)])
2151 2154 except:
2152 2155 # Show the ugly traceback if the matcher causes an
2153 2156 # exception, but do NOT crash the kernel!
2154 2157 sys.excepthook(*sys.exc_info())
2155 2158 else:
2156 2159 for matcher in self.matchers:
2157 2160 matches = [(m, matcher.__qualname__)
2158 2161 for m in matcher(text)]
2159 2162 if matches:
2160 2163 break
2161 2164
2162 2165 seen = set()
2163 2166 filtered_matches = set()
2164 2167 for m in matches:
2165 2168 t, c = m
2166 2169 if t not in seen:
2167 2170 filtered_matches.add(m)
2168 2171 seen.add(t)
2169 2172
2170 2173 _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0]))
2171 2174
2172 2175 custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []]
2173 2176
2174 2177 _filtered_matches = custom_res or _filtered_matches
2175 2178
2176 2179 _filtered_matches = _filtered_matches[:MATCHES_LIMIT]
2177 2180 _matches = [m[0] for m in _filtered_matches]
2178 2181 origins = [m[1] for m in _filtered_matches]
2179 2182
2180 2183 self.matches = _matches
2181 2184
2182 2185 return _CompleteResult(text, _matches, origins, completions)
2183 2186
2184 2187 def fwd_unicode_match(self, text:str) -> Tuple[str, Sequence[str]]:
2185 2188 """
2186 2189 Forward match a string starting with a backslash with a list of
2187 2190 potential Unicode completions.
2188 2191
2189 2192 Will compute list list of Unicode character names on first call and cache it.
2190 2193
2191 2194 Returns
2192 2195 -------
2193 2196 At tuple with:
2194 2197 - matched text (empty if no matches)
2195 2198 - list of potential completions, empty tuple otherwise)
2196 2199 """
2197 2200 # TODO: self.unicode_names is here a list we traverse each time with ~100k elements.
2198 2201 # We could do a faster match using a Trie.
2199 2202
2200 2203 # Using pygtrie the following seem to work:
2201 2204
2202 2205 # s = PrefixSet()
2203 2206
2204 2207 # for c in range(0,0x10FFFF + 1):
2205 2208 # try:
2206 2209 # s.add(unicodedata.name(chr(c)))
2207 2210 # except ValueError:
2208 2211 # pass
2209 2212 # [''.join(k) for k in s.iter(prefix)]
2210 2213
2211 2214 # But need to be timed and adds an extra dependency.
2212 2215
2213 2216 slashpos = text.rfind('\\')
2214 2217 # if text starts with slash
2215 2218 if slashpos > -1:
2216 2219 # PERF: It's important that we don't access self._unicode_names
2217 2220 # until we're inside this if-block. _unicode_names is lazily
2218 2221 # initialized, and it takes a user-noticeable amount of time to
2219 2222 # initialize it, so we don't want to initialize it unless we're
2220 2223 # actually going to use it.
2221 2224 s = text[slashpos+1:]
2222 2225 candidates = [x for x in self.unicode_names if x.startswith(s)]
2223 2226 if candidates:
2224 2227 return s, candidates
2225 2228 else:
2226 2229 return '', ()
2227 2230
2228 2231 # if text does not start with slash
2229 2232 else:
2230 2233 return '', ()
2231 2234
2232 2235 @property
2233 2236 def unicode_names(self) -> List[str]:
2234 2237 """List of names of unicode code points that can be completed.
2235 2238
2236 2239 The list is lazily initialized on first access.
2237 2240 """
2238 2241 if self._unicode_names is None:
2239 2242 names = []
2240 2243 for c in range(0,0x10FFFF + 1):
2241 2244 try:
2242 2245 names.append(unicodedata.name(chr(c)))
2243 2246 except ValueError:
2244 2247 pass
2245 2248 self._unicode_names = _unicode_name_compute(_UNICODE_RANGES)
2246 2249
2247 2250 return self._unicode_names
2248 2251
2249 2252 def _unicode_name_compute(ranges:List[Tuple[int,int]]) -> List[str]:
2250 2253 names = []
2251 2254 for start,stop in ranges:
2252 2255 for c in range(start, stop) :
2253 2256 try:
2254 2257 names.append(unicodedata.name(chr(c)))
2255 2258 except ValueError:
2256 2259 pass
2257 2260 return names
@@ -1,437 +1,365 b''
1 1 """Nose Plugin that supports IPython doctests.
2 2
3 3 Limitations:
4 4
5 5 - When generating examples for use as doctests, make sure that you have
6 6 pretty-printing OFF. This can be done either by setting the
7 7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
8 8 by interactively disabling it with %Pprint. This is required so that IPython
9 9 output matches that of normal Python, which is used by doctest for internal
10 10 execution.
11 11
12 12 - Do not rely on specific prompt numbers for results (such as using
13 13 '_34==True', for example). For IPython tests run via an external process the
14 14 prompt numbers may be different, and IPython tests run as normal python code
15 15 won't even have these special _NN variables set at all.
16 16 """
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Module imports
20 20
21 21 # From the standard library
22 22 import doctest
23 import inspect
24 23 import logging
25 24 import os
26 25 import re
27 26
28 27 from testpath import modified_env
29 28
30 29 #-----------------------------------------------------------------------------
31 30 # Module globals and other constants
32 31 #-----------------------------------------------------------------------------
33 32
34 33 log = logging.getLogger(__name__)
35 34
36 35
37 36 #-----------------------------------------------------------------------------
38 37 # Classes and functions
39 38 #-----------------------------------------------------------------------------
40 39
41 40 class DocTestSkip(object):
42 41 """Object wrapper for doctests to be skipped."""
43 42
44 43 ds_skip = """Doctest to skip.
45 44 >>> 1 #doctest: +SKIP
46 45 """
47 46
48 47 def __init__(self,obj):
49 48 self.obj = obj
50 49
51 50 def __getattribute__(self,key):
52 51 if key == '__doc__':
53 52 return DocTestSkip.ds_skip
54 53 else:
55 54 return getattr(object.__getattribute__(self,'obj'),key)
56 55
57 # Modified version of the one in the stdlib, that fixes a python bug (doctests
58 # not found in extension modules, http://bugs.python.org/issue3158)
59 class DocTestFinder(doctest.DocTestFinder):
60
61 def _from_module(self, module, object):
62 """
63 Return true if the given object is defined in the given
64 module.
65 """
66 if module is None:
67 return True
68 elif inspect.isfunction(object):
69 return module.__dict__ is object.__globals__
70 elif inspect.isbuiltin(object):
71 return module.__name__ == object.__module__
72 elif inspect.isclass(object):
73 return module.__name__ == object.__module__
74 elif inspect.ismethod(object):
75 # This one may be a bug in cython that fails to correctly set the
76 # __module__ attribute of methods, but since the same error is easy
77 # to make by extension code writers, having this safety in place
78 # isn't such a bad idea
79 return module.__name__ == object.__self__.__class__.__module__
80 elif inspect.getmodule(object) is not None:
81 return module is inspect.getmodule(object)
82 elif hasattr(object, '__module__'):
83 return module.__name__ == object.__module__
84 elif isinstance(object, property):
85 return True # [XX] no way not be sure.
86 elif inspect.ismethoddescriptor(object):
87 # Unbound PyQt signals reach this point in Python 3.4b3, and we want
88 # to avoid throwing an error. See also http://bugs.python.org/issue3158
89 return False
90 else:
91 raise ValueError("object must be a class or function, got %r" % object)
92 56
57 class DocTestFinder(doctest.DocTestFinder):
93 58 def _find(self, tests, obj, name, module, source_lines, globs, seen):
94 59 """
95 60 Find tests for the given object and any contained objects, and
96 61 add them to `tests`.
97 62 """
98 63 print('_find for:', obj, name, module) # dbg
99 64 if bool(getattr(obj, "__skip_doctest__", False)):
100 65 #print 'SKIPPING DOCTEST FOR:',obj # dbg
101 66 obj = DocTestSkip(obj)
102 67
103 doctest.DocTestFinder._find(self,tests, obj, name, module,
104 source_lines, globs, seen)
105
106 # Below we re-run pieces of the above method with manual modifications,
107 # because the original code is buggy and fails to correctly identify
108 # doctests in extension modules.
109
110 # Local shorthands
111 from inspect import isroutine, isclass
112
113 # Look for tests in a module's contained objects.
114 if inspect.ismodule(obj) and self._recurse:
115 for valname, val in obj.__dict__.items():
116 valname1 = '%s.%s' % (name, valname)
117 if ( (isroutine(val) or isclass(val))
118 and self._from_module(module, val) ):
119
120 self._find(tests, val, valname1, module, source_lines,
121 globs, seen)
122
123 # Look for tests in a class's contained objects.
124 if inspect.isclass(obj) and self._recurse:
125 #print 'RECURSE into class:',obj # dbg
126 for valname, val in obj.__dict__.items():
127 # Special handling for staticmethod/classmethod.
128 if isinstance(val, staticmethod):
129 val = getattr(obj, valname)
130 if isinstance(val, classmethod):
131 val = getattr(obj, valname).__func__
132
133 # Recurse to methods, properties, and nested classes.
134 if ((inspect.isfunction(val) or inspect.isclass(val) or
135 inspect.ismethod(val) or
136 isinstance(val, property)) and
137 self._from_module(module, val)):
138 valname = '%s.%s' % (name, valname)
139 self._find(tests, val, valname, module, source_lines,
140 globs, seen)
68 super()._find(tests, obj, name, module, source_lines, globs, seen)
141 69
142 70
143 71 class IPDoctestOutputChecker(doctest.OutputChecker):
144 72 """Second-chance checker with support for random tests.
145 73
146 74 If the default comparison doesn't pass, this checker looks in the expected
147 75 output string for flags that tell us to ignore the output.
148 76 """
149 77
150 78 random_re = re.compile(r'#\s*random\s+')
151 79
152 80 def check_output(self, want, got, optionflags):
153 81 """Check output, accepting special markers embedded in the output.
154 82
155 83 If the output didn't pass the default validation but the special string
156 84 '#random' is included, we accept it."""
157 85
158 86 # Let the original tester verify first, in case people have valid tests
159 87 # that happen to have a comment saying '#random' embedded in.
160 88 ret = doctest.OutputChecker.check_output(self, want, got,
161 89 optionflags)
162 90 if not ret and self.random_re.search(want):
163 91 #print >> sys.stderr, 'RANDOM OK:',want # dbg
164 92 return True
165 93
166 94 return ret
167 95
168 96
169 97 # A simple subclassing of the original with a different class name, so we can
170 98 # distinguish and treat differently IPython examples from pure python ones.
171 99 class IPExample(doctest.Example): pass
172 100
173 101
174 102 class IPExternalExample(doctest.Example):
175 103 """Doctest examples to be run in an external process."""
176 104
177 105 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
178 106 options=None):
179 107 # Parent constructor
180 108 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
181 109
182 110 # An EXTRA newline is needed to prevent pexpect hangs
183 111 self.source += '\n'
184 112
185 113
186 114 class IPDocTestParser(doctest.DocTestParser):
187 115 """
188 116 A class used to parse strings containing doctest examples.
189 117
190 118 Note: This is a version modified to properly recognize IPython input and
191 119 convert any IPython examples into valid Python ones.
192 120 """
193 121 # This regular expression is used to find doctest examples in a
194 122 # string. It defines three groups: `source` is the source code
195 123 # (including leading indentation and prompts); `indent` is the
196 124 # indentation of the first (PS1) line of the source code; and
197 125 # `want` is the expected output (including leading indentation).
198 126
199 127 # Classic Python prompts or default IPython ones
200 128 _PS1_PY = r'>>>'
201 129 _PS2_PY = r'\.\.\.'
202 130
203 131 _PS1_IP = r'In\ \[\d+\]:'
204 132 _PS2_IP = r'\ \ \ \.\.\.+:'
205 133
206 134 _RE_TPL = r'''
207 135 # Source consists of a PS1 line followed by zero or more PS2 lines.
208 136 (?P<source>
209 137 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
210 138 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
211 139 \n? # a newline
212 140 # Want consists of any non-blank lines that do not start with PS1.
213 141 (?P<want> (?:(?![ ]*$) # Not a blank line
214 142 (?![ ]*%s) # Not a line starting with PS1
215 143 (?![ ]*%s) # Not a line starting with PS2
216 144 .*$\n? # But any other line
217 145 )*)
218 146 '''
219 147
220 148 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
221 149 re.MULTILINE | re.VERBOSE)
222 150
223 151 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
224 152 re.MULTILINE | re.VERBOSE)
225 153
226 154 # Mark a test as being fully random. In this case, we simply append the
227 155 # random marker ('#random') to each individual example's output. This way
228 156 # we don't need to modify any other code.
229 157 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
230 158
231 159 # Mark tests to be executed in an external process - currently unsupported.
232 160 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
233 161
234 162 def ip2py(self,source):
235 163 """Convert input IPython source into valid Python."""
236 164 block = _ip.input_transformer_manager.transform_cell(source)
237 165 if len(block.splitlines()) == 1:
238 166 return _ip.prefilter(block)
239 167 else:
240 168 return block
241 169
242 170 def parse(self, string, name='<string>'):
243 171 """
244 172 Divide the given string into examples and intervening text,
245 173 and return them as a list of alternating Examples and strings.
246 174 Line numbers for the Examples are 0-based. The optional
247 175 argument `name` is a name identifying this string, and is only
248 176 used for error messages.
249 177 """
250 178
251 179 #print 'Parse string:\n',string # dbg
252 180
253 181 string = string.expandtabs()
254 182 # If all lines begin with the same indentation, then strip it.
255 183 min_indent = self._min_indent(string)
256 184 if min_indent > 0:
257 185 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
258 186
259 187 output = []
260 188 charno, lineno = 0, 0
261 189
262 190 # We make 'all random' tests by adding the '# random' mark to every
263 191 # block of output in the test.
264 192 if self._RANDOM_TEST.search(string):
265 193 random_marker = '\n# random'
266 194 else:
267 195 random_marker = ''
268 196
269 197 # Whether to convert the input from ipython to python syntax
270 198 ip2py = False
271 199 # Find all doctest examples in the string. First, try them as Python
272 200 # examples, then as IPython ones
273 201 terms = list(self._EXAMPLE_RE_PY.finditer(string))
274 202 if terms:
275 203 # Normal Python example
276 204 #print '-'*70 # dbg
277 205 #print 'PyExample, Source:\n',string # dbg
278 206 #print '-'*70 # dbg
279 207 Example = doctest.Example
280 208 else:
281 209 # It's an ipython example. Note that IPExamples are run
282 210 # in-process, so their syntax must be turned into valid python.
283 211 # IPExternalExamples are run out-of-process (via pexpect) so they
284 212 # don't need any filtering (a real ipython will be executing them).
285 213 terms = list(self._EXAMPLE_RE_IP.finditer(string))
286 214 if self._EXTERNAL_IP.search(string):
287 215 #print '-'*70 # dbg
288 216 #print 'IPExternalExample, Source:\n',string # dbg
289 217 #print '-'*70 # dbg
290 218 Example = IPExternalExample
291 219 else:
292 220 #print '-'*70 # dbg
293 221 #print 'IPExample, Source:\n',string # dbg
294 222 #print '-'*70 # dbg
295 223 Example = IPExample
296 224 ip2py = True
297 225
298 226 for m in terms:
299 227 # Add the pre-example text to `output`.
300 228 output.append(string[charno:m.start()])
301 229 # Update lineno (lines before this example)
302 230 lineno += string.count('\n', charno, m.start())
303 231 # Extract info from the regexp match.
304 232 (source, options, want, exc_msg) = \
305 233 self._parse_example(m, name, lineno,ip2py)
306 234
307 235 # Append the random-output marker (it defaults to empty in most
308 236 # cases, it's only non-empty for 'all-random' tests):
309 237 want += random_marker
310 238
311 239 if Example is IPExternalExample:
312 240 options[doctest.NORMALIZE_WHITESPACE] = True
313 241 want += '\n'
314 242
315 243 # Create an Example, and add it to the list.
316 244 if not self._IS_BLANK_OR_COMMENT(source):
317 245 output.append(Example(source, want, exc_msg,
318 246 lineno=lineno,
319 247 indent=min_indent+len(m.group('indent')),
320 248 options=options))
321 249 # Update lineno (lines inside this example)
322 250 lineno += string.count('\n', m.start(), m.end())
323 251 # Update charno.
324 252 charno = m.end()
325 253 # Add any remaining post-example text to `output`.
326 254 output.append(string[charno:])
327 255 return output
328 256
329 257 def _parse_example(self, m, name, lineno,ip2py=False):
330 258 """
331 259 Given a regular expression match from `_EXAMPLE_RE` (`m`),
332 260 return a pair `(source, want)`, where `source` is the matched
333 261 example's source code (with prompts and indentation stripped);
334 262 and `want` is the example's expected output (with indentation
335 263 stripped).
336 264
337 265 `name` is the string's name, and `lineno` is the line number
338 266 where the example starts; both are used for error messages.
339 267
340 268 Optional:
341 269 `ip2py`: if true, filter the input via IPython to convert the syntax
342 270 into valid python.
343 271 """
344 272
345 273 # Get the example's indentation level.
346 274 indent = len(m.group('indent'))
347 275
348 276 # Divide source into lines; check that they're properly
349 277 # indented; and then strip their indentation & prompts.
350 278 source_lines = m.group('source').split('\n')
351 279
352 280 # We're using variable-length input prompts
353 281 ps1 = m.group('ps1')
354 282 ps2 = m.group('ps2')
355 283 ps1_len = len(ps1)
356 284
357 285 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
358 286 if ps2:
359 287 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
360 288
361 289 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
362 290
363 291 if ip2py:
364 292 # Convert source input from IPython into valid Python syntax
365 293 source = self.ip2py(source)
366 294
367 295 # Divide want into lines; check that it's properly indented; and
368 296 # then strip the indentation. Spaces before the last newline should
369 297 # be preserved, so plain rstrip() isn't good enough.
370 298 want = m.group('want')
371 299 want_lines = want.split('\n')
372 300 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
373 301 del want_lines[-1] # forget final newline & spaces after it
374 302 self._check_prefix(want_lines, ' '*indent, name,
375 303 lineno + len(source_lines))
376 304
377 305 # Remove ipython output prompt that might be present in the first line
378 306 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
379 307
380 308 want = '\n'.join([wl[indent:] for wl in want_lines])
381 309
382 310 # If `want` contains a traceback message, then extract it.
383 311 m = self._EXCEPTION_RE.match(want)
384 312 if m:
385 313 exc_msg = m.group('msg')
386 314 else:
387 315 exc_msg = None
388 316
389 317 # Extract options from the source.
390 318 options = self._find_options(source, name, lineno)
391 319
392 320 return source, options, want, exc_msg
393 321
394 322 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
395 323 """
396 324 Given the lines of a source string (including prompts and
397 325 leading indentation), check to make sure that every prompt is
398 326 followed by a space character. If any line is not followed by
399 327 a space character, then raise ValueError.
400 328
401 329 Note: IPython-modified version which takes the input prompt length as a
402 330 parameter, so that prompts of variable length can be dealt with.
403 331 """
404 332 space_idx = indent+ps1_len
405 333 min_len = space_idx+1
406 334 for i, line in enumerate(lines):
407 335 if len(line) >= min_len and line[space_idx] != ' ':
408 336 raise ValueError('line %r of the docstring for %s '
409 337 'lacks blank after %s: %r' %
410 338 (lineno+i+1, name,
411 339 line[indent:space_idx], line))
412 340
413 341
414 342 SKIP = doctest.register_optionflag('SKIP')
415 343
416 344
417 345 class IPDocTestRunner(doctest.DocTestRunner,object):
418 346 """Test runner that synchronizes the IPython namespace with test globals.
419 347 """
420 348
421 349 def run(self, test, compileflags=None, out=None, clear_globs=True):
422 350
423 351 # Hack: ipython needs access to the execution context of the example,
424 352 # so that it can propagate user variables loaded by %run into
425 353 # test.globs. We put them here into our modified %run as a function
426 354 # attribute. Our new %run will then only make the namespace update
427 355 # when called (rather than unconditionally updating test.globs here
428 356 # for all examples, most of which won't be calling %run anyway).
429 357 #_ip._ipdoctest_test_globs = test.globs
430 358 #_ip._ipdoctest_test_filename = test.filename
431 359
432 360 test.globs.update(_ip.user_ns)
433 361
434 362 # Override terminal size to standardise traceback format
435 363 with modified_env({'COLUMNS': '80', 'LINES': '24'}):
436 364 return super(IPDocTestRunner,self).run(test,
437 365 compileflags,out,clear_globs)
General Comments 0
You need to be logged in to leave comments. Login now