##// END OF EJS Templates
typos
Matthias Bussonnier -
Show More
@@ -1,638 +1,637 b''
1 1 """Input transformer machinery to support IPython special syntax.
2 2
3 3 This includes the machinery to recognise and transform ``%magic`` commands,
4 4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 5
6 6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7 7 deprecated in 7.0.
8 8 """
9 9
10 10 # Copyright (c) IPython Development Team.
11 11 # Distributed under the terms of the Modified BSD License.
12 12
13 13 from codeop import compile_command
14 14 import re
15 15 import tokenize
16 16 from typing import List, Tuple
17 17 import warnings
18 18
19 19 _indent_re = re.compile(r'^[ \t]+')
20 20
21 21 def leading_indent(lines):
22 22 """Remove leading indentation.
23 23
24 24 If the first line starts with a spaces or tabs, the same whitespace will be
25 25 removed from each following line in the cell.
26 26 """
27 27 m = _indent_re.match(lines[0])
28 28 if not m:
29 29 return lines
30 30 space = m.group(0)
31 31 n = len(space)
32 32 return [l[n:] if l.startswith(space) else l
33 33 for l in lines]
34 34
35 35 class PromptStripper:
36 36 """Remove matching input prompts from a block of input.
37 37
38 38 Parameters
39 39 ----------
40 40 prompt_re : regular expression
41 41 A regular expression matching any input prompt (including continuation,
42 42 e.g. ``...``)
43 43 initial_re : regular expression, optional
44 44 A regular expression matching only the initial prompt, but not continuation.
45 45 If no initial expression is given, prompt_re will be used everywhere.
46 46 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
47 47 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
48 48
49 49 If initial_re and prompt_re differ,
50 50 only initial_re will be tested against the first line.
51 51 If any prompt is found on the first two lines,
52 52 prompts will be stripped from the rest of the block.
53 53 """
54 54 def __init__(self, prompt_re, initial_re=None):
55 55 self.prompt_re = prompt_re
56 56 self.initial_re = initial_re or prompt_re
57 57
58 58 def _strip(self, lines):
59 59 return [self.prompt_re.sub('', l, count=1) for l in lines]
60 60
61 61 def __call__(self, lines):
62 62 if self.initial_re.match(lines[0]) or \
63 63 (len(lines) > 1 and self.prompt_re.match(lines[1])):
64 64 return self._strip(lines)
65 65 return lines
66 66
67 67 classic_prompt = PromptStripper(
68 68 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
69 69 initial_re=re.compile(r'^>>>( |$)')
70 70 )
71 71
72 72 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
73 73
74 74 def cell_magic(lines):
75 75 if not lines[0].startswith('%%'):
76 76 return lines
77 77 if re.match('%%\w+\?', lines[0]):
78 78 # This case will be handled by help_end
79 79 return lines
80 80 magic_name, _, first_line = lines[0][2:-1].partition(' ')
81 81 body = ''.join(lines[1:])
82 82 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
83 83 % (magic_name, first_line, body)]
84 84
85 85
86 86 def _find_assign_op(token_line):
87 87 """Get the index of the first assignment in the line ('=' not inside brackets)
88 88
89 89 Note: We don't try to support multiple special assignment (a = b = %foo)
90 90 """
91 91 paren_level = 0
92 92 for i, ti in enumerate(token_line):
93 93 s = ti.string
94 94 if s == '=' and paren_level == 0:
95 95 return i
96 96 if s in '([{':
97 97 paren_level += 1
98 98 elif s in ')]}':
99 99 if paren_level > 0:
100 100 paren_level -= 1
101 101
102 102 def find_end_of_continued_line(lines, start_line: int):
103 103 """Find the last line of a line explicitly extended using backslashes.
104 104
105 105 Uses 0-indexed line numbers.
106 106 """
107 107 end_line = start_line
108 108 while lines[end_line].endswith('\\\n'):
109 109 end_line += 1
110 110 if end_line >= len(lines):
111 111 break
112 112 return end_line
113 113
114 114 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
115 115 """Assemble a single line from multiple continued line pieces
116 116
117 117 Continued lines are lines ending in ``\``, and the line following the last
118 118 ``\`` in the block.
119 119
120 120 For example, this code continues over multiple lines::
121 121
122 122 if (assign_ix is not None) \
123 123 and (len(line) >= assign_ix + 2) \
124 124 and (line[assign_ix+1].string == '%') \
125 125 and (line[assign_ix+2].type == tokenize.NAME):
126 126
127 127 This statement contains four continued line pieces.
128 128 Assembling these pieces into a single line would give::
129 129
130 130 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
131 131
132 132 This uses 0-indexed line numbers. *start* is (lineno, colno).
133 133
134 134 Used to allow ``%magic`` and ``!system`` commands to be continued over
135 135 multiple lines.
136 136 """
137 137 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
138 138 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
139 139 + [parts[-1][:-1]]) # Strip newline from last line
140 140
141 141 class TokenTransformBase:
142 142 """Base class for transformations which examine tokens.
143 143
144 144 Special syntax should not be transformed when it occurs inside strings or
145 145 comments. This is hard to reliably avoid with regexes. The solution is to
146 146 tokenise the code as Python, and recognise the special syntax in the tokens.
147 147
148 148 IPython's special syntax is not valid Python syntax, so tokenising may go
149 149 wrong after the special syntax starts. These classes therefore find and
150 150 transform *one* instance of special syntax at a time into regular Python
151 151 syntax. After each transformation, tokens are regenerated to find the next
152 152 piece of special syntax.
153 153
154 154 Subclasses need to implement one class method (find)
155 155 and one regular method (transform).
156 156
157 157 The priority attribute can select which transformation to apply if multiple
158 158 transformers match in the same place. Lower numbers have higher priority.
159 159 This allows "%magic?" to be turned into a help call rather than a magic call.
160 160 """
161 161 # Lower numbers -> higher priority (for matches in the same location)
162 162 priority = 10
163 163
164 164 def sortby(self):
165 165 return self.start_line, self.start_col, self.priority
166 166
167 167 def __init__(self, start):
168 168 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
169 169 self.start_col = start[1]
170 170
171 171 @classmethod
172 172 def find(cls, tokens_by_line):
173 173 """Find one instance of special syntax in the provided tokens.
174 174
175 175 Tokens are grouped into logical lines for convenience,
176 176 so it is easy to e.g. look at the first token of each line.
177 177 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
178 178
179 179 This should return an instance of its class, pointing to the start
180 180 position it has found, or None if it found no match.
181 181 """
182 182 raise NotImplementedError
183 183
184 184 def transform(self, lines: List[str]):
185 185 """Transform one instance of special syntax found by ``find()``
186 186
187 187 Takes a list of strings representing physical lines,
188 188 returns a similar list of transformed lines.
189 189 """
190 190 raise NotImplementedError
191 191
192 192 class MagicAssign(TokenTransformBase):
193 193 """Transformer for assignments from magics (a = %foo)"""
194 194 @classmethod
195 195 def find(cls, tokens_by_line):
196 196 """Find the first magic assignment (a = %foo) in the cell.
197 197 """
198 198 for line in tokens_by_line:
199 199 assign_ix = _find_assign_op(line)
200 200 if (assign_ix is not None) \
201 201 and (len(line) >= assign_ix + 2) \
202 202 and (line[assign_ix+1].string == '%') \
203 203 and (line[assign_ix+2].type == tokenize.NAME):
204 204 return cls(line[assign_ix+1].start)
205 205
206 206 def transform(self, lines: List[str]):
207 207 """Transform a magic assignment found by the ``find()`` classmethod.
208 208 """
209 209 start_line, start_col = self.start_line, self.start_col
210 210 lhs = lines[start_line][:start_col]
211 211 end_line = find_end_of_continued_line(lines, start_line)
212 212 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
213 213 assert rhs.startswith('%'), rhs
214 214 magic_name, _, args = rhs[1:].partition(' ')
215 215
216 216 lines_before = lines[:start_line]
217 217 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
218 218 new_line = lhs + call + '\n'
219 219 lines_after = lines[end_line+1:]
220 220
221 221 return lines_before + [new_line] + lines_after
222 222
223 223
224 224 class SystemAssign(TokenTransformBase):
225 225 """Transformer for assignments from system commands (a = !foo)"""
226 226 @classmethod
227 227 def find(cls, tokens_by_line):
228 228 """Find the first system assignment (a = !foo) in the cell.
229 229 """
230 230 for line in tokens_by_line:
231 231 assign_ix = _find_assign_op(line)
232 232 if (assign_ix is not None) \
233 233 and (len(line) >= assign_ix + 2) \
234 234 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
235 235 ix = assign_ix + 1
236 236
237 237 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
238 238 if line[ix].string == '!':
239 239 return cls(line[ix].start)
240 240 elif not line[ix].string.isspace():
241 241 break
242 242 ix += 1
243 243
244 244 def transform(self, lines: List[str]):
245 245 """Transform a system assignment found by the ``find()`` classmethod.
246 246 """
247 247 start_line, start_col = self.start_line, self.start_col
248 248
249 249 lhs = lines[start_line][:start_col]
250 250 end_line = find_end_of_continued_line(lines, start_line)
251 251 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
252 252 assert rhs.startswith('!'), rhs
253 253 cmd = rhs[1:]
254 254
255 255 lines_before = lines[:start_line]
256 256 call = "get_ipython().getoutput({!r})".format(cmd)
257 257 new_line = lhs + call + '\n'
258 258 lines_after = lines[end_line + 1:]
259 259
260 260 return lines_before + [new_line] + lines_after
261 261
262 262 # The escape sequences that define the syntax transformations IPython will
263 263 # apply to user input. These can NOT be just changed here: many regular
264 264 # expressions and other parts of the code may use their hardcoded values, and
265 265 # for all intents and purposes they constitute the 'IPython syntax', so they
266 266 # should be considered fixed.
267 267
268 268 ESC_SHELL = '!' # Send line to underlying system shell
269 269 ESC_SH_CAP = '!!' # Send line to system shell and capture output
270 270 ESC_HELP = '?' # Find information about object
271 271 ESC_HELP2 = '??' # Find extra-detailed information about object
272 272 ESC_MAGIC = '%' # Call magic function
273 273 ESC_MAGIC2 = '%%' # Call cell-magic function
274 274 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
275 275 ESC_QUOTE2 = ';' # Quote all args as a single string, call
276 276 ESC_PAREN = '/' # Call first argument with rest of line as arguments
277 277
278 278 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
279 279 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
280 280
281 281 def _make_help_call(target, esc, next_input=None):
282 282 """Prepares a pinfo(2)/psearch call from a target name and the escape
283 283 (i.e. ? or ??)"""
284 284 method = 'pinfo2' if esc == '??' \
285 285 else 'psearch' if '*' in target \
286 286 else 'pinfo'
287 287 arg = " ".join([method, target])
288 288 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
289 289 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
290 290 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
291 291 if next_input is None:
292 292 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
293 293 else:
294 294 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
295 295 (next_input, t_magic_name, t_magic_arg_s)
296 296
297 297 def _tr_help(content):
298 298 """Translate lines escaped with: ?
299 299
300 300 A naked help line should fire the intro help screen (shell.show_usage())
301 301 """
302 302 if not content:
303 303 return 'get_ipython().show_usage()'
304 304
305 305 return _make_help_call(content, '?')
306 306
307 307 def _tr_help2(content):
308 308 """Translate lines escaped with: ??
309 309
310 310 A naked help line should fire the intro help screen (shell.show_usage())
311 311 """
312 312 if not content:
313 313 return 'get_ipython().show_usage()'
314 314
315 315 return _make_help_call(content, '??')
316 316
317 317 def _tr_magic(content):
318 318 "Translate lines escaped with a percent sign: %"
319 319 name, _, args = content.partition(' ')
320 320 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
321 321
322 322 def _tr_quote(content):
323 323 "Translate lines escaped with a comma: ,"
324 324 name, _, args = content.partition(' ')
325 325 return '%s("%s")' % (name, '", "'.join(args.split()) )
326 326
327 327 def _tr_quote2(content):
328 328 "Translate lines escaped with a semicolon: ;"
329 329 name, _, args = content.partition(' ')
330 330 return '%s("%s")' % (name, args)
331 331
332 332 def _tr_paren(content):
333 333 "Translate lines escaped with a slash: /"
334 334 name, _, args = content.partition(' ')
335 335 return '%s(%s)' % (name, ", ".join(args.split()))
336 336
337 337 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
338 338 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
339 339 ESC_HELP : _tr_help,
340 340 ESC_HELP2 : _tr_help2,
341 341 ESC_MAGIC : _tr_magic,
342 342 ESC_QUOTE : _tr_quote,
343 343 ESC_QUOTE2 : _tr_quote2,
344 344 ESC_PAREN : _tr_paren }
345 345
346 346 class EscapedCommand(TokenTransformBase):
347 347 """Transformer for escaped commands like %foo, !foo, or /foo"""
348 348 @classmethod
349 349 def find(cls, tokens_by_line):
350 350 """Find the first escaped command (%foo, !foo, etc.) in the cell.
351 351 """
352 352 for line in tokens_by_line:
353 353 ix = 0
354 354 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
355 355 ix += 1
356 356 if line[ix].string in ESCAPE_SINGLES:
357 357 return cls(line[ix].start)
358 358
359 359 def transform(self, lines):
360 360 """Transform an escaped line found by the ``find()`` classmethod.
361 361 """
362 362 start_line, start_col = self.start_line, self.start_col
363 363
364 364 indent = lines[start_line][:start_col]
365 365 end_line = find_end_of_continued_line(lines, start_line)
366 366 line = assemble_continued_line(lines, (start_line, start_col), end_line)
367 367
368 368 if line[:2] in ESCAPE_DOUBLES:
369 369 escape, content = line[:2], line[2:]
370 370 else:
371 371 escape, content = line[:1], line[1:]
372 372 call = tr[escape](content)
373 373
374 374 lines_before = lines[:start_line]
375 375 new_line = indent + call + '\n'
376 376 lines_after = lines[end_line + 1:]
377 377
378 378 return lines_before + [new_line] + lines_after
379 379
380 380 _help_end_re = re.compile(r"""(%{0,2}
381 381 [a-zA-Z_*][\w*]* # Variable name
382 382 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
383 383 )
384 384 (\?\??)$ # ? or ??
385 385 """,
386 386 re.VERBOSE)
387 387
388 388 class HelpEnd(TokenTransformBase):
389 389 """Transformer for help syntax: obj? and obj??"""
390 390 # This needs to be higher priority (lower number) than EscapedCommand so
391 391 # that inspecting magics (%foo?) works.
392 392 priority = 5
393 393
394 394 def __init__(self, start, q_locn):
395 395 super().__init__(start)
396 396 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
397 397 self.q_col = q_locn[1]
398 398
399 399 @classmethod
400 400 def find(cls, tokens_by_line):
401 401 """Find the first help command (foo?) in the cell.
402 402 """
403 403 for line in tokens_by_line:
404 404 # Last token is NEWLINE; look at last but one
405 405 if len(line) > 2 and line[-2].string == '?':
406 406 # Find the first token that's not INDENT/DEDENT
407 407 ix = 0
408 408 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
409 409 ix += 1
410 410 return cls(line[ix].start, line[-2].start)
411 411
412 412 def transform(self, lines):
413 413 """Transform a help command found by the ``find()`` classmethod.
414 414 """
415 415 piece = ''.join(lines[self.start_line:self.q_line+1])
416 416 indent, content = piece[:self.start_col], piece[self.start_col:]
417 417 lines_before = lines[:self.start_line]
418 418 lines_after = lines[self.q_line + 1:]
419 419
420 420 m = _help_end_re.search(content)
421 421 assert m is not None, content
422 422 target = m.group(1)
423 423 esc = m.group(3)
424 424
425 425 # If we're mid-command, put it back on the next prompt for the user.
426 426 next_input = None
427 427 if (not lines_before) and (not lines_after) \
428 428 and content.strip() != m.group(0):
429 429 next_input = content.rstrip('?\n')
430 430
431 431 call = _make_help_call(target, esc, next_input=next_input)
432 432 new_line = indent + call + '\n'
433 433
434 434 return lines_before + [new_line] + lines_after
435 435
436 436 def make_tokens_by_line(lines):
437 437 """Tokenize a series of lines and group tokens by line.
438 438
439 439 The tokens for a multiline Python string or expression are
440 440 grouped as one line.
441 441 """
442 442 # NL tokens are used inside multiline expressions, but also after blank
443 443 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
444 444 # We want to group the former case together but split the latter, so we
445 445 # track parentheses level, similar to the internals of tokenize.
446 446 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
447 447 tokens_by_line = [[]]
448 448 parenlev = 0
449 449 try:
450 450 for token in tokenize.generate_tokens(iter(lines).__next__):
451 451 tokens_by_line[-1].append(token)
452 452 if (token.type == NEWLINE) \
453 453 or ((token.type == NL) and (parenlev <= 0)):
454 454 tokens_by_line.append([])
455 455 elif token.string in {'(', '[', '{'}:
456 456 parenlev += 1
457 457 elif token.string in {')', ']', '}'}:
458 458 if parenlev > 0:
459 459 parenlev -= 1
460 460 except tokenize.TokenError:
461 461 # Input ended in a multiline string or expression. That's OK for us.
462 462 pass
463 463
464 464 return tokens_by_line
465 465
466 466 def show_linewise_tokens(s: str):
467 467 """For investigation and debugging"""
468 468 if not s.endswith('\n'):
469 469 s += '\n'
470 470 lines = s.splitlines(keepends=True)
471 471 for line in make_tokens_by_line(lines):
472 472 print("Line -------")
473 473 for tokinfo in line:
474 474 print(" ", tokinfo)
475 475
476 476 # Arbitrary limit to prevent getting stuck in infinite loops
477 477 TRANSFORM_LOOP_LIMIT = 500
478 478
479 479 class TransformerManager:
480 480 """Applies various transformations to a cell or code block.
481 481
482 482 The key methods for external use are ``transform_cell()``
483 483 and ``check_complete()``.
484 484 """
485 485 def __init__(self):
486 486 self.cleanup_transforms = [
487 487 leading_indent,
488 488 classic_prompt,
489 489 ipython_prompt,
490 490 ]
491 491 self.line_transforms = [
492 492 cell_magic,
493 493 ]
494 494 self.token_transformers = [
495 495 MagicAssign,
496 496 SystemAssign,
497 497 EscapedCommand,
498 498 HelpEnd,
499 499 ]
500 500
501 501 def do_one_token_transform(self, lines):
502 502 """Find and run the transform earliest in the code.
503 503
504 504 Returns (changed, lines).
505 505
506 506 This method is called repeatedly until changed is False, indicating
507 507 that all available transformations are complete.
508 508
509 509 The tokens following IPython special syntax might not be valid, so
510 510 the transformed code is retokenised every time to identify the next
511 511 piece of special syntax. Hopefully long code cells are mostly valid
512 512 Python, not using lots of IPython special syntax, so this shouldn't be
513 513 a performance issue.
514 514 """
515 515 tokens_by_line = make_tokens_by_line(lines)
516 516 candidates = []
517 517 for transformer_cls in self.token_transformers:
518 518 transformer = transformer_cls.find(tokens_by_line)
519 519 if transformer:
520 520 candidates.append(transformer)
521 521
522 522 if not candidates:
523 523 # Nothing to transform
524 524 return False, lines
525 525
526 526 transformer = min(candidates, key=TokenTransformBase.sortby)
527 527 return True, transformer.transform(lines)
528 528
529 529 def do_token_transforms(self, lines):
530 530 for _ in range(TRANSFORM_LOOP_LIMIT):
531 531 changed, lines = self.do_one_token_transform(lines)
532 532 if not changed:
533 533 return lines
534 534
535 535 raise RuntimeError("Input transformation still changing after "
536 536 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
537 537
538 538 def transform_cell(self, cell: str) -> str:
539 539 """Transforms a cell of input code"""
540 540 if not cell.endswith('\n'):
541 541 cell += '\n' # Ensure the cell has a trailing newline
542 542 lines = cell.splitlines(keepends=True)
543 543 for transform in self.cleanup_transforms + self.line_transforms:
544 #print(transform, lines)
545 544 lines = transform(lines)
546 545
547 546 lines = self.do_token_transforms(lines)
548 547 return ''.join(lines)
549 548
550 549 def check_complete(self, cell: str):
551 550 """Return whether a block of code is ready to execute, or should be continued
552 551
553 552 Parameters
554 553 ----------
555 554 source : string
556 555 Python input code, which can be multiline.
557 556
558 557 Returns
559 558 -------
560 559 status : str
561 560 One of 'complete', 'incomplete', or 'invalid' if source is not a
562 561 prefix of valid code.
563 562 indent_spaces : int or None
564 563 The number of spaces by which to indent the next line of code. If
565 564 status is not 'incomplete', this is None.
566 565 """
567 566 if not cell.endswith('\n'):
568 567 cell += '\n' # Ensure the cell has a trailing newline
569 568 lines = cell.splitlines(keepends=True)
570 569 if lines[-1][:-1].endswith('\\'):
571 570 # Explicit backslash continuation
572 571 return 'incomplete', find_last_indent(lines)
573 572
574 573 try:
575 574 for transform in self.cleanup_transforms:
576 575 lines = transform(lines)
577 576 except SyntaxError:
578 577 return 'invalid', None
579 578
580 579 if lines[0].startswith('%%'):
581 580 # Special case for cell magics - completion marked by blank line
582 581 if lines[-1].strip():
583 582 return 'incomplete', find_last_indent(lines)
584 583 else:
585 584 return 'complete', None
586 585
587 586 try:
588 587 for transform in self.line_transforms:
589 588 lines = transform(lines)
590 589 lines = self.do_token_transforms(lines)
591 590 except SyntaxError:
592 591 return 'invalid', None
593 592
594 593 tokens_by_line = make_tokens_by_line(lines)
595 594 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
596 595 # We're in a multiline string or expression
597 596 return 'incomplete', find_last_indent(lines)
598 597
599 598 # Find the last token on the previous line that's not NEWLINE or COMMENT
600 599 toks_last_line = tokens_by_line[-2]
601 600 ix = len(toks_last_line) - 1
602 601 while ix >= 0 and toks_last_line[ix].type in {tokenize.NEWLINE,
603 602 tokenize.COMMENT}:
604 603 ix -= 1
605 604
606 605 if toks_last_line[ix].string == ':':
607 606 # The last line starts a block (e.g. 'if foo:')
608 607 ix = 0
609 608 while toks_last_line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
610 609 ix += 1
611 610 indent = toks_last_line[ix].start[1]
612 611 return 'incomplete', indent + 4
613 612
614 613 # If there's a blank line at the end, assume we're ready to execute.
615 614 if not lines[-1].strip():
616 615 return 'complete', None
617 616
618 617 # At this point, our checks think the code is complete (or invalid).
619 618 # We'll use codeop.compile_command to check this with the real parser.
620 619
621 620 try:
622 621 with warnings.catch_warnings():
623 622 warnings.simplefilter('error', SyntaxWarning)
624 623 res = compile_command(''.join(lines), symbol='exec')
625 624 except (SyntaxError, OverflowError, ValueError, TypeError,
626 625 MemoryError, SyntaxWarning):
627 626 return 'invalid', None
628 627 else:
629 628 if res is None:
630 629 return 'incomplete', find_last_indent(lines)
631 630 return 'complete', None
632 631
633 632
634 633 def find_last_indent(lines):
635 634 m = _indent_re.match(lines[-1])
636 635 if not m:
637 636 return 0
638 637 return len(m.group(0).replace('\t', ' '*4))
@@ -1,222 +1,222 b''
1 1 .. _autoawait:
2 2
3 3 Asynchronous in REPL: Autoawait
4 4 ===============================
5 5
6 6 .. note::
7 7
8 This feature is experimental and behavior can change betwen python and
8 This feature is experimental and behavior can change between python and
9 9 IPython version without prior deprecation.
10 10
11 11 Starting with IPython 7.0, and when user Python 3.6 and above, IPython offer the
12 12 ability to run asynchronous code from the REPL. Constructs which are
13 13 :exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython.
14 14
15 15 The example given here are for terminal IPython, running async code in a
16 16 notebook interface or any other frontend using the Jupyter protocol will need to
17 17 use a newer version of IPykernel. The details of how async code runs in
18 18 IPykernel will differ between IPython, IPykernel and their versions.
19 19
20 20 When a supported library is used, IPython will automatically allow Futures and
21 21 Coroutines in the REPL to be ``await`` ed. This will happen if an :ref:`await
22 22 <await>` (or any other async constructs like async-with, async-for) is use at
23 23 top level scope, or if any structure valid only in `async def
24 24 <https://docs.python.org/3/reference/compound_stmts.html#async-def>`_ function
25 25 context are present. For example, the following being a syntax error in the
26 26 Python REPL::
27 27
28 28 Python 3.6.0
29 29 [GCC 4.2.1]
30 30 Type "help", "copyright", "credits" or "license" for more information.
31 31 >>> import aiohttp
32 32 >>> result = aiohttp.get('https://api.github.com')
33 33 >>> response = await result
34 34 File "<stdin>", line 1
35 35 response = await result
36 36 ^
37 37 SyntaxError: invalid syntax
38 38
39 39 Should behave as expected in the IPython REPL::
40 40
41 41 Python 3.6.0
42 42 Type 'copyright', 'credits' or 'license' for more information
43 43 IPython 7.0.0 -- An enhanced Interactive Python. Type '?' for help.
44 44
45 45 In [1]: import aiohttp
46 46 ...: result = aiohttp.get('https://api.github.com')
47 47
48 48 In [2]: response = await result
49 49 <pause for a few 100s ms>
50 50
51 51 In [3]: await response.json()
52 52 Out[3]:
53 53 {'authorizations_url': 'https://api.github.com/authorizations',
54 54 'code_search_url': 'https://api.github.com/search/code?q={query}...',
55 55 ...
56 56 }
57 57
58 58
59 59 You can use the ``c.InteractiveShell.autoawait`` configuration option and set it
60 60 to :any:`False` to deactivate automatic wrapping of asynchronous code. You can also
61 61 use the :magic:`%autoawait` magic to toggle the behavior at runtime::
62 62
63 63 In [1]: %autoawait False
64 64
65 65 In [2]: %autoawait
66 66 IPython autoawait is `Off`, and set to use `asyncio`
67 67
68 68
69 69
70 70 By default IPython will assume integration with Python's provided
71 71 :mod:`asyncio`, but integration with other libraries is provided. In particular
72 72 we provide experimental integration with the ``curio`` and ``trio`` library.
73 73
74 74 You can switch current integration by using the
75 75 ``c.InteractiveShell.loop_runner`` option or the ``autoawait <name
76 76 integration>`` magic.
77 77
78 78 For example::
79 79
80 80 In [1]: %autoawait trio
81 81
82 82 In [2]: import trio
83 83
84 84 In [3]: async def child(i):
85 85 ...: print(" child %s goes to sleep"%i)
86 86 ...: await trio.sleep(2)
87 87 ...: print(" child %s wakes up"%i)
88 88
89 89 In [4]: print('parent start')
90 90 ...: async with trio.open_nursery() as n:
91 91 ...: for i in range(5):
92 92 ...: n.spawn(child, i)
93 93 ...: print('parent end')
94 94 parent start
95 95 child 2 goes to sleep
96 96 child 0 goes to sleep
97 97 child 3 goes to sleep
98 98 child 1 goes to sleep
99 99 child 4 goes to sleep
100 100 <about 2 seconds pause>
101 101 child 2 wakes up
102 102 child 1 wakes up
103 103 child 0 wakes up
104 104 child 3 wakes up
105 105 child 4 wakes up
106 106 parent end
107 107
108 108
109 109 In the above example, ``async with`` at top level scope is a syntax error in
110 110 Python.
111 111
112 112 Using this mode can have unexpected consequences if used in interaction with
113 113 other features of IPython and various registered extensions. In particular if you
114 114 are a direct or indirect user of the AST transformers, these may not apply to
115 115 your code.
116 116
117 117 When using command line IPython, the default loop (or runner) does not process
118 118 in the background, so top level asynchronous code must finish for the REPL to
119 119 allow you to enter more code. As with usual Python semantic, the awaitables are
120 120 started only when awaited for the first time. That is to say, in first example,
121 121 no network request is done between ``In[1]`` and ``In[2]``.
122 122
123 123
124 124 Effects on IPython.embed()
125 125 ==========================
126 126
127 127 IPython core being asynchronous, the use of ``IPython.embed()`` will now require
128 128 a loop to run. By default IPython will use a fake coroutine runner which should
129 129 allow ``IPython.embed()`` to be nested. Though this will prevent usage of the
130 130 ``autoawait`` feature when using IPython embed.
131 131
132 132 You can set explicitly a coroutine runner for ``embed()`` if you desire to run
133 133 asynchronous code, the exact behavior is though undefined.
134 134
135 135 Effects on Magics
136 136 =================
137 137
138 138 A couple of magics (``%%timeit``, ``%timeit``, ``%%time``, ``%%prun``) have not
139 139 yet been updated to work with asynchronous code and will raise syntax errors
140 140 when trying to use top-level ``await``. We welcome any contribution to help fix
141 141 those, and extra cases we haven't caught yet. We hope for better support in Cor
142 142 Python for top-level Async code.
143 143
144 144 Internals
145 145 =========
146 146
147 147 As running asynchronous code is not supported in interactive REPL (as of Python
148 148 3.7) we have to rely to a number of complex workaround and heuristic to allow
149 149 this to happen. It is interesting to understand how this works in order to
150 150 comprehend potential bugs, or provide a custom runner.
151 151
152 152 Among the many approaches that are at our disposition, we find only one that
153 153 suited out need. Under the hood we use the code object from a async-def function
154 154 and run it in global namespace after modifying it to not create a new
155 155 ``locals()`` scope::
156 156
157 157 async def inner_async():
158 158 locals().update(**global_namespace)
159 159 #
160 160 # here is user code
161 161 #
162 162 return last_user_statement
163 163 codeobj = modify(inner_async.__code__)
164 164 coroutine = eval(codeobj, user_ns)
165 165 display(loop_runner(coroutine))
166 166
167 167
168 168
169 169 The first thing you'll notice is that unlike classical ``exec``, there is only
170 170 one namespace. Second, user code runs in a function scope, and not a module
171 171 scope.
172 172
173 173 On top of the above there are significant modification to the AST of
174 174 ``function``, and ``loop_runner`` can be arbitrary complex. So there is a
175 175 significant overhead to this kind of code.
176 176
177 177 By default the generated coroutine function will be consumed by Asyncio's
178 178 ``loop_runner = asyncio.get_evenloop().run_until_complete()`` method if
179 179 ``async`` mode is deemed necessary, otherwise the coroutine will just be
180 180 exhausted in a simple runner. It is though possible to change the default
181 181 runner.
182 182
183 183 A loop runner is a *synchronous* function responsible from running a coroutine
184 184 object.
185 185
186 186 The runner is responsible from ensuring that ``coroutine`` run to completion,
187 187 and should return the result of executing the coroutine. Let's write a
188 188 runner for ``trio`` that print a message when used as an exercise, ``trio`` is
189 189 special as it usually prefer to run a function object and make a coroutine by
190 190 itself, we can get around this limitation by wrapping it in an async-def without
191 191 parameters and passing this value to ``trio``::
192 192
193 193
194 194 In [1]: import trio
195 195 ...: from types import CoroutineType
196 196 ...:
197 197 ...: def trio_runner(coro:CoroutineType):
198 198 ...: print('running asynchronous code')
199 199 ...: async def corowrap(coro):
200 200 ...: return await coro
201 201 ...: return trio.run(corowrap, coro)
202 202
203 203 We can set it up by passing it to ``%autoawait``::
204 204
205 205 In [2]: %autoawait trio_runner
206 206
207 207 In [3]: async def async_hello(name):
208 208 ...: await trio.sleep(1)
209 209 ...: print(f'Hello {name} world !')
210 210 ...: await trio.sleep(1)
211 211
212 212 In [4]: await async_hello('async')
213 213 running asynchronous code
214 214 Hello async world !
215 215
216 216
217 217 Asynchronous programming in python (and in particular in the REPL) is still a
218 218 relatively young subject. We expect some code to not behave as you expect, so
219 219 feel free to contribute improvements to this codebase and give us feedback.
220 220
221 221 We invite you to thoroughly test this feature and report any unexpected behavior
222 222 as well as propose any improvement.
@@ -1,68 +1,68 b''
1 1 .. Developers should add in this file, during each release cycle, information
2 2 .. about important changes they've made, in a summary format that's meant for
3 3 .. end users. For each release we normally have three sections: features, bug
4 4 .. fixes and api breakage.
5 5 .. Please remember to credit the authors of the contributions by name,
6 6 .. especially when they are new users or developers who do not regularly
7 7 .. participate in IPython's development.
8 8
9 9 .. _whatsnew_index:
10 10
11 11 =====================
12 12 What's new in IPython
13 13 =====================
14 14
15 15 ..
16 this will appear in the docs if we are nto releasing a versin (ie is
17 `_version_extra` in release.py is empty stringA
16 this will appear in the docs if we are not releasing a versin (ie is
17 `_version_extra` in release.py is empty string
18 18
19 19 .. only:: ipydev
20 20
21 Developpement version in-progress features:
21 Development version in-progress features:
22 22
23 23 .. toctree::
24 24
25 25 development
26 26
27 27 ..
28 28 this make a hidden toctree that avoid sphinx to complain about documents
29 29 included nowhere when building docs for stable
30 30
31 31 .. only:: ipystable
32 32
33 33 .. toctree::
34 34 :hidden:
35 35
36 36 development
37 37
38 38 This section documents the changes that have been made in various versions of
39 39 IPython. Users should consult these pages to learn about new features, bug
40 40 fixes and backwards incompatibilities. Developers should summarize the
41 41 development work they do here in a user friendly format.
42 42
43 43 .. toctree::
44 44 :maxdepth: 1
45 45
46 46 version7
47 47 version6
48 48 github-stats-6
49 49 version5
50 50 github-stats-5
51 51 version4
52 52 github-stats-4
53 53 version3
54 54 github-stats-3
55 55 version3_widget_migration
56 56 version2.0
57 57 github-stats-2.0
58 58 version1.0
59 59 github-stats-1.0
60 60 version0.13
61 61 github-stats-0.13
62 62 version0.12
63 63 github-stats-0.12
64 64 version0.11
65 65 github-stats-0.11
66 66 version0.10
67 67 version0.9
68 68 version0.8
@@ -1,670 +1,670 b''
1 1 =============
2 2 0.13 Series
3 3 =============
4 4
5 5 Release 0.13
6 6 ============
7 7
8 8 IPython 0.13 contains several major new features, as well as a large amount of
9 9 bug and regression fixes. The previous version (0.12) was released on December
10 10 19 2011, and in this development cycle we had:
11 11
12 12 - ~6 months of work.
13 13 - 373 pull requests merged.
14 14 - 742 issues closed (non-pull requests).
15 15 - contributions from 62 authors.
16 16 - 1760 commits.
17 17 - a diff of 114226 lines.
18 18
19 19 The amount of work included in this release is so large, that we can only cover
20 20 here the main highlights; please see our :ref:`detailed release statistics
21 21 <issues_list_013>` for links to every issue and pull request closed on GitHub
22 22 as well as a full list of individual contributors.
23 23
24 24
25 25 Major Notebook improvements: new user interface and more
26 26 --------------------------------------------------------
27 27
28 28 The IPython Notebook, which has proven since its release to be wildly popular,
29 29 has seen a massive amount of work in this release cycle, leading to a
30 30 significantly improved user experience as well as many new features.
31 31
32 32 The first user-visible change is a reorganization of the user interface; the
33 33 left panel has been removed and was replaced by a real menu system and a
34 34 toolbar with icons. Both the toolbar and the header above the menu can be
35 35 collapsed to leave an unobstructed working area:
36 36
37 37 .. image:: ../_images/ipy_013_notebook_spectrogram.png
38 38 :width: 460px
39 39 :alt: New user interface for Notebook
40 40 :align: center
41 41 :target: ../_images/ipy_013_notebook_spectrogram.png
42 42
43 43 The notebook handles very long outputs much better than before (this was a
44 44 serious usability issue when running processes that generated massive amounts
45 45 of output). Now, in the presence of outputs longer than ~100 lines, the
46 46 notebook will automatically collapse to a scrollable area and the entire left
47 47 part of this area controls the display: one click in this area will expand the
48 48 output region completely, and a double-click will hide it completely. This
49 49 figure shows both the scrolled and hidden modes:
50 50
51 51 .. image:: ../_images/ipy_013_notebook_long_out.png
52 52 :width: 460px
53 53 :alt: Scrolling and hiding of long output in the notebook.
54 54 :align: center
55 55 :target: ../_images/ipy_013_notebook_long_out.png
56 56
57 57 .. note::
58 58
59 59 The auto-folding of long outputs is disabled in Firefox due to bugs in its
60 60 scrolling behavior. See :ghpull:`2047` for details.
61 61
62 62 Uploading notebooks to the dashboard is now easier: in addition to drag and
63 63 drop (which can be finicky sometimes), you can now click on the upload text and
64 64 use a regular file dialog box to select notebooks to upload. Furthermore, the
65 65 notebook dashboard now auto-refreshes its contents and offers buttons to shut
66 66 down any running kernels (:ghpull:`1739`):
67 67
68 68 .. image:: ../_images/ipy_013_dashboard.png
69 69 :width: 460px
70 70 :alt: Improved dashboard
71 71 :align: center
72 72 :target: ../_images/ipy_013_dashboard.png
73 73
74 74
75 75 Cluster management
76 76 ~~~~~~~~~~~~~~~~~~
77 77
78 78 The notebook dashboard can now also start and stop clusters, thanks to a new
79 79 tab in the dashboard user interface:
80 80
81 81 .. image:: ../_images/ipy_013_dashboard_cluster.png
82 82 :width: 460px
83 83 :alt: Cluster management from the notebook dashboard
84 84 :align: center
85 85 :target: ../_images/ipy_013_dashboard_cluster.png
86 86
87 87 This interface allows, for each profile you have configured, to start and stop
88 88 a cluster (and optionally override the default number of engines corresponding
89 89 to that configuration). While this hides all error reporting, once you have a
90 90 configuration that you know works smoothly, it is a very convenient interface
91 91 for controlling your parallel resources.
92 92
93 93
94 94 New notebook format
95 95 ~~~~~~~~~~~~~~~~~~~
96 96
97 97 The notebooks saved now use version 3 of our format, which supports heading
98 98 levels as well as the concept of 'raw' text cells that are not rendered as
99 99 Markdown. These will be useful with converters_ we are developing, to pass raw
100 100 markup (say LaTeX). That conversion code is still under heavy development and
101 101 not quite ready for prime time, but we welcome help on this front so that we
102 102 can merge it for full production use as soon as possible.
103 103
104 104 .. _converters: https://github.com/ipython/nbconvert
105 105
106 106 .. note::
107 107
108 108 v3 notebooks can *not* be read by older versions of IPython, but we provide
109 109 a `simple script`_ that you can use in case you need to export a v3
110 110 notebook to share with a v2 user.
111 111
112 112 .. _simple script: https://gist.github.com/1935808
113 113
114 114
115 115 JavaScript refactoring
116 116 ~~~~~~~~~~~~~~~~~~~~~~
117 117
118 118 All the client-side JavaScript has been decoupled to ease reuse of parts of the
119 119 machinery without having to build a full-blown notebook. This will make it much
120 120 easier to communicate with an IPython kernel from existing web pages and to
121 121 integrate single cells into other sites, without loading the full notebook
122 122 document-like UI. :ghpull:`1711`.
123 123
124 124 This refactoring also enables the possibility of writing dynamic javascript
125 125 widgets that are returned from Python code and that present an interactive view
126 126 to the user, with callbacks in Javascript executing calls to the Kernel. This
127 127 will enable many interactive elements to be added by users in notebooks.
128 128
129 129 An example of this capability has been provided as a proof of concept in
130 130 :file:`examples/widgets` that lets you directly communicate with one or more
131 131 parallel engines, acting as a mini-console for parallel debugging and
132 132 introspection.
133 133
134 134
135 135 Improved tooltips
136 136 ~~~~~~~~~~~~~~~~~
137 137
138 138 The object tooltips have gained some new functionality. By pressing tab several
139 139 times, you can expand them to see more of a docstring, keep them visible as you
140 140 fill in a function's parameters, or transfer the information to the pager at the
141 141 bottom of the screen. For the details, look at the example notebook
142 142 :file:`01_notebook_introduction.ipynb`.
143 143
144 144 .. figure:: ../_images/ipy_013_notebook_tooltip.png
145 145 :width: 460px
146 146 :alt: Improved tooltips in the notebook.
147 147 :align: center
148 148 :target: ../_images/ipy_013_notebook_tooltip.png
149 149
150 150 The new notebook tooltips.
151 151
152 152 Other improvements to the Notebook
153 153 ----------------------------------
154 154
155 155 These are some other notable small improvements to the notebook, in addition to
156 156 many bug fixes and minor changes to add polish and robustness throughout:
157 157
158 * The notebook pager (the area at the bottom) is now resizeable by dragging its
158 * The notebook pager (the area at the bottom) is now Resizable by dragging its
159 159 divider handle, a feature that had been requested many times by just about
160 160 anyone who had used the notebook system. :ghpull:`1705`.
161 161
162 162 * It is now possible to open notebooks directly from the command line; for
163 163 example: ``ipython notebook path/`` will automatically set ``path/`` as the
164 164 notebook directory, and ``ipython notebook path/foo.ipynb`` will further
165 165 start with the ``foo.ipynb`` notebook opened. :ghpull:`1686`.
166 166
167 167 * If a notebook directory is specified with ``--notebook-dir`` (or with the
168 168 corresponding configuration flag ``NotebookManager.notebook_dir``), all
169 169 kernels start in this directory.
170 170
171 171 * Fix codemirror clearing of cells with ``Ctrl-Z``; :ghpull:`1965`.
172 172
173 173 * Text (markdown) cells now line wrap correctly in the notebook, making them
174 174 much easier to edit :ghpull:`1330`.
175 175
176 176 * PNG and JPEG figures returned from plots can be interactively resized in the
177 177 notebook, by dragging them from their lower left corner. :ghpull:`1832`.
178 178
179 179 * Clear ``In []`` prompt numbers on "Clear All Output". For more
180 180 version-control-friendly ``.ipynb`` files, we now strip all prompt numbers
181 181 when doing a "Clear all output". This reduces the amount of noise in
182 182 commit-to-commit diffs that would otherwise show the (highly variable) prompt
183 183 number changes. :ghpull:`1621`.
184 184
185 185 * The notebook server now requires *two* consecutive ``Ctrl-C`` within 5
186 186 seconds (or an interactive confirmation) to terminate operation. This makes
187 187 it less likely that you will accidentally kill a long-running server by
188 188 typing ``Ctrl-C`` in the wrong terminal. :ghpull:`1609`.
189 189
190 190 * Using ``Ctrl-S`` (or ``Cmd-S`` on a Mac) actually saves the notebook rather
191 191 than providing the fairly useless browser html save dialog. :ghpull:`1334`.
192 192
193 193 * Allow accessing local files from the notebook (in urls), by serving any local
194 194 file as the url ``files/<relativepath>``. This makes it possible to, for
195 195 example, embed local images in a notebook. :ghpull:`1211`.
196 196
197 197
198 198 Cell magics
199 199 -----------
200 200
201 201 We have completely refactored the magic system, finally moving the magic
202 202 objects to standalone, independent objects instead of being the mixin class
203 203 we'd had since the beginning of IPython (:ghpull:`1732`). Now, a separate base
204 204 class is provided in :class:`IPython.core.magic.Magics` that users can subclass
205 205 to create their own magics. Decorators are also provided to create magics from
206 206 simple functions without the need for object orientation. Please see the
207 207 :ref:`magic` docs for further details.
208 208
209 209 All builtin magics now exist in a few subclasses that group together related
210 210 functionality, and the new :mod:`IPython.core.magics` package has been created
211 211 to organize this into smaller files.
212 212
213 213 This cleanup was the last major piece of deep refactoring needed from the
214 214 original 2001 codebase.
215 215
216 216 We have also introduced a new type of magic function, prefixed with `%%`
217 217 instead of `%`, which operates at the whole-cell level. A cell magic receives
218 218 two arguments: the line it is called on (like a line magic) and the body of the
219 219 cell below it.
220 220
221 221 Cell magics are most natural in the notebook, but they also work in the
222 222 terminal and qt console, with the usual approach of using a blank line to
223 223 signal cell termination.
224 224
225 225 For example, to time the execution of several statements::
226 226
227 227 %%timeit x = 0 # setup
228 228 for i in range(100000):
229 229 x += i**2
230 230
231 231 This is particularly useful to integrate code in another language, and cell
232 232 magics already exist for shell scripts, Cython, R and Octave. Using ``%%script
233 233 /usr/bin/foo``, you can run a cell in any interpreter that accepts code via
234 234 stdin.
235 235
236 236 Another handy cell magic makes it easy to write short text files: ``%%file
237 237 ~/save/to/here.txt``.
238 238
239 239 The following cell magics are now included by default; all those that use
240 240 special interpreters (Perl, Ruby, bash, etc.) assume you have the requisite
241 241 interpreter installed:
242 242
243 243 * ``%%!``: run cell body with the underlying OS shell; this is similar to
244 244 prefixing every line in the cell with ``!``.
245 245
246 246 * ``%%bash``: run cell body under bash.
247 247
248 248 * ``%%capture``: capture the output of the code in the cell (and stderr as
249 249 well). Useful to run codes that produce too much output that you don't even
250 250 want scrolled.
251 251
252 252 * ``%%file``: save cell body as a file.
253 253
254 254 * ``%%perl``: run cell body using Perl.
255 255
256 256 * ``%%prun``: run cell body with profiler (cell extension of ``%prun``).
257 257
258 258 * ``%%python3``: run cell body using Python 3.
259 259
260 260 * ``%%ruby``: run cell body using Ruby.
261 261
262 262 * ``%%script``: run cell body with the script specified in the first line.
263 263
264 264 * ``%%sh``: run cell body using sh.
265 265
266 266 * ``%%sx``: run cell with system shell and capture process output (cell
267 267 extension of ``%sx``).
268 268
269 269 * ``%%system``: run cell with system shell (``%%!`` is an alias to this).
270 270
271 271 * ``%%timeit``: time the execution of the cell (extension of ``%timeit``).
272 272
273 273 This is what some of the script-related magics look like in action:
274 274
275 275 .. image:: ../_images/ipy_013_notebook_script_cells.png
276 276 :width: 460px
277 277 :alt: Cluster management from the notebook dashboard
278 278 :align: center
279 279 :target: ../_images/ipy_013_notebook_script_cells.png
280 280
281 281 In addition, we have also a number of :ref:`extensions <extensions_overview>`
282 282 that provide specialized magics. These typically require additional software
283 283 to run and must be manually loaded via ``%load_ext <extension name>``, but are
284 284 extremely useful. The following extensions are provided:
285 285
286 286 **Cython magics** (extension ``cythonmagic``)
287 287 This extension provides magics to automatically build and compile Python
288 288 extension modules using the Cython_ language. You must install Cython
289 289 separately, as well as a C compiler, for this to work. The examples
290 290 directory in the source distribution ships with a full notebook
291 291 demonstrating these capabilities:
292 292
293 293 .. image:: ../_images/ipy_013_notebook_cythonmagic.png
294 294 :width: 460px
295 295 :alt: Cython magic
296 296 :align: center
297 297 :target: ../_images/ipy_013_notebook_cythonmagic.png
298 298
299 299 .. _cython: http://cython.org
300 300
301 301 **Octave magics** (extension ``octavemagic``)
302 302 This extension provides several magics that support calling code written in
303 303 the Octave_ language for numerical computing. You can execute single-lines
304 304 or whole blocks of Octave code, capture both output and figures inline
305 305 (just like matplotlib plots), and have variables automatically converted
306 306 between the two languages. To use this extension, you must have Octave
307 307 installed as well as the oct2py_ package. The examples
308 308 directory in the source distribution ships with a full notebook
309 309 demonstrating these capabilities:
310 310
311 311 .. image:: ../_images/ipy_013_notebook_octavemagic.png
312 312 :width: 460px
313 313 :alt: Octave magic
314 314 :align: center
315 315 :target: ../_images/ipy_013_notebook_octavemagic.png
316 316
317 317 .. _octave: http://www.gnu.org/software/octave
318 318 .. _oct2py: http://pypi.python.org/pypi/oct2py
319 319
320 320 **R magics** (extension ``rmagic``)
321 321 This extension provides several magics that support calling code written in
322 322 the R_ language for statistical data analysis. You can execute
323 323 single-lines or whole blocks of R code, capture both output and figures
324 324 inline (just like matplotlib plots), and have variables automatically
325 325 converted between the two languages. To use this extension, you must have
326 326 R installed as well as the rpy2_ package that bridges Python and R. The
327 327 examples directory in the source distribution ships with a full notebook
328 328 demonstrating these capabilities:
329 329
330 330 .. image:: ../_images/ipy_013_notebook_rmagic.png
331 331 :width: 460px
332 332 :alt: R magic
333 333 :align: center
334 334 :target: ../_images/ipy_013_notebook_rmagic.png
335 335
336 336 .. _R: http://www.r-project.org
337 337 .. _rpy2: http://rpy.sourceforge.net/rpy2.html
338 338
339 339
340 340 Tab completer improvements
341 341 --------------------------
342 342
343 343 Useful tab-completion based on live inspection of objects is one of the most
344 344 popular features of IPython. To make this process even more user-friendly, the
345 345 completers of both the Qt console and the Notebook have been reworked.
346 346
347 347 The Qt console comes with a new ncurses-like tab completer, activated by
348 348 default, which lets you cycle through the available completions by pressing tab,
349 349 or select a completion with the arrow keys (:ghpull:`1851`).
350 350
351 351 .. figure:: ../_images/ipy_013_qtconsole_completer.png
352 352 :width: 460px
353 353 :alt: ncurses-like completer, with highlighted selection.
354 354 :align: center
355 355 :target: ../_images/ipy_013_qtconsole_completer.png
356 356
357 357 The new improved Qt console's ncurses-like completer allows to easily
358 358 navigate thought long list of completions.
359 359
360 360 In the notebook, completions are now sourced both from object introspection and
361 361 analysis of surrounding code, so limited completions can be offered for
362 362 variables defined in the current cell, or while the kernel is busy
363 363 (:ghpull:`1711`).
364 364
365 365
366 366 We have implemented a new configurable flag to control tab completion on
367 367 modules that provide the ``__all__`` attribute::
368 368
369 369 IPCompleter.limit_to__all__= Boolean
370 370
371 371 This instructs the completer to honor ``__all__`` for the completion.
372 372 Specifically, when completing on ``object.<tab>``, if True: only those names
373 373 in ``obj.__all__`` will be included. When False [default]: the ``__all__``
374 374 attribute is ignored. :ghpull:`1529`.
375 375
376 376
377 377 Improvements to the Qt console
378 378 ------------------------------
379 379
380 380 The Qt console continues to receive improvements and refinements, despite the
381 381 fact that it is by now a fairly mature and robust component. Lots of small
382 382 polish has gone into it, here are a few highlights:
383 383
384 384 * A number of changes were made to the underlying code for easier integration
385 385 into other projects such as Spyder_ (:ghpull:`2007`, :ghpull:`2024`).
386 386
387 387 * Improved menus with a new Magic menu that is organized by magic groups (this
388 388 was made possible by the reorganization of the magic system
389 389 internals). :ghpull:`1782`.
390 390
391 391 * Allow for restarting kernels without clearing the qtconsole, while leaving a
392 392 visible indication that the kernel has restarted. :ghpull:`1681`.
393 393
394 394 * Allow the native display of jpeg images in the qtconsole. :ghpull:`1643`.
395 395
396 396 .. _spyder: https://code.google.com/p/spyderlib
397 397
398 398
399 399
400 400 Parallel
401 401 --------
402 402
403 403 The parallel tools have been improved and fine-tuned on multiple fronts. Now,
404 404 the creation of an :class:`IPython.parallel.Client` object automatically
405 405 activates a line and cell magic function ``px`` that sends its code to all the
406 406 engines. Further magics can be easily created with the :meth:`.Client.activate`
407 407 method, to conveniently execute code on any subset of engines. :ghpull:`1893`.
408 408
409 409 The ``%%px`` cell magic can also be given an optional targets argument, as well
410 410 as a ``--out`` argument for storing its output.
411 411
412 412 A new magic has also been added, ``%pxconfig``, that lets you configure various
413 413 defaults of the parallel magics. As usual, type ``%pxconfig?`` for details.
414 414
415 415 The exception reporting in parallel contexts has been improved to be easier to
416 416 read. Now, IPython directly reports the remote exceptions without showing any
417 417 of the internal execution parts:
418 418
419 419 .. image:: ../_images/ipy_013_par_tb.png
420 420 :width: 460px
421 421 :alt: Improved parallel exceptions.
422 422 :align: center
423 423 :target: ../_images/ipy_013_par_tb.png
424 424
425 425 The parallel tools now default to using ``NoDB`` as the storage backend for
426 426 intermediate results. This means that the default usage case will have a
427 427 significantly reduced memory footprint, though certain advanced features are
428 428 not available with this backend.
429 429
430 430 The parallel magics now display all output, so you can do parallel plotting or
431 431 other actions with complex display. The ``px`` magic has now both line and cell
432 432 modes, and in cell mode finer control has been added about how to collate
433 433 output from multiple engines. :ghpull:`1768`.
434 434
435 435 There have also been incremental improvements to the SSH launchers:
436 436
437 437 * add to_send/fetch steps for moving connection files around.
438 438
439 439 * add SSHProxyEngineSetLauncher, for invoking to `ipcluster engines` on a
440 440 remote host. This can be used to start a set of engines via PBS/SGE/MPI
441 441 *remotely*.
442 442
443 443 This makes the SSHLauncher usable on machines without shared filesystems.
444 444
445 445 A number of 'sugar' methods/properties were added to AsyncResult that are
446 446 quite useful (:ghpull:`1548`) for everday work:
447 447
448 448 * ``ar.wall_time`` = received - submitted
449 449 * ``ar.serial_time`` = sum of serial computation time
450 450 * ``ar.elapsed`` = time since submission (wall_time if done)
451 451 * ``ar.progress`` = (int) number of sub-tasks that have completed
452 452 * ``len(ar)`` = # of tasks
453 453 * ``ar.wait_interactive()``: prints progress
454 454
455 455 Added :meth:`.Client.spin_thread` / :meth:`~.Client.stop_spin_thread` for
456 456 running spin in a background thread, to keep zmq queue clear. This can be used
457 457 to ensure that timing information is as accurate as possible (at the cost of
458 458 having a background thread active).
459 459
460 460 Set TaskScheduler.hwm default to 1 instead of 0. 1 has more
461 461 predictable/intuitive behavior, if often slower, and thus a more logical
462 462 default. Users whose workloads require maximum throughput and are largely
463 463 homogeneous in time per task can make the optimization themselves, but now the
464 464 behavior will be less surprising to new users. :ghpull:`1294`.
465 465
466 466
467 467 Kernel/Engine unification
468 468 -------------------------
469 469
470 470 This is mostly work 'under the hood', but it is actually a *major* achievement
471 471 for the project that has deep implications in the long term: at last, we have
472 472 unified the main object that executes as the user's interactive shell (which we
473 473 refer to as the *IPython kernel*) with the objects that run in all the worker
474 474 nodes of the parallel computing facilities (the *IPython engines*). Ever since
475 475 the first implementation of IPython's parallel code back in 2006, we had wanted
476 476 to have these two roles be played by the same machinery, but a number of
477 477 technical reasons had prevented that from being true.
478 478
479 479 In this release we have now merged them, and this has a number of important
480 480 consequences:
481 481
482 482 * It is now possible to connect any of our clients (qtconsole or terminal
483 483 console) to any individual parallel engine, with the *exact* behavior of
484 484 working at a 'regular' IPython console/qtconsole. This makes debugging,
485 485 plotting, etc. in parallel scenarios vastly easier.
486 486
487 487 * Parallel engines can always execute arbitrary 'IPython code', that is, code
488 488 that has magics, shell extensions, etc. In combination with the ``%%px``
489 489 magics, it is thus extremely natural for example to send to all engines a
490 490 block of Cython or R code to be executed via the new Cython and R magics. For
491 491 example, this snippet would send the R block to all active engines in a
492 492 cluster::
493 493
494 494 %%px
495 495 %%R
496 496 ... R code goes here
497 497
498 498 * It is possible to embed not only an interactive shell with the
499 499 :func:`IPython.embed` call as always, but now you can also embed a *kernel*
500 500 with :func:`IPython.embed_kernel()`. Embedding an IPython kernel in an
501 501 application is useful when you want to use :func:`IPython.embed` but don't
502 502 have a terminal attached on stdin and stdout.
503 503
504 504 * The new :func:`IPython.parallel.bind_kernel` allows you to promote Engines to
505 505 listening Kernels, and connect QtConsoles to an Engine and debug it
506 506 directly.
507 507
508 508 In addition, having a single core object through our entire architecture also
509 509 makes the project conceptually cleaner, easier to maintain and more robust.
510 510 This took a lot of work to get in place, but we are thrilled to have this major
511 511 piece of architecture finally where we'd always wanted it to be.
512 512
513 513
514 514 Official Public API
515 515 -------------------
516 516
517 517 We have begun organizing our API for easier public use, with an eye towards an
518 518 official IPython 1.0 release which will firmly maintain this API compatible for
519 519 its entire lifecycle. There is now an :mod:`IPython.display` module that
520 520 aggregates all display routines, and the :mod:`traitlets.config` namespace has
521 521 all public configuration tools. We will continue improving our public API
522 522 layout so that users only need to import names one level deeper than the main
523 523 ``IPython`` package to access all public namespaces.
524 524
525 525
526 526 IPython notebook file icons
527 527 ---------------------------
528 528
529 529 The directory ``docs/resources`` in the source distribution contains SVG and
530 530 PNG versions of our file icons, as well as an ``Info.plist.example`` file with
531 531 instructions to install them on Mac OSX. This is a first draft of our icons,
532 532 and we encourage contributions from users with graphic talent to improve them
533 533 in the future.
534 534
535 535
536 536 New top-level `locate` command
537 537 ------------------------------
538 538
539 539 Add `locate` entry points; these would be useful for quickly locating IPython
540 540 directories and profiles from other (non-Python) applications. :ghpull:`1762`.
541 541
542 542 Examples::
543 543
544 544 $> ipython locate
545 545 /Users/me/.ipython
546 546
547 547 $> ipython locate profile foo
548 548 /Users/me/.ipython/profile_foo
549 549
550 550 $> ipython locate profile
551 551 /Users/me/.ipython/profile_default
552 552
553 553 $> ipython locate profile dne
554 554 [ProfileLocate] Profile u'dne' not found.
555 555
556 556
557 557 Other new features and improvements
558 558 -----------------------------------
559 559
560 560 * **%install_ext**: A new magic function to install an IPython extension from
561 561 a URL. E.g. ``%install_ext
562 562 https://bitbucket.org/birkenfeld/ipython-physics/raw/default/physics.py``.
563 563
564 564 * The ``%loadpy`` magic is no longer restricted to Python files, and has been
565 565 renamed ``%load``. The old name remains as an alias.
566 566
567 567 * New command line arguments will help external programs find IPython folders:
568 568 ``ipython locate`` finds the user's IPython directory, and ``ipython locate
569 569 profile foo`` finds the folder for the 'foo' profile (if it exists).
570 570
571 571 * The :envvar:`IPYTHON_DIR` environment variable, introduced in the Great
572 572 Reorganization of 0.11 and existing only in versions 0.11-0.13, has been
573 573 deprecated. As described in :ghpull:`1167`, the complexity and confusion of
574 574 migrating to this variable is not worth the aesthetic improvement. Please use
575 575 the historical :envvar:`IPYTHONDIR` environment variable instead.
576 576
577 577 * The default value of *interactivity* passed from
578 578 :meth:`~IPython.core.interactiveshell.InteractiveShell.run_cell` to
579 579 :meth:`~IPython.core.interactiveshell.InteractiveShell.run_ast_nodes`
580 580 is now configurable.
581 581
582 582 * New ``%alias_magic`` function to conveniently create aliases of existing
583 583 magics, if you prefer to have shorter names for personal use.
584 584
585 585 * We ship unminified versions of the JavaScript libraries we use, to better
586 586 comply with Debian's packaging policies.
587 587
588 588 * Simplify the information presented by ``obj?/obj??`` to eliminate a few
589 589 redundant fields when possible. :ghpull:`2038`.
590 590
591 591 * Improved continuous integration for IPython. We now have automated test runs
592 592 on `Shining Panda <https://jenkins.shiningpanda.com/ipython>`_ and `Travis-CI
593 593 <http://travis-ci.org/#!/ipython/ipython>`_, as well as `Tox support
594 594 <http://tox.testrun.org>`_.
595 595
596 596 * The `vim-ipython`_ functionality (externally developed) has been updated to
597 597 the latest version.
598 598
599 599 .. _vim-ipython: https://github.com/ivanov/vim-ipython
600 600
601 601 * The ``%save`` magic now has a ``-f`` flag to force overwriting, which makes
602 602 it much more usable in the notebook where it is not possible to reply to
603 603 interactive questions from the kernel. :ghpull:`1937`.
604 604
605 605 * Use dvipng to format sympy.Matrix, enabling display of matrices in the Qt
606 606 console with the sympy printing extension. :ghpull:`1861`.
607 607
608 608 * Our messaging protocol now has a reasonable test suite, helping ensure that
609 609 we don't accidentally deviate from the spec and possibly break third-party
610 610 applications that may have been using it. We encourage users to contribute
611 611 more stringent tests to this part of the test suite. :ghpull:`1627`.
612 612
613 613 * Use LaTeX to display, on output, various built-in types with the SymPy
614 614 printing extension. :ghpull:`1399`.
615 615
616 616 * Add Gtk3 event loop integration and example. :ghpull:`1588`.
617 617
618 618 * ``clear_output`` improvements, which allow things like progress bars and other
619 619 simple animations to work well in the notebook (:ghpull:`1563`):
620 620
621 621 * `clear_output()` clears the line, even in terminal IPython, the QtConsole
622 622 and plain Python as well, by printing `\r` to streams.
623 623
624 624 * `clear_output()` avoids the flicker in the notebook by adding a delay,
625 625 and firing immediately upon the next actual display message.
626 626
627 627 * `display_javascript` hides its `output_area` element, so using display to
628 628 run a bunch of javascript doesn't result in ever-growing vertical space.
629 629
630 630 * Add simple support for running inside a virtualenv. While this doesn't
631 631 supplant proper installation (as users should do), it helps ad-hoc calling of
632 632 IPython from inside a virtualenv. :ghpull:`1388`.
633 633
634 634
635 635 Major Bugs fixed
636 636 ----------------
637 637
638 638 In this cycle, we have :ref:`closed over 740 issues <issues_list_013>`, but a
639 639 few major ones merit special mention:
640 640
641 641 * The ``%pastebin`` magic has been updated to point to gist.github.com, since
642 642 unfortunately http://paste.pocoo.org has closed down. We also added a -d flag
643 643 for the user to provide a gist description string. :ghpull:`1670`.
644 644
645 645 * Fix ``%paste`` that would reject certain valid inputs. :ghpull:`1258`.
646 646
647 647 * Fix sending and receiving of Numpy structured arrays (those with composite
648 648 dtypes, often used as recarrays). :ghpull:`2034`.
649 649
650 650 * Reconnect when the websocket connection closes unexpectedly. :ghpull:`1577`.
651 651
652 652 * Fix truncated representation of objects in the debugger by showing at least
653 653 80 characters' worth of information. :ghpull:`1793`.
654 654
655 655 * Fix logger to be Unicode-aware: logging could crash ipython if there was
656 656 unicode in the input. :ghpull:`1792`.
657 657
658 658 * Fix images missing from XML/SVG export in the Qt console. :ghpull:`1449`.
659 659
660 660 * Fix deepreload on Python 3. :ghpull:`1625`, as well as having a much cleaner
661 661 and more robust implementation of deepreload in general. :ghpull:`1457`.
662 662
663 663
664 664 Backwards incompatible changes
665 665 ------------------------------
666 666
667 667 * The exception :exc:`IPython.core.error.TryNext` previously accepted
668 668 arguments and keyword arguments to be passed to the next implementation
669 669 of the hook. This feature was removed as it made error message propagation
670 670 difficult and violated the principle of loose coupling.
@@ -1,391 +1,391 b''
1 1 ============
2 2 3.x Series
3 3 ============
4 4
5 5 IPython 3.2.3
6 6 =============
7 7
8 8 Fixes compatibility with Python 3.4.4.
9 9
10 10 IPython 3.2.2
11 11 =============
12 12
13 13 Address vulnerabilities when files have maliciously crafted filenames (CVE-2015-6938),
14 14 or vulnerability when opening text files with malicious binary content (CVE pending).
15 15
16 16 Users are **strongly** encouraged to upgrade immediately.
17 17 There are also a few small unicode and nbconvert-related fixes.
18 18
19 19
20 20 IPython 3.2.1
21 21 =============
22 22
23 23 IPython 3.2.1 is a small bugfix release, primarily for cross-site security fixes in the notebook.
24 24 Users are **strongly** encouraged to upgrade immediately.
25 25 There are also a few small unicode and nbconvert-related fixes.
26 26
27 27 See :ref:`issues_list_3` for details.
28 28
29 29
30 30 IPython 3.2
31 31 ===========
32 32
33 33 IPython 3.2 contains important security fixes. Users are **strongly** encouraged to upgrade immediately.
34 34
35 35 Highlights:
36 36
37 37 - Address cross-site scripting vulnerabilities CVE-2015-4706, CVE-2015-4707
38 38 - A security improvement that set the secure attribute to login cookie to prevent them to be sent over http
39 39 - Revert the face color of matplotlib axes in the inline backend to not be transparent.
40 40 - Enable mathjax safe mode by default
41 41 - Fix XSS vulnerability in JSON error messages
42 42 - Various widget-related fixes
43 43
44 44 See :ref:`issues_list_3` for details.
45 45
46 46
47 47 IPython 3.1
48 48 ===========
49 49
50 50 Released April 3, 2015
51 51
52 52 The first 3.x bugfix release, with 33 contributors and 344 commits.
53 53 This primarily includes bugfixes to notebook layout and focus problems.
54 54
55 55
56 56 Highlights:
57 57
58 58 - Various focus jumping and scrolling fixes in the notebook.
59 59 - Various message ordering and widget fixes in the notebook.
60 60 - Images in markdown and output are confined to the notebook width.
61 61 An `.unconfined` CSS class is added to disable this behavior per-image.
62 62 The resize handle on output images is removed.
63 63 - Improved ordering of tooltip content for Python functions, putting the signature at the top.
64 64 - Fix UnicodeErrors when displaying some objects with unicode reprs on Python 2.
65 65 - Set the kernel's working directory to the notebook directory when running ``nbconvert --execute``,
66 66 so that behavior matches the live notebook.
67 67 - Allow setting custom SSL options for the tornado server with ``NotebookApp.ssl_options``,
68 68 and protect against POODLE with default settings by disabling SSLv3.
69 69 - Fix memory leak in the IPython.parallel Controller on Python 3.
70 70
71 71
72 72 See :ref:`issues_list_3` for details.
73 73
74 74
75 75 Release 3.0
76 76 ===========
77 77
78 78 Released February 27, 2015
79 79
80 80 This is a really big release. Over 150 contributors, and almost 6000 commits in a bit under a year.
81 81 Support for languages other than Python is greatly improved,
82 82 notebook UI has been significantly redesigned,
83 83 and a lot of improvement has happened in the experimental interactive widgets.
84 84 The message protocol and document format have both been updated,
85 85 while maintaining better compatibility with previous versions than prior updates.
86 86 The notebook webapp now enables editing of any text file, and even
87 87 a web-based terminal (on Unix platforms).
88 88
89 89 3.x will be the last monolithic release of IPython,
90 90 as the next release cycle will see the growing project split into its Python-specific and language-agnostic components.
91 91 Language-agnostic projects (notebook, qtconsole, etc.) will move under the umbrella of the new Project Jupyter name,
92 92 while Python-specific projects (interactive Python shell, Python kernel, IPython.parallel)
93 93 will remain under IPython, and be split into a few smaller packages.
94 94 To reflect this, IPython is in a bit of a transition state.
95 95 The logo on the notebook is now the Jupyter logo.
96 96 When installing kernels system-wide, they go in a `jupyter` directory.
97 97 We are going to do our best to ease this transition for users and developers.
98 98
99 99 Big changes are ahead.
100 100
101 101
102 102 Using different kernels
103 103 -----------------------
104 104
105 105 .. image:: ../_images/kernel_selector_screenshot.png
106 106 :alt: Screenshot of 'new' dropdown showing different kernels
107 107 :align: center
108 108
109 109 You can now choose a kernel for a notebook within the user interface, rather
110 110 than starting up a separate notebook server for each kernel you want to use. The
111 111 syntax highlighting adapts to match the language you're working in.
112 112
113 113 Information about the kernel is stored in the notebook file, so when you open a
114 114 notebook, it will automatically start the correct kernel.
115 115
116 116 It is also easier to use the Qt console and the terminal console with other
117 117 kernels, using the --kernel flag::
118 118
119 119 ipython qtconsole --kernel bash
120 120 ipython console --kernel bash
121 121
122 122 # To list available kernels
123 123 ipython kernelspec list
124 124
125 125 Kernel authors should see :ref:`kernelspecs` for how to register their kernels
126 126 with IPython so that these mechanisms work.
127 127
128 128 Typing unicode identifiers
129 129 --------------------------
130 130
131 131 .. image:: /_images/unicode_completion.png
132 132
133 133 Complex expressions can be much cleaner when written with a wider choice of
134 134 characters. Python 3 allows unicode identifiers, and IPython 3 makes it easier
135 135 to type those, using a feature from Julia. Type a backslash followed by a LaTeX
136 136 style short name, such as ``\alpha``. Press tab, and it will turn into Ξ±.
137 137
138 138 Widget migration guide
139 139 ----------------------
140 140 The widget framework has a lot of backwards incompatible changes.
141 141 For information about migrating widget notebooks and custom widgets to 3.0 refer
142 142 to the :doc:`widget migration guide<version3_widget_migration>`.
143 143
144 144 Other new features
145 145 ------------------
146 146
147 147 * :class:`~.TextWidget` and :class:`~.TextareaWidget` objects now include a
148 148 ``placeholder`` attribute, for displaying placeholder text before the
149 149 user has typed anything.
150 150
151 151 * The :magic:`load` magic can now find the source for objects in the user namespace.
152 152 To enable searching the namespace, use the ``-n`` option.
153 153
154 154 .. sourcecode:: ipython
155 155
156 156 In [1]: %load -n my_module.some_function
157 157
158 158 * :class:`~.DirectView` objects have a new :meth:`~.DirectView.use_cloudpickle`
159 159 method, which works like ``view.use_dill()``, but causes the ``cloudpickle``
160 160 module from PiCloud's `cloud`__ library to be used rather than dill or the
161 161 builtin pickle module.
162 162
163 163 __ https://pypi.python.org/pypi/cloud
164 164
165 165 * Added a .ipynb exporter to nbconvert. It can be used by passing `--to notebook`
166 166 as a commandline argument to nbconvert.
167 167
168 168 * New nbconvert preprocessor called :class:`~.ClearOutputPreprocessor`. This
169 169 clears the output from IPython notebooks.
170 170
171 171 * New preprocessor for nbconvert that executes all the code cells in a notebook.
172 172 To run a notebook and save its output in a new notebook::
173 173
174 174 ipython nbconvert InputNotebook --ExecutePreprocessor.enabled=True --to notebook --output Executed
175 175
176 176 * Consecutive stream (stdout/stderr) output is merged into a single output
177 177 in the notebook document.
178 178 Previously, all output messages were preserved as separate output fields in the JSON.
179 179 Now, the same merge is applied to the stored output as the displayed output,
180 180 improving document load time for notebooks with many small outputs.
181 181
182 182 * ``NotebookApp.webapp_settings`` is deprecated and replaced with
183 183 the more informatively named ``NotebookApp.tornado_settings``.
184 184
185 * Using :magic:`timeit` prints warnings if there is atleast a 4x difference in timings
185 * Using :magic:`timeit` prints warnings if there is at least a 4x difference in timings
186 186 between the slowest and fastest runs, since this might meant that the multiple
187 187 runs are not independent of one another.
188 188
189 189 * It's now possible to provide mechanisms to integrate IPython with other event
190 190 loops, in addition to the ones we already support. This lets you run GUI code
191 191 in IPython with an interactive prompt, and to embed the IPython
192 192 kernel in GUI applications. See :doc:`/config/eventloops` for details. As part
193 193 of this, the direct ``enable_*`` and ``disable_*`` functions for various GUIs
194 194 in :mod:`IPython.lib.inputhook` have been deprecated in favour of
195 195 :meth:`~.InputHookManager.enable_gui` and :meth:`~.InputHookManager.disable_gui`.
196 196
197 197 * A ``ScrollManager`` was added to the notebook. The ``ScrollManager`` controls how the notebook document is scrolled using keyboard. Users can inherit from the ``ScrollManager`` or ``TargetScrollManager`` to customize how their notebook scrolls. The default ``ScrollManager`` is the ``SlideScrollManager``, which tries to scroll to the nearest slide or sub-slide cell.
198 198
199 199 * The function :func:`~IPython.html.widgets.interaction.interact_manual` has been
200 200 added which behaves similarly to :func:`~IPython.html.widgets.interaction.interact`,
201 201 but adds a button to explicitly run the interacted-with function, rather than
202 202 doing it automatically for every change of the parameter widgets. This should
203 203 be useful for long-running functions.
204 204
205 205 * The ``%cython`` magic is now part of the Cython module. Use `%load_ext Cython` with a version of Cython >= 0.21 to have access to the magic now.
206 206
207 207 * The Notebook application now offers integrated terminals on Unix platforms,
208 208 intended for when it is used on a remote server. To enable these, install
209 209 the ``terminado`` Python package.
210 210
211 211 * The Notebook application can now edit any plain text files, via a full-page CodeMirror instance.
212 212
213 213 * Setting the default highlighting language for nbconvert with the config option
214 214 ``NbConvertBase.default_language`` is deprecated. Nbconvert now respects
215 215 metadata stored in the :ref:`kernel spec <kernelspecs>`.
216 216
217 217 * IPython can now be configured systemwide, with files in :file:`/etc/ipython`
218 218 or :file:`/usr/local/etc/ipython` on Unix systems,
219 219 or :file:`{%PROGRAMDATA%}\\ipython` on Windows.
220 220
221 221 * Added support for configurable user-supplied `Jinja
222 222 <http://jinja.pocoo.org/>`_ HTML templates for the notebook. Paths to
223 223 directories containing template files can be specified via
224 224 ``NotebookApp.extra_template_paths``. User-supplied template directories
225 225 searched first by the notebook, making it possible to replace existing
226 226 templates with your own files.
227 227
228 228 For example, to replace the notebook's built-in ``error.html`` with your own,
229 229 create a directory like ``/home/my_templates`` and put your override template
230 230 at ``/home/my_templates/error.html``. To start the notebook with your custom
231 231 error page enabled, you would run::
232 232
233 233 ipython notebook '--extra_template_paths=["/home/my_templates/"]'
234 234
235 235 It's also possible to override a template while also `inheriting
236 236 <http://jinja.pocoo.org/docs/dev/templates/#template-inheritance>`_ from that
237 237 template, by prepending ``templates/`` to the ``{% extends %}`` target of
238 238 your child template. This is useful when you only want to override a
239 239 specific block of a template. For example, to add additional CSS to the
240 240 built-in ``error.html``, you might create an override that looks like::
241 241
242 242 {% extends "templates/error.html" %}
243 243
244 244 {% block stylesheet %}
245 245 {{super()}}
246 246 <style type="text/css">
247 247 /* My Awesome CSS */
248 248 </style>
249 249 {% endblock %}
250 250
251 251 * Added a widget persistence API. This allows you to persist your notebooks interactive widgets.
252 252 Two levels of control are provided:
253 253 1. Higher level- ``WidgetManager.set_state_callbacks`` allows you to register callbacks for loading and saving widget state. The callbacks you register are automatically called when necessary.
254 254 2. Lower level- the ``WidgetManager`` Javascript class now has ``get_state`` and ``set_state`` methods that allow you to get and set the state of the widget runtime.
255 255
256 256 Example code for persisting your widget state to session data::
257 257
258 258 %%javascript
259 259 require(['widgets/js/manager'], function(manager) {
260 260 manager.WidgetManager.set_state_callbacks(function() { // Load
261 261 return JSON.parse(sessionStorage.widgets_state || '{}');
262 262 }, function(state) { // Save
263 263 sessionStorage.widgets_state = JSON.stringify(state);
264 264 });
265 265 });
266 266
267 267 * Enhanced support for :magic:`env` magic. As before, :magic:`env` with no
268 268 arguments displays all environment variables and values. Additionally,
269 269 :magic:`env` can be used to get or set individual environment variables. To
270 270 display an individual value, use the `%env var` syntax. To set a value, use
271 271 `env var val` or `env var=val`. Python value expansion using `$` works as usual.
272 272
273 273
274 274 Backwards incompatible changes
275 275 ------------------------------
276 276
277 277 * The :ref:`message protocol <messaging>` has been updated from version 4 to version 5.
278 278 Adapters are included, so IPython frontends can still talk to kernels that
279 279 implement protocol version 4.
280 280
281 281 * The notebook format has been updated from version 3 to version 4.
282 282 Read-only support for v4 notebooks has been backported to IPython 2.4.
283 283 Notable changes:
284 284
285 285 * heading cells are removed in favor or markdown headings
286 286 * notebook outputs and output messages are more consistent with each other
287 287 * use :func:`IPython.nbformat.read` and :func:`~IPython.nbformat.write`
288 288 to read and write notebook files
289 289 instead of the deprecated :mod:`IPython.nbformat.current` APIs.
290 290
291 291 You can downgrade a notebook to v3 via ``nbconvert``::
292 292
293 293 ipython nbconvert --to notebook --nbformat 3 <notebook>
294 294
295 295 which will create :file:`notebook.v3.ipynb`, a copy of the notebook in v3 format.
296 296
297 297 * :func:`IPython.core.oinspect.getsource` call specification has changed:
298 298
299 299 * `oname` keyword argument has been added for property source formatting
300 300 * `is_binary` keyword argument has been dropped, passing ``True`` had
301 301 previously short-circuited the function to return ``None`` unconditionally
302 302
303 303 * Removed the octavemagic extension: it is now available as ``oct2py.ipython``.
304 304
305 305 * Creating PDFs with LaTeX no longer uses a post processor.
306 306 Use `nbconvert --to pdf` instead of `nbconvert --to latex --post pdf`.
307 307
308 308 * Used https://github.com/jdfreder/bootstrap2to3 to migrate the Notebook to Bootstrap 3.
309 309
310 310 Additional changes:
311 311
312 312 - Set `.tab-content .row` `0px;` left and right margin (bootstrap default is `-15px;`)
313 313 - Removed `height: @btn_mini_height;` from `.list_header>div, .list_item>div` in `tree.less`
314 314 - Set `#header` div `margin-bottom: 0px;`
315 315 - Set `#menus` to `float: left;`
316 316 - Set `#maintoolbar .navbar-text` to `float: none;`
317 317 - Added no-padding convenience class.
318 318 - Set border of #maintoolbar to 0px
319 319
320 320 * Accessing the `container` DOM object when displaying javascript has been
321 321 deprecated in IPython 2.0 in favor of accessing `element`. Starting with
322 322 IPython 3.0 trying to access `container` will raise an error in browser
323 323 javascript console.
324 324
325 325 * ``IPython.utils.py3compat.open`` was removed: :func:`io.open` provides all
326 326 the same functionality.
327 327
328 328 * The NotebookManager and ``/api/notebooks`` service has been replaced by
329 329 a more generic ContentsManager and ``/api/contents`` service,
330 330 which supports all kinds of files.
331 331 * The Dashboard now lists all files, not just notebooks and directories.
332 332 * The ``--script`` hook for saving notebooks to Python scripts is removed,
333 333 use :samp:`ipython nbconvert --to python {notebook}` instead.
334 334
335 335 * The ``rmagic`` extension is deprecated, as it is now part of rpy2. See
336 336 :mod:`rpy2.ipython.rmagic`.
337 337
338 338 * :meth:`~.KernelManager.start_kernel` and :meth:`~.KernelManager.format_kernel_cmd`
339 339 no longer accept a ``executable`` parameter. Use the kernelspec machinery instead.
340 340
341 341 * The widget classes have been renamed from `*Widget` to `*`. The old names are
342 342 still functional, but are deprecated. i.e. `IntSliderWidget` has been renamed
343 343 to `IntSlider`.
344 344 * The ContainerWidget was renamed to Box and no longer defaults as a flexible
345 345 box in the web browser. A new FlexBox widget was added, which allows you to
346 346 use the flexible box model.
347 347
348 348 * The notebook now uses a single websocket at `/kernels/<kernel-id>/channels` instead of separate
349 349 `/kernels/<kernel-id>/{shell|iopub|stdin}` channels. Messages on each channel are identified by a
350 350 `channel` key in the message dict, for both send and recv.
351 351
352 352
353 353 Content Security Policy
354 354 ```````````````````````
355 355
356 356 The Content Security Policy is a web standard for adding a layer of security to
357 357 detect and mitigate certain classes of attacks, including Cross Site Scripting
358 358 (XSS) and data injection attacks. This was introduced into the notebook to
359 359 ensure that the IPython Notebook and its APIs (by default) can only be embedded
360 360 in an iframe on the same origin.
361 361
362 362 Override ``headers['Content-Security-Policy']`` within your notebook
363 363 configuration to extend for alternate domains and security settings.::
364 364
365 365 c.NotebookApp.tornado_settings = {
366 366 'headers': {
367 367 'Content-Security-Policy': "frame-ancestors 'self'"
368 368 }
369 369 }
370 370
371 371 Example policies::
372 372
373 373 Content-Security-Policy: default-src 'self' https://*.jupyter.org
374 374
375 375 Matches embeddings on any subdomain of jupyter.org, so long as they are served
376 376 over SSL.
377 377
378 378 There is a `report-uri <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-uri>`_ endpoint available for logging CSP violations, located at
379 379 ``/api/security/csp-report``. To use it, set ``report-uri`` as part of the CSP::
380 380
381 381 c.NotebookApp.tornado_settings = {
382 382 'headers': {
383 383 'Content-Security-Policy': "frame-ancestors 'self'; report-uri /api/security/csp-report"
384 384 }
385 385 }
386 386
387 387 It simply provides the CSP report as a warning in IPython's logs. The default
388 388 CSP sets this report-uri relative to the ``base_url`` (not shown above).
389 389
390 390 For a more thorough and accurate guide on Content Security Policies, check out
391 391 `MDN's Using Content Security Policy <https://developer.mozilla.org/en-US/docs/Web/Security/CSP/Using_Content_Security_Policy>`_ for more examples.
@@ -1,215 +1,215 b''
1 1 ============
2 2 7.x Series
3 3 ============
4 4
5 5 .. _whatsnew700:
6 6
7 7 IPython 7.0.0
8 8 =============
9 9
10 10 .. warning::
11 11
12 12 IPython 7.0 is currently in Beta, Feedback on API/changes and
13 13 addition/updates to this cahngelog are welcomed.
14 14
15 15 Released .... ...., 2017
16 16
17 17 IPython 7 include major features improvement as you can read in the following
18 18 changelog. This is also the second major version of IPython to stop support only
19 19 Python 3 – starting at Python 3.4. Python 2 is still still community supported
20 20 on the bugfix only 5.x branch, but we remind you that Python 2 EOL is Jan 1st
21 21 2020.
22 22
23 23 We were able to backport bug fixes to the 5.x branch thanks to our backport bot which
24 24 backported more than `70 Pull-Requests
25 25 <https://github.com/ipython/ipython/pulls?page=3&q=is%3Apr+sort%3Aupdated-desc+author%3Aapp%2Fmeeseeksdev++5.x&utf8=%E2%9C%93>`_, but there are still many PRs that required manually work, and this is an area of the project were you can easily contribute by looking for `PRs still needed backport <https://github.com/ipython/ipython/issues?q=label%3A%22Still+Needs+Manual+Backport%22+is%3Aclosed+sort%3Aupdated-desc>`_
26 26
27 27 IPython 6.x branch will likely not see any further release unless we critical
28 28 bugs are found.
29 29
30 30 Make sure you have pip > 9.0 before upgrading. You should be able to update by simply runngin
31 31
32 32 .. code::
33 33
34 34 pip install ipython --upgrade
35 35
36 36 Or if you have conda installed:
37 37
38 38 .. code::
39 39
40 40 conda install ipython
41 41
42 42
43 43
44 44 Prompt Toolkit 2.0
45 45 ------------------
46 46
47 47 IPython 7.0+ now use ``prompt_toolkit 2.0``, if you still need to use earlier
48 48 ``prompt_toolkit`` version you may need to pin IPython to ``<7.0``.
49 49
50 50 Autowait: Asynchronous REPL
51 51 ---------------------------
52 52
53 53 Staring with IPython 7.0 and on Python 3.6+, IPython can automatically await
54 54 code at top level, you should not need to access an event loop or runner
55 55 yourself. To know more read the :ref:`autoawait` section of our docs, see
56 56 :ghpull:`11265` or try the following code::
57 57
58 58 Python 3.6.0
59 59 Type 'copyright', 'credits' or 'license' for more information
60 60 IPython 7.0.0 -- An enhanced Interactive Python. Type '?' for help.
61 61
62 62 In [1]: import aiohttp
63 63 ...: result = aiohttp.get('https://api.github.com')
64 64
65 65 In [2]: response = await result
66 66 <pause for a few 100s ms>
67 67
68 68 In [3]: await response.json()
69 69 Out[3]:
70 70 {'authorizations_url': 'https://api.github.com/authorizations',
71 71 'code_search_url': 'https://api.github.com/search/code?q={query}{&page,per_page,sort,order}',
72 72 ...
73 73 }
74 74
75 75 .. note::
76 76
77 77 Async integration is experimental code, behavior may change or be removed
78 78 between Python and IPython versions without warnings.
79 79
80 80 Integration is by default with `asyncio`, but other libraries can be configured,
81 81 like ``curio`` or ``trio``, to improve concurrency in the REPL::
82 82
83 83 In [1]: %autoawait trio
84 84
85 85 In [2]: import trio
86 86
87 87 In [3]: async def child(i):
88 88 ...: print(" child %s goes to sleep"%i)
89 89 ...: await trio.sleep(2)
90 90 ...: print(" child %s wakes up"%i)
91 91
92 92 In [4]: print('parent start')
93 93 ...: async with trio.open_nursery() as n:
94 94 ...: for i in range(3):
95 95 ...: n.spawn(child, i)
96 96 ...: print('parent end')
97 97 parent start
98 98 child 2 goes to sleep
99 99 child 0 goes to sleep
100 100 child 1 goes to sleep
101 101 <about 2 seconds pause>
102 102 child 2 wakes up
103 103 child 1 wakes up
104 104 child 0 wakes up
105 105 parent end
106 106
107 107 See :ref:`autoawait` for more information.
108 108
109 109
110 110 Asynchronous code in a Notebook interface or any other frontend using the
111 111 Jupyter Protocol will need further updates of the IPykernel package.
112 112
113 113 Non-Asynchronous code
114 114 ~~~~~~~~~~~~~~~~~~~~~
115 115
116 116 As the internal API of IPython are now asynchronous, IPython need to run under
117 117 an even loop. In order to allow many workflow, (like using the ``%run`` magic,
118 118 or copy_pasting code that explicitly starts/stop event loop), when top-level code
119 119 is detected as not being asynchronous, IPython code is advanced via a
120 120 pseudo-synchronous runner, and will not may not advance pending tasks.
121 121
122 122 Change to Nested Embed
123 123 ~~~~~~~~~~~~~~~~~~~~~~
124 124
125 125 The introduction of the ability to run async code had some effect on the
126 126 ``IPython.embed()`` API. By default embed will not allow you to run asynchronous
127 127 code unless a event loop is specified.
128 128
129 129 Effects on Magics
130 130 ~~~~~~~~~~~~~~~~~
131 131
132 132 Some magics will not work with Async, and will need updates. Contribution
133 133 welcome.
134 134
135 135 Expected Future changes
136 136 ~~~~~~~~~~~~~~~~~~~~~~~
137 137
138 138 We expect more internal but public IPython function to become ``async``, and
139 139 will likely end up having a persisting event loop while IPython is running.
140 140
141 141 Thanks
142 142 ~~~~~~
143 143
144 144 This took more than a year in the making, and the code was rebased a number of
145 145 time leading to commit authorship that may have been lost in the final
146 146 Pull-Request. Huge thanks to many people for contribution, discussion, code,
147 147 documentation, use-case: dalejung, danielballan, ellisonbg, fperez, gnestor,
148 148 minrk, njsmith, pganssle, tacaswell, takluyver , vidartf ... And many other.
149 149
150 150
151 Autoreload Improvment
152 ---------------------
151 Autoreload Improvement
152 ----------------------
153 153
154 154 The magic ``%autoreload 2`` now captures new methods added to classes. Earlier, only methods existing as of the initial import were being tracked and updated.
155 155
156 156 This new feature helps dual environment development - Jupyter+IDE - where the code gradually moves from notebook cells to package files, as it gets structured.
157 157
158 158 **Example**: An instance of the class `MyClass` will be able to access the method `cube()` after it is uncommented and the file `file1.py` saved on disk.
159 159
160 160
161 161 ..code::
162 162
163 163 # notebook
164 164
165 165 from mymodule import MyClass
166 166 first = MyClass(5)
167 167
168 168 .. code::
169 169
170 170 # mymodule/file1.py
171 171
172 172 class MyClass:
173 173
174 174 def __init__(self, a=10):
175 175 self.a = a
176 176
177 177 def square(self):
178 178 print('compute square')
179 179 return self.a*self.a
180 180
181 181 # def cube(self):
182 182 # print('compute cube')
183 183 # return self.a*self.a*self.a
184 184
185 185
186 186
187 187
188 188 Misc
189 189 ----
190 190
191 191 The autoindent feature that was deprecated in 5.x was re-enabled and
192 192 un-deprecated in :ghpull:`11257`
193 193
194 194 Make ``%run -n -i ...`` work correctly. Earlier, if ``%run`` was passed both arguments, ``-n`` would be silently ignored. See :ghpull:`10308`
195 195
196 196
197 197
198 198
199 199
200 200 Deprecations
201 201 ------------
202 202
203 203 A couple of unused function and methods have been deprecated and will be removed
204 204 in future versions:
205 205
206 206 - ``IPython.utils.io.raw_print_err``
207 207 - ``IPython.utils.io.raw_print``
208 208
209 209
210 210 Backwards incompatible changes
211 211 ------------------------------
212 212
213 213 * The API for transforming input before it is parsed as Python code has been
214 214 completely redesigned, and any custom input transformations will need to be
215 215 rewritten. See :doc:`/config/inputtransforms` for details of the new API.
General Comments 0
You need to be logged in to leave comments. Login now