Refactor of coloring and traceback mechanism.

This should not change behavior be should slightly clean the code to be
a bit more pythonic, and start to introduce methods that avoid side
effect (that is to say return values instead of directly writing to
stdout). I keep it relatively small and simple for now.

My goal is to slowly change the traceback mechanisme to separate buiding
the data structure from rendering it to potentially introduce richer
tracebacks.  Locally I've also started from the other side (re build a
traceback rendered from scratch) to see what's needed.

IPython/core/ultratb.py

@@ -374,16 +374,32 b' def _fixed_getinnerframes(etb, context=1, tb_offset=0):'
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 
-def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,  _line_format=(lambda x,_:x,None)):
+def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):
+    """
+    Format tracebacks lines with pointing arrow, leading numbers...
+
+    Parameters
+    ==========
+
+    lnum: int
+    index: int
+    lines: list[string]
+    Colors:
+        ColorScheme used.
+    lvals: bytes
+        Values of local variables, already colored, to inject just after the error line.
+    _line_format: f (str) -> (str, bool)
+        return (colorized version of str, failure to do so)
+    """
     numbers_width = INDENT_SIZE - 1
     res = []
-    i = lnum - index
 
-    for line in lines:
+    for i,line in enumerate(lines, lnum-index):
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str')
-        if not err: line = new_line
+        if not err:
+            line = new_line
 
         if i == lnum:
             # This is the line with the error
@@ -399,7 +415,6 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals=None, _line_forma'
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
-        i = i + 1
     return res
 
 def is_recursion_error(etype, value, records):
@@ -869,7 +884,7 b' class VerboseTB(TBTools):'
 
         file = py3compat.cast_unicode(file, util_path.fs_encoding)
         link = tpl_link % util_path.compress_user(file)
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
+        args, varargs, varkw, locals_ = inspect.getargvalues(frame)
 
         if func == '?':
             call = ''
@@ -879,7 +894,7 b' class VerboseTB(TBTools):'
             try:
                 call = tpl_call % (func, inspect.formatargvalues(args,
                                                                  varargs, varkw,
-                                                                 locals, formatvalue=var_repr))
+                                                                 locals_, formatvalue=var_repr))
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
@@ -968,14 +983,15 b' class VerboseTB(TBTools):'
         unique_names = uniq_stable(names)
 
         # Start loop over vars
-        lvals = []
+        lvals = ''
+        lvals_list = []
         if self.include_vars:
             for name_full in unique_names:
                 name_base = name_full.split('.', 1)[0]
                 if name_base in frame.f_code.co_varnames:
-                    if name_base in locals:
+                    if name_base in locals_:
                         try:
-                            value = repr(eval(name_full, locals))
+                            value = repr(eval(name_full, locals_))
                         except:
                             value = undefined
                     else:
@@ -990,11 +1006,9 b' class VerboseTB(TBTools):'
                     else:
                         value = undefined
                     name = tpl_global_var % name_full
-                lvals.append(tpl_name_val % (name, value))
-        if lvals:
-            lvals = '%s%s' % (indent, em_normal.join(lvals))
-        else:
-            lvals = ''
+                lvals_list.append(tpl_name_val % (name, value))
+        if lvals_list:
+            lvals = '%s%s' % (indent, em_normal.join(lvals_list))
 
         level = '%s %s\n' % (link, call)
 

IPython/utils/PyColorize.py

@@ -213,7 +213,7 b' class Parser(Colorable):'
 
         string_output = 0
         if out == 'str' or self.out == 'str' or \
-           isinstance(self.out,StringIO):
+           isinstance(self.out, StringIO):
             # XXX - I don't really like this state handling logic, but at this
             # point I don't want to make major changes, so adding the
             # isinstance() check is the simplest I can do to ensure correct
@@ -223,6 +223,8 b' class Parser(Colorable):'
             string_output = 1
         elif out is not None:
             self.out = out
+        else:
+            raise ValueError('`out` or `self.out` should be file-like or the value `"str"`')
 
         # Fast return of the unmodified input for NoColor scheme
         if self.style == 'NoColor':
@@ -275,12 +277,13 b' class Parser(Colorable):'
             return (output, error)
         return (None, error)
 
-    def __call__(self, toktype, toktext, start_pos, end_pos, line):
-        """ Token handler, with syntax highlighting."""
+    def _inner_call_(self, toktype, toktext, start_pos, end_pos, line):
+        """like call but write to a temporary buffer"""
+        buff = StringIO()
         (srow,scol) = start_pos
         (erow,ecol) = end_pos
         colors = self.colors
-        owrite = self.out.write
+        owrite = buff.write
 
         # line separator, so this works across platforms
         linesep = os.linesep
@@ -297,7 +300,8 b' class Parser(Colorable):'
         # skip indenting tokens
         if toktype in [token.INDENT, token.DEDENT]:
             self.pos = newpos
-            return
+            buff.seek(0)
+            return buff.read()
 
         # map token type to a color group
         if token.LPAR <= toktype <= token.OP:
@@ -316,3 +320,12 b' class Parser(Colorable):'
 
         # send text
         owrite('%s%s%s' % (color,toktext,colors.normal))
+        buff.seek(0)
+        return buff.read()
+
+
+    def __call__(self, toktype, toktext, start_pos, end_pos, line):
+        """ Token handler, with syntax highlighting."""
+        self.out.write(
+            self._inner_call_(toktype, toktext, start_pos, end_pos, line))
+