Revert "Cleanup Python 2 compact from Lexers"

IPython/lib/lexers.py

@@ -1,549 +1,540 b''
 # -*- coding: utf-8 -*-
 """
 Defines a variety of Pygments lexers for highlighting IPython code.
 
 This includes:
 
-    IPython3Lexer
-        Lexer for pure IPython (python + magic/shell commands)
+    IPythonLexer, IPython3Lexer
+        Lexers for pure IPython (python + magic/shell commands)
 
     IPythonPartialTracebackLexer, IPythonTracebackLexer
-        The partial traceback lexer reads everything but the Python code
-        appearing in a traceback.
-        The full lexer combines the partial lexer with the IPython3Lexer.
+        Supports 2.x and 3.x via keyword `python3`.  The partial traceback
+        lexer reads everything but the Python code appearing in a traceback.
+        The full lexer combines the partial lexer with an IPython lexer.
 
     IPythonConsoleLexer
         A lexer for IPython console sessions, with support for tracebacks.
 
     IPyLexer
         A friendly lexer which examines the first line of text and from it,
         decides whether to use an IPython lexer or an IPython console lexer.
         This is probably the only lexer that needs to be explicitly added
         to Pygments.
 
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 # Standard library
 import re
 
 # Third party
 from pygments.lexers import (
-    BashLexer,
-    HtmlLexer,
-    JavascriptLexer,
-    RubyLexer,
-    PerlLexer,
-    Python3Lexer,
-    TexLexer,
-)
+    BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
+    Python3Lexer, TexLexer)
 from pygments.lexer import (
-    Lexer,
-    DelegatingLexer,
-    RegexLexer,
-    do_insertions,
-    bygroups,
-    using,
-    inherit,
+    Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
 )
 from pygments.token import (
     Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
 )
 from pygments.util import get_bool_opt
 
 # Local
 
 line_re = re.compile('.*?\n')
 
-__all__ = [
-    "IPython3Lexer",
-    "IPythonPartialTracebackLexer",
-    "IPythonTracebackLexer",
-    "IPythonConsoleLexer",
-    "IPyLexer",
-]
+__all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
+           'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
+           'IPythonConsoleLexer', 'IPyLexer']
 
 
-class IPython3Lexer(Python3Lexer):
-    """IPython3 Lexer"""
+def build_ipy_lexer(python3):
+    """Builds IPython lexers depending on the value of `python3`.
 
-    name = "IPython3"
-    aliases = ["ipython3"]
+    The lexer inherits from an appropriate Python lexer and then adds
+    information about IPython specific keywords (i.e. magic commands,
+    shell commands, etc.)
 
-    tokens = {
-        "root": [
-            (
-                r"(?s)(\s*)(%%capture)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%debug)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?is)(\s*)(%%html)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(HtmlLexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%javascript)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(JavascriptLexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%js)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(JavascriptLexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%latex)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(TexLexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%perl)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(PerlLexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%prun)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%pypy)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%python)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%python3)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%ruby)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(RubyLexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%time)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%timeit)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%writefile)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (
-                r"(?s)(\s*)(%%file)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(Python3Lexer)),
-            ),
-            (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
-            (
-                r"(?s)(^\s*)(%%!)([^\n]*\n)(.*)",
-                bygroups(Text, Operator, Text, using(BashLexer)),
-            ),
-            (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)),
-            (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)),
-            (
-                r"(%)(sx|sc|system)(.*)(\n)",
-                bygroups(Operator, Keyword, using(BashLexer), Text),
-            ),
-            (r"(%)(\w+)(.*\n)", bygroups(Operator, Keyword, Text)),
-            (r"^(!!)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)),
-            (r"(!)(?!=)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)),
-            (r"^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)", bygroups(Text, Operator, Text)),
-            (r"(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$", bygroups(Text, Operator, Text)),
-            inherit,
-        ]
-    }
+    Parameters
+    ----------
+    python3 : bool
+        If `True`, then build an IPython lexer from a Python 3 lexer.
+
+    """
+    # It would be nice to have a single IPython lexer class which takes
+    # a boolean `python3`.  But since there are two Python lexer classes,
+    # we will also have two IPython lexer classes.
+    if python3:
+        PyLexer = Python3Lexer
+        name = 'IPython3'
+        aliases = ['ipython3']
+        doc = """IPython3 Lexer"""
+    else:
+        PyLexer = PythonLexer
+        name = 'IPython'
+        aliases = ['ipython2', 'ipython']
+        doc = """IPython Lexer"""
+
+    ipython_tokens = [
+       (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
+        (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
+        (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
+        (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
+        (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
+        (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
+        (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
+        (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
+        (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
+        (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
+        (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
+        (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
+        (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
+                                                using(BashLexer), Text)),
+        (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
+        (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+        (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+        (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
+        (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
+    ]
+
+    tokens = PyLexer.tokens.copy()
+    tokens['root'] = ipython_tokens + tokens['root']
+
+    attrs = {'name': name, 'aliases': aliases, 'filenames': [],
+             '__doc__': doc, 'tokens': tokens}
+
+    return type(name, (PyLexer,), attrs)
+
+
+IPython3Lexer = build_ipy_lexer(python3=True)
+IPythonLexer = build_ipy_lexer(python3=False)
 
 
 class IPythonPartialTracebackLexer(RegexLexer):
     """
     Partial lexer for IPython tracebacks.
 
     Handles all the non-python output.
 
     """
     name = 'IPython Partial Traceback'
 
     tokens = {
         'root': [
             # Tracebacks for syntax errors have a different style.
             # For both types of tracebacks, we mark the first line with
             # Generic.Traceback.  For syntax errors, we mark the filename
             # as we mark the filenames for non-syntax tracebacks.
             #
             # These two regexps define how IPythonConsoleLexer finds a
             # traceback.
             #
             ## Non-syntax traceback
             (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
             ## Syntax traceback
             (r'^(  File)(.*)(, line )(\d+\n)',
              bygroups(Generic.Traceback, Name.Namespace,
                       Generic.Traceback, Literal.Number.Integer)),
 
             # (Exception Identifier)(Whitespace)(Traceback Message)
             (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
              bygroups(Name.Exception, Generic.Whitespace, Text)),
             # (Module/Filename)(Text)(Callee)(Function Signature)
             # Better options for callee and function signature?
             (r'(.*)( in )(.*)(\(.*\)\n)',
              bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
             # Regular line: (Whitespace)(Line Number)(Python Code)
             (r'(\s*?)(\d+)(.*?\n)',
              bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
             # Emphasized line: (Arrow)(Line Number)(Python Code)
             # Using Exception token so arrow color matches the Exception.
             (r'(-*>?\s?)(\d+)(.*?\n)',
              bygroups(Name.Exception, Literal.Number.Integer, Other)),
             # (Exception Identifier)(Message)
             (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
              bygroups(Name.Exception, Text)),
             # Tag everything else as Other, will be handled later.
             (r'.*\n', Other),
         ],
     }
 
 
 class IPythonTracebackLexer(DelegatingLexer):
     """
     IPython traceback lexer.
 
     For doctests, the tracebacks can be snipped as much as desired with the
     exception to the lines that designate a traceback. For non-syntax error
     tracebacks, this is the line of hyphens. For syntax error tracebacks,
     this is the line which lists the File and line number.
 
     """
-
-    # The lexer inherits from DelegatingLexer.  The "root" lexer is the
-    # IPython3 lexer. First, we parse with the partial IPython traceback lexer.
+    # The lexer inherits from DelegatingLexer.  The "root" lexer is an
+    # appropriate IPython lexer, which depends on the value of the boolean
+    # `python3`.  First, we parse with the partial IPython traceback lexer.
     # Then, any code marked with the "Other" token is delegated to the root
     # lexer.
     #
     name = 'IPython Traceback'
     aliases = ['ipythontb']
 
     def __init__(self, **options):
         """
         A subclass of `DelegatingLexer` which delegates to the appropriate to either IPyLexer,
         IPythonPartialTracebackLexer.
         """
         # note we need a __init__ doc, as otherwise it inherits the doc from the super class
         # which will fail the documentation build as it references section of the pygments docs that
         # do not exists when building IPython's docs.
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipython3tb']
+        else:
+            self.aliases = ['ipython2tb', 'ipythontb']
 
-        super().__init__(IPython3Lexer, IPythonPartialTracebackLexer, **options)
+        if self.python3:
+            IPyLexer = IPython3Lexer
+        else:
+            IPyLexer = IPythonLexer
 
+        DelegatingLexer.__init__(self, IPyLexer,
+                                 IPythonPartialTracebackLexer, **options)
 
 class IPythonConsoleLexer(Lexer):
     """
     An IPython console lexer for IPython code-blocks and doctests, such as:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: a = 'foo'
 
             In [2]: a
             Out[2]: 'foo'
 
             In [3]: print(a)
             foo
 
 
     Support is also provided for IPython exceptions:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: raise Exception
             Traceback (most recent call last):
             ...
             Exception
 
     """
     name = 'IPython console session'
     aliases = ['ipythonconsole']
     mimetypes = ['text/x-ipython-console']
 
     # The regexps used to determine what is input and what is output.
     # The default prompts for IPython are:
     #
     #    in           = 'In [#]: '
     #    continuation = '   .D.: '
     #    template     = 'Out[#]: '
     #
-    # Where '#' is the 'prompt number' or 'execution count' and 'D'
-    # D is a number of dots  matching the width of the execution count
+    # Where '#' is the 'prompt number' or 'execution count' and 'D' 
+    # D is a number of dots  matching the width of the execution count 
     #
     in1_regex = r'In \[[0-9]+\]: '
     in2_regex = r'   \.\.+\.: '
     out_regex = r'Out\[[0-9]+\]: '
 
     #: The regex to determine when a traceback starts.
     ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
 
     def __init__(self, **options):
         """Initialize the IPython console lexer.
 
         Parameters
         ----------
+        python3 : bool
+            If `True`, then the console inputs are parsed using a Python 3
+            lexer. Otherwise, they are parsed using a Python 2 lexer.
         in1_regex : RegexObject
             The compiled regular expression used to detect the start
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         in2_regex : RegexObject
             The compiled regular expression used to detect the continuation
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         out_regex : RegexObject
             The compiled regular expression used to detect outputs. If `None`,
             then the default output prompt is assumed.
 
         """
-        self.aliases = ["ipython3console"]
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipython3console']
+        else:
+            self.aliases = ['ipython2console', 'ipythonconsole']
 
         in1_regex = options.get('in1_regex', self.in1_regex)
         in2_regex = options.get('in2_regex', self.in2_regex)
         out_regex = options.get('out_regex', self.out_regex)
 
         # So that we can work with input and output prompts which have been
         # rstrip'd (possibly by editors) we also need rstrip'd variants. If
         # we do not do this, then such prompts will be tagged as 'output'.
         # The reason can't just use the rstrip'd variants instead is because
         # we want any whitespace associated with the prompt to be inserted
         # with the token. This allows formatted code to be modified so as hide
         # the appearance of prompts, with the whitespace included. One example
         # use of this is in copybutton.js from the standard lib Python docs.
         in1_regex_rstrip = in1_regex.rstrip() + '\n'
         in2_regex_rstrip = in2_regex.rstrip() + '\n'
         out_regex_rstrip = out_regex.rstrip() + '\n'
 
         # Compile and save them all.
         attrs = ['in1_regex', 'in2_regex', 'out_regex',
                  'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
         for attr in attrs:
             self.__setattr__(attr, re.compile(locals()[attr]))
 
         Lexer.__init__(self, **options)
 
-        self.pylexer = IPython3Lexer(**options)
-        self.tblexer = IPythonTracebackLexer(**options)
+        if self.python3:
+            pylexer = IPython3Lexer
+            tblexer = IPythonTracebackLexer
+        else:
+            pylexer = IPythonLexer
+            tblexer = IPythonTracebackLexer
+
+        self.pylexer = pylexer(**options)
+        self.tblexer = tblexer(**options)
 
         self.reset()
 
     def reset(self):
         self.mode = 'output'
         self.index = 0
         self.buffer = u''
         self.insertions = []
 
     def buffered_tokens(self):
         """
         Generator of unprocessed tokens after doing insertions and before
         changing to a new state.
 
         """
         if self.mode == 'output':
             tokens = [(0, Generic.Output, self.buffer)]
         elif self.mode == 'input':
             tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
         else: # traceback
             tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
 
         for i, t, v in do_insertions(self.insertions, tokens):
             # All token indexes are relative to the buffer.
             yield self.index + i, t, v
 
         # Clear it all
         self.index += len(self.buffer)
         self.buffer = u''
         self.insertions = []
 
     def get_mci(self, line):
         """
         Parses the line and returns a 3-tuple: (mode, code, insertion).
 
         `mode` is the next mode (or state) of the lexer, and is always equal
         to 'input', 'output', or 'tb'.
 
         `code` is a portion of the line that should be added to the buffer
         corresponding to the next mode and eventually lexed by another lexer.
         For example, `code` could be Python code if `mode` were 'input'.
 
         `insertion` is a 3-tuple (index, token, text) representing an
         unprocessed "token" that will be inserted into the stream of tokens
         that are created from the buffer once we change modes. This is usually
         the input or output prompt.
 
         In general, the next mode depends on current mode and on the contents
         of `line`.
 
         """
         # To reduce the number of regex match checks, we have multiple
         # 'if' blocks instead of 'if-elif' blocks.
 
         # Check for possible end of input
         in2_match = self.in2_regex.match(line)
         in2_match_rstrip = self.in2_regex_rstrip.match(line)
         if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
            in2_match_rstrip:
             end_input = True
         else:
             end_input = False
         if end_input and self.mode != 'tb':
             # Only look for an end of input when not in tb mode.
             # An ellipsis could appear within the traceback.
             mode = 'output'
             code = u''
             insertion = (0, Generic.Prompt, line)
             return mode, code, insertion
 
         # Check for output prompt
         out_match = self.out_regex.match(line)
         out_match_rstrip = self.out_regex_rstrip.match(line)
         if out_match or out_match_rstrip:
             mode = 'output'
             if out_match:
                 idx = out_match.end()
             else:
                 idx = out_match_rstrip.end()
             code = line[idx:]
             # Use the 'heading' token for output.  We cannot use Generic.Error
             # since it would conflict with exceptions.
             insertion = (0, Generic.Heading, line[:idx])
             return mode, code, insertion
 
 
         # Check for input or continuation prompt (non stripped version)
         in1_match = self.in1_regex.match(line)
         if in1_match or (in2_match and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match:
                 idx = in1_match.end()
             else: # in2_match
                 idx = in2_match.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for input or continuation prompt (stripped version)
         in1_match_rstrip = self.in1_regex_rstrip.match(line)
         if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match_rstrip:
                 idx = in1_match_rstrip.end()
             else: # in2_match
                 idx = in2_match_rstrip.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for traceback
         if self.ipytb_start.match(line):
             mode = 'tb'
             code = line
             insertion = None
             return mode, code, insertion
 
         # All other stuff...
         if self.mode in ('input', 'output'):
             # We assume all other text is output. Multiline input that
             # does not use the continuation marker cannot be detected.
             # For example, the 3 in the following is clearly output:
             #
             #    In [1]: print 3
             #    3
             #
             # But the following second line is part of the input:
             #
             #    In [2]: while True:
             #        print True
             #
             # In both cases, the 2nd line will be 'output'.
             #
             mode = 'output'
         else:
             mode = 'tb'
 
         code = line
         insertion = None
 
         return mode, code, insertion
 
     def get_tokens_unprocessed(self, text):
         self.reset()
         for match in line_re.finditer(text):
             line = match.group()
             mode, code, insertion = self.get_mci(line)
 
             if mode != self.mode:
                 # Yield buffered tokens before transitioning to new mode.
                 for token in self.buffered_tokens():
                     yield token
                 self.mode = mode
 
             if insertion:
                 self.insertions.append((len(self.buffer), [insertion]))
             self.buffer += code
 
         for token in self.buffered_tokens():
             yield token
 
 class IPyLexer(Lexer):
     r"""
     Primary lexer for all IPython-like code.
 
     This is a simple helper lexer.  If the first line of the text begins with
     "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
     lexer. If not, then the entire text is parsed with an IPython lexer.
 
     The goal is to reduce the number of lexers that are registered
     with Pygments.
 
     """
     name = 'IPy session'
     aliases = ['ipy']
 
     def __init__(self, **options):
         """
         Create a new IPyLexer instance which dispatch to either an
-        IPythonConsoleLexer (if In prompts are present) or and IPython3Lexer (if
+        IPythonCOnsoleLexer (if In prompts are present) or and IPythonLexer (if
         In prompts are not present).
         """
         # init docstring is necessary for docs not to fail to build do to parent
         # docs referenceing a section in pygments docs.
-        self.aliases = ["ipy3"]
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipy3']
+        else:
+            self.aliases = ['ipy2', 'ipy']
 
         Lexer.__init__(self, **options)
 
-        self.IPythonLexer = IPython3Lexer(**options)
+        self.IPythonLexer = IPythonLexer(**options)
         self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
 
     def get_tokens_unprocessed(self, text):
         # Search for the input prompt anywhere...this allows code blocks to
         # begin with comments as well.
         if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
             lex = self.IPythonConsoleLexer
         else:
             lex = self.IPythonLexer
         for token in lex.get_tokens_unprocessed(text):
             yield token
+

IPython/lib/tests/test_lexers.py

@@ -1,184 +1,184 b''
 """Test lexers module"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from unittest import TestCase
 from pygments import __version__ as pygments_version
 from pygments.token import Token
 from pygments.lexers import BashLexer
 
 from .. import lexers
 
 pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14)
 
 
 class TestLexers(TestCase):
     """Collection of lexers tests"""
     def setUp(self):
-        self.lexer = lexers.IPython3Lexer()
+        self.lexer = lexers.IPythonLexer()
         self.bash_lexer = BashLexer()
 
-    def testIPython3Lexer(self):
+    def testIPythonLexer(self):
         fragment = '!echo $HOME\n'
         bash_tokens = [
             (Token.Operator, '!'),
         ]
         bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:]))
         ipylex_token = list(self.lexer.get_tokens(fragment))
         assert bash_tokens[:-1] == ipylex_token[:-1]
 
         fragment_2 = "!" + fragment
         tokens_2 = [
             (Token.Operator, '!!'),
         ] + bash_tokens[1:]
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = '\t %%!\n' + fragment[1:]
         tokens_2 = [
             (Token.Text, '\t '),
             (Token.Operator, '%%!'),
             (Token.Text, '\n'),
         ] + bash_tokens[1:]
         assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = 'x = ' + fragment
         tokens_2 = [
             (Token.Name, 'x'),
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
         ] + bash_tokens
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x, = ' + fragment
         tokens_2 = [
             (Token.Name, 'x'),
             (Token.Punctuation, ','),
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
         ] + bash_tokens
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x, = %sx ' + fragment[1:]
         tokens_2 = [
             (Token.Name, 'x'),
             (Token.Punctuation, ','),
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
             (Token.Operator, '%'),
             (Token.Keyword, 'sx'),
             (Token.Text, ' '),
         ] + bash_tokens[1:]
         if tokens_2[7] == (Token.Text, " ") and pyg214:  # pygments 2.14+
             tokens_2[7] = (Token.Text.Whitespace, " ")
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'f = %R function () {}\n'
         tokens_2 = [
             (Token.Name, 'f'),
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
             (Token.Operator, '%'),
             (Token.Keyword, 'R'),
             (Token.Text, ' function () {}\n'),
         ]
         assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = '\t%%xyz\n$foo\n'
         tokens_2 = [
             (Token.Text, '\t'),
             (Token.Operator, '%%'),
             (Token.Keyword, 'xyz'),
             (Token.Text, '\n$foo\n'),
         ]
         assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = '%system?\n'
         tokens_2 = [
             (Token.Operator, '%'),
             (Token.Keyword, 'system'),
             (Token.Operator, '?'),
             (Token.Text, '\n'),
         ]
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x != y\n'
         tokens_2 = [
             (Token.Name, 'x'),
             (Token.Text, ' '),
             (Token.Operator, '!='),
             (Token.Text, ' '),
             (Token.Name, 'y'),
             (Token.Text, '\n'),
         ]
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = ' ?math.sin\n'
         tokens_2 = [
             (Token.Text, ' '),
             (Token.Operator, '?'),
             (Token.Text, 'math.sin'),
             (Token.Text, '\n'),
         ]
         assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment = ' *int*?\n'
         tokens = [
             (Token.Text, ' *int*'),
             (Token.Operator, '?'),
             (Token.Text, '\n'),
         ]
         assert tokens == list(self.lexer.get_tokens(fragment))
 
         fragment = '%%writefile -a foo.py\nif a == b:\n    pass'
         tokens = [
             (Token.Operator, '%%writefile'),
             (Token.Text, ' -a foo.py\n'),
             (Token.Keyword, 'if'),
             (Token.Text, ' '),
             (Token.Name, 'a'),
             (Token.Text, ' '),
             (Token.Operator, '=='),
             (Token.Text, ' '),
             (Token.Name, 'b'),
             (Token.Punctuation, ':'),
             (Token.Text, '\n'),
             (Token.Text, '    '),
             (Token.Keyword, 'pass'),
             (Token.Text, '\n'),
         ]
         if tokens[10] == (Token.Text, "\n") and pyg214:  # pygments 2.14+
             tokens[10] = (Token.Text.Whitespace, "\n")
         assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1]
 
         fragment = '%%timeit\nmath.sin(0)'
         tokens = [
             (Token.Operator, '%%timeit\n'),
             (Token.Name, 'math'),
             (Token.Operator, '.'),
             (Token.Name, 'sin'),
             (Token.Punctuation, '('),
             (Token.Literal.Number.Integer, '0'),
             (Token.Punctuation, ')'),
             (Token.Text, '\n'),
         ]
 
         fragment = '%%HTML\n<div>foo</div>'
         tokens = [
             (Token.Operator, '%%HTML'),
             (Token.Text, '\n'),
             (Token.Punctuation, '<'),
             (Token.Name.Tag, 'div'),
             (Token.Punctuation, '>'),
             (Token.Text, 'foo'),
             (Token.Punctuation, '<'),
             (Token.Punctuation, '/'),
             (Token.Name.Tag, 'div'),
             (Token.Punctuation, '>'),
             (Token.Text, '\n'),
         ]
         assert tokens == list(self.lexer.get_tokens(fragment))

IPython/lib/tests/test_pygments.py

@@ -1,24 +1,26 b''
 from typing import List
 
 import pytest
 import pygments.lexers
 import pygments.lexer
 
-from IPython.lib.lexers import IPythonConsoleLexer, IPython3Lexer
+from IPython.lib.lexers import IPythonConsoleLexer, IPythonLexer, IPython3Lexer
 
 #: the human-readable names of the IPython lexers with ``entry_points``
-EXPECTED_LEXER_NAMES = [cls.name for cls in [IPythonConsoleLexer, IPython3Lexer]]
+EXPECTED_LEXER_NAMES = [
+    cls.name for cls in [IPythonConsoleLexer, IPythonLexer, IPython3Lexer]
+]
 
 
 @pytest.fixture
 def all_pygments_lexer_names() -> List[str]:
     """Get all lexer names registered in pygments."""
     return {l[0] for l in pygments.lexers.get_all_lexers()}
 
 
 @pytest.mark.parametrize("expected_lexer", EXPECTED_LEXER_NAMES)
 def test_pygments_entry_points(
     expected_lexer: str, all_pygments_lexer_names: List[str]
 ) -> None:
     """Check whether the ``entry_points`` for ``pygments.lexers`` are correct."""
     assert expected_lexer in all_pygments_lexer_names

IPython/sphinxext/ipython_console_highlighting.py

@@ -1,24 +1,28 b''
 """
 reST directive for syntax-highlighting ipython interactive sessions.
 
 """
 
 from sphinx import highlighting
 from IPython.lib.lexers import IPyLexer
 
 def setup(app):
     """Setup as a sphinx extension."""
 
     # This is only a lexer, so adding it below to pygments appears sufficient.
     # But if somebody knows what the right API usage should be to do that via
     # sphinx, by all means fix it here.  At least having this setup.py
     # suppresses the sphinx warning we'd get without it.
     metadata = {'parallel_read_safe': True, 'parallel_write_safe': True}
     return metadata
 
 # Register the extension as a valid pygments lexer.
 # Alternatively, we could register the lexer with pygments instead. This would
 # require using setuptools entrypoints: http://pygments.org/docs/plugins
 
-highlighting.lexers["ipython"] = IPyLexer()
-highlighting.lexers["ipython3"] = IPyLexer()
+ipy2 = IPyLexer(python3=False)
+ipy3 = IPyLexer(python3=True)
+
+highlighting.lexers['ipython'] = ipy2
+highlighting.lexers['ipython2'] = ipy2
+highlighting.lexers['ipython3'] = ipy3

docs/source/development/lexer.rst

@@ -1,62 +1,64 b''
 .. _console_lexer:
 
 New IPython Console Lexer
 -------------------------
 
 .. versionadded:: 2.0.0
 
 The IPython console lexer has been rewritten and now supports tracebacks
 and customized input/output prompts. An entire suite of lexers is now
 available at :mod:`IPython.lib.lexers`. These include:
 
-IPython3Lexer
-  Lexer for pure IPython  (python 3 + magic/shell commands)
+IPythonLexer & IPython3Lexer
+  Lexers for pure IPython (python + magic/shell commands)
 
 IPythonPartialTracebackLexer & IPythonTracebackLexer
-  The partial traceback lexer reads everything but the Python code
-  appearing in a traceback. The full lexer combines the partial lexer
-  with the IPython3Lexer.
+  Supports 2.x and 3.x via the keyword `python3`. The partial traceback
+  lexer reads everything but the Python code appearing in a traceback.
+  The full lexer combines the partial lexer with an IPython lexer.
 
 IPythonConsoleLexer
-  A lexer for python 3 IPython console sessions, with support for tracebacks.
+  A lexer for IPython console sessions, with support for tracebacks.
+  Supports 2.x and 3.x via the keyword `python3`.
 
 IPyLexer
   A friendly lexer which examines the first line of text and from it,
   decides whether to use an IPython lexer or an IPython console lexer.
+  Supports 2.x and 3.x via the keyword `python3`.
 
 Previously, the :class:`IPythonConsoleLexer` class was available at
 :mod:`IPython.sphinxext.ipython_console_hightlight`.  It was inserted
 into Pygments' list of available lexers under the name `ipython`.  It should
 be mentioned that this name is inaccurate, since an IPython console session
 is not the same as IPython code (which itself is a superset of the Python
 language).
 
 Now, the Sphinx extension inserts two console lexers into Pygments' list of
 available lexers. Both are IPyLexer instances under the names: `ipython` and
 `ipython3`. Although the names can be confusing (as mentioned above), their
 continued use is, in part, to maintain backwards compatibility and to
 aid typical usage. If a project needs to make Pygments aware of more than just
 the IPyLexer class, then one should not make the IPyLexer class available under
 the name `ipython` and use `ipy` or some other non-conflicting value.
 
 Code blocks such as:
 
 .. code-block:: rst
 
     .. code-block:: ipython
 
         In [1]: 2**2
     Out[1]: 4
 
     will continue to work as before, but now, they will also properly highlight
 tracebacks.  For pure IPython code, the same lexer will also work:
 
 .. code-block:: rst
 
     .. code-block:: ipython
 
         x = ''.join(map(str, range(10)))
     !echo $x
 
     Since the first line of the block did not begin with a standard IPython console
 prompt, the entire block is assumed to consist of IPython code instead.

setup.py

@@ -1,157 +1,158 b''
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
 Under Posix environments it works like a typical setup.py script.
 Under Windows, the command sdist is not supported, since IPython
 requires utilities which are not available under Windows."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008-2011, IPython Development Team.
 #  Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.rst, distributed with this software.
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 # **Python version check**
 #
 # This check is also made in IPython/__init__, don't forget to update both when
 # changing Python version requirements.
 if sys.version_info < (3, 9):
     pip_message = 'This may be due to an out of date pip. Make sure you have pip >= 9.0.1.'
     try:
         import pip
         pip_version = tuple([int(x) for x in pip.__version__.split('.')[:3]])
         if pip_version < (9, 0, 1) :
             pip_message = 'Your pip version is out of date, please install pip >= 9.0.1. '\
             'pip {} detected.'.format(pip.__version__)
         else:
             # pip is new enough - it must be something else
             pip_message = ''
     except Exception:
         pass
 
 
     error = """
 IPython 8.13+ supports Python 3.9 and above, following NEP 29.
 IPython 8.0-8.12 supports Python 3.8 and above, following NEP 29.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
 Python 3.3 and 3.4 were supported up to IPython 6.x.
 Python 3.5 was supported with IPython 7.0 to 7.9.
 Python 3.6 was supported with IPython up to 7.16.
 Python 3.7 was still supported with the 7.x branch.
 
 See IPython `README.rst` file for more information:
 
     https://github.com/ipython/ipython/blob/main/README.rst
 
 Python {py} detected.
 {pip}
 """.format(
         py=sys.version_info, pip=pip_message
     )
 
     print(error, file=sys.stderr)
     sys.exit(1)
 
 # At least we're on the python version we need, move on.
 
 from setuptools import setup
 
 # Our own imports
 sys.path.insert(0, ".")
 
 from setupbase import target_update, find_entry_points
 
 from setupbase import (
     setup_args,
     check_package_data_first,
     find_data_files,
     git_prebuild,
     install_symlinked,
     install_lib_symlink,
     install_scripts_for_symlink,
     unsymlink,
 )
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
 
 if os.name in ('nt','dos'):
     os_name = 'windows'
 else:
     os_name = os.name
 
 # Under Windows, 'sdist' has not been supported.  Now that the docs build with
 # Sphinx it might work, but let's not turn it on until someone confirms that it
 # actually works.
 if os_name == 'windows' and 'sdist' in sys.argv:
     print('The sdist command is not available under Windows.  Exiting.')
     sys.exit(1)
 
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
 
 # update the manuals when building a source dist
 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
 
     # List of things to be updated. Each entry is a triplet of args for
     # target_update()
     to_update = [
         (
             "docs/man/ipython.1.gz",
             ["docs/man/ipython.1"],
             "cd docs/man && python -m gzip --best ipython.1",
         ),
     ]
 
 
     [ target_update(*t) for t in to_update ]
 
 #---------------------------------------------------------------------------
 # Find all the packages, package data, and data_files
 #---------------------------------------------------------------------------
 
 data_files = find_data_files()
 
 setup_args['data_files'] = data_files
 
 #---------------------------------------------------------------------------
 # custom distutils commands
 #---------------------------------------------------------------------------
 # imports here, so they are after setuptools import if there was one
 from setuptools.command.sdist import sdist
 
 setup_args['cmdclass'] = {
     'build_py': \
             check_package_data_first(git_prebuild('IPython')),
     'sdist' : git_prebuild('IPython', sdist),
     'symlink': install_symlinked,
     'install_lib_symlink': install_lib_symlink,
     'install_scripts_sym': install_scripts_for_symlink,
     'unsymlink': unsymlink,
 }
 
 setup_args["entry_points"] = {
     "console_scripts": find_entry_points(),
     "pygments.lexers": [
         "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer",
+        "ipython = IPython.lib.lexers:IPythonLexer",
         "ipython3 = IPython.lib.lexers:IPython3Lexer",
     ],
 }
 
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
 
 if __name__ == "__main__":
     setup(**setup_args)