From 56f48b87bedd0a9baf238619099260e5e652bb4a 2023-10-30 11:54:29 From: Matthias Bussonnier Date: 2023-10-30 11:54:29 Subject: [PATCH] Revert "Cleanup Python 2 compact from Lexers" --- diff --git a/IPython/lib/lexers.py b/IPython/lib/lexers.py index a675606..42d5b7a 100644 --- a/IPython/lib/lexers.py +++ b/IPython/lib/lexers.py @@ -4,13 +4,13 @@ Defines a variety of Pygments lexers for highlighting IPython code. This includes: - IPython3Lexer - Lexer for pure IPython (python + magic/shell commands) + IPythonLexer, IPython3Lexer + Lexers for pure IPython (python + magic/shell commands) IPythonPartialTracebackLexer, IPythonTracebackLexer - The partial traceback lexer reads everything but the Python code - appearing in a traceback. - The full lexer combines the partial lexer with the IPython3Lexer. + Supports 2.x and 3.x via keyword `python3`. The partial traceback + lexer reads everything but the Python code appearing in a traceback. + The full lexer combines the partial lexer with an IPython lexer. IPythonConsoleLexer A lexer for IPython console sessions, with support for tracebacks. @@ -35,22 +35,10 @@ import re # Third party from pygments.lexers import ( - BashLexer, - HtmlLexer, - JavascriptLexer, - RubyLexer, - PerlLexer, - Python3Lexer, - TexLexer, -) + BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer, + Python3Lexer, TexLexer) from pygments.lexer import ( - Lexer, - DelegatingLexer, - RegexLexer, - do_insertions, - bygroups, - using, - inherit, + Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using, ) from pygments.token import ( Generic, Keyword, Literal, Name, Operator, Other, Text, Error, @@ -61,106 +49,80 @@ from pygments.util import get_bool_opt line_re = re.compile('.*?\n') -__all__ = [ - "IPython3Lexer", - "IPythonPartialTracebackLexer", - "IPythonTracebackLexer", - "IPythonConsoleLexer", - "IPyLexer", -] +__all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer', + 'IPythonPartialTracebackLexer', 'IPythonTracebackLexer', + 'IPythonConsoleLexer', 'IPyLexer'] -class IPython3Lexer(Python3Lexer): - """IPython3 Lexer""" +def build_ipy_lexer(python3): + """Builds IPython lexers depending on the value of `python3`. - name = "IPython3" - aliases = ["ipython3"] + The lexer inherits from an appropriate Python lexer and then adds + information about IPython specific keywords (i.e. magic commands, + shell commands, etc.) - tokens = { - "root": [ - ( - r"(?s)(\s*)(%%capture)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%debug)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?is)(\s*)(%%html)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(HtmlLexer)), - ), - ( - r"(?s)(\s*)(%%javascript)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(JavascriptLexer)), - ), - ( - r"(?s)(\s*)(%%js)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(JavascriptLexer)), - ), - ( - r"(?s)(\s*)(%%latex)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(TexLexer)), - ), - ( - r"(?s)(\s*)(%%perl)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(PerlLexer)), - ), - ( - r"(?s)(\s*)(%%prun)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%pypy)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%python)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%python3)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%ruby)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(RubyLexer)), - ), - ( - r"(?s)(\s*)(%%time)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%timeit)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%writefile)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - ( - r"(?s)(\s*)(%%file)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(Python3Lexer)), - ), - (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)), - ( - r"(?s)(^\s*)(%%!)([^\n]*\n)(.*)", - bygroups(Text, Operator, Text, using(BashLexer)), - ), - (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), - (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), - ( - r"(%)(sx|sc|system)(.*)(\n)", - bygroups(Operator, Keyword, using(BashLexer), Text), - ), - (r"(%)(\w+)(.*\n)", bygroups(Operator, Keyword, Text)), - (r"^(!!)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)), - (r"(!)(?!=)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)), - (r"^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)", bygroups(Text, Operator, Text)), - (r"(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$", bygroups(Text, Operator, Text)), - inherit, - ] - } + Parameters + ---------- + python3 : bool + If `True`, then build an IPython lexer from a Python 3 lexer. + + """ + # It would be nice to have a single IPython lexer class which takes + # a boolean `python3`. But since there are two Python lexer classes, + # we will also have two IPython lexer classes. + if python3: + PyLexer = Python3Lexer + name = 'IPython3' + aliases = ['ipython3'] + doc = """IPython3 Lexer""" + else: + PyLexer = PythonLexer + name = 'IPython' + aliases = ['ipython2', 'ipython'] + doc = """IPython Lexer""" + + ipython_tokens = [ + (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))), + (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), + (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), + (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))), + (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))), + (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))), + (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), + (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))), + (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), + (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)), + (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))), + (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), + (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), + (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword, + using(BashLexer), Text)), + (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)), + (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), + (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), + (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)), + (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)), + ] + + tokens = PyLexer.tokens.copy() + tokens['root'] = ipython_tokens + tokens['root'] + + attrs = {'name': name, 'aliases': aliases, 'filenames': [], + '__doc__': doc, 'tokens': tokens} + + return type(name, (PyLexer,), attrs) + + +IPython3Lexer = build_ipy_lexer(python3=True) +IPythonLexer = build_ipy_lexer(python3=False) class IPythonPartialTracebackLexer(RegexLexer): @@ -222,9 +184,9 @@ class IPythonTracebackLexer(DelegatingLexer): this is the line which lists the File and line number. """ - - # The lexer inherits from DelegatingLexer. The "root" lexer is the - # IPython3 lexer. First, we parse with the partial IPython traceback lexer. + # The lexer inherits from DelegatingLexer. The "root" lexer is an + # appropriate IPython lexer, which depends on the value of the boolean + # `python3`. First, we parse with the partial IPython traceback lexer. # Then, any code marked with the "Other" token is delegated to the root # lexer. # @@ -239,9 +201,19 @@ class IPythonTracebackLexer(DelegatingLexer): # note we need a __init__ doc, as otherwise it inherits the doc from the super class # which will fail the documentation build as it references section of the pygments docs that # do not exists when building IPython's docs. + self.python3 = get_bool_opt(options, 'python3', False) + if self.python3: + self.aliases = ['ipython3tb'] + else: + self.aliases = ['ipython2tb', 'ipythontb'] - super().__init__(IPython3Lexer, IPythonPartialTracebackLexer, **options) + if self.python3: + IPyLexer = IPython3Lexer + else: + IPyLexer = IPythonLexer + DelegatingLexer.__init__(self, IPyLexer, + IPythonPartialTracebackLexer, **options) class IPythonConsoleLexer(Lexer): """ @@ -283,8 +255,8 @@ class IPythonConsoleLexer(Lexer): # continuation = ' .D.: ' # template = 'Out[#]: ' # - # Where '#' is the 'prompt number' or 'execution count' and 'D' - # D is a number of dots matching the width of the execution count + # Where '#' is the 'prompt number' or 'execution count' and 'D' + # D is a number of dots matching the width of the execution count # in1_regex = r'In \[[0-9]+\]: ' in2_regex = r' \.\.+\.: ' @@ -298,6 +270,9 @@ class IPythonConsoleLexer(Lexer): Parameters ---------- + python3 : bool + If `True`, then the console inputs are parsed using a Python 3 + lexer. Otherwise, they are parsed using a Python 2 lexer. in1_regex : RegexObject The compiled regular expression used to detect the start of inputs. Although the IPython configuration setting may have a @@ -313,7 +288,11 @@ class IPythonConsoleLexer(Lexer): then the default output prompt is assumed. """ - self.aliases = ["ipython3console"] + self.python3 = get_bool_opt(options, 'python3', False) + if self.python3: + self.aliases = ['ipython3console'] + else: + self.aliases = ['ipython2console', 'ipythonconsole'] in1_regex = options.get('in1_regex', self.in1_regex) in2_regex = options.get('in2_regex', self.in2_regex) @@ -339,8 +318,15 @@ class IPythonConsoleLexer(Lexer): Lexer.__init__(self, **options) - self.pylexer = IPython3Lexer(**options) - self.tblexer = IPythonTracebackLexer(**options) + if self.python3: + pylexer = IPython3Lexer + tblexer = IPythonTracebackLexer + else: + pylexer = IPythonLexer + tblexer = IPythonTracebackLexer + + self.pylexer = pylexer(**options) + self.tblexer = tblexer(**options) self.reset() @@ -526,16 +512,20 @@ class IPyLexer(Lexer): def __init__(self, **options): """ Create a new IPyLexer instance which dispatch to either an - IPythonConsoleLexer (if In prompts are present) or and IPython3Lexer (if + IPythonCOnsoleLexer (if In prompts are present) or and IPythonLexer (if In prompts are not present). """ # init docstring is necessary for docs not to fail to build do to parent # docs referenceing a section in pygments docs. - self.aliases = ["ipy3"] + self.python3 = get_bool_opt(options, 'python3', False) + if self.python3: + self.aliases = ['ipy3'] + else: + self.aliases = ['ipy2', 'ipy'] Lexer.__init__(self, **options) - self.IPythonLexer = IPython3Lexer(**options) + self.IPythonLexer = IPythonLexer(**options) self.IPythonConsoleLexer = IPythonConsoleLexer(**options) def get_tokens_unprocessed(self, text): @@ -547,3 +537,4 @@ class IPyLexer(Lexer): lex = self.IPythonLexer for token in lex.get_tokens_unprocessed(text): yield token + diff --git a/IPython/lib/tests/test_lexers.py b/IPython/lib/tests/test_lexers.py index 52c786a..000b8fe 100644 --- a/IPython/lib/tests/test_lexers.py +++ b/IPython/lib/tests/test_lexers.py @@ -16,10 +16,10 @@ pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14) class TestLexers(TestCase): """Collection of lexers tests""" def setUp(self): - self.lexer = lexers.IPython3Lexer() + self.lexer = lexers.IPythonLexer() self.bash_lexer = BashLexer() - def testIPython3Lexer(self): + def testIPythonLexer(self): fragment = '!echo $HOME\n' bash_tokens = [ (Token.Operator, '!'), diff --git a/IPython/lib/tests/test_pygments.py b/IPython/lib/tests/test_pygments.py index 224965a..877b422 100644 --- a/IPython/lib/tests/test_pygments.py +++ b/IPython/lib/tests/test_pygments.py @@ -4,10 +4,12 @@ import pytest import pygments.lexers import pygments.lexer -from IPython.lib.lexers import IPythonConsoleLexer, IPython3Lexer +from IPython.lib.lexers import IPythonConsoleLexer, IPythonLexer, IPython3Lexer #: the human-readable names of the IPython lexers with ``entry_points`` -EXPECTED_LEXER_NAMES = [cls.name for cls in [IPythonConsoleLexer, IPython3Lexer]] +EXPECTED_LEXER_NAMES = [ + cls.name for cls in [IPythonConsoleLexer, IPythonLexer, IPython3Lexer] +] @pytest.fixture diff --git a/IPython/sphinxext/ipython_console_highlighting.py b/IPython/sphinxext/ipython_console_highlighting.py index a72c506..b93a151 100644 --- a/IPython/sphinxext/ipython_console_highlighting.py +++ b/IPython/sphinxext/ipython_console_highlighting.py @@ -20,5 +20,9 @@ def setup(app): # Alternatively, we could register the lexer with pygments instead. This would # require using setuptools entrypoints: http://pygments.org/docs/plugins -highlighting.lexers["ipython"] = IPyLexer() -highlighting.lexers["ipython3"] = IPyLexer() +ipy2 = IPyLexer(python3=False) +ipy3 = IPyLexer(python3=True) + +highlighting.lexers['ipython'] = ipy2 +highlighting.lexers['ipython2'] = ipy2 +highlighting.lexers['ipython3'] = ipy3 diff --git a/docs/source/development/lexer.rst b/docs/source/development/lexer.rst index ffa249a..2bacdd7 100644 --- a/docs/source/development/lexer.rst +++ b/docs/source/development/lexer.rst @@ -9,20 +9,22 @@ The IPython console lexer has been rewritten and now supports tracebacks and customized input/output prompts. An entire suite of lexers is now available at :mod:`IPython.lib.lexers`. These include: -IPython3Lexer - Lexer for pure IPython (python 3 + magic/shell commands) +IPythonLexer & IPython3Lexer + Lexers for pure IPython (python + magic/shell commands) IPythonPartialTracebackLexer & IPythonTracebackLexer - The partial traceback lexer reads everything but the Python code - appearing in a traceback. The full lexer combines the partial lexer - with the IPython3Lexer. + Supports 2.x and 3.x via the keyword `python3`. The partial traceback + lexer reads everything but the Python code appearing in a traceback. + The full lexer combines the partial lexer with an IPython lexer. IPythonConsoleLexer - A lexer for python 3 IPython console sessions, with support for tracebacks. + A lexer for IPython console sessions, with support for tracebacks. + Supports 2.x and 3.x via the keyword `python3`. IPyLexer A friendly lexer which examines the first line of text and from it, decides whether to use an IPython lexer or an IPython console lexer. + Supports 2.x and 3.x via the keyword `python3`. Previously, the :class:`IPythonConsoleLexer` class was available at :mod:`IPython.sphinxext.ipython_console_hightlight`. It was inserted diff --git a/setup.py b/setup.py index 7a2ea72..dfe21ac 100644 --- a/setup.py +++ b/setup.py @@ -145,6 +145,7 @@ setup_args["entry_points"] = { "console_scripts": find_entry_points(), "pygments.lexers": [ "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer", + "ipython = IPython.lib.lexers:IPythonLexer", "ipython3 = IPython.lib.lexers:IPython3Lexer", ], }