Move IPython lexers module to lib

Closes gh-7941

I've left a backwards compatibility module in
IPython.nbconvert.utils.lexers - I don't know if anyone is importing it
directly, but since we put it in entry points, we should probably
consider it public API.

IPython/lib/lexers.py

@@ -0,0 +1,512 b''
+# -*- coding: utf-8 -*-
+"""
+Defines a variety of Pygments lexers for highlighting IPython code.
+
+This includes:
+
+    IPythonLexer, IPython3Lexer
+        Lexers for pure IPython (python + magic/shell commands)
+
+    IPythonPartialTracebackLexer, IPythonTracebackLexer
+        Supports 2.x and 3.x via keyword `python3`.  The partial traceback
+        lexer reads everything but the Python code appearing in a traceback.
+        The full lexer combines the partial lexer with an IPython lexer.
+
+    IPythonConsoleLexer
+        A lexer for IPython console sessions, with support for tracebacks.
+
+    IPyLexer
+        A friendly lexer which examines the first line of text and from it,
+        decides whether to use an IPython lexer or an IPython console lexer.
+        This is probably the only lexer that needs to be explicitly added
+        to Pygments.
+
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+# Standard library
+import re
+
+# Third party
+from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
+from pygments.lexer import (
+    Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
+)
+from pygments.token import (
+    Comment, Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
+)
+from pygments.util import get_bool_opt
+
+# Local
+from IPython.testing.skipdoctest import skip_doctest
+
+line_re = re.compile('.*?\n')
+
+__all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
+           'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
+           'IPythonConsoleLexer', 'IPyLexer']
+
+ipython_tokens = [
+  (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
+  (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
+  (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
+  (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
+  (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
+                                       using(BashLexer), Text)),
+  (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
+  (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
+]
+
+def build_ipy_lexer(python3):
+    """Builds IPython lexers depending on the value of `python3`.
+
+    The lexer inherits from an appropriate Python lexer and then adds
+    information about IPython specific keywords (i.e. magic commands,
+    shell commands, etc.)
+
+    Parameters
+    ----------
+    python3 : bool
+        If `True`, then build an IPython lexer from a Python 3 lexer.
+
+    """
+    # It would be nice to have a single IPython lexer class which takes
+    # a boolean `python3`.  But since there are two Python lexer classes,
+    # we will also have two IPython lexer classes.
+    if python3:
+        PyLexer = Python3Lexer
+        clsname = 'IPython3Lexer'
+        name = 'IPython3'
+        aliases = ['ipython3']
+        doc = """IPython3 Lexer"""
+    else:
+        PyLexer = PythonLexer
+        clsname = 'IPythonLexer'
+        name = 'IPython'
+        aliases = ['ipython2', 'ipython']
+        doc = """IPython Lexer"""
+
+    tokens = PyLexer.tokens.copy()
+    tokens['root'] = ipython_tokens + tokens['root']
+
+    attrs = {'name': name, 'aliases': aliases, 'filenames': [],
+             '__doc__': doc, 'tokens': tokens}
+
+    return type(name, (PyLexer,), attrs)
+
+
+IPython3Lexer = build_ipy_lexer(python3=True)
+IPythonLexer = build_ipy_lexer(python3=False)
+
+
+class IPythonPartialTracebackLexer(RegexLexer):
+    """
+    Partial lexer for IPython tracebacks.
+
+    Handles all the non-python output. This works for both Python 2.x and 3.x.
+
+    """
+    name = 'IPython Partial Traceback'
+
+    tokens = {
+        'root': [
+            # Tracebacks for syntax errors have a different style.
+            # For both types of tracebacks, we mark the first line with
+            # Generic.Traceback.  For syntax errors, we mark the filename
+            # as we mark the filenames for non-syntax tracebacks.
+            #
+            # These two regexps define how IPythonConsoleLexer finds a
+            # traceback.
+            #
+            ## Non-syntax traceback
+            (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
+            ## Syntax traceback
+            (r'^(  File)(.*)(, line )(\d+\n)',
+             bygroups(Generic.Traceback, Name.Namespace,
+                      Generic.Traceback, Literal.Number.Integer)),
+
+            # (Exception Identifier)(Whitespace)(Traceback Message)
+            (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
+             bygroups(Name.Exception, Generic.Whitespace, Text)),
+            # (Module/Filename)(Text)(Callee)(Function Signature)
+            # Better options for callee and function signature?
+            (r'(.*)( in )(.*)(\(.*\)\n)',
+             bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
+            # Regular line: (Whitespace)(Line Number)(Python Code)
+            (r'(\s*?)(\d+)(.*?\n)',
+             bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
+            # Emphasized line: (Arrow)(Line Number)(Python Code)
+            # Using Exception token so arrow color matches the Exception.
+            (r'(-*>?\s?)(\d+)(.*?\n)',
+             bygroups(Name.Exception, Literal.Number.Integer, Other)),
+            # (Exception Identifier)(Message)
+            (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
+             bygroups(Name.Exception, Text)),
+            # Tag everything else as Other, will be handled later.
+            (r'.*\n', Other),
+        ],
+    }
+
+
+class IPythonTracebackLexer(DelegatingLexer):
+    """
+    IPython traceback lexer.
+
+    For doctests, the tracebacks can be snipped as much as desired with the
+    exception to the lines that designate a traceback. For non-syntax error
+    tracebacks, this is the line of hyphens. For syntax error tracebacks,
+    this is the line which lists the File and line number.
+
+    """
+    # The lexer inherits from DelegatingLexer.  The "root" lexer is an
+    # appropriate IPython lexer, which depends on the value of the boolean
+    # `python3`.  First, we parse with the partial IPython traceback lexer.
+    # Then, any code marked with the "Other" token is delegated to the root
+    # lexer.
+    #
+    name = 'IPython Traceback'
+    aliases = ['ipythontb']
+
+    def __init__(self, **options):
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipython3tb']
+        else:
+            self.aliases = ['ipython2tb', 'ipythontb']
+
+        if self.python3:
+            IPyLexer = IPython3Lexer
+        else:
+            IPyLexer = IPythonLexer
+
+        DelegatingLexer.__init__(self, IPyLexer,
+                                 IPythonPartialTracebackLexer, **options)
+
+@skip_doctest
+class IPythonConsoleLexer(Lexer):
+    """
+    An IPython console lexer for IPython code-blocks and doctests, such as:
+
+    .. code-block:: rst
+
+        .. code-block:: ipythonconsole
+
+            In [1]: a = 'foo'
+
+            In [2]: a
+            Out[2]: 'foo'
+
+            In [3]: print a
+            foo
+
+            In [4]: 1 / 0
+
+
+    Support is also provided for IPython exceptions:
+
+    .. code-block:: rst
+
+        .. code-block:: ipythonconsole
+
+            In [1]: raise Exception
+
+            ---------------------------------------------------------------------------
+            Exception                                 Traceback (most recent call last)
+            <ipython-input-1-fca2ab0ca76b> in <module>()
+            ----> 1 raise Exception
+
+            Exception:
+
+    """
+    name = 'IPython console session'
+    aliases = ['ipythonconsole']
+    mimetypes = ['text/x-ipython-console']
+
+    # The regexps used to determine what is input and what is output.
+    # The default prompts for IPython are:
+    #
+    #     c.PromptManager.in_template  = 'In [\#]: '
+    #     c.PromptManager.in2_template = '   .\D.: '
+    #     c.PromptManager.out_template = 'Out[\#]: '
+    #
+    in1_regex = r'In \[[0-9]+\]: '
+    in2_regex = r'   \.\.+\.: '
+    out_regex = r'Out\[[0-9]+\]: '
+
+    #: The regex to determine when a traceback starts.
+    ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
+
+    def __init__(self, **options):
+        """Initialize the IPython console lexer.
+
+        Parameters
+        ----------
+        python3 : bool
+            If `True`, then the console inputs are parsed using a Python 3
+            lexer. Otherwise, they are parsed using a Python 2 lexer.
+        in1_regex : RegexObject
+            The compiled regular expression used to detect the start
+            of inputs. Although the IPython configuration setting may have a
+            trailing whitespace, do not include it in the regex. If `None`,
+            then the default input prompt is assumed.
+        in2_regex : RegexObject
+            The compiled regular expression used to detect the continuation
+            of inputs. Although the IPython configuration setting may have a
+            trailing whitespace, do not include it in the regex. If `None`,
+            then the default input prompt is assumed.
+        out_regex : RegexObject
+            The compiled regular expression used to detect outputs. If `None`,
+            then the default output prompt is assumed.
+
+        """
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipython3console']
+        else:
+            self.aliases = ['ipython2console', 'ipythonconsole']
+
+        in1_regex = options.get('in1_regex', self.in1_regex)
+        in2_regex = options.get('in2_regex', self.in2_regex)
+        out_regex = options.get('out_regex', self.out_regex)
+
+        # So that we can work with input and output prompts which have been
+        # rstrip'd (possibly by editors) we also need rstrip'd variants. If
+        # we do not do this, then such prompts will be tagged as 'output'.
+        # The reason can't just use the rstrip'd variants instead is because
+        # we want any whitespace associated with the prompt to be inserted
+        # with the token. This allows formatted code to be modified so as hide
+        # the appearance of prompts, with the whitespace included. One example
+        # use of this is in copybutton.js from the standard lib Python docs.
+        in1_regex_rstrip = in1_regex.rstrip() + '\n'
+        in2_regex_rstrip = in2_regex.rstrip() + '\n'
+        out_regex_rstrip = out_regex.rstrip() + '\n'
+
+        # Compile and save them all.
+        attrs = ['in1_regex', 'in2_regex', 'out_regex',
+                 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
+        for attr in attrs:
+            self.__setattr__(attr, re.compile(locals()[attr]))
+
+        Lexer.__init__(self, **options)
+
+        if self.python3:
+            pylexer = IPython3Lexer
+            tblexer = IPythonTracebackLexer
+        else:
+            pylexer = IPythonLexer
+            tblexer = IPythonTracebackLexer
+
+        self.pylexer = pylexer(**options)
+        self.tblexer = tblexer(**options)
+
+        self.reset()
+
+    def reset(self):
+        self.mode = 'output'
+        self.index = 0
+        self.buffer = u''
+        self.insertions = []
+
+    def buffered_tokens(self):
+        """
+        Generator of unprocessed tokens after doing insertions and before
+        changing to a new state.
+
+        """
+        if self.mode == 'output':
+            tokens = [(0, Generic.Output, self.buffer)]
+        elif self.mode == 'input':
+            tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
+        else: # traceback
+            tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
+
+        for i, t, v in do_insertions(self.insertions, tokens):
+            # All token indexes are relative to the buffer.
+            yield self.index + i, t, v
+
+        # Clear it all
+        self.index += len(self.buffer)
+        self.buffer = u''
+        self.insertions = []
+
+    def get_mci(self, line):
+        """
+        Parses the line and returns a 3-tuple: (mode, code, insertion).
+
+        `mode` is the next mode (or state) of the lexer, and is always equal
+        to 'input', 'output', or 'tb'.
+
+        `code` is a portion of the line that should be added to the buffer
+        corresponding to the next mode and eventually lexed by another lexer.
+        For example, `code` could be Python code if `mode` were 'input'.
+
+        `insertion` is a 3-tuple (index, token, text) representing an
+        unprocessed "token" that will be inserted into the stream of tokens
+        that are created from the buffer once we change modes. This is usually
+        the input or output prompt.
+
+        In general, the next mode depends on current mode and on the contents
+        of `line`.
+
+        """
+        # To reduce the number of regex match checks, we have multiple
+        # 'if' blocks instead of 'if-elif' blocks.
+
+        # Check for possible end of input
+        in2_match = self.in2_regex.match(line)
+        in2_match_rstrip = self.in2_regex_rstrip.match(line)
+        if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
+           in2_match_rstrip:
+            end_input = True
+        else:
+            end_input = False
+        if end_input and self.mode != 'tb':
+            # Only look for an end of input when not in tb mode.
+            # An ellipsis could appear within the traceback.
+            mode = 'output'
+            code = u''
+            insertion = (0, Generic.Prompt, line)
+            return mode, code, insertion
+
+        # Check for output prompt
+        out_match = self.out_regex.match(line)
+        out_match_rstrip = self.out_regex_rstrip.match(line)
+        if out_match or out_match_rstrip:
+            mode = 'output'
+            if out_match:
+                idx = out_match.end()
+            else:
+                idx = out_match_rstrip.end()
+            code = line[idx:]
+            # Use the 'heading' token for output.  We cannot use Generic.Error
+            # since it would conflict with exceptions.
+            insertion = (0, Generic.Heading, line[:idx])
+            return mode, code, insertion
+
+
+        # Check for input or continuation prompt (non stripped version)
+        in1_match = self.in1_regex.match(line)
+        if in1_match or (in2_match and self.mode != 'tb'):
+            # New input or when not in tb, continued input.
+            # We do not check for continued input when in tb since it is
+            # allowable to replace a long stack with an ellipsis.
+            mode = 'input'
+            if in1_match:
+                idx = in1_match.end()
+            else: # in2_match
+                idx = in2_match.end()
+            code = line[idx:]
+            insertion = (0, Generic.Prompt, line[:idx])
+            return mode, code, insertion
+
+        # Check for input or continuation prompt (stripped version)
+        in1_match_rstrip = self.in1_regex_rstrip.match(line)
+        if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
+            # New input or when not in tb, continued input.
+            # We do not check for continued input when in tb since it is
+            # allowable to replace a long stack with an ellipsis.
+            mode = 'input'
+            if in1_match_rstrip:
+                idx = in1_match_rstrip.end()
+            else: # in2_match
+                idx = in2_match_rstrip.end()
+            code = line[idx:]
+            insertion = (0, Generic.Prompt, line[:idx])
+            return mode, code, insertion
+
+        # Check for traceback
+        if self.ipytb_start.match(line):
+            mode = 'tb'
+            code = line
+            insertion = None
+            return mode, code, insertion
+
+        # All other stuff...
+        if self.mode in ('input', 'output'):
+            # We assume all other text is output. Multiline input that
+            # does not use the continuation marker cannot be detected.
+            # For example, the 3 in the following is clearly output:
+            #
+            #    In [1]: print 3
+            #    3
+            #
+            # But the following second line is part of the input:
+            #
+            #    In [2]: while True:
+            #        print True
+            #
+            # In both cases, the 2nd line will be 'output'.
+            #
+            mode = 'output'
+        else:
+            mode = 'tb'
+
+        code = line
+        insertion = None
+
+        return mode, code, insertion
+
+    def get_tokens_unprocessed(self, text):
+        self.reset()
+        for match in line_re.finditer(text):
+            line = match.group()
+            mode, code, insertion = self.get_mci(line)
+
+            if mode != self.mode:
+                # Yield buffered tokens before transitioning to new mode.
+                for token in self.buffered_tokens():
+                    yield token
+                self.mode = mode
+
+            if insertion:
+                self.insertions.append((len(self.buffer), [insertion]))
+            self.buffer += code
+        else:
+            for token in self.buffered_tokens():
+                yield token
+
+class IPyLexer(Lexer):
+    """
+    Primary lexer for all IPython-like code.
+
+    This is a simple helper lexer.  If the first line of the text begins with
+    "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
+    lexer. If not, then the entire text is parsed with an IPython lexer.
+
+    The goal is to reduce the number of lexers that are registered
+    with Pygments.
+
+    """
+    name = 'IPy session'
+    aliases = ['ipy']
+
+    def __init__(self, **options):
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipy3']
+        else:
+            self.aliases = ['ipy2', 'ipy']
+
+        Lexer.__init__(self, **options)
+
+        self.IPythonLexer = IPythonLexer(**options)
+        self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
+
+    def get_tokens_unprocessed(self, text):
+        # Search for the input prompt anywhere...this allows code blocks to
+        # begin with comments as well.
+        if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
+            lex = self.IPythonConsoleLexer
+        else:
+            lex = self.IPythonLexer
+        for token in lex.get_tokens_unprocessed(text):
+            yield token
+

IPython/nbconvert/filters/highlight.py

@@ -1,135 +1,135 b''
 """
 Module containing filter functions that allow code to be highlighted
 from within Jinja templates.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 # pygments must not be imported at the module level
 # because errors should be raised at runtime if it's actually needed,
 # not import time, when it may not be needed.
 
 from IPython.nbconvert.utils.base import NbConvertBase
 from warnings import warn
 
 MULTILINE_OUTPUTS = ['text', 'html', 'svg', 'latex', 'javascript', 'json']
 
 __all__ = [
     'Highlight2HTML',
     'Highlight2Latex'
 ]
 
 class Highlight2HTML(NbConvertBase):
     def __init__(self, pygments_lexer=None, **kwargs):
         self.pygments_lexer = pygments_lexer or 'ipython3'
         super(Highlight2HTML, self).__init__(**kwargs)
 
     def _default_language_changed(self, name, old, new):
         warn('Setting default_language in config is deprecated, '
              'please use language_info metadata instead.')
         self.pygments_lexer = new
 
     def __call__(self, source, language=None, metadata=None):
         """
         Return a syntax-highlighted version of the input source as html output.
 
         Parameters
         ----------
         source : str
             source of the cell to highlight
         language : str
             language to highlight the syntax of
         metadata : NotebookNode cell metadata
             metadata of the cell to highlight
         """
         from pygments.formatters import HtmlFormatter
 
         if not language:
             language=self.pygments_lexer
 
         return _pygments_highlight(source if len(source) > 0 else ' ',
                                    # needed to help post processors:
                                    HtmlFormatter(cssclass=" highlight hl-"+language),
                                    language, metadata)
 
 
 class Highlight2Latex(NbConvertBase):
     def __init__(self, pygments_lexer=None, **kwargs):
         self.pygments_lexer = pygments_lexer or 'ipython3'
         super(Highlight2Latex, self).__init__(**kwargs)
 
     def _default_language_changed(self, name, old, new):
         warn('Setting default_language in config is deprecated, '
              'please use language_info metadata instead.')
         self.pygments_lexer = new
 
     def __call__(self, source, language=None, metadata=None, strip_verbatim=False):
         """
         Return a syntax-highlighted version of the input source as latex output.
 
         Parameters
         ----------
         source : str
             source of the cell to highlight
         language : str
             language to highlight the syntax of
         metadata : NotebookNode cell metadata
             metadata of the cell to highlight
         strip_verbatim : bool
             remove the Verbatim environment that pygments provides by default
         """
         from pygments.formatters import LatexFormatter
         if not language:
             language=self.pygments_lexer
 
         latex = _pygments_highlight(source, LatexFormatter(), language, metadata)
         if strip_verbatim:
             latex = latex.replace(r'\begin{Verbatim}[commandchars=\\\{\}]' + '\n', '')
             return latex.replace('\n\\end{Verbatim}\n', '')
         else:
             return latex
 
 
 
 def _pygments_highlight(source, output_formatter, language='ipython', metadata=None):
     """
     Return a syntax-highlighted version of the input source
 
     Parameters
     ----------
     source : str
         source of the cell to highlight
     output_formatter : Pygments formatter
     language : str
         language to highlight the syntax of
     metadata : NotebookNode cell metadata
         metadata of the cell to highlight
     """
     from pygments import highlight
     from pygments.lexers import get_lexer_by_name
     from pygments.util import ClassNotFound
-    from IPython.nbconvert.utils.lexers import IPythonLexer, IPython3Lexer
+    from IPython.lib.lexers import IPythonLexer, IPython3Lexer
 
     # If the cell uses a magic extension language,
     # use the magic language instead.
     if language.startswith('ipython') \
         and metadata \
         and 'magics_language' in metadata:
 
         language = metadata['magics_language']
 
     if language == 'ipython2':
         lexer = IPythonLexer()
     elif language == 'ipython3':
         lexer = IPython3Lexer()
     else:
         try:
             lexer = get_lexer_by_name(language, stripall=True)
         except ClassNotFound:
             warn("No lexer found for language %r. Treating as plain text." % language)
             from pygments.lexers.special import TextLexer
             lexer = TextLexer()
 
 
     return highlight(source, lexer, output_formatter)

IPython/nbconvert/utils/lexers.py

@@ -1,508 +1,3 b''
-# -*- coding: utf-8 -*-
-"""
-Defines a variety of Pygments lexers for highlighting IPython code.
-
-This includes:
-
-    IPythonLexer, IPython3Lexer
-        Lexers for pure IPython (python + magic/shell commands)
-
-    IPythonPartialTracebackLexer, IPythonTracebackLexer
-        Supports 2.x and 3.x via keyword `python3`.  The partial traceback
-        lexer reads everything but the Python code appearing in a traceback.
-        The full lexer combines the partial lexer with an IPython lexer.
-
-    IPythonConsoleLexer
-        A lexer for IPython console sessions, with support for tracebacks.
-
-    IPyLexer
-        A friendly lexer which examines the first line of text and from it,
-        decides whether to use an IPython lexer or an IPython console lexer.
-        This is probably the only lexer that needs to be explicitly added
-        to Pygments.
-
-"""
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, the IPython Development Team.
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
-
-# Standard library
-import re
-
-# Third party
-from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
-from pygments.lexer import (
-    Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
-)
-from pygments.token import (
-    Comment, Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
-)
-from pygments.util import get_bool_opt
-
-# Local
-from IPython.testing.skipdoctest import skip_doctest
-
-line_re = re.compile('.*?\n')
-
-ipython_tokens = [
-  (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
-  (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
-  (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
-  (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
-  (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
-                                       using(BashLexer), Text)),
-  (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
-  (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
-  (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
-  (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
-]
-
-def build_ipy_lexer(python3):
-    """Builds IPython lexers depending on the value of `python3`.
-
-    The lexer inherits from an appropriate Python lexer and then adds
-    information about IPython specific keywords (i.e. magic commands,
-    shell commands, etc.)
-
-    Parameters
-    ----------
-    python3 : bool
-        If `True`, then build an IPython lexer from a Python 3 lexer.
-
-    """
-    # It would be nice to have a single IPython lexer class which takes
-    # a boolean `python3`.  But since there are two Python lexer classes,
-    # we will also have two IPython lexer classes.
-    if python3:
-        PyLexer = Python3Lexer
-        clsname = 'IPython3Lexer'
-        name = 'IPython3'
-        aliases = ['ipython3']
-        doc = """IPython3 Lexer"""
-    else:
-        PyLexer = PythonLexer
-        clsname = 'IPythonLexer'
-        name = 'IPython'
-        aliases = ['ipython2', 'ipython']
-        doc = """IPython Lexer"""
-
-    tokens = PyLexer.tokens.copy()
-    tokens['root'] = ipython_tokens + tokens['root']
-
-    attrs = {'name': name, 'aliases': aliases, 'filenames': [],
-             '__doc__': doc, 'tokens': tokens}
-
-    return type(name, (PyLexer,), attrs)
-
-
-IPython3Lexer = build_ipy_lexer(python3=True)
-IPythonLexer = build_ipy_lexer(python3=False)
-
-
-class IPythonPartialTracebackLexer(RegexLexer):
-    """
-    Partial lexer for IPython tracebacks.
-
-    Handles all the non-python output. This works for both Python 2.x and 3.x.
-
-    """
-    name = 'IPython Partial Traceback'
-
-    tokens = {
-        'root': [
-            # Tracebacks for syntax errors have a different style.
-            # For both types of tracebacks, we mark the first line with
-            # Generic.Traceback.  For syntax errors, we mark the filename
-            # as we mark the filenames for non-syntax tracebacks.
-            #
-            # These two regexps define how IPythonConsoleLexer finds a
-            # traceback.
-            #
-            ## Non-syntax traceback
-            (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
-            ## Syntax traceback
-            (r'^(  File)(.*)(, line )(\d+\n)',
-             bygroups(Generic.Traceback, Name.Namespace,
-                      Generic.Traceback, Literal.Number.Integer)),
-
-            # (Exception Identifier)(Whitespace)(Traceback Message)
-            (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
-             bygroups(Name.Exception, Generic.Whitespace, Text)),
-            # (Module/Filename)(Text)(Callee)(Function Signature)
-            # Better options for callee and function signature?
-            (r'(.*)( in )(.*)(\(.*\)\n)',
-             bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
-            # Regular line: (Whitespace)(Line Number)(Python Code)
-            (r'(\s*?)(\d+)(.*?\n)',
-             bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
-            # Emphasized line: (Arrow)(Line Number)(Python Code)
-            # Using Exception token so arrow color matches the Exception.
-            (r'(-*>?\s?)(\d+)(.*?\n)',
-             bygroups(Name.Exception, Literal.Number.Integer, Other)),
-            # (Exception Identifier)(Message)
-            (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
-             bygroups(Name.Exception, Text)),
-            # Tag everything else as Other, will be handled later.
-            (r'.*\n', Other),
-        ],
-    }
-
-
-class IPythonTracebackLexer(DelegatingLexer):
-    """
-    IPython traceback lexer.
-
-    For doctests, the tracebacks can be snipped as much as desired with the
-    exception to the lines that designate a traceback. For non-syntax error
-    tracebacks, this is the line of hyphens. For syntax error tracebacks,
-    this is the line which lists the File and line number.
-
-    """
-    # The lexer inherits from DelegatingLexer.  The "root" lexer is an
-    # appropriate IPython lexer, which depends on the value of the boolean
-    # `python3`.  First, we parse with the partial IPython traceback lexer.
-    # Then, any code marked with the "Other" token is delegated to the root
-    # lexer.
-    #
-    name = 'IPython Traceback'
-    aliases = ['ipythontb']
-
-    def __init__(self, **options):
-        self.python3 = get_bool_opt(options, 'python3', False)
-        if self.python3:
-            self.aliases = ['ipython3tb']
-        else:
-            self.aliases = ['ipython2tb', 'ipythontb']
-
-        if self.python3:
-            IPyLexer = IPython3Lexer
-        else:
-            IPyLexer = IPythonLexer
-
-        DelegatingLexer.__init__(self, IPyLexer,
-                                 IPythonPartialTracebackLexer, **options)
-
-@skip_doctest
-class IPythonConsoleLexer(Lexer):
-    """
-    An IPython console lexer for IPython code-blocks and doctests, such as:
-
-    .. code-block:: rst
-
-        .. code-block:: ipythonconsole
-
-            In [1]: a = 'foo'
-
-            In [2]: a
-            Out[2]: 'foo'
-
-            In [3]: print a
-            foo
-
-            In [4]: 1 / 0
-
-
-    Support is also provided for IPython exceptions:
-
-    .. code-block:: rst
-
-        .. code-block:: ipythonconsole
-
-            In [1]: raise Exception
-
-            ---------------------------------------------------------------------------
-            Exception                                 Traceback (most recent call last)
-            <ipython-input-1-fca2ab0ca76b> in <module>()
-            ----> 1 raise Exception
-
-            Exception:
-
-    """
-    name = 'IPython console session'
-    aliases = ['ipythonconsole']
-    mimetypes = ['text/x-ipython-console']
-
-    # The regexps used to determine what is input and what is output.
-    # The default prompts for IPython are:
-    #
-    #     c.PromptManager.in_template  = 'In [\#]: '
-    #     c.PromptManager.in2_template = '   .\D.: '
-    #     c.PromptManager.out_template = 'Out[\#]: '
-    #
-    in1_regex = r'In \[[0-9]+\]: '
-    in2_regex = r'   \.\.+\.: '
-    out_regex = r'Out\[[0-9]+\]: '
-
-    #: The regex to determine when a traceback starts.
-    ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
-
-    def __init__(self, **options):
-        """Initialize the IPython console lexer.
-
-        Parameters
-        ----------
-        python3 : bool
-            If `True`, then the console inputs are parsed using a Python 3
-            lexer. Otherwise, they are parsed using a Python 2 lexer.
-        in1_regex : RegexObject
-            The compiled regular expression used to detect the start
-            of inputs. Although the IPython configuration setting may have a
-            trailing whitespace, do not include it in the regex. If `None`,
-            then the default input prompt is assumed.
-        in2_regex : RegexObject
-            The compiled regular expression used to detect the continuation
-            of inputs. Although the IPython configuration setting may have a
-            trailing whitespace, do not include it in the regex. If `None`,
-            then the default input prompt is assumed.
-        out_regex : RegexObject
-            The compiled regular expression used to detect outputs. If `None`,
-            then the default output prompt is assumed.
-
-        """
-        self.python3 = get_bool_opt(options, 'python3', False)
-        if self.python3:
-            self.aliases = ['ipython3console']
-        else:
-            self.aliases = ['ipython2console', 'ipythonconsole']
-
-        in1_regex = options.get('in1_regex', self.in1_regex)
-        in2_regex = options.get('in2_regex', self.in2_regex)
-        out_regex = options.get('out_regex', self.out_regex)
-
-        # So that we can work with input and output prompts which have been
-        # rstrip'd (possibly by editors) we also need rstrip'd variants. If
-        # we do not do this, then such prompts will be tagged as 'output'.
-        # The reason can't just use the rstrip'd variants instead is because
-        # we want any whitespace associated with the prompt to be inserted
-        # with the token. This allows formatted code to be modified so as hide
-        # the appearance of prompts, with the whitespace included. One example
-        # use of this is in copybutton.js from the standard lib Python docs.
-        in1_regex_rstrip = in1_regex.rstrip() + '\n'
-        in2_regex_rstrip = in2_regex.rstrip() + '\n'
-        out_regex_rstrip = out_regex.rstrip() + '\n'
-
-        # Compile and save them all.
-        attrs = ['in1_regex', 'in2_regex', 'out_regex',
-                 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
-        for attr in attrs:
-            self.__setattr__(attr, re.compile(locals()[attr]))
-
-        Lexer.__init__(self, **options)
-
-        if self.python3:
-            pylexer = IPython3Lexer
-            tblexer = IPythonTracebackLexer
-        else:
-            pylexer = IPythonLexer
-            tblexer = IPythonTracebackLexer
-
-        self.pylexer = pylexer(**options)
-        self.tblexer = tblexer(**options)
-
-        self.reset()
-
-    def reset(self):
-        self.mode = 'output'
-        self.index = 0
-        self.buffer = u''
-        self.insertions = []
-
-    def buffered_tokens(self):
-        """
-        Generator of unprocessed tokens after doing insertions and before
-        changing to a new state.
-
-        """
-        if self.mode == 'output':
-            tokens = [(0, Generic.Output, self.buffer)]
-        elif self.mode == 'input':
-            tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
-        else: # traceback
-            tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
-
-        for i, t, v in do_insertions(self.insertions, tokens):
-            # All token indexes are relative to the buffer.
-            yield self.index + i, t, v
-
-        # Clear it all
-        self.index += len(self.buffer)
-        self.buffer = u''
-        self.insertions = []
-
-    def get_mci(self, line):
-        """
-        Parses the line and returns a 3-tuple: (mode, code, insertion).
-
-        `mode` is the next mode (or state) of the lexer, and is always equal
-        to 'input', 'output', or 'tb'.
-
-        `code` is a portion of the line that should be added to the buffer
-        corresponding to the next mode and eventually lexed by another lexer.
-        For example, `code` could be Python code if `mode` were 'input'.
-
-        `insertion` is a 3-tuple (index, token, text) representing an
-        unprocessed "token" that will be inserted into the stream of tokens
-        that are created from the buffer once we change modes. This is usually
-        the input or output prompt.
-
-        In general, the next mode depends on current mode and on the contents
-        of `line`.
-
-        """
-        # To reduce the number of regex match checks, we have multiple
-        # 'if' blocks instead of 'if-elif' blocks.
-
-        # Check for possible end of input
-        in2_match = self.in2_regex.match(line)
-        in2_match_rstrip = self.in2_regex_rstrip.match(line)
-        if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
-           in2_match_rstrip:
-            end_input = True
-        else:
-            end_input = False
-        if end_input and self.mode != 'tb':
-            # Only look for an end of input when not in tb mode.
-            # An ellipsis could appear within the traceback.
-            mode = 'output'
-            code = u''
-            insertion = (0, Generic.Prompt, line)
-            return mode, code, insertion
-
-        # Check for output prompt
-        out_match = self.out_regex.match(line)
-        out_match_rstrip = self.out_regex_rstrip.match(line)
-        if out_match or out_match_rstrip:
-            mode = 'output'
-            if out_match:
-                idx = out_match.end()
-            else:
-                idx = out_match_rstrip.end()
-            code = line[idx:]
-            # Use the 'heading' token for output.  We cannot use Generic.Error
-            # since it would conflict with exceptions.
-            insertion = (0, Generic.Heading, line[:idx])
-            return mode, code, insertion
-
-
-        # Check for input or continuation prompt (non stripped version)
-        in1_match = self.in1_regex.match(line)
-        if in1_match or (in2_match and self.mode != 'tb'):
-            # New input or when not in tb, continued input.
-            # We do not check for continued input when in tb since it is
-            # allowable to replace a long stack with an ellipsis.
-            mode = 'input'
-            if in1_match:
-                idx = in1_match.end()
-            else: # in2_match
-                idx = in2_match.end()
-            code = line[idx:]
-            insertion = (0, Generic.Prompt, line[:idx])
-            return mode, code, insertion
-
-        # Check for input or continuation prompt (stripped version)
-        in1_match_rstrip = self.in1_regex_rstrip.match(line)
-        if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
-            # New input or when not in tb, continued input.
-            # We do not check for continued input when in tb since it is
-            # allowable to replace a long stack with an ellipsis.
-            mode = 'input'
-            if in1_match_rstrip:
-                idx = in1_match_rstrip.end()
-            else: # in2_match
-                idx = in2_match_rstrip.end()
-            code = line[idx:]
-            insertion = (0, Generic.Prompt, line[:idx])
-            return mode, code, insertion
-
-        # Check for traceback
-        if self.ipytb_start.match(line):
-            mode = 'tb'
-            code = line
-            insertion = None
-            return mode, code, insertion
-
-        # All other stuff...
-        if self.mode in ('input', 'output'):
-            # We assume all other text is output. Multiline input that
-            # does not use the continuation marker cannot be detected.
-            # For example, the 3 in the following is clearly output:
-            #
-            #    In [1]: print 3
-            #    3
-            #
-            # But the following second line is part of the input:
-            #
-            #    In [2]: while True:
-            #        print True
-            #
-            # In both cases, the 2nd line will be 'output'.
-            #
-            mode = 'output'
-        else:
-            mode = 'tb'
-
-        code = line
-        insertion = None
-
-        return mode, code, insertion
-
-    def get_tokens_unprocessed(self, text):
-        self.reset()
-        for match in line_re.finditer(text):
-            line = match.group()
-            mode, code, insertion = self.get_mci(line)
-
-            if mode != self.mode:
-                # Yield buffered tokens before transitioning to new mode.
-                for token in self.buffered_tokens():
-                    yield token
-                self.mode = mode
-
-            if insertion:
-                self.insertions.append((len(self.buffer), [insertion]))
-            self.buffer += code
-        else:
-            for token in self.buffered_tokens():
-                yield token
-
-class IPyLexer(Lexer):
-    """
-    Primary lexer for all IPython-like code.
-
-    This is a simple helper lexer.  If the first line of the text begins with
-    "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
-    lexer. If not, then the entire text is parsed with an IPython lexer.
-
-    The goal is to reduce the number of lexers that are registered
-    with Pygments.
-
-    """
-    name = 'IPy session'
-    aliases = ['ipy']
-
-    def __init__(self, **options):
-        self.python3 = get_bool_opt(options, 'python3', False)
-        if self.python3:
-            self.aliases = ['ipy3']
-        else:
-            self.aliases = ['ipy2', 'ipy']
-
-        Lexer.__init__(self, **options)
-
-        self.IPythonLexer = IPythonLexer(**options)
-        self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
-
-    def get_tokens_unprocessed(self, text):
-        # Search for the input prompt anywhere...this allows code blocks to
-        # begin with comments as well.
-        if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
-            lex = self.IPythonConsoleLexer
-        else:
-            lex = self.IPythonLexer
-        for token in lex.get_tokens_unprocessed(text):
-            yield token
+"""Deprecated; import from IPython.lib.lexers instead."""
 
+from IPython.lib.lexers import *

IPython/sphinxext/ipython_console_highlighting.py

@@ -1,27 +1,27 b''
 """
 reST directive for syntax-highlighting ipython interactive sessions.
 
 """
 
 from sphinx import highlighting
-from ..nbconvert.utils.lexers import IPyLexer
+from IPython.lib.lexers import IPyLexer
 
 def setup(app):
     """Setup as a sphinx extension."""
 
     # This is only a lexer, so adding it below to pygments appears sufficient.
     # But if somebody knows what the right API usage should be to do that via
     # sphinx, by all means fix it here.  At least having this setup.py
     # suppresses the sphinx warning we'd get without it.
     pass
 
 # Register the extension as a valid pygments lexer.
 # Alternatively, we could register the lexer with pygments instead. This would
 # require using setuptools entrypoints: http://pygments.org/docs/plugins
 
 ipy2 = IPyLexer(python3=False)
 ipy3 = IPyLexer(python3=True)
 
 highlighting.lexers['ipython'] = ipy2
 highlighting.lexers['ipython2'] = ipy2
 highlighting.lexers['ipython3'] = ipy3

docs/autogen_api.py

@@ -1,79 +1,81 b''
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 # stdlib imports
 import os
 import sys
 
 # local imports
 sys.path.append(os.path.abspath('sphinxext'))
 from apigen import ApiDocWriter
 
 #*****************************************************************************
 if __name__ == '__main__':
     pjoin = os.path.join
     package = 'IPython'
     outdir = pjoin('source','api','generated')
     docwriter = ApiDocWriter(package,rst_extension='.rst')
     # You have to escape the . here because . is a special char for regexps.
     # You must do make clean if you change this!
     docwriter.package_skip_patterns += [r'\.external$',
                                         # Extensions are documented elsewhere.
                                         r'\.extensions',
                                         r'\.config\.profile',
                                         # Old nbformat versions
                                         r'\.nbformat\.v[1-2]',
                                         # Public API for this is in kernel.zmq.eventloops
                                         r'\.kernel\.zmq\.gui',
                                         # Magics are documented separately
                                         r'\.core\.magics',
                                         ]
 
     # The inputhook* modules often cause problems on import, such as trying to
     # load incompatible Qt bindings. It's easiest to leave them all out. The
     # main API is in the inputhook module, which is documented.
     docwriter.module_skip_patterns += [ r'\.lib\.inputhook.+',
                                         r'\.ipdoctest',
                                         r'\.testing\.plugin',
                                         # This just prints a deprecation msg:
                                         r'\.frontend$',
                                         # Deprecated:
                                         r'\.core\.magics\.deprecated',
+                                        # Backwards compat import for lib.lexers
+                                        r'\.nbconvert\.utils\.lexers',
                                         # We document this manually.
                                         r'\.utils\.py3compat',
                                         # These are exposed by nbformat
                                         r'\.nbformat\.convert',
                                         r'\.nbformat\.validator',
                                         r'\.nbformat\.notebooknode',
                                         # Deprecated
                                         r'\.nbformat\.current',
                                         # Exposed by nbformat.vN
                                         r'\.nbformat\.v[3-4]\.nbbase',
                                         # These are exposed in display
                                         r'\.core\.display',
                                         r'\.lib\.display',
                                         # This isn't actually a module
                                         r'\.html\.tasks',
                                         # This is private
                                         r'\.html\.allow76'
                                         ]
 
     # These modules import functions and classes from other places to expose
     # them as part of the public API. They must have __all__ defined. The
     # non-API modules they import from should be excluded by the skip patterns
     # above.
     docwriter.names_from__all__.update({
         'IPython.nbformat',
         'IPython.nbformat.v3',
         'IPython.nbformat.v4',
         'IPython.display',
     })
     
     # Now, generate the outputs
     docwriter.write_api_docs(outdir)
     # Write index with .txt extension - we can include it, but Sphinx won't try
     # to compile it
     docwriter.write_index(outdir, 'gen.txt',
                           relative_to = pjoin('source','api')
                           )
     print ('%d files written' % len(docwriter.written_modules))

docs/source/development/lexer.rst

@@ -1,64 +1,64 b''
 .. _console_lexer:
 
 New IPython Console Lexer
 -------------------------
 
 .. versionadded:: 2.0.0
 
 The IPython console lexer has been rewritten and now supports tracebacks
 and customized input/output prompts. An entire suite of lexers is now
-available at :mod:`IPython.nbconvert.utils.lexers`. These include:
+available at :mod:`IPython.lib.lexers`. These include:
 
 IPythonLexer & IPython3Lexer
   Lexers for pure IPython (python + magic/shell commands)
 
 IPythonPartialTracebackLexer & IPythonTracebackLexer
   Supports 2.x and 3.x via the keyword `python3`. The partial traceback
   lexer reads everything but the Python code appearing in a traceback.
   The full lexer combines the partial lexer with an IPython lexer.
 
 IPythonConsoleLexer
   A lexer for IPython console sessions, with support for tracebacks.
   Supports 2.x and 3.x via the keyword `python3`.
 
 IPyLexer
   A friendly lexer which examines the first line of text and from it,
   decides whether to use an IPython lexer or an IPython console lexer.
   Supports 2.x and 3.x via the keyword `python3`.
 
 Previously, the :class:`IPythonConsoleLexer` class was available at
 :mod:`IPython.sphinxext.ipython_console_hightlight`.  It was inserted
 into Pygments' list of available lexers under the name `ipython`.  It should
 be mentioned that this name is inaccurate, since an IPython console session
 is not the same as IPython code (which itself is a superset of the Python
 language).
 
 Now, the Sphinx extension inserts two console lexers into Pygments' list of
 available lexers. Both are IPyLexer instances under the names: `ipython` and
 `ipython3`. Although the names can be confusing (as mentioned above), their
 continued use is, in part, to maintain backwards compatibility and to
 aid typical usage. If a project needs to make Pygments aware of more than just
 the IPyLexer class, then one should not make the IPyLexer class available under
 the name `ipython` and use `ipy` or some other non-conflicting value.
 
 Code blocks such as:
 
 .. code-block:: rst
 
     .. code-block:: ipython
 
         In [1]: 2**2
     Out[1]: 4
 
     will continue to work as before, but now, they will also properly highlight
 tracebacks.  For pure IPython code, the same lexer will also work:
 
 .. code-block:: rst
 
     .. code-block:: ipython
 
         x = ''.join(map(str, range(10)))
     !echo $x
 
     Since the first line of the block did not begin with a standard IPython console
 prompt, the entire block is assumed to consist of IPython code instead.

setup.py

@@ -1,342 +1,342 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
 Under Posix environments it works like a typical setup.py script.
 Under Windows, the command sdist is not supported, since IPython
 requires utilities which are not available under Windows."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008-2011, IPython Development Team.
 #  Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.rst, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Minimal Python version sanity check
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import sys
 
 # This check is also made in IPython/__init__, don't forget to update both when
 # changing Python version requirements.
 v = sys.version_info
 if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,3)):
     error = "ERROR: IPython requires Python version 2.7 or 3.3 or above."
     print(error, file=sys.stderr)
     sys.exit(1)
 
 PY3 = (sys.version_info[0] >= 3)
 
 # At least we're on the python version we need, move on.
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Stdlib imports
 import os
 import shutil
 
 from glob import glob
 
 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
 # update it when the contents of directories change.
 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
 
 from distutils.core import setup
 
 # Our own imports
 from setupbase import target_update
 
 from setupbase import (
     setup_args,
     find_packages,
     find_package_data,
     check_package_data_first,
     find_entry_points,
     build_scripts_entrypt,
     find_data_files,
     check_for_dependencies,
     git_prebuild,
     check_submodule_status,
     update_submodules,
     require_submodules,
     UpdateSubmodules,
     get_bdist_wheel,
     CompileCSS,
     JavascriptVersion,
     css_js_prerelease,
     install_symlinked,
     install_lib_symlink,
     install_scripts_for_symlink,
     unsymlink,
 )
 from setupext import setupext
 
 isfile = os.path.isfile
 pjoin = os.path.join
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
 
 if os.name in ('nt','dos'):
     os_name = 'windows'
 else:
     os_name = os.name
 
 # Under Windows, 'sdist' has not been supported.  Now that the docs build with
 # Sphinx it might work, but let's not turn it on until someone confirms that it
 # actually works.
 if os_name == 'windows' and 'sdist' in sys.argv:
     print('The sdist command is not available under Windows.  Exiting.')
     sys.exit(1)
 
 #-------------------------------------------------------------------------------
 # Make sure we aren't trying to run without submodules
 #-------------------------------------------------------------------------------
 here = os.path.abspath(os.path.dirname(__file__))
 
 def require_clean_submodules():
     """Check on git submodules before distutils can do anything
 
     Since distutils cannot be trusted to update the tree
     after everything has been set in motion,
     this is not a distutils command.
     """
     # PACKAGERS: Add a return here to skip checks for git submodules
     
     # don't do anything if nothing is actually supposed to happen
     for do_nothing in ('-h', '--help', '--help-commands', 'clean', 'submodule'):
         if do_nothing in sys.argv:
             return
 
     status = check_submodule_status(here)
 
     if status == "missing":
         print("checking out submodules for the first time")
         update_submodules(here)
     elif status == "unclean":
         print('\n'.join([
             "Cannot build / install IPython with unclean submodules",
             "Please update submodules with",
             "    python setup.py submodule",
             "or",
             "    git submodule update",
             "or commit any submodule changes you have made."
         ]))
         sys.exit(1)
 
 require_clean_submodules()
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
 
 # update the manuals when building a source dist
 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
 
     # List of things to be updated. Each entry is a triplet of args for
     # target_update()
     to_update = [
                   # FIXME - Disabled for now: we need to redo an automatic way
                   # of generating the magic info inside the rst.
                   #('docs/magic.tex',
                   #['IPython/Magic.py'],
                   #"cd doc && ./update_magic.sh" ),
 
                  ('docs/man/ipcluster.1.gz',
                   ['docs/man/ipcluster.1'],
                   'cd docs/man && gzip -9c ipcluster.1 > ipcluster.1.gz'),
 
                  ('docs/man/ipcontroller.1.gz',
                   ['docs/man/ipcontroller.1'],
                   'cd docs/man && gzip -9c ipcontroller.1 > ipcontroller.1.gz'),
 
                  ('docs/man/ipengine.1.gz',
                   ['docs/man/ipengine.1'],
                   'cd docs/man && gzip -9c ipengine.1 > ipengine.1.gz'),
 
                  ('docs/man/ipython.1.gz',
                   ['docs/man/ipython.1'],
                   'cd docs/man && gzip -9c ipython.1 > ipython.1.gz'),
 
                  ]
 
 
     [ target_update(*t) for t in to_update ]
 
 #---------------------------------------------------------------------------
 # Find all the packages, package data, and data_files
 #---------------------------------------------------------------------------
 
 packages = find_packages()
 package_data = find_package_data()
 
 data_files = find_data_files()
 
 setup_args['packages'] = packages
 setup_args['package_data'] = package_data
 setup_args['data_files'] = data_files
 
 #---------------------------------------------------------------------------
 # custom distutils commands
 #---------------------------------------------------------------------------
 # imports here, so they are after setuptools import if there was one
 from distutils.command.sdist import sdist
 from distutils.command.upload import upload
 
 class UploadWindowsInstallers(upload):
 
     description = "Upload Windows installers to PyPI (only used from tools/release_windows.py)"
     user_options = upload.user_options + [
         ('files=', 'f', 'exe file (or glob) to upload')
     ]
     def initialize_options(self):
         upload.initialize_options(self)
         meta = self.distribution.metadata
         base = '{name}-{version}'.format(
             name=meta.get_name(),
             version=meta.get_version()
         )
         self.files = os.path.join('dist', '%s.*.exe' % base)
 
     def run(self):
         for dist_file in glob(self.files):
             self.upload_file('bdist_wininst', 'any', dist_file)
 
 setup_args['cmdclass'] = {
     'build_py': css_js_prerelease(
             check_package_data_first(git_prebuild('IPython'))),
     'sdist' : css_js_prerelease(git_prebuild('IPython', sdist)),
     'upload_wininst' : UploadWindowsInstallers,
     'submodule' : UpdateSubmodules,
     'css' : CompileCSS,
     'symlink': install_symlinked,
     'install_lib_symlink': install_lib_symlink,
     'install_scripts_sym': install_scripts_for_symlink,
     'unsymlink': unsymlink,
     'jsversion' : JavascriptVersion,
 }
 
 #---------------------------------------------------------------------------
 # Handle scripts, dependencies, and setuptools specific things
 #---------------------------------------------------------------------------
 
 # For some commands, use setuptools.  Note that we do NOT list install here!
 # If you want a setuptools-enhanced install, just run 'setupegg.py install'
 needs_setuptools = set(('develop', 'release', 'bdist_egg', 'bdist_rpm',
            'bdist', 'bdist_dumb', 'bdist_wininst', 'bdist_wheel',
            'egg_info', 'easy_install', 'upload', 'install_egg_info',
             ))
 
 if len(needs_setuptools.intersection(sys.argv)) > 0:
     import setuptools
 
 # This dict is used for passing extra arguments that are setuptools
 # specific to setup
 setuptools_extra_args = {}
 
 # setuptools requirements
 
 pyzmq = 'pyzmq>=13'
 
 extras_require = dict(
     parallel = [pyzmq],
     qtconsole = [pyzmq, 'pygments'],
     doc = ['Sphinx>=1.1', 'numpydoc'],
     test = ['nose>=0.10.1', 'requests'],
     terminal = [],
     nbformat = ['jsonschema>=2.0'],
     notebook = ['tornado>=4.0', pyzmq, 'jinja2', 'pygments', 'mistune>=0.5'],
     nbconvert = ['pygments', 'jinja2', 'mistune>=0.3.1']
 )
 
 if not sys.platform.startswith('win'):
     extras_require['notebook'].append('terminado>=0.3.3')
 
 if sys.version_info < (3, 3):
     extras_require['test'].append('mock')
 
 extras_require['notebook'].extend(extras_require['nbformat'])
 extras_require['nbconvert'].extend(extras_require['nbformat'])
 
 install_requires = []
 
 # add readline
 if sys.platform == 'darwin':
     if 'bdist_wheel' in sys.argv[1:] or not setupext.check_for_readline():
         install_requires.append('gnureadline')
 elif sys.platform.startswith('win'):
     extras_require['terminal'].append('pyreadline>=2.0')
 
 everything = set()
 for deps in extras_require.values():
     everything.update(deps)
 extras_require['all'] = everything
 
 if 'setuptools' in sys.modules:
     # setup.py develop should check for submodules
     from setuptools.command.develop import develop
     setup_args['cmdclass']['develop'] = require_submodules(develop)
     setup_args['cmdclass']['bdist_wheel'] = css_js_prerelease(get_bdist_wheel())
     
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
         'console_scripts': find_entry_points(),
         'pygments.lexers': [
-            'ipythonconsole = IPython.nbconvert.utils.lexers:IPythonConsoleLexer',
-            'ipython = IPython.nbconvert.utils.lexers:IPythonLexer',
-            'ipython3 = IPython.nbconvert.utils.lexers:IPython3Lexer',
+            'ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer',
+            'ipython = IPython.lib.lexers:IPythonLexer',
+            'ipython3 = IPython.lib.lexers:IPython3Lexer',
         ],
     }
     setup_args['extras_require'] = extras_require
     requires = setup_args['install_requires'] = install_requires
 
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
     # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
     # doesn't find them.
     if 'bdist_wininst' in sys.argv:
         if len(sys.argv) > 2 and \
                ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print("ERROR: bdist_wininst must be run alone. Exiting.", file=sys.stderr)
             sys.exit(1)
         setup_args['data_files'].append(
             ['Scripts', ('scripts/ipython.ico', 'scripts/ipython_nb.ico')])
         setup_args['scripts'] = [pjoin('scripts','ipython_win_post_install.py')]
         setup_args['options'] = {"bdist_wininst":
                                  {"install_script":
                                   "ipython_win_post_install.py"}}
 
 else:
     # If we are installing without setuptools, call this function which will
     # check for dependencies an inform the user what is needed.  This is
     # just to make life easy for users.
     for install_cmd in ('install', 'symlink'):
         if install_cmd in sys.argv:
             check_for_dependencies()
             break
     # scripts has to be a non-empty list, or install_scripts isn't called
     setup_args['scripts'] = [e.split('=')[0].strip() for e in find_entry_points()]
 
     setup_args['cmdclass']['build_scripts'] = build_scripts_entrypt
 
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
 
 setup_args.update(setuptools_extra_args)
 
 def main():
     setup(**setup_args)
 
 if __name__ == '__main__':
     main()