From b2c6426f4be190f1cb68df249d81f4ee52a2b8fd 2015-01-25 12:28:32
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: 2015-01-25 12:28:32
Subject: [PATCH] Merge pull request #7556 from abalkin/issue-7548-2

Issue 7548: Better handling of line magics in nbconvert
---

diff --git a/IPython/nbconvert/utils/lexers.py b/IPython/nbconvert/utils/lexers.py
index c795900..79f7eb8 100644
--- a/IPython/nbconvert/utils/lexers.py
+++ b/IPython/nbconvert/utils/lexers.py
@@ -49,10 +49,15 @@ from IPython.testing.skipdoctest import skip_doctest
 line_re = re.compile('.*?\n')
 
 ipython_tokens = [
-  (r'(\%+)(\w+)\s+(\.*)(\n)', bygroups(Operator, Keyword,
+  (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
+  (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
+  (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
+  (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
+  (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
                                        using(BashLexer), Text)),
-  (r'(\%+)(\w+)\b', bygroups(Operator, Keyword)),
-  (r'^(!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
+  (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'(!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
 ]
 
 def build_ipy_lexer(python3):
diff --git a/IPython/nbconvert/utils/tests/test_lexers.py b/IPython/nbconvert/utils/tests/test_lexers.py
new file mode 100644
index 0000000..a18f892
--- /dev/null
+++ b/IPython/nbconvert/utils/tests/test_lexers.py
@@ -0,0 +1,111 @@
+"""Test lexers module"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2014 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from pygments.token import Token
+
+from IPython.nbconvert.tests.base import TestsBase
+from .. import lexers
+
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+class TestLexers(TestsBase):
+    """Collection of lexers tests"""
+    def setUp(self):
+        self.lexer = lexers.IPythonLexer()
+
+    def testIPythonLexer(self):
+        fragment = '!echo $HOME\n'
+        tokens = [
+            (Token.Operator, '!'),
+            (Token.Name.Builtin, 'echo'),
+            (Token.Text, ' '),
+            (Token.Name.Variable, '$HOME'),
+            (Token.Text, '\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+        fragment_2 = '!' + fragment
+        tokens_2 = [
+            (Token.Operator, '!!'),
+        ] + tokens[1:]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = '\t %%!\n' + fragment[1:]
+        tokens_2 = [
+            (Token.Text, '\t '),
+            (Token.Operator, '%%!'),
+            (Token.Text, '\n'),
+        ] + tokens[1:]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'x = ' + fragment
+        tokens_2 = [
+            (Token.Name, 'x'),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+        ] + tokens
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'x, = ' + fragment
+        tokens_2 = [
+            (Token.Name, 'x'),
+            (Token.Punctuation, ','),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+        ] + tokens
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'x, = %sx ' + fragment[1:]
+        tokens_2 = [
+            (Token.Name, 'x'),
+            (Token.Punctuation, ','),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+            (Token.Operator, '%'),
+            (Token.Keyword, 'sx'),
+            (Token.Text, ' '),
+        ] + tokens[1:]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'f = %R function () {}\n'
+        tokens_2 = [
+            (Token.Name, 'f'),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+            (Token.Operator, '%'),
+            (Token.Keyword, 'R'),
+            (Token.Text, ' function () {}\n'),
+        ]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = '\t%%xyz\n$foo\n'
+        tokens_2 = [
+            (Token.Text, '\t'),
+            (Token.Operator, '%%'),
+            (Token.Keyword, 'xyz'),
+            (Token.Text, '\n$foo\n'),
+        ]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = '%system?\n'
+        tokens_2 = [
+            (Token.Operator, '%'),
+            (Token.Keyword, 'system'),
+            (Token.Operator, '?'),
+            (Token.Text, '\n'),
+        ]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))