From 79516ce71e7eb89b4e2c4811344c09947ef615ec 2015-02-09 21:55:40
From: Min RK <benjaminrk@gmail.com>
Date: 2015-02-09 21:55:40
Subject: [PATCH] Backport PR #6878: Update pygments monkeypatch for compatibility with Pygments 2.0

Closes gh-6877

I spent a while looking for a better way to achive this, to get rid of the monkeypatch, but none occurred to me, so I went with Carlos' suggestion of just updating it.

---

diff --git a/IPython/qt/console/pygments_highlighter.py b/IPython/qt/console/pygments_highlighter.py
index 4044be7..35b6255 100644
--- a/IPython/qt/console/pygments_highlighter.py
+++ b/IPython/qt/console/pygments_highlighter.py
@@ -12,6 +12,11 @@ def get_tokens_unprocessed(self, text, stack=('root',)):
     """ Split ``text`` into (tokentype, text) pairs.
 
         Monkeypatched to store the final stack on the object itself.
+
+        The `text` parameter this gets passed is only the current line, so to
+        highlight things like multiline strings correctly, we need to retrieve
+        the state from the previous line (this is done in PygmentsHighlighter,
+        below), and use it to continue processing the current line.
     """
     pos = 0
     tokendefs = self._tokens
@@ -24,11 +29,12 @@ def get_tokens_unprocessed(self, text, stack=('root',)):
         for rexmatch, action, new_state in statetokens:
             m = rexmatch(text, pos)
             if m:
-                if type(action) is _TokenType:
-                    yield pos, action, m.group()
-                else:
-                    for item in action(self, m):
-                        yield item
+                if action is not None:
+                    if type(action) is _TokenType:
+                        yield pos, action, m.group()
+                    else:
+                        for item in action(self, m):
+                            yield item
                 pos = m.end()
                 if new_state is not None:
                     # state transition