From ed7f35f8b721d4b4dcafea173ce724bee25704c7 2023-01-03 10:57:26
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: 2023-01-03 10:57:26
Subject: [PATCH] Fix tests for pygments > 2.14

Pygments 2.14+  have the bash lexer return some tokens as
Text.Whitespace instead of Text, this update the test to support this.

---

diff --git a/IPython/lib/tests/test_lexers.py b/IPython/lib/tests/test_lexers.py
index efa00d6..000b8fe 100644
--- a/IPython/lib/tests/test_lexers.py
+++ b/IPython/lib/tests/test_lexers.py
@@ -4,11 +4,14 @@
 # Distributed under the terms of the Modified BSD License.
 
 from unittest import TestCase
+from pygments import __version__ as pygments_version
 from pygments.token import Token
 from pygments.lexers import BashLexer
 
 from .. import lexers
 
+pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14)
+
 
 class TestLexers(TestCase):
     """Collection of lexers tests"""
@@ -18,25 +21,26 @@ class TestLexers(TestCase):
 
     def testIPythonLexer(self):
         fragment = '!echo $HOME\n'
-        tokens = [
+        bash_tokens = [
             (Token.Operator, '!'),
         ]
-        tokens.extend(self.bash_lexer.get_tokens(fragment[1:]))
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:]))
+        ipylex_token = list(self.lexer.get_tokens(fragment))
+        assert bash_tokens[:-1] == ipylex_token[:-1]
 
-        fragment_2 = '!' + fragment
+        fragment_2 = "!" + fragment
         tokens_2 = [
             (Token.Operator, '!!'),
-        ] + tokens[1:]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens[1:]
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = '\t %%!\n' + fragment[1:]
         tokens_2 = [
             (Token.Text, '\t '),
             (Token.Operator, '%%!'),
             (Token.Text, '\n'),
-        ] + tokens[1:]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens[1:]
+        assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = 'x = ' + fragment
         tokens_2 = [
@@ -44,8 +48,8 @@ class TestLexers(TestCase):
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
-        ] + tokens
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x, = ' + fragment
         tokens_2 = [
@@ -54,8 +58,8 @@ class TestLexers(TestCase):
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
-        ] + tokens
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x, = %sx ' + fragment[1:]
         tokens_2 = [
@@ -67,8 +71,10 @@ class TestLexers(TestCase):
             (Token.Operator, '%'),
             (Token.Keyword, 'sx'),
             (Token.Text, ' '),
-        ] + tokens[1:]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens[1:]
+        if tokens_2[7] == (Token.Text, " ") and pyg214:  # pygments 2.14+
+            tokens_2[7] = (Token.Text.Whitespace, " ")
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'f = %R function () {}\n'
         tokens_2 = [
@@ -80,7 +86,7 @@ class TestLexers(TestCase):
             (Token.Keyword, 'R'),
             (Token.Text, ' function () {}\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = '\t%%xyz\n$foo\n'
         tokens_2 = [
@@ -89,7 +95,7 @@ class TestLexers(TestCase):
             (Token.Keyword, 'xyz'),
             (Token.Text, '\n$foo\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = '%system?\n'
         tokens_2 = [
@@ -98,7 +104,7 @@ class TestLexers(TestCase):
             (Token.Operator, '?'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x != y\n'
         tokens_2 = [
@@ -109,7 +115,7 @@ class TestLexers(TestCase):
             (Token.Name, 'y'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = ' ?math.sin\n'
         tokens_2 = [
@@ -118,7 +124,7 @@ class TestLexers(TestCase):
             (Token.Text, 'math.sin'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment = ' *int*?\n'
         tokens = [
@@ -126,7 +132,7 @@ class TestLexers(TestCase):
             (Token.Operator, '?'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        assert tokens == list(self.lexer.get_tokens(fragment))
 
         fragment = '%%writefile -a foo.py\nif a == b:\n    pass'
         tokens = [
@@ -145,7 +151,9 @@ class TestLexers(TestCase):
             (Token.Keyword, 'pass'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        if tokens[10] == (Token.Text, "\n") and pyg214:  # pygments 2.14+
+            tokens[10] = (Token.Text.Whitespace, "\n")
+        assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1]
 
         fragment = '%%timeit\nmath.sin(0)'
         tokens = [
@@ -173,4 +181,4 @@ class TestLexers(TestCase):
             (Token.Punctuation, '>'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        assert tokens == list(self.lexer.get_tokens(fragment))