Backport PR #12208: support for unicode identifiers

IPython/core/completerlib.py

@@ -52,7 +52,7 b' TIMEOUT_STORAGE = 2'
 TIMEOUT_GIVEUP = 20
 
 # Regular expression for the python import statement
-import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)'
+import_re = re.compile(r'(?P<name>[^\W\d]\w*?)'
                        r'(?P<package>[/\\]__init__)?'
                        r'(?P<suffix>%s)$' %
                        r'|'.join(re.escape(s) for s in _suffixes))

IPython/core/inputtransformer.py

@@ -278,8 +278,8 b' def escaped_commands(line):'
 _initial_space_re = re.compile(r'\s*')
 
 _help_end_re = re.compile(r"""(%{0,2}
-                              [a-zA-Z_*][\w*]*        # Variable name
-                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              (?!\d)[\w*]+            # Variable name
+                              (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,

IPython/core/inputtransformer2.py

@@ -405,8 +405,8 b' class EscapedCommand(TokenTransformBase):'
         return lines_before + [new_line] + lines_after
 
 _help_end_re = re.compile(r"""(%{0,2}
-                              [a-zA-Z_*][\w*]*        # Variable name
-                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              (?!\d)[\w*]+            # Variable name
+                              (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,

IPython/core/magics/namespace.py

@@ -208,12 +208,6 b' class NamespaceMagics(Magics):'
         
           %psearch -l            list all available object types
         """
-        try:
-            parameter_s.encode('ascii')
-        except UnicodeEncodeError:
-            print('Python identifiers can only contain ascii characters.')
-            return
-
         # default namespaces to be searched
         def_search = ['user_local', 'user_global', 'builtin']
 

IPython/core/prefilter.py

@@ -37,7 +37,7 b' class PrefilterError(Exception):'
 
 
 # RegExp to identify potential function names
-re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
+re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$')
 
 # RegExp to exclude strings with this start from autocalling.  In
 # particular, all binary operators should be excluded, so that if foo is

IPython/core/tests/test_inputtransformer.py

@@ -113,6 +113,7 b' syntax = \\'
         (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"),
         (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"),
         (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
+        (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"),
         (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"),
         (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
         (u'a = abc?', "get_ipython().set_next_input('a = abc');"

IPython/core/tests/test_inputtransformer2.py

@@ -119,6 +119,11 b' b) = zip?'
 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
 )
 
+HELP_UNICODE = (
+    ["π.foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"]
+)
+
 
 def null_cleanup_transformer(lines):
     """
@@ -223,6 +228,9 b' def test_transform_help():'
     tf = ipt2.HelpEnd((1, 0), (2, 8))
     nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
 
+    tf = ipt2.HelpEnd((1, 0), (1, 0))
+    nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2])
+
 def test_find_assign_op_dedent():
     """
     be careful that empty token like dedent are not counted as parens

IPython/core/tests/test_magic.py

@@ -625,6 +625,8 b' def doctest_precision():'
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
+    with tt.AssertPrints("π.is_integer"):
+        _ip.run_cell("π = 3.14;\nπ.is_integ*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""

IPython/core/tests/test_prefilter.py

@@ -115,3 +115,13 b' def test_prefilter_attribute_errors():'
     finally:
         del ip.user_ns['x']
         ip.magic('autocall 0')
+
+
+def test_autocall_should_support_unicode():
+    ip.magic('autocall 2')
+    ip.user_ns['π'] = lambda x: x
+    try:
+        nt.assert_equal(ip.prefilter('π 3'),'π(3)')
+    finally:
+        ip.magic('autocall 0')
+        del ip.user_ns['π']