support for unicode identifiers

This rewrites some of the regular expressions that are used to match
Python identifiers, so that they are unicode compatible. In Python 3,
identifiers can contain unicode characters as long as the first
character is not numeric.

Examples for the changes:

• inputtransformer:

```
In [1]: π = 3.14
In [2]: π.is_integer?
Object `is_integer` not found.
```

----------
• namespace:

```
π.is_integ*?
```
or
```
In [1]: %psearch π.is_integ
Python identifiers can only contain ascii characters.
```

----------
• prefilter:

```
%autocall 1
φ = float
get_ipython().prefilter("φ 3")  # should be 'φ(3)', but returns 'φ 3'
```

----------
• completerlib:

If there is a file e.g. named `π.py` in the current directory, then

```
import IPython
IPython.core.completerlib.module_list('.')  # should contain module 'π'
```

IPython/core/completerlib.py

@@ -52,7 +52,7 b' TIMEOUT_STORAGE = 2'
 TIMEOUT_GIVEUP = 20
 
 # Regular expression for the python import statement
-import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)'
+import_re = re.compile(r'(?P<name>[^\W\d]\w*?)'
                        r'(?P<package>[/\\]__init__)?'
                        r'(?P<suffix>%s)$' %
                        r'|'.join(re.escape(s) for s in _suffixes))

IPython/core/inputtransformer.py

@@ -278,8 +278,8 b' def escaped_commands(line):'
 _initial_space_re = re.compile(r'\s*')
 
 _help_end_re = re.compile(r"""(%{0,2}
-                              [a-zA-Z_*][\w*]*        # Variable name
-                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              (?!\d)[\w*]+            # Variable name
+                              (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,

IPython/core/inputtransformer2.py

@@ -405,8 +405,8 b' class EscapedCommand(TokenTransformBase):'
         return lines_before + [new_line] + lines_after
 
 _help_end_re = re.compile(r"""(%{0,2}
-                              [a-zA-Z_*][\w*]*        # Variable name
-                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              (?!\d)[\w*]+            # Variable name
+                              (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,

IPython/core/magics/namespace.py

@@ -208,12 +208,6 b' class NamespaceMagics(Magics):'
         
           %psearch -l            list all available object types
         """
-        try:
-            parameter_s.encode('ascii')
-        except UnicodeEncodeError:
-            print('Python identifiers can only contain ascii characters.')
-            return
-
         # default namespaces to be searched
         def_search = ['user_local', 'user_global', 'builtin']
 

IPython/core/prefilter.py

@@ -37,7 +37,7 b' class PrefilterError(Exception):'
 
 
 # RegExp to identify potential function names
-re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
+re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$')
 
 # RegExp to exclude strings with this start from autocalling.  In
 # particular, all binary operators should be excluded, so that if foo is

IPython/core/tests/test_inputtransformer.py

@@ -113,6 +113,7 b' syntax = \\'
         (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"),
         (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"),
         (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
+        (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"),
         (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"),
         (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
         (u'a = abc?', "get_ipython().set_next_input('a = abc');"

IPython/core/tests/test_inputtransformer2.py

@@ -119,6 +119,11 b' b) = zip?'
 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
 )
 
+HELP_UNICODE = (
+    ["π.foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"]
+)
+
 
 def null_cleanup_transformer(lines):
     """
@@ -223,6 +228,9 b' def test_transform_help():'
     tf = ipt2.HelpEnd((1, 0), (2, 8))
     nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
 
+    tf = ipt2.HelpEnd((1, 0), (1, 0))
+    nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2])
+
 def test_find_assign_op_dedent():
     """
     be careful that empty token like dedent are not counted as parens

IPython/core/tests/test_magic.py

@@ -602,6 +602,8 b' def doctest_precision():'
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
+    with tt.AssertPrints("π.is_integer"):
+        _ip.run_cell("π = 3.14;\nπ.is_integ*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""

IPython/core/tests/test_prefilter.py

@@ -115,3 +115,13 b' def test_prefilter_attribute_errors():'
     finally:
         del ip.user_ns['x']
         ip.magic('autocall 0')
+
+
+def test_autocall_should_support_unicode():
+    ip.magic('autocall 2')
+    ip.user_ns['π'] = lambda x: x
+    try:
+        nt.assert_equal(ip.prefilter('π 3'),'π(3)')
+    finally:
+        ip.magic('autocall 0')
+        del ip.user_ns['π']