##// END OF EJS Templates
Merge branch 'unicode-issues'
Thomas Kluyver -
r3459:a6a9d580 merge
parent child Browse files
Show More
@@ -0,0 +1,68 b''
1 # coding: utf-8
2 """Tests for IPython.core.application"""
3
4 import os
5 import tempfile
6
7 from IPython.core.application import Application
8
9 def test_unicode_cwd():
10 """Check that IPython starts with non-ascii characters in the path."""
11 wd = tempfile.mkdtemp(suffix=u"€")
12
13 old_wd = os.getcwdu()
14 os.chdir(wd)
15 #raise Exception(repr(os.getcwd()))
16 try:
17 app = Application()
18 # The lines below are copied from Application.initialize()
19 app.create_default_config()
20 app.log_default_config()
21 app.set_default_config_log_level()
22
23 # Find resources needed for filesystem access, using information from
24 # the above two
25 app.find_ipython_dir()
26 app.find_resources()
27 app.find_config_file_name()
28 app.find_config_file_paths()
29
30 # File-based config
31 app.pre_load_file_config()
32 app.load_file_config(suppress_errors=False)
33 finally:
34 os.chdir(old_wd)
35
36 def test_unicode_ipdir():
37 """Check that IPython starts with non-ascii characters in the IP dir."""
38 ipdir = tempfile.mkdtemp(suffix=u"€")
39
40 # Create the config file, so it tries to load it.
41 with open(os.path.join(ipdir, 'ipython_config.py'), "w") as f:
42 pass
43
44 old_ipdir1 = os.environ.pop("IPYTHONDIR", None)
45 old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
46 os.environ["IPYTHONDIR"] = ipdir.encode("utf-8")
47 try:
48 app = Application()
49 # The lines below are copied from Application.initialize()
50 app.create_default_config()
51 app.log_default_config()
52 app.set_default_config_log_level()
53
54 # Find resources needed for filesystem access, using information from
55 # the above two
56 app.find_ipython_dir()
57 app.find_resources()
58 app.find_config_file_name()
59 app.find_config_file_paths()
60
61 # File-based config
62 app.pre_load_file_config()
63 app.load_file_config(suppress_errors=False)
64 finally:
65 if old_ipdir1:
66 os.environ["IPYTHONDIR"] = old_ipdir1
67 if old_ipdir2:
68 os.environ["IPYTHONDIR"] = old_ipdir2
@@ -285,7 +285,9 b' class PyFileConfigLoader(FileConfigLoader):'
285 return self.config
285 return self.config
286
286
287 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
287 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
288 execfile(self.full_filename, namespace)
288 fs_encoding = sys.getfilesystemencoding() or 'ascii'
289 conf_filename = self.full_filename.encode(fs_encoding)
290 execfile(conf_filename, namespace)
289
291
290 def _convert_to_config(self):
292 def _convert_to_config(self):
291 if self.data is None:
293 if self.data is None:
@@ -353,18 +353,22 b' class Application(object):'
353 # our shipped copies of builtin profiles even if they don't have them
353 # our shipped copies of builtin profiles even if they don't have them
354 # in their local ipython directory.
354 # in their local ipython directory.
355 prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
355 prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
356 self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
356 self.config_file_paths = (os.getcwdu(), self.ipython_dir, prof_dir)
357
357
358 def pre_load_file_config(self):
358 def pre_load_file_config(self):
359 """Do actions before the config file is loaded."""
359 """Do actions before the config file is loaded."""
360 pass
360 pass
361
361
362 def load_file_config(self):
362 def load_file_config(self, suppress_errors=True):
363 """Load the config file.
363 """Load the config file.
364
364
365 This tries to load the config file from disk. If successful, the
365 This tries to load the config file from disk. If successful, the
366 ``CONFIG_FILE`` config variable is set to the resolved config file
366 ``CONFIG_FILE`` config variable is set to the resolved config file
367 location. If not successful, an empty config is used.
367 location. If not successful, an empty config is used.
368
369 By default, errors in loading config are handled, and a warning
370 printed on screen. For testing, the suppress_errors option is set
371 to False, so errors will make tests fail.
368 """
372 """
369 self.log.debug("Attempting to load config file: %s" %
373 self.log.debug("Attempting to load config file: %s" %
370 self.config_file_name)
374 self.config_file_name)
@@ -380,6 +384,8 b' class Application(object):'
380 self.config_file_name, exc_info=True)
384 self.config_file_name, exc_info=True)
381 self.file_config = Config()
385 self.file_config = Config()
382 except:
386 except:
387 if not suppress_errors: # For testing purposes
388 raise
383 self.log.warn("Error loading config file: %s" %
389 self.log.warn("Error loading config file: %s" %
384 self.config_file_name, exc_info=True)
390 self.config_file_name, exc_info=True)
385 self.file_config = Config()
391 self.file_config = Config()
@@ -38,8 +38,10 b' import time'
38
38
39 def code_name(code, number=0):
39 def code_name(code, number=0):
40 """ Compute a (probably) unique name for code for caching.
40 """ Compute a (probably) unique name for code for caching.
41
42 This now expects code to be unicode.
41 """
43 """
42 hash_digest = hashlib.md5(code).hexdigest()
44 hash_digest = hashlib.md5(code.encode("utf-8")).hexdigest()
43 # Include the number and 12 characters of the hash in the name. It's
45 # Include the number and 12 characters of the hash in the name. It's
44 # pretty much impossible that in a single session we'll have collisions
46 # pretty much impossible that in a single session we'll have collisions
45 # even with truncated hashes, and the full one makes tracebacks too long
47 # even with truncated hashes, and the full one makes tracebacks too long
@@ -66,6 +66,7 b' from __future__ import print_function'
66 # Imports
66 # Imports
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68 # stdlib
68 # stdlib
69 import ast
69 import codeop
70 import codeop
70 import re
71 import re
71 import sys
72 import sys
@@ -185,9 +186,6 b' def split_blocks(python):'
185 commands : list of str
186 commands : list of str
186 Separate commands that can be exec'ed independently.
187 Separate commands that can be exec'ed independently.
187 """
188 """
188
189 import compiler
190
191 # compiler.parse treats trailing spaces after a newline as a
189 # compiler.parse treats trailing spaces after a newline as a
192 # SyntaxError. This is different than codeop.CommandCompiler, which
190 # SyntaxError. This is different than codeop.CommandCompiler, which
193 # will compile the trailng spaces just fine. We simply strip any
191 # will compile the trailng spaces just fine. We simply strip any
@@ -197,22 +195,15 b' def split_blocks(python):'
197 python_ori = python # save original in case we bail on error
195 python_ori = python # save original in case we bail on error
198 python = python.strip()
196 python = python.strip()
199
197
200 # The compiler module does not like unicode. We need to convert
201 # it encode it:
202 if isinstance(python, unicode):
203 # Use the utf-8-sig BOM so the compiler detects this a UTF-8
204 # encode string.
205 python = '\xef\xbb\xbf' + python.encode('utf-8')
206
207 # The compiler module will parse the code into an abstract syntax tree.
198 # The compiler module will parse the code into an abstract syntax tree.
208 # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
199 # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
209 try:
200 try:
210 ast = compiler.parse(python)
201 code_ast = ast.parse(python)
211 except:
202 except:
212 return [python_ori]
203 return [python_ori]
213
204
214 # Uncomment to help debug the ast tree
205 # Uncomment to help debug the ast tree
215 # for n in ast.node:
206 # for n in code_ast.body:
216 # print n.lineno,'->',n
207 # print n.lineno,'->',n
217
208
218 # Each separate command is available by iterating over ast.node. The
209 # Each separate command is available by iterating over ast.node. The
@@ -223,14 +214,7 b' def split_blocks(python):'
223 # other situations that cause Discard nodes that shouldn't be discarded.
214 # other situations that cause Discard nodes that shouldn't be discarded.
224 # We might eventually discover other cases where lineno is None and have
215 # We might eventually discover other cases where lineno is None and have
225 # to put in a more sophisticated test.
216 # to put in a more sophisticated test.
226 linenos = [x.lineno-1 for x in ast.node if x.lineno is not None]
217 linenos = [x.lineno-1 for x in code_ast.body if x.lineno is not None]
227
228 # When we have a bare string as the first statement, it does not end up as
229 # a Discard Node in the AST as we might expect. Instead, it gets interpreted
230 # as the docstring of the module. Check for this case and prepend 0 (the
231 # first line number) to the list of linenos to account for it.
232 if ast.doc is not None:
233 linenos.insert(0, 0)
234
218
235 # When we finally get the slices, we will need to slice all the way to
219 # When we finally get the slices, we will need to slice all the way to
236 # the end even though we don't have a line number for it. Fortunately,
220 # the end even though we don't have a line number for it. Fortunately,
@@ -614,7 +598,7 b' class InputSplitter(object):'
614 setattr(self, store, self._set_source(buffer))
598 setattr(self, store, self._set_source(buffer))
615
599
616 def _set_source(self, buffer):
600 def _set_source(self, buffer):
617 return ''.join(buffer).encode(self.encoding)
601 return u''.join(buffer)
618
602
619
603
620 #-----------------------------------------------------------------------------
604 #-----------------------------------------------------------------------------
@@ -1550,12 +1550,14 b' class InteractiveShell(Configurable, Magic):'
1550 # otherwise we end up with a monster history after a while:
1550 # otherwise we end up with a monster history after a while:
1551 readline.set_history_length(self.history_length)
1551 readline.set_history_length(self.history_length)
1552
1552
1553 stdin_encoding = sys.stdin.encoding or "utf-8"
1554
1553 # Load the last 1000 lines from history
1555 # Load the last 1000 lines from history
1554 for _, _, cell in self.history_manager.get_tail(1000,
1556 for _, _, cell in self.history_manager.get_tail(1000,
1555 include_latest=True):
1557 include_latest=True):
1556 if cell.strip(): # Ignore blank lines
1558 if cell.strip(): # Ignore blank lines
1557 for line in cell.splitlines():
1559 for line in cell.splitlines():
1558 readline.add_history(line)
1560 readline.add_history(line.encode(stdin_encoding))
1559
1561
1560 # Configure auto-indent for all platforms
1562 # Configure auto-indent for all platforms
1561 self.set_autoindent(self.autoindent)
1563 self.set_autoindent(self.autoindent)
@@ -2106,7 +2108,6 b' class InteractiveShell(Configurable, Magic):'
2106 cell = self.prefilter_manager.prefilter_line(blocks[0])
2108 cell = self.prefilter_manager.prefilter_line(blocks[0])
2107 blocks = self.input_splitter.split_blocks(cell)
2109 blocks = self.input_splitter.split_blocks(cell)
2108
2110
2109
2110 # Store the 'ipython' version of the cell as well, since that's what
2111 # Store the 'ipython' version of the cell as well, since that's what
2111 # needs to go into the translated history and get executed (the
2112 # needs to go into the translated history and get executed (the
2112 # original cell may contain non-python syntax).
2113 # original cell may contain non-python syntax).
@@ -2246,7 +2247,7 b' class InteractiveShell(Configurable, Magic):'
2246 else:
2247 else:
2247 usource = source
2248 usource = source
2248
2249
2249 if 0: # dbg
2250 if False: # dbg
2250 print 'Source:', repr(source) # dbg
2251 print 'Source:', repr(source) # dbg
2251 print 'USource:', repr(usource) # dbg
2252 print 'USource:', repr(usource) # dbg
2252 print 'type:', type(source) # dbg
2253 print 'type:', type(source) # dbg
@@ -2063,7 +2063,8 b' Currently the magic system has the following functions:\\n"""'
2063 return
2063 return
2064 cmds = self.extract_input_lines(ranges, 'r' in opts)
2064 cmds = self.extract_input_lines(ranges, 'r' in opts)
2065 with open(fname,'w') as f:
2065 with open(fname,'w') as f:
2066 f.write(cmds)
2066 f.write("# coding: utf-8\n")
2067 f.write(cmds.encode("utf-8"))
2067 print 'The following commands were written to file `%s`:' % fname
2068 print 'The following commands were written to file `%s`:' % fname
2068 print cmds
2069 print cmds
2069
2070
@@ -1,3 +1,4 b''
1 # coding: utf-8
1 """Tests for the compilerop module.
2 """Tests for the compilerop module.
2 """
3 """
3 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
@@ -15,6 +16,7 b' from __future__ import print_function'
15
16
16 # Stdlib imports
17 # Stdlib imports
17 import linecache
18 import linecache
19 import sys
18
20
19 # Third-party imports
21 # Third-party imports
20 import nose.tools as nt
22 import nose.tools as nt
@@ -46,6 +48,16 b' def test_compiler():'
46 cp('x=1', 'single')
48 cp('x=1', 'single')
47 nt.assert_true(len(linecache.cache) > ncache)
49 nt.assert_true(len(linecache.cache) > ncache)
48
50
51 def setUp():
52 # Check we're in a proper Python 2 environment (some imports, such
53 # as GTK, can change the default encoding, which can hide bugs.)
54 nt.assert_equal(sys.getdefaultencoding(), "ascii")
55
56 def test_compiler_unicode():
57 cp = compilerop.CachingCompiler()
58 ncache = len(linecache.cache)
59 cp(u"t = 'žćčšđ'", "single")
60 nt.assert_true(len(linecache.cache) > ncache)
49
61
50 def test_compiler_check_cache():
62 def test_compiler_check_cache():
51 """Test the compiler properly manages the cache.
63 """Test the compiler properly manages the cache.
@@ -1,3 +1,4 b''
1 # coding: utf-8
1 """Tests for the IPython tab-completion machinery.
2 """Tests for the IPython tab-completion machinery.
2 """
3 """
3 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
@@ -16,8 +17,10 b' import nose.tools as nt'
16 from IPython.utils.tempdir import TemporaryDirectory
17 from IPython.utils.tempdir import TemporaryDirectory
17 from IPython.core.history import HistoryManager, extract_hist_ranges
18 from IPython.core.history import HistoryManager, extract_hist_ranges
18
19
19 def test_history():
20 def setUp():
21 nt.assert_equal(sys.getdefaultencoding(), "ascii")
20
22
23 def test_history():
21 ip = get_ipython()
24 ip = get_ipython()
22 with TemporaryDirectory() as tmpdir:
25 with TemporaryDirectory() as tmpdir:
23 #tmpdir = '/software/temp'
26 #tmpdir = '/software/temp'
@@ -32,7 +35,7 b' def test_history():'
32 ip.history_manager.init_db() # Has to be called after changing file
35 ip.history_manager.init_db() # Has to be called after changing file
33 ip.history_manager.reset()
36 ip.history_manager.reset()
34 print 'test',histfile
37 print 'test',histfile
35 hist = ['a=1', 'def f():\n test = 1\n return test', 'b=2']
38 hist = ['a=1', 'def f():\n test = 1\n return test', u"b='€Æ¾÷ß'"]
36 for i, h in enumerate(hist, start=1):
39 for i, h in enumerate(hist, start=1):
37 ip.history_manager.store_inputs(i, h)
40 ip.history_manager.store_inputs(i, h)
38
41
@@ -82,7 +85,8 b' def test_history():'
82 testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
85 testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
83 ip.magic_save(testfilename + " ~1/1-3")
86 ip.magic_save(testfilename + " ~1/1-3")
84 testfile = open(testfilename, "r")
87 testfile = open(testfilename, "r")
85 nt.assert_equal(testfile.read(), "\n".join(hist))
88 nt.assert_equal(testfile.read().decode("utf-8"),
89 "# coding: utf-8\n" + "\n".join(hist))
86
90
87 # Duplicate line numbers - check that it doesn't crash, and
91 # Duplicate line numbers - check that it doesn't crash, and
88 # gets a new session
92 # gets a new session
@@ -92,6 +96,7 b' def test_history():'
92 # Restore history manager
96 # Restore history manager
93 ip.history_manager = hist_manager_ori
97 ip.history_manager = hist_manager_ori
94
98
99
95 def test_extract_hist_ranges():
100 def test_extract_hist_ranges():
96 instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5"
101 instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5"
97 expected = [(0, 1, 2), # 0 == current session
102 expected = [(0, 1, 2), # 0 == current session
@@ -364,7 +364,7 b' class InputSplitterTestCase(unittest.TestCase):'
364 def test_unicode(self):
364 def test_unicode(self):
365 self.isp.push(u"Pérez")
365 self.isp.push(u"Pérez")
366 self.isp.push(u'\xc3\xa9')
366 self.isp.push(u'\xc3\xa9')
367 self.isp.push("u'\xc3\xa9'")
367 self.isp.push(u"u'\xc3\xa9'")
368
368
369 class InteractiveLoopTestCase(unittest.TestCase):
369 class InteractiveLoopTestCase(unittest.TestCase):
370 """Tests for an interactive loop like a python shell.
370 """Tests for an interactive loop like a python shell.
@@ -293,9 +293,9 b' def test_parse_options():'
293
293
294 def test_dirops():
294 def test_dirops():
295 """Test various directory handling operations."""
295 """Test various directory handling operations."""
296 curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
296 curpath = lambda :os.path.splitdrive(os.getcwdu())[1].replace('\\','/')
297
297
298 startdir = os.getcwd()
298 startdir = os.getcwdu()
299 ipdir = _ip.ipython_dir
299 ipdir = _ip.ipython_dir
300 try:
300 try:
301 _ip.magic('cd "%s"' % ipdir)
301 _ip.magic('cd "%s"' % ipdir)
@@ -105,8 +105,6 b" have['zope.interface'] = test_for('zope.interface')"
105 have['twisted'] = test_for('twisted')
105 have['twisted'] = test_for('twisted')
106 have['foolscap'] = test_for('foolscap')
106 have['foolscap'] = test_for('foolscap')
107 have['pexpect'] = test_for('pexpect')
107 have['pexpect'] = test_for('pexpect')
108 have['gtk'] = test_for('gtk')
109 have['gobject'] = test_for('gobject')
110
108
111 #-----------------------------------------------------------------------------
109 #-----------------------------------------------------------------------------
112 # Functions and classes
110 # Functions and classes
@@ -171,7 +169,8 b' def make_exclude():'
171 if not have['wx']:
169 if not have['wx']:
172 exclusions.append(ipjoin('lib', 'inputhookwx'))
170 exclusions.append(ipjoin('lib', 'inputhookwx'))
173
171
174 if not have['gtk'] or not have['gobject']:
172 # We do this unconditionally, so that the test suite doesn't import
173 # gtk, changing the default encoding and masking some unicode bugs.
175 exclusions.append(ipjoin('lib', 'inputhookgtk'))
174 exclusions.append(ipjoin('lib', 'inputhookgtk'))
176
175
177 # These have to be skipped on win32 because the use echo, rm, cd, etc.
176 # These have to be skipped on win32 because the use echo, rm, cd, etc.
General Comments 0
You need to be logged in to leave comments. Login now