##// END OF EJS Templates
Clean up converted code....
Thomas Kluyver -
Show More
@@ -1,221 +1,221 b''
1 """Logger class for IPython's logging facilities.
1 """Logger class for IPython's logging facilities.
2 """
2 """
3 from __future__ import print_function
3 from __future__ import print_function
4
4
5 #*****************************************************************************
5 #*****************************************************************************
6 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
6 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
7 # Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
7 # Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #*****************************************************************************
11 #*****************************************************************************
12
12
13 #****************************************************************************
13 #****************************************************************************
14 # Modules and globals
14 # Modules and globals
15
15
16 # Python standard modules
16 # Python standard modules
17 import glob
17 import glob
18 import io
18 import io
19 import os
19 import os
20 import time
20 import time
21
21
22 from IPython.utils.py3compat import str_to_unicode
22 from IPython.utils.py3compat import str_to_unicode
23
23
24 #****************************************************************************
24 #****************************************************************************
25 # FIXME: This class isn't a mixin anymore, but it still needs attributes from
25 # FIXME: This class isn't a mixin anymore, but it still needs attributes from
26 # ipython and does input cache management. Finish cleanup later...
26 # ipython and does input cache management. Finish cleanup later...
27
27
28 class Logger(object):
28 class Logger(object):
29 """A Logfile class with different policies for file creation"""
29 """A Logfile class with different policies for file creation"""
30
30
31 def __init__(self, home_dir, logfname='Logger.log', loghead=u'',
31 def __init__(self, home_dir, logfname='Logger.log', loghead=u'',
32 logmode='over'):
32 logmode='over'):
33
33
34 # this is the full ipython instance, we need some attributes from it
34 # this is the full ipython instance, we need some attributes from it
35 # which won't exist until later. What a mess, clean up later...
35 # which won't exist until later. What a mess, clean up later...
36 self.home_dir = home_dir
36 self.home_dir = home_dir
37
37
38 self.logfname = logfname
38 self.logfname = logfname
39 self.loghead = loghead
39 self.loghead = loghead
40 self.logmode = logmode
40 self.logmode = logmode
41 self.logfile = None
41 self.logfile = None
42
42
43 # Whether to log raw or processed input
43 # Whether to log raw or processed input
44 self.log_raw_input = False
44 self.log_raw_input = False
45
45
46 # whether to also log output
46 # whether to also log output
47 self.log_output = False
47 self.log_output = False
48
48
49 # whether to put timestamps before each log entry
49 # whether to put timestamps before each log entry
50 self.timestamp = False
50 self.timestamp = False
51
51
52 # activity control flags
52 # activity control flags
53 self.log_active = False
53 self.log_active = False
54
54
55 # logmode is a validated property
55 # logmode is a validated property
56 def _set_mode(self,mode):
56 def _set_mode(self,mode):
57 if mode not in ['append','backup','global','over','rotate']:
57 if mode not in ['append','backup','global','over','rotate']:
58 raise ValueError('invalid log mode %s given' % mode)
58 raise ValueError('invalid log mode %s given' % mode)
59 self._logmode = mode
59 self._logmode = mode
60
60
61 def _get_mode(self):
61 def _get_mode(self):
62 return self._logmode
62 return self._logmode
63
63
64 logmode = property(_get_mode,_set_mode)
64 logmode = property(_get_mode,_set_mode)
65
65
66 def logstart(self, logfname=None, loghead=None, logmode=None,
66 def logstart(self, logfname=None, loghead=None, logmode=None,
67 log_output=False, timestamp=False, log_raw_input=False):
67 log_output=False, timestamp=False, log_raw_input=False):
68 """Generate a new log-file with a default header.
68 """Generate a new log-file with a default header.
69
69
70 Raises RuntimeError if the log has already been started"""
70 Raises RuntimeError if the log has already been started"""
71
71
72 if self.logfile is not None:
72 if self.logfile is not None:
73 raise RuntimeError('Log file is already active: %s' %
73 raise RuntimeError('Log file is already active: %s' %
74 self.logfname)
74 self.logfname)
75
75
76 # The parameters can override constructor defaults
76 # The parameters can override constructor defaults
77 if logfname is not None: self.logfname = logfname
77 if logfname is not None: self.logfname = logfname
78 if loghead is not None: self.loghead = loghead
78 if loghead is not None: self.loghead = loghead
79 if logmode is not None: self.logmode = logmode
79 if logmode is not None: self.logmode = logmode
80
80
81 # Parameters not part of the constructor
81 # Parameters not part of the constructor
82 self.timestamp = timestamp
82 self.timestamp = timestamp
83 self.log_output = log_output
83 self.log_output = log_output
84 self.log_raw_input = log_raw_input
84 self.log_raw_input = log_raw_input
85
85
86 # init depending on the log mode requested
86 # init depending on the log mode requested
87 isfile = os.path.isfile
87 isfile = os.path.isfile
88 logmode = self.logmode
88 logmode = self.logmode
89
89
90 if logmode == 'append':
90 if logmode == 'append':
91 self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
91 self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
92
92
93 elif logmode == 'backup':
93 elif logmode == 'backup':
94 if isfile(self.logfname):
94 if isfile(self.logfname):
95 backup_logname = self.logfname+'~'
95 backup_logname = self.logfname+'~'
96 # Manually remove any old backup, since os.rename may fail
96 # Manually remove any old backup, since os.rename may fail
97 # under Windows.
97 # under Windows.
98 if isfile(backup_logname):
98 if isfile(backup_logname):
99 os.remove(backup_logname)
99 os.remove(backup_logname)
100 os.rename(self.logfname,backup_logname)
100 os.rename(self.logfname,backup_logname)
101 self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
101 self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
102
102
103 elif logmode == 'global':
103 elif logmode == 'global':
104 self.logfname = os.path.join(self.home_dir,self.logfname)
104 self.logfname = os.path.join(self.home_dir,self.logfname)
105 self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
105 self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
106
106
107 elif logmode == 'over':
107 elif logmode == 'over':
108 if isfile(self.logfname):
108 if isfile(self.logfname):
109 os.remove(self.logfname)
109 os.remove(self.logfname)
110 self.logfile = io.open(self.logfname,'w', encoding='utf-8')
110 self.logfile = io.open(self.logfname,'w', encoding='utf-8')
111
111
112 elif logmode == 'rotate':
112 elif logmode == 'rotate':
113 if isfile(self.logfname):
113 if isfile(self.logfname):
114 if isfile(self.logfname+'.001~'):
114 if isfile(self.logfname+'.001~'):
115 old = glob.glob(self.logfname+'.*~')
115 old = glob.glob(self.logfname+'.*~')
116 old.sort()
116 old.sort()
117 old.reverse()
117 old.reverse()
118 for f in old:
118 for f in old:
119 root, ext = os.path.splitext(f)
119 root, ext = os.path.splitext(f)
120 num = int(ext[1:-1])+1
120 num = int(ext[1:-1])+1
121 os.rename(f, root+'.'+repr(num).zfill(3)+'~')
121 os.rename(f, root+'.'+repr(num).zfill(3)+'~')
122 os.rename(self.logfname, self.logfname+'.001~')
122 os.rename(self.logfname, self.logfname+'.001~')
123 self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
123 self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
124
124
125 if logmode != 'append':
125 if logmode != 'append':
126 self.logfile.write(self.loghead)
126 self.logfile.write(self.loghead)
127
127
128 self.logfile.flush()
128 self.logfile.flush()
129 self.log_active = True
129 self.log_active = True
130
130
131 def switch_log(self,val):
131 def switch_log(self,val):
132 """Switch logging on/off. val should be ONLY a boolean."""
132 """Switch logging on/off. val should be ONLY a boolean."""
133
133
134 if val not in [False,True,0,1]:
134 if val not in [False,True,0,1]:
135 raise ValueError('Call switch_log ONLY with a boolean argument, '
135 raise ValueError('Call switch_log ONLY with a boolean argument, '
136 'not with: %s' % val)
136 'not with: %s' % val)
137
137
138 label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}
138 label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}
139
139
140 if self.logfile is None:
140 if self.logfile is None:
141 print("""
141 print("""
142 Logging hasn't been started yet (use logstart for that).
142 Logging hasn't been started yet (use logstart for that).
143
143
144 %logon/%logoff are for temporarily starting and stopping logging for a logfile
144 %logon/%logoff are for temporarily starting and stopping logging for a logfile
145 which already exists. But you must first start the logging process with
145 which already exists. But you must first start the logging process with
146 %logstart (optionally giving a logfile name).""")
146 %logstart (optionally giving a logfile name).""")
147
147
148 else:
148 else:
149 if self.log_active == val:
149 if self.log_active == val:
150 print('Logging is already',label[val])
150 print('Logging is already',label[val])
151 else:
151 else:
152 print('Switching logging',label[val])
152 print('Switching logging',label[val])
153 self.log_active = not self.log_active
153 self.log_active = not self.log_active
154 self.log_active_out = self.log_active
154 self.log_active_out = self.log_active
155
155
156 def logstate(self):
156 def logstate(self):
157 """Print a status message about the logger."""
157 """Print a status message about the logger."""
158 if self.logfile is None:
158 if self.logfile is None:
159 print('Logging has not been activated.')
159 print('Logging has not been activated.')
160 else:
160 else:
161 state = self.log_active and 'active' or 'temporarily suspended'
161 state = self.log_active and 'active' or 'temporarily suspended'
162 print('Filename :',self.logfname)
162 print('Filename :', self.logfname)
163 print('Mode :',self.logmode)
163 print('Mode :', self.logmode)
164 print('Output logging :',self.log_output)
164 print('Output logging :', self.log_output)
165 print('Raw input log :',self.log_raw_input)
165 print('Raw input log :', self.log_raw_input)
166 print('Timestamping :',self.timestamp)
166 print('Timestamping :', self.timestamp)
167 print('State :',state)
167 print('State :', state)
168
168
169 def log(self, line_mod, line_ori):
169 def log(self, line_mod, line_ori):
170 """Write the sources to a log.
170 """Write the sources to a log.
171
171
172 Inputs:
172 Inputs:
173
173
174 - line_mod: possibly modified input, such as the transformations made
174 - line_mod: possibly modified input, such as the transformations made
175 by input prefilters or input handlers of various kinds. This should
175 by input prefilters or input handlers of various kinds. This should
176 always be valid Python.
176 always be valid Python.
177
177
178 - line_ori: unmodified input line from the user. This is not
178 - line_ori: unmodified input line from the user. This is not
179 necessarily valid Python.
179 necessarily valid Python.
180 """
180 """
181
181
182 # Write the log line, but decide which one according to the
182 # Write the log line, but decide which one according to the
183 # log_raw_input flag, set when the log is started.
183 # log_raw_input flag, set when the log is started.
184 if self.log_raw_input:
184 if self.log_raw_input:
185 self.log_write(line_ori)
185 self.log_write(line_ori)
186 else:
186 else:
187 self.log_write(line_mod)
187 self.log_write(line_mod)
188
188
189 def log_write(self, data, kind='input'):
189 def log_write(self, data, kind='input'):
190 """Write data to the log file, if active"""
190 """Write data to the log file, if active"""
191
191
192 #print 'data: %r' % data # dbg
192 #print 'data: %r' % data # dbg
193 if self.log_active and data:
193 if self.log_active and data:
194 write = self.logfile.write
194 write = self.logfile.write
195 if kind=='input':
195 if kind=='input':
196 if self.timestamp:
196 if self.timestamp:
197 write(str_to_unicode(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
197 write(str_to_unicode(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
198 time.localtime())))
198 time.localtime())))
199 write(data)
199 write(data)
200 elif kind=='output' and self.log_output:
200 elif kind=='output' and self.log_output:
201 odata = u'\n'.join([u'#[Out]# %s' % s
201 odata = u'\n'.join([u'#[Out]# %s' % s
202 for s in data.splitlines()])
202 for s in data.splitlines()])
203 write(u'%s\n' % odata)
203 write(u'%s\n' % odata)
204 self.logfile.flush()
204 self.logfile.flush()
205
205
206 def logstop(self):
206 def logstop(self):
207 """Fully stop logging and close log file.
207 """Fully stop logging and close log file.
208
208
209 In order to start logging again, a new logstart() call needs to be
209 In order to start logging again, a new logstart() call needs to be
210 made, possibly (though not necessarily) with a new filename, mode and
210 made, possibly (though not necessarily) with a new filename, mode and
211 other options."""
211 other options."""
212
212
213 if self.logfile is not None:
213 if self.logfile is not None:
214 self.logfile.close()
214 self.logfile.close()
215 self.logfile = None
215 self.logfile = None
216 else:
216 else:
217 print("Logging hadn't been started.")
217 print("Logging hadn't been started.")
218 self.log_active = False
218 self.log_active = False
219
219
220 # For backwards compatibility, in case anyone was using this.
220 # For backwards compatibility, in case anyone was using this.
221 close_log = logstop
221 close_log = logstop
@@ -1,740 +1,740 b''
1 """Implementation of magic functions for interaction with the OS.
1 """Implementation of magic functions for interaction with the OS.
2
2
3 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
3 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
4 builtin.
4 builtin.
5 """
5 """
6 from __future__ import print_function
6 from __future__ import print_function
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (c) 2012 The IPython Development Team.
8 # Copyright (c) 2012 The IPython Development Team.
9 #
9 #
10 # Distributed under the terms of the Modified BSD License.
10 # Distributed under the terms of the Modified BSD License.
11 #
11 #
12 # The full license is in the file COPYING.txt, distributed with this software.
12 # The full license is in the file COPYING.txt, distributed with this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 # Stdlib
19 # Stdlib
20 import io
20 import io
21 import os
21 import os
22 import re
22 import re
23 import sys
23 import sys
24 from pprint import pformat
24 from pprint import pformat
25
25
26 # Our own packages
26 # Our own packages
27 from IPython.core import magic_arguments
27 from IPython.core import magic_arguments
28 from IPython.core import oinspect
28 from IPython.core import oinspect
29 from IPython.core import page
29 from IPython.core import page
30 from IPython.core.alias import AliasError, Alias
30 from IPython.core.alias import AliasError, Alias
31 from IPython.core.error import UsageError
31 from IPython.core.error import UsageError
32 from IPython.core.magic import (
32 from IPython.core.magic import (
33 Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
33 Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
34 )
34 )
35 from IPython.testing.skipdoctest import skip_doctest
35 from IPython.testing.skipdoctest import skip_doctest
36 from IPython.utils.openpy import source_to_unicode
36 from IPython.utils.openpy import source_to_unicode
37 from IPython.utils.path import unquote_filename
37 from IPython.utils.path import unquote_filename
38 from IPython.utils.process import abbrev_cwd
38 from IPython.utils.process import abbrev_cwd
39 from IPython.utils.py3compat import unicode_type
39 from IPython.utils.py3compat import unicode_type
40 from IPython.utils.terminal import set_term_title
40 from IPython.utils.terminal import set_term_title
41
41
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43 # Magic implementation classes
43 # Magic implementation classes
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45 @magics_class
45 @magics_class
46 class OSMagics(Magics):
46 class OSMagics(Magics):
47 """Magics to interact with the underlying OS (shell-type functionality).
47 """Magics to interact with the underlying OS (shell-type functionality).
48 """
48 """
49
49
50 @skip_doctest
50 @skip_doctest
51 @line_magic
51 @line_magic
52 def alias(self, parameter_s=''):
52 def alias(self, parameter_s=''):
53 """Define an alias for a system command.
53 """Define an alias for a system command.
54
54
55 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
55 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
56
56
57 Then, typing 'alias_name params' will execute the system command 'cmd
57 Then, typing 'alias_name params' will execute the system command 'cmd
58 params' (from your underlying operating system).
58 params' (from your underlying operating system).
59
59
60 Aliases have lower precedence than magic functions and Python normal
60 Aliases have lower precedence than magic functions and Python normal
61 variables, so if 'foo' is both a Python variable and an alias, the
61 variables, so if 'foo' is both a Python variable and an alias, the
62 alias can not be executed until 'del foo' removes the Python variable.
62 alias can not be executed until 'del foo' removes the Python variable.
63
63
64 You can use the %l specifier in an alias definition to represent the
64 You can use the %l specifier in an alias definition to represent the
65 whole line when the alias is called. For example::
65 whole line when the alias is called. For example::
66
66
67 In [2]: alias bracket echo "Input in brackets: <%l>"
67 In [2]: alias bracket echo "Input in brackets: <%l>"
68 In [3]: bracket hello world
68 In [3]: bracket hello world
69 Input in brackets: <hello world>
69 Input in brackets: <hello world>
70
70
71 You can also define aliases with parameters using %s specifiers (one
71 You can also define aliases with parameters using %s specifiers (one
72 per parameter)::
72 per parameter)::
73
73
74 In [1]: alias parts echo first %s second %s
74 In [1]: alias parts echo first %s second %s
75 In [2]: %parts A B
75 In [2]: %parts A B
76 first A second B
76 first A second B
77 In [3]: %parts A
77 In [3]: %parts A
78 Incorrect number of arguments: 2 expected.
78 Incorrect number of arguments: 2 expected.
79 parts is an alias to: 'echo first %s second %s'
79 parts is an alias to: 'echo first %s second %s'
80
80
81 Note that %l and %s are mutually exclusive. You can only use one or
81 Note that %l and %s are mutually exclusive. You can only use one or
82 the other in your aliases.
82 the other in your aliases.
83
83
84 Aliases expand Python variables just like system calls using ! or !!
84 Aliases expand Python variables just like system calls using ! or !!
85 do: all expressions prefixed with '$' get expanded. For details of
85 do: all expressions prefixed with '$' get expanded. For details of
86 the semantic rules, see PEP-215:
86 the semantic rules, see PEP-215:
87 http://www.python.org/peps/pep-0215.html. This is the library used by
87 http://www.python.org/peps/pep-0215.html. This is the library used by
88 IPython for variable expansion. If you want to access a true shell
88 IPython for variable expansion. If you want to access a true shell
89 variable, an extra $ is necessary to prevent its expansion by
89 variable, an extra $ is necessary to prevent its expansion by
90 IPython::
90 IPython::
91
91
92 In [6]: alias show echo
92 In [6]: alias show echo
93 In [7]: PATH='A Python string'
93 In [7]: PATH='A Python string'
94 In [8]: show $PATH
94 In [8]: show $PATH
95 A Python string
95 A Python string
96 In [9]: show $$PATH
96 In [9]: show $$PATH
97 /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
97 /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
98
98
99 You can use the alias facility to acess all of $PATH. See the %rehash
99 You can use the alias facility to acess all of $PATH. See the %rehash
100 and %rehashx functions, which automatically create aliases for the
100 and %rehashx functions, which automatically create aliases for the
101 contents of your $PATH.
101 contents of your $PATH.
102
102
103 If called with no parameters, %alias prints the current alias table."""
103 If called with no parameters, %alias prints the current alias table."""
104
104
105 par = parameter_s.strip()
105 par = parameter_s.strip()
106 if not par:
106 if not par:
107 aliases = sorted(self.shell.alias_manager.aliases)
107 aliases = sorted(self.shell.alias_manager.aliases)
108 # stored = self.shell.db.get('stored_aliases', {} )
108 # stored = self.shell.db.get('stored_aliases', {} )
109 # for k, v in stored:
109 # for k, v in stored:
110 # atab.append(k, v[0])
110 # atab.append(k, v[0])
111
111
112 print("Total number of aliases:", len(aliases))
112 print("Total number of aliases:", len(aliases))
113 sys.stdout.flush()
113 sys.stdout.flush()
114 return aliases
114 return aliases
115
115
116 # Now try to define a new one
116 # Now try to define a new one
117 try:
117 try:
118 alias,cmd = par.split(None, 1)
118 alias,cmd = par.split(None, 1)
119 except TypeError:
119 except TypeError:
120 print((oinspect.getdoc(self.alias)))
120 print(oinspect.getdoc(self.alias))
121 return
121 return
122
122
123 try:
123 try:
124 self.shell.alias_manager.define_alias(alias, cmd)
124 self.shell.alias_manager.define_alias(alias, cmd)
125 except AliasError as e:
125 except AliasError as e:
126 print(e)
126 print(e)
127 # end magic_alias
127 # end magic_alias
128
128
129 @line_magic
129 @line_magic
130 def unalias(self, parameter_s=''):
130 def unalias(self, parameter_s=''):
131 """Remove an alias"""
131 """Remove an alias"""
132
132
133 aname = parameter_s.strip()
133 aname = parameter_s.strip()
134 try:
134 try:
135 self.shell.alias_manager.undefine_alias(aname)
135 self.shell.alias_manager.undefine_alias(aname)
136 except ValueError as e:
136 except ValueError as e:
137 print(e)
137 print(e)
138 return
138 return
139
139
140 stored = self.shell.db.get('stored_aliases', {} )
140 stored = self.shell.db.get('stored_aliases', {} )
141 if aname in stored:
141 if aname in stored:
142 print("Removing %stored alias",aname)
142 print("Removing %stored alias",aname)
143 del stored[aname]
143 del stored[aname]
144 self.shell.db['stored_aliases'] = stored
144 self.shell.db['stored_aliases'] = stored
145
145
146 @line_magic
146 @line_magic
147 def rehashx(self, parameter_s=''):
147 def rehashx(self, parameter_s=''):
148 """Update the alias table with all executable files in $PATH.
148 """Update the alias table with all executable files in $PATH.
149
149
150 This version explicitly checks that every entry in $PATH is a file
150 This version explicitly checks that every entry in $PATH is a file
151 with execute access (os.X_OK), so it is much slower than %rehash.
151 with execute access (os.X_OK), so it is much slower than %rehash.
152
152
153 Under Windows, it checks executability as a match against a
153 Under Windows, it checks executability as a match against a
154 '|'-separated string of extensions, stored in the IPython config
154 '|'-separated string of extensions, stored in the IPython config
155 variable win_exec_ext. This defaults to 'exe|com|bat'.
155 variable win_exec_ext. This defaults to 'exe|com|bat'.
156
156
157 This function also resets the root module cache of module completer,
157 This function also resets the root module cache of module completer,
158 used on slow filesystems.
158 used on slow filesystems.
159 """
159 """
160 from IPython.core.alias import InvalidAliasError
160 from IPython.core.alias import InvalidAliasError
161
161
162 # for the benefit of module completer in ipy_completers.py
162 # for the benefit of module completer in ipy_completers.py
163 del self.shell.db['rootmodules_cache']
163 del self.shell.db['rootmodules_cache']
164
164
165 path = [os.path.abspath(os.path.expanduser(p)) for p in
165 path = [os.path.abspath(os.path.expanduser(p)) for p in
166 os.environ.get('PATH','').split(os.pathsep)]
166 os.environ.get('PATH','').split(os.pathsep)]
167 path = filter(os.path.isdir,path)
167 path = filter(os.path.isdir,path)
168
168
169 syscmdlist = []
169 syscmdlist = []
170 # Now define isexec in a cross platform manner.
170 # Now define isexec in a cross platform manner.
171 if os.name == 'posix':
171 if os.name == 'posix':
172 isexec = lambda fname:os.path.isfile(fname) and \
172 isexec = lambda fname:os.path.isfile(fname) and \
173 os.access(fname,os.X_OK)
173 os.access(fname,os.X_OK)
174 else:
174 else:
175 try:
175 try:
176 winext = os.environ['pathext'].replace(';','|').replace('.','')
176 winext = os.environ['pathext'].replace(';','|').replace('.','')
177 except KeyError:
177 except KeyError:
178 winext = 'exe|com|bat|py'
178 winext = 'exe|com|bat|py'
179 if 'py' not in winext:
179 if 'py' not in winext:
180 winext += '|py'
180 winext += '|py'
181 execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
181 execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
182 isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
182 isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
183 savedir = os.getcwdu()
183 savedir = os.getcwdu()
184
184
185 # Now walk the paths looking for executables to alias.
185 # Now walk the paths looking for executables to alias.
186 try:
186 try:
187 # write the whole loop for posix/Windows so we don't have an if in
187 # write the whole loop for posix/Windows so we don't have an if in
188 # the innermost part
188 # the innermost part
189 if os.name == 'posix':
189 if os.name == 'posix':
190 for pdir in path:
190 for pdir in path:
191 os.chdir(pdir)
191 os.chdir(pdir)
192 for ff in os.listdir(pdir):
192 for ff in os.listdir(pdir):
193 if isexec(ff):
193 if isexec(ff):
194 try:
194 try:
195 # Removes dots from the name since ipython
195 # Removes dots from the name since ipython
196 # will assume names with dots to be python.
196 # will assume names with dots to be python.
197 if not self.shell.alias_manager.is_alias(ff):
197 if not self.shell.alias_manager.is_alias(ff):
198 self.shell.alias_manager.define_alias(
198 self.shell.alias_manager.define_alias(
199 ff.replace('.',''), ff)
199 ff.replace('.',''), ff)
200 except InvalidAliasError:
200 except InvalidAliasError:
201 pass
201 pass
202 else:
202 else:
203 syscmdlist.append(ff)
203 syscmdlist.append(ff)
204 else:
204 else:
205 no_alias = Alias.blacklist
205 no_alias = Alias.blacklist
206 for pdir in path:
206 for pdir in path:
207 os.chdir(pdir)
207 os.chdir(pdir)
208 for ff in os.listdir(pdir):
208 for ff in os.listdir(pdir):
209 base, ext = os.path.splitext(ff)
209 base, ext = os.path.splitext(ff)
210 if isexec(ff) and base.lower() not in no_alias:
210 if isexec(ff) and base.lower() not in no_alias:
211 if ext.lower() == '.exe':
211 if ext.lower() == '.exe':
212 ff = base
212 ff = base
213 try:
213 try:
214 # Removes dots from the name since ipython
214 # Removes dots from the name since ipython
215 # will assume names with dots to be python.
215 # will assume names with dots to be python.
216 self.shell.alias_manager.define_alias(
216 self.shell.alias_manager.define_alias(
217 base.lower().replace('.',''), ff)
217 base.lower().replace('.',''), ff)
218 except InvalidAliasError:
218 except InvalidAliasError:
219 pass
219 pass
220 syscmdlist.append(ff)
220 syscmdlist.append(ff)
221 self.shell.db['syscmdlist'] = syscmdlist
221 self.shell.db['syscmdlist'] = syscmdlist
222 finally:
222 finally:
223 os.chdir(savedir)
223 os.chdir(savedir)
224
224
225 @skip_doctest
225 @skip_doctest
226 @line_magic
226 @line_magic
227 def pwd(self, parameter_s=''):
227 def pwd(self, parameter_s=''):
228 """Return the current working directory path.
228 """Return the current working directory path.
229
229
230 Examples
230 Examples
231 --------
231 --------
232 ::
232 ::
233
233
234 In [9]: pwd
234 In [9]: pwd
235 Out[9]: '/home/tsuser/sprint/ipython'
235 Out[9]: '/home/tsuser/sprint/ipython'
236 """
236 """
237 return os.getcwdu()
237 return os.getcwdu()
238
238
239 @skip_doctest
239 @skip_doctest
240 @line_magic
240 @line_magic
241 def cd(self, parameter_s=''):
241 def cd(self, parameter_s=''):
242 """Change the current working directory.
242 """Change the current working directory.
243
243
244 This command automatically maintains an internal list of directories
244 This command automatically maintains an internal list of directories
245 you visit during your IPython session, in the variable _dh. The
245 you visit during your IPython session, in the variable _dh. The
246 command %dhist shows this history nicely formatted. You can also
246 command %dhist shows this history nicely formatted. You can also
247 do 'cd -<tab>' to see directory history conveniently.
247 do 'cd -<tab>' to see directory history conveniently.
248
248
249 Usage:
249 Usage:
250
250
251 cd 'dir': changes to directory 'dir'.
251 cd 'dir': changes to directory 'dir'.
252
252
253 cd -: changes to the last visited directory.
253 cd -: changes to the last visited directory.
254
254
255 cd -<n>: changes to the n-th directory in the directory history.
255 cd -<n>: changes to the n-th directory in the directory history.
256
256
257 cd --foo: change to directory that matches 'foo' in history
257 cd --foo: change to directory that matches 'foo' in history
258
258
259 cd -b <bookmark_name>: jump to a bookmark set by %bookmark
259 cd -b <bookmark_name>: jump to a bookmark set by %bookmark
260 (note: cd <bookmark_name> is enough if there is no
260 (note: cd <bookmark_name> is enough if there is no
261 directory <bookmark_name>, but a bookmark with the name exists.)
261 directory <bookmark_name>, but a bookmark with the name exists.)
262 'cd -b <tab>' allows you to tab-complete bookmark names.
262 'cd -b <tab>' allows you to tab-complete bookmark names.
263
263
264 Options:
264 Options:
265
265
266 -q: quiet. Do not print the working directory after the cd command is
266 -q: quiet. Do not print the working directory after the cd command is
267 executed. By default IPython's cd command does print this directory,
267 executed. By default IPython's cd command does print this directory,
268 since the default prompts do not display path information.
268 since the default prompts do not display path information.
269
269
270 Note that !cd doesn't work for this purpose because the shell where
270 Note that !cd doesn't work for this purpose because the shell where
271 !command runs is immediately discarded after executing 'command'.
271 !command runs is immediately discarded after executing 'command'.
272
272
273 Examples
273 Examples
274 --------
274 --------
275 ::
275 ::
276
276
277 In [10]: cd parent/child
277 In [10]: cd parent/child
278 /home/tsuser/parent/child
278 /home/tsuser/parent/child
279 """
279 """
280
280
281 oldcwd = os.getcwdu()
281 oldcwd = os.getcwdu()
282 numcd = re.match(r'(-)(\d+)$',parameter_s)
282 numcd = re.match(r'(-)(\d+)$',parameter_s)
283 # jump in directory history by number
283 # jump in directory history by number
284 if numcd:
284 if numcd:
285 nn = int(numcd.group(2))
285 nn = int(numcd.group(2))
286 try:
286 try:
287 ps = self.shell.user_ns['_dh'][nn]
287 ps = self.shell.user_ns['_dh'][nn]
288 except IndexError:
288 except IndexError:
289 print('The requested directory does not exist in history.')
289 print('The requested directory does not exist in history.')
290 return
290 return
291 else:
291 else:
292 opts = {}
292 opts = {}
293 elif parameter_s.startswith('--'):
293 elif parameter_s.startswith('--'):
294 ps = None
294 ps = None
295 fallback = None
295 fallback = None
296 pat = parameter_s[2:]
296 pat = parameter_s[2:]
297 dh = self.shell.user_ns['_dh']
297 dh = self.shell.user_ns['_dh']
298 # first search only by basename (last component)
298 # first search only by basename (last component)
299 for ent in reversed(dh):
299 for ent in reversed(dh):
300 if pat in os.path.basename(ent) and os.path.isdir(ent):
300 if pat in os.path.basename(ent) and os.path.isdir(ent):
301 ps = ent
301 ps = ent
302 break
302 break
303
303
304 if fallback is None and pat in ent and os.path.isdir(ent):
304 if fallback is None and pat in ent and os.path.isdir(ent):
305 fallback = ent
305 fallback = ent
306
306
307 # if we have no last part match, pick the first full path match
307 # if we have no last part match, pick the first full path match
308 if ps is None:
308 if ps is None:
309 ps = fallback
309 ps = fallback
310
310
311 if ps is None:
311 if ps is None:
312 print("No matching entry in directory history")
312 print("No matching entry in directory history")
313 return
313 return
314 else:
314 else:
315 opts = {}
315 opts = {}
316
316
317
317
318 else:
318 else:
319 #turn all non-space-escaping backslashes to slashes,
319 #turn all non-space-escaping backslashes to slashes,
320 # for c:\windows\directory\names\
320 # for c:\windows\directory\names\
321 parameter_s = re.sub(r'\\(?! )','/', parameter_s)
321 parameter_s = re.sub(r'\\(?! )','/', parameter_s)
322 opts,ps = self.parse_options(parameter_s,'qb',mode='string')
322 opts,ps = self.parse_options(parameter_s,'qb',mode='string')
323 # jump to previous
323 # jump to previous
324 if ps == '-':
324 if ps == '-':
325 try:
325 try:
326 ps = self.shell.user_ns['_dh'][-2]
326 ps = self.shell.user_ns['_dh'][-2]
327 except IndexError:
327 except IndexError:
328 raise UsageError('%cd -: No previous directory to change to.')
328 raise UsageError('%cd -: No previous directory to change to.')
329 # jump to bookmark if needed
329 # jump to bookmark if needed
330 else:
330 else:
331 if not os.path.isdir(ps) or 'b' in opts:
331 if not os.path.isdir(ps) or 'b' in opts:
332 bkms = self.shell.db.get('bookmarks', {})
332 bkms = self.shell.db.get('bookmarks', {})
333
333
334 if ps in bkms:
334 if ps in bkms:
335 target = bkms[ps]
335 target = bkms[ps]
336 print('(bookmark:%s) -> %s' % (ps, target))
336 print('(bookmark:%s) -> %s' % (ps, target))
337 ps = target
337 ps = target
338 else:
338 else:
339 if 'b' in opts:
339 if 'b' in opts:
340 raise UsageError("Bookmark '%s' not found. "
340 raise UsageError("Bookmark '%s' not found. "
341 "Use '%%bookmark -l' to see your bookmarks." % ps)
341 "Use '%%bookmark -l' to see your bookmarks." % ps)
342
342
343 # strip extra quotes on Windows, because os.chdir doesn't like them
343 # strip extra quotes on Windows, because os.chdir doesn't like them
344 ps = unquote_filename(ps)
344 ps = unquote_filename(ps)
345 # at this point ps should point to the target dir
345 # at this point ps should point to the target dir
346 if ps:
346 if ps:
347 try:
347 try:
348 os.chdir(os.path.expanduser(ps))
348 os.chdir(os.path.expanduser(ps))
349 if hasattr(self.shell, 'term_title') and self.shell.term_title:
349 if hasattr(self.shell, 'term_title') and self.shell.term_title:
350 set_term_title('IPython: ' + abbrev_cwd())
350 set_term_title('IPython: ' + abbrev_cwd())
351 except OSError:
351 except OSError:
352 print(sys.exc_info()[1])
352 print(sys.exc_info()[1])
353 else:
353 else:
354 cwd = os.getcwdu()
354 cwd = os.getcwdu()
355 dhist = self.shell.user_ns['_dh']
355 dhist = self.shell.user_ns['_dh']
356 if oldcwd != cwd:
356 if oldcwd != cwd:
357 dhist.append(cwd)
357 dhist.append(cwd)
358 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
358 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
359
359
360 else:
360 else:
361 os.chdir(self.shell.home_dir)
361 os.chdir(self.shell.home_dir)
362 if hasattr(self.shell, 'term_title') and self.shell.term_title:
362 if hasattr(self.shell, 'term_title') and self.shell.term_title:
363 set_term_title('IPython: ' + '~')
363 set_term_title('IPython: ' + '~')
364 cwd = os.getcwdu()
364 cwd = os.getcwdu()
365 dhist = self.shell.user_ns['_dh']
365 dhist = self.shell.user_ns['_dh']
366
366
367 if oldcwd != cwd:
367 if oldcwd != cwd:
368 dhist.append(cwd)
368 dhist.append(cwd)
369 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
369 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
370 if not 'q' in opts and self.shell.user_ns['_dh']:
370 if not 'q' in opts and self.shell.user_ns['_dh']:
371 print(self.shell.user_ns['_dh'][-1])
371 print(self.shell.user_ns['_dh'][-1])
372
372
373
373
374 @line_magic
374 @line_magic
375 def env(self, parameter_s=''):
375 def env(self, parameter_s=''):
376 """List environment variables."""
376 """List environment variables."""
377
377
378 return dict(os.environ)
378 return dict(os.environ)
379
379
380 @line_magic
380 @line_magic
381 def pushd(self, parameter_s=''):
381 def pushd(self, parameter_s=''):
382 """Place the current dir on stack and change directory.
382 """Place the current dir on stack and change directory.
383
383
384 Usage:\\
384 Usage:\\
385 %pushd ['dirname']
385 %pushd ['dirname']
386 """
386 """
387
387
388 dir_s = self.shell.dir_stack
388 dir_s = self.shell.dir_stack
389 tgt = os.path.expanduser(unquote_filename(parameter_s))
389 tgt = os.path.expanduser(unquote_filename(parameter_s))
390 cwd = os.getcwdu().replace(self.shell.home_dir,'~')
390 cwd = os.getcwdu().replace(self.shell.home_dir,'~')
391 if tgt:
391 if tgt:
392 self.cd(parameter_s)
392 self.cd(parameter_s)
393 dir_s.insert(0,cwd)
393 dir_s.insert(0,cwd)
394 return self.shell.magic('dirs')
394 return self.shell.magic('dirs')
395
395
396 @line_magic
396 @line_magic
397 def popd(self, parameter_s=''):
397 def popd(self, parameter_s=''):
398 """Change to directory popped off the top of the stack.
398 """Change to directory popped off the top of the stack.
399 """
399 """
400 if not self.shell.dir_stack:
400 if not self.shell.dir_stack:
401 raise UsageError("%popd on empty stack")
401 raise UsageError("%popd on empty stack")
402 top = self.shell.dir_stack.pop(0)
402 top = self.shell.dir_stack.pop(0)
403 self.cd(top)
403 self.cd(top)
404 print("popd ->",top)
404 print("popd ->",top)
405
405
406 @line_magic
406 @line_magic
407 def dirs(self, parameter_s=''):
407 def dirs(self, parameter_s=''):
408 """Return the current directory stack."""
408 """Return the current directory stack."""
409
409
410 return self.shell.dir_stack
410 return self.shell.dir_stack
411
411
412 @line_magic
412 @line_magic
413 def dhist(self, parameter_s=''):
413 def dhist(self, parameter_s=''):
414 """Print your history of visited directories.
414 """Print your history of visited directories.
415
415
416 %dhist -> print full history\\
416 %dhist -> print full history\\
417 %dhist n -> print last n entries only\\
417 %dhist n -> print last n entries only\\
418 %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
418 %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
419
419
420 This history is automatically maintained by the %cd command, and
420 This history is automatically maintained by the %cd command, and
421 always available as the global list variable _dh. You can use %cd -<n>
421 always available as the global list variable _dh. You can use %cd -<n>
422 to go to directory number <n>.
422 to go to directory number <n>.
423
423
424 Note that most of time, you should view directory history by entering
424 Note that most of time, you should view directory history by entering
425 cd -<TAB>.
425 cd -<TAB>.
426
426
427 """
427 """
428
428
429 dh = self.shell.user_ns['_dh']
429 dh = self.shell.user_ns['_dh']
430 if parameter_s:
430 if parameter_s:
431 try:
431 try:
432 args = map(int,parameter_s.split())
432 args = map(int,parameter_s.split())
433 except:
433 except:
434 self.arg_err(self.dhist)
434 self.arg_err(self.dhist)
435 return
435 return
436 if len(args) == 1:
436 if len(args) == 1:
437 ini,fin = max(len(dh)-(args[0]),0),len(dh)
437 ini,fin = max(len(dh)-(args[0]),0),len(dh)
438 elif len(args) == 2:
438 elif len(args) == 2:
439 ini,fin = args
439 ini,fin = args
440 fin = min(fin, len(dh))
440 fin = min(fin, len(dh))
441 else:
441 else:
442 self.arg_err(self.dhist)
442 self.arg_err(self.dhist)
443 return
443 return
444 else:
444 else:
445 ini,fin = 0,len(dh)
445 ini,fin = 0,len(dh)
446 print('Directory history (kept in _dh)')
446 print('Directory history (kept in _dh)')
447 for i in range(ini, fin):
447 for i in range(ini, fin):
448 print("%d: %s" % (i, dh[i]))
448 print("%d: %s" % (i, dh[i]))
449
449
450 @skip_doctest
450 @skip_doctest
451 @line_magic
451 @line_magic
452 def sc(self, parameter_s=''):
452 def sc(self, parameter_s=''):
453 """Shell capture - run shell command and capture output (DEPRECATED use !).
453 """Shell capture - run shell command and capture output (DEPRECATED use !).
454
454
455 DEPRECATED. Suboptimal, retained for backwards compatibility.
455 DEPRECATED. Suboptimal, retained for backwards compatibility.
456
456
457 You should use the form 'var = !command' instead. Example:
457 You should use the form 'var = !command' instead. Example:
458
458
459 "%sc -l myfiles = ls ~" should now be written as
459 "%sc -l myfiles = ls ~" should now be written as
460
460
461 "myfiles = !ls ~"
461 "myfiles = !ls ~"
462
462
463 myfiles.s, myfiles.l and myfiles.n still apply as documented
463 myfiles.s, myfiles.l and myfiles.n still apply as documented
464 below.
464 below.
465
465
466 --
466 --
467 %sc [options] varname=command
467 %sc [options] varname=command
468
468
469 IPython will run the given command using commands.getoutput(), and
469 IPython will run the given command using commands.getoutput(), and
470 will then update the user's interactive namespace with a variable
470 will then update the user's interactive namespace with a variable
471 called varname, containing the value of the call. Your command can
471 called varname, containing the value of the call. Your command can
472 contain shell wildcards, pipes, etc.
472 contain shell wildcards, pipes, etc.
473
473
474 The '=' sign in the syntax is mandatory, and the variable name you
474 The '=' sign in the syntax is mandatory, and the variable name you
475 supply must follow Python's standard conventions for valid names.
475 supply must follow Python's standard conventions for valid names.
476
476
477 (A special format without variable name exists for internal use)
477 (A special format without variable name exists for internal use)
478
478
479 Options:
479 Options:
480
480
481 -l: list output. Split the output on newlines into a list before
481 -l: list output. Split the output on newlines into a list before
482 assigning it to the given variable. By default the output is stored
482 assigning it to the given variable. By default the output is stored
483 as a single string.
483 as a single string.
484
484
485 -v: verbose. Print the contents of the variable.
485 -v: verbose. Print the contents of the variable.
486
486
487 In most cases you should not need to split as a list, because the
487 In most cases you should not need to split as a list, because the
488 returned value is a special type of string which can automatically
488 returned value is a special type of string which can automatically
489 provide its contents either as a list (split on newlines) or as a
489 provide its contents either as a list (split on newlines) or as a
490 space-separated string. These are convenient, respectively, either
490 space-separated string. These are convenient, respectively, either
491 for sequential processing or to be passed to a shell command.
491 for sequential processing or to be passed to a shell command.
492
492
493 For example::
493 For example::
494
494
495 # Capture into variable a
495 # Capture into variable a
496 In [1]: sc a=ls *py
496 In [1]: sc a=ls *py
497
497
498 # a is a string with embedded newlines
498 # a is a string with embedded newlines
499 In [2]: a
499 In [2]: a
500 Out[2]: 'setup.py\\nwin32_manual_post_install.py'
500 Out[2]: 'setup.py\\nwin32_manual_post_install.py'
501
501
502 # which can be seen as a list:
502 # which can be seen as a list:
503 In [3]: a.l
503 In [3]: a.l
504 Out[3]: ['setup.py', 'win32_manual_post_install.py']
504 Out[3]: ['setup.py', 'win32_manual_post_install.py']
505
505
506 # or as a whitespace-separated string:
506 # or as a whitespace-separated string:
507 In [4]: a.s
507 In [4]: a.s
508 Out[4]: 'setup.py win32_manual_post_install.py'
508 Out[4]: 'setup.py win32_manual_post_install.py'
509
509
510 # a.s is useful to pass as a single command line:
510 # a.s is useful to pass as a single command line:
511 In [5]: !wc -l $a.s
511 In [5]: !wc -l $a.s
512 146 setup.py
512 146 setup.py
513 130 win32_manual_post_install.py
513 130 win32_manual_post_install.py
514 276 total
514 276 total
515
515
516 # while the list form is useful to loop over:
516 # while the list form is useful to loop over:
517 In [6]: for f in a.l:
517 In [6]: for f in a.l:
518 ...: !wc -l $f
518 ...: !wc -l $f
519 ...:
519 ...:
520 146 setup.py
520 146 setup.py
521 130 win32_manual_post_install.py
521 130 win32_manual_post_install.py
522
522
523 Similarly, the lists returned by the -l option are also special, in
523 Similarly, the lists returned by the -l option are also special, in
524 the sense that you can equally invoke the .s attribute on them to
524 the sense that you can equally invoke the .s attribute on them to
525 automatically get a whitespace-separated string from their contents::
525 automatically get a whitespace-separated string from their contents::
526
526
527 In [7]: sc -l b=ls *py
527 In [7]: sc -l b=ls *py
528
528
529 In [8]: b
529 In [8]: b
530 Out[8]: ['setup.py', 'win32_manual_post_install.py']
530 Out[8]: ['setup.py', 'win32_manual_post_install.py']
531
531
532 In [9]: b.s
532 In [9]: b.s
533 Out[9]: 'setup.py win32_manual_post_install.py'
533 Out[9]: 'setup.py win32_manual_post_install.py'
534
534
535 In summary, both the lists and strings used for output capture have
535 In summary, both the lists and strings used for output capture have
536 the following special attributes::
536 the following special attributes::
537
537
538 .l (or .list) : value as list.
538 .l (or .list) : value as list.
539 .n (or .nlstr): value as newline-separated string.
539 .n (or .nlstr): value as newline-separated string.
540 .s (or .spstr): value as space-separated string.
540 .s (or .spstr): value as space-separated string.
541 """
541 """
542
542
543 opts,args = self.parse_options(parameter_s, 'lv')
543 opts,args = self.parse_options(parameter_s, 'lv')
544 # Try to get a variable name and command to run
544 # Try to get a variable name and command to run
545 try:
545 try:
546 # the variable name must be obtained from the parse_options
546 # the variable name must be obtained from the parse_options
547 # output, which uses shlex.split to strip options out.
547 # output, which uses shlex.split to strip options out.
548 var,_ = args.split('=', 1)
548 var,_ = args.split('=', 1)
549 var = var.strip()
549 var = var.strip()
550 # But the command has to be extracted from the original input
550 # But the command has to be extracted from the original input
551 # parameter_s, not on what parse_options returns, to avoid the
551 # parameter_s, not on what parse_options returns, to avoid the
552 # quote stripping which shlex.split performs on it.
552 # quote stripping which shlex.split performs on it.
553 _,cmd = parameter_s.split('=', 1)
553 _,cmd = parameter_s.split('=', 1)
554 except ValueError:
554 except ValueError:
555 var,cmd = '',''
555 var,cmd = '',''
556 # If all looks ok, proceed
556 # If all looks ok, proceed
557 split = 'l' in opts
557 split = 'l' in opts
558 out = self.shell.getoutput(cmd, split=split)
558 out = self.shell.getoutput(cmd, split=split)
559 if 'v' in opts:
559 if 'v' in opts:
560 print('%s ==\n%s' % (var, pformat(out)))
560 print('%s ==\n%s' % (var, pformat(out)))
561 if var:
561 if var:
562 self.shell.user_ns.update({var:out})
562 self.shell.user_ns.update({var:out})
563 else:
563 else:
564 return out
564 return out
565
565
566 @line_cell_magic
566 @line_cell_magic
567 def sx(self, line='', cell=None):
567 def sx(self, line='', cell=None):
568 """Shell execute - run shell command and capture output (!! is short-hand).
568 """Shell execute - run shell command and capture output (!! is short-hand).
569
569
570 %sx command
570 %sx command
571
571
572 IPython will run the given command using commands.getoutput(), and
572 IPython will run the given command using commands.getoutput(), and
573 return the result formatted as a list (split on '\\n'). Since the
573 return the result formatted as a list (split on '\\n'). Since the
574 output is _returned_, it will be stored in ipython's regular output
574 output is _returned_, it will be stored in ipython's regular output
575 cache Out[N] and in the '_N' automatic variables.
575 cache Out[N] and in the '_N' automatic variables.
576
576
577 Notes:
577 Notes:
578
578
579 1) If an input line begins with '!!', then %sx is automatically
579 1) If an input line begins with '!!', then %sx is automatically
580 invoked. That is, while::
580 invoked. That is, while::
581
581
582 !ls
582 !ls
583
583
584 causes ipython to simply issue system('ls'), typing::
584 causes ipython to simply issue system('ls'), typing::
585
585
586 !!ls
586 !!ls
587
587
588 is a shorthand equivalent to::
588 is a shorthand equivalent to::
589
589
590 %sx ls
590 %sx ls
591
591
592 2) %sx differs from %sc in that %sx automatically splits into a list,
592 2) %sx differs from %sc in that %sx automatically splits into a list,
593 like '%sc -l'. The reason for this is to make it as easy as possible
593 like '%sc -l'. The reason for this is to make it as easy as possible
594 to process line-oriented shell output via further python commands.
594 to process line-oriented shell output via further python commands.
595 %sc is meant to provide much finer control, but requires more
595 %sc is meant to provide much finer control, but requires more
596 typing.
596 typing.
597
597
598 3) Just like %sc -l, this is a list with special attributes:
598 3) Just like %sc -l, this is a list with special attributes:
599 ::
599 ::
600
600
601 .l (or .list) : value as list.
601 .l (or .list) : value as list.
602 .n (or .nlstr): value as newline-separated string.
602 .n (or .nlstr): value as newline-separated string.
603 .s (or .spstr): value as whitespace-separated string.
603 .s (or .spstr): value as whitespace-separated string.
604
604
605 This is very useful when trying to use such lists as arguments to
605 This is very useful when trying to use such lists as arguments to
606 system commands."""
606 system commands."""
607
607
608 if cell is None:
608 if cell is None:
609 # line magic
609 # line magic
610 return self.shell.getoutput(line)
610 return self.shell.getoutput(line)
611 else:
611 else:
612 opts,args = self.parse_options(line, '', 'out=')
612 opts,args = self.parse_options(line, '', 'out=')
613 output = self.shell.getoutput(cell)
613 output = self.shell.getoutput(cell)
614 out_name = opts.get('out', opts.get('o'))
614 out_name = opts.get('out', opts.get('o'))
615 if out_name:
615 if out_name:
616 self.shell.user_ns[out_name] = output
616 self.shell.user_ns[out_name] = output
617 else:
617 else:
618 return output
618 return output
619
619
620 system = line_cell_magic('system')(sx)
620 system = line_cell_magic('system')(sx)
621 bang = cell_magic('!')(sx)
621 bang = cell_magic('!')(sx)
622
622
623 @line_magic
623 @line_magic
624 def bookmark(self, parameter_s=''):
624 def bookmark(self, parameter_s=''):
625 """Manage IPython's bookmark system.
625 """Manage IPython's bookmark system.
626
626
627 %bookmark <name> - set bookmark to current dir
627 %bookmark <name> - set bookmark to current dir
628 %bookmark <name> <dir> - set bookmark to <dir>
628 %bookmark <name> <dir> - set bookmark to <dir>
629 %bookmark -l - list all bookmarks
629 %bookmark -l - list all bookmarks
630 %bookmark -d <name> - remove bookmark
630 %bookmark -d <name> - remove bookmark
631 %bookmark -r - remove all bookmarks
631 %bookmark -r - remove all bookmarks
632
632
633 You can later on access a bookmarked folder with::
633 You can later on access a bookmarked folder with::
634
634
635 %cd -b <name>
635 %cd -b <name>
636
636
637 or simply '%cd <name>' if there is no directory called <name> AND
637 or simply '%cd <name>' if there is no directory called <name> AND
638 there is such a bookmark defined.
638 there is such a bookmark defined.
639
639
640 Your bookmarks persist through IPython sessions, but they are
640 Your bookmarks persist through IPython sessions, but they are
641 associated with each profile."""
641 associated with each profile."""
642
642
643 opts,args = self.parse_options(parameter_s,'drl',mode='list')
643 opts,args = self.parse_options(parameter_s,'drl',mode='list')
644 if len(args) > 2:
644 if len(args) > 2:
645 raise UsageError("%bookmark: too many arguments")
645 raise UsageError("%bookmark: too many arguments")
646
646
647 bkms = self.shell.db.get('bookmarks',{})
647 bkms = self.shell.db.get('bookmarks',{})
648
648
649 if 'd' in opts:
649 if 'd' in opts:
650 try:
650 try:
651 todel = args[0]
651 todel = args[0]
652 except IndexError:
652 except IndexError:
653 raise UsageError(
653 raise UsageError(
654 "%bookmark -d: must provide a bookmark to delete")
654 "%bookmark -d: must provide a bookmark to delete")
655 else:
655 else:
656 try:
656 try:
657 del bkms[todel]
657 del bkms[todel]
658 except KeyError:
658 except KeyError:
659 raise UsageError(
659 raise UsageError(
660 "%%bookmark -d: Can't delete bookmark '%s'" % todel)
660 "%%bookmark -d: Can't delete bookmark '%s'" % todel)
661
661
662 elif 'r' in opts:
662 elif 'r' in opts:
663 bkms = {}
663 bkms = {}
664 elif 'l' in opts:
664 elif 'l' in opts:
665 bks = bkms.keys()
665 bks = bkms.keys()
666 bks.sort()
666 bks.sort()
667 if bks:
667 if bks:
668 size = max(map(len, bks))
668 size = max(map(len, bks))
669 else:
669 else:
670 size = 0
670 size = 0
671 fmt = '%-'+str(size)+'s -> %s'
671 fmt = '%-'+str(size)+'s -> %s'
672 print('Current bookmarks:')
672 print('Current bookmarks:')
673 for bk in bks:
673 for bk in bks:
674 print(fmt % (bk, bkms[bk]))
674 print(fmt % (bk, bkms[bk]))
675 else:
675 else:
676 if not args:
676 if not args:
677 raise UsageError("%bookmark: You must specify the bookmark name")
677 raise UsageError("%bookmark: You must specify the bookmark name")
678 elif len(args)==1:
678 elif len(args)==1:
679 bkms[args[0]] = os.getcwdu()
679 bkms[args[0]] = os.getcwdu()
680 elif len(args)==2:
680 elif len(args)==2:
681 bkms[args[0]] = args[1]
681 bkms[args[0]] = args[1]
682 self.shell.db['bookmarks'] = bkms
682 self.shell.db['bookmarks'] = bkms
683
683
684 @line_magic
684 @line_magic
685 def pycat(self, parameter_s=''):
685 def pycat(self, parameter_s=''):
686 """Show a syntax-highlighted file through a pager.
686 """Show a syntax-highlighted file through a pager.
687
687
688 This magic is similar to the cat utility, but it will assume the file
688 This magic is similar to the cat utility, but it will assume the file
689 to be Python source and will show it with syntax highlighting.
689 to be Python source and will show it with syntax highlighting.
690
690
691 This magic command can either take a local filename, an url,
691 This magic command can either take a local filename, an url,
692 an history range (see %history) or a macro as argument ::
692 an history range (see %history) or a macro as argument ::
693
693
694 %pycat myscript.py
694 %pycat myscript.py
695 %pycat 7-27
695 %pycat 7-27
696 %pycat myMacro
696 %pycat myMacro
697 %pycat http://www.example.com/myscript.py
697 %pycat http://www.example.com/myscript.py
698 """
698 """
699 if not parameter_s:
699 if not parameter_s:
700 raise UsageError('Missing filename, URL, input history range, '
700 raise UsageError('Missing filename, URL, input history range, '
701 'or macro.')
701 'or macro.')
702
702
703 try :
703 try :
704 cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
704 cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
705 except (ValueError, IOError):
705 except (ValueError, IOError):
706 print("Error: no such file, variable, URL, history range or macro")
706 print("Error: no such file, variable, URL, history range or macro")
707 return
707 return
708
708
709 page.page(self.shell.pycolorize(source_to_unicode(cont)))
709 page.page(self.shell.pycolorize(source_to_unicode(cont)))
710
710
711 @magic_arguments.magic_arguments()
711 @magic_arguments.magic_arguments()
712 @magic_arguments.argument(
712 @magic_arguments.argument(
713 '-a', '--append', action='store_true', default=False,
713 '-a', '--append', action='store_true', default=False,
714 help='Append contents of the cell to an existing file. '
714 help='Append contents of the cell to an existing file. '
715 'The file will be created if it does not exist.'
715 'The file will be created if it does not exist.'
716 )
716 )
717 @magic_arguments.argument(
717 @magic_arguments.argument(
718 'filename', type=unicode_type,
718 'filename', type=unicode_type,
719 help='file to write'
719 help='file to write'
720 )
720 )
721 @cell_magic
721 @cell_magic
722 def writefile(self, line, cell):
722 def writefile(self, line, cell):
723 """Write the contents of the cell to a file.
723 """Write the contents of the cell to a file.
724
724
725 The file will be overwritten unless the -a (--append) flag is specified.
725 The file will be overwritten unless the -a (--append) flag is specified.
726 """
726 """
727 args = magic_arguments.parse_argstring(self.writefile, line)
727 args = magic_arguments.parse_argstring(self.writefile, line)
728 filename = os.path.expanduser(unquote_filename(args.filename))
728 filename = os.path.expanduser(unquote_filename(args.filename))
729
729
730 if os.path.exists(filename):
730 if os.path.exists(filename):
731 if args.append:
731 if args.append:
732 print("Appending to %s" % filename)
732 print("Appending to %s" % filename)
733 else:
733 else:
734 print("Overwriting %s" % filename)
734 print("Overwriting %s" % filename)
735 else:
735 else:
736 print("Writing %s" % filename)
736 print("Writing %s" % filename)
737
737
738 mode = 'a' if args.append else 'w'
738 mode = 'a' if args.append else 'w'
739 with io.open(filename, mode, encoding='utf-8') as f:
739 with io.open(filename, mode, encoding='utf-8') as f:
740 f.write(cell)
740 f.write(cell)
@@ -1,144 +1,143 b''
1 """Implementation of magic functions for matplotlib/pylab support.
1 """Implementation of magic functions for matplotlib/pylab support.
2 """
2 """
3 from __future__ import print_function
3 from __future__ import print_function
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (c) 2012 The IPython Development Team.
5 # Copyright (c) 2012 The IPython Development Team.
6 #
6 #
7 # Distributed under the terms of the Modified BSD License.
7 # Distributed under the terms of the Modified BSD License.
8 #
8 #
9 # The full license is in the file COPYING.txt, distributed with this software.
9 # The full license is in the file COPYING.txt, distributed with this software.
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Imports
13 # Imports
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15
15
16 # Our own packages
16 # Our own packages
17 from IPython.config.application import Application
17 from IPython.config.application import Application
18 from IPython.core import magic_arguments
18 from IPython.core import magic_arguments
19 from IPython.core.magic import Magics, magics_class, line_magic
19 from IPython.core.magic import Magics, magics_class, line_magic
20 from IPython.testing.skipdoctest import skip_doctest
20 from IPython.testing.skipdoctest import skip_doctest
21 from IPython.utils.warn import warn
21 from IPython.utils.warn import warn
22 from IPython.core.pylabtools import backends
22 from IPython.core.pylabtools import backends
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Magic implementation classes
25 # Magic implementation classes
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 magic_gui_arg = magic_arguments.argument(
28 magic_gui_arg = magic_arguments.argument(
29 'gui', nargs='?',
29 'gui', nargs='?',
30 help="""Name of the matplotlib backend to use %s.
30 help="""Name of the matplotlib backend to use %s.
31 If given, the corresponding matplotlib backend is used,
31 If given, the corresponding matplotlib backend is used,
32 otherwise it will be matplotlib's default
32 otherwise it will be matplotlib's default
33 (which you can set in your matplotlib config file).
33 (which you can set in your matplotlib config file).
34 """ % str(tuple(sorted(backends.keys())))
34 """ % str(tuple(sorted(backends.keys())))
35 )
35 )
36
36
37
37
38 @magics_class
38 @magics_class
39 class PylabMagics(Magics):
39 class PylabMagics(Magics):
40 """Magics related to matplotlib's pylab support"""
40 """Magics related to matplotlib's pylab support"""
41
41
42 @skip_doctest
42 @skip_doctest
43 @line_magic
43 @line_magic
44 @magic_arguments.magic_arguments()
44 @magic_arguments.magic_arguments()
45 @magic_gui_arg
45 @magic_gui_arg
46 def matplotlib(self, line=''):
46 def matplotlib(self, line=''):
47 """Set up matplotlib to work interactively.
47 """Set up matplotlib to work interactively.
48
48
49 This function lets you activate matplotlib interactive support
49 This function lets you activate matplotlib interactive support
50 at any point during an IPython session.
50 at any point during an IPython session.
51 It does not import anything into the interactive namespace.
51 It does not import anything into the interactive namespace.
52
52
53 If you are using the inline matplotlib backend for embedded figures,
53 If you are using the inline matplotlib backend for embedded figures,
54 you can adjust its behavior via the %config magic::
54 you can adjust its behavior via the %config magic::
55
55
56 # enable SVG figures, necessary for SVG+XHTML export in the qtconsole
56 # enable SVG figures, necessary for SVG+XHTML export in the qtconsole
57 In [1]: %config InlineBackend.figure_format = 'svg'
57 In [1]: %config InlineBackend.figure_format = 'svg'
58
58
59 # change the behavior of closing all figures at the end of each
59 # change the behavior of closing all figures at the end of each
60 # execution (cell), or allowing reuse of active figures across
60 # execution (cell), or allowing reuse of active figures across
61 # cells:
61 # cells:
62 In [2]: %config InlineBackend.close_figures = False
62 In [2]: %config InlineBackend.close_figures = False
63
63
64 Examples
64 Examples
65 --------
65 --------
66 In this case, where the MPL default is TkAgg::
66 In this case, where the MPL default is TkAgg::
67
67
68 In [2]: %matplotlib
68 In [2]: %matplotlib
69 Using matplotlib backend: TkAgg
69 Using matplotlib backend: TkAgg
70
70
71 But you can explicitly request a different backend::
71 But you can explicitly request a different backend::
72
72
73 In [3]: %matplotlib qt
73 In [3]: %matplotlib qt
74 """
74 """
75 args = magic_arguments.parse_argstring(self.matplotlib, line)
75 args = magic_arguments.parse_argstring(self.matplotlib, line)
76 gui, backend = self.shell.enable_matplotlib(args.gui)
76 gui, backend = self.shell.enable_matplotlib(args.gui)
77 self._show_matplotlib_backend(args.gui, backend)
77 self._show_matplotlib_backend(args.gui, backend)
78
78
79 @skip_doctest
79 @skip_doctest
80 @line_magic
80 @line_magic
81 @magic_arguments.magic_arguments()
81 @magic_arguments.magic_arguments()
82 @magic_arguments.argument(
82 @magic_arguments.argument(
83 '--no-import-all', action='store_true', default=None,
83 '--no-import-all', action='store_true', default=None,
84 help="""Prevent IPython from performing ``import *`` into the interactive namespace.
84 help="""Prevent IPython from performing ``import *`` into the interactive namespace.
85
85
86 You can govern the default behavior of this flag with the
86 You can govern the default behavior of this flag with the
87 InteractiveShellApp.pylab_import_all configurable.
87 InteractiveShellApp.pylab_import_all configurable.
88 """
88 """
89 )
89 )
90 @magic_gui_arg
90 @magic_gui_arg
91 def pylab(self, line=''):
91 def pylab(self, line=''):
92 """Load numpy and matplotlib to work interactively.
92 """Load numpy and matplotlib to work interactively.
93
93
94 This function lets you activate pylab (matplotlib, numpy and
94 This function lets you activate pylab (matplotlib, numpy and
95 interactive support) at any point during an IPython session.
95 interactive support) at any point during an IPython session.
96
96
97 %pylab makes the following imports::
97 %pylab makes the following imports::
98
98
99 import numpy
99 import numpy
100 import matplotlib
100 import matplotlib
101 from matplotlib import pylab, mlab, pyplot
101 from matplotlib import pylab, mlab, pyplot
102 np = numpy
102 np = numpy
103 plt = pyplot
103 plt = pyplot
104
104
105 from IPython.display import display
105 from IPython.display import display
106 from IPython.core.pylabtools import figsize, getfigs
106 from IPython.core.pylabtools import figsize, getfigs
107
107
108 from pylab import *
108 from pylab import *
109 from numpy import *
109 from numpy import *
110
110
111 If you pass `--no-import-all`, the last two `*` imports will be excluded.
111 If you pass `--no-import-all`, the last two `*` imports will be excluded.
112
112
113 See the %matplotlib magic for more details about activating matplotlib
113 See the %matplotlib magic for more details about activating matplotlib
114 without affecting the interactive namespace.
114 without affecting the interactive namespace.
115 """
115 """
116 args = magic_arguments.parse_argstring(self.pylab, line)
116 args = magic_arguments.parse_argstring(self.pylab, line)
117 if args.no_import_all is None:
117 if args.no_import_all is None:
118 # get default from Application
118 # get default from Application
119 if Application.initialized():
119 if Application.initialized():
120 app = Application.instance()
120 app = Application.instance()
121 try:
121 try:
122 import_all = app.pylab_import_all
122 import_all = app.pylab_import_all
123 except AttributeError:
123 except AttributeError:
124 import_all = True
124 import_all = True
125 else:
125 else:
126 # nothing specified, no app - default True
126 # nothing specified, no app - default True
127 import_all = True
127 import_all = True
128 else:
128 else:
129 # invert no-import flag
129 # invert no-import flag
130 import_all = not args.no_import_all
130 import_all = not args.no_import_all
131
131
132 gui, backend, clobbered = self.shell.enable_pylab(args.gui, import_all=import_all)
132 gui, backend, clobbered = self.shell.enable_pylab(args.gui, import_all=import_all)
133 self._show_matplotlib_backend(args.gui, backend)
133 self._show_matplotlib_backend(args.gui, backend)
134 print ("Populating the interactive namespace from numpy and matplotlib")
134 print ("Populating the interactive namespace from numpy and matplotlib")
135 if clobbered:
135 if clobbered:
136 warn("pylab import has clobbered these variables: %s" % clobbered +
136 warn("pylab import has clobbered these variables: %s" % clobbered +
137 "\n`%pylab --no-import-all` prevents importing * from pylab and numpy"
137 "\n`%pylab --no-import-all` prevents importing * from pylab and numpy"
138 )
138 )
139
139
140 def _show_matplotlib_backend(self, gui, backend):
140 def _show_matplotlib_backend(self, gui, backend):
141 """show matplotlib message backend message"""
141 """show matplotlib message backend message"""
142 if not gui or gui == 'auto':
142 if not gui or gui == 'auto':
143 print(("Using matplotlib backend: %s" % backend))
143 print("Using matplotlib backend: %s" % backend)
144
@@ -1,340 +1,340 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Pylab (matplotlib) support utilities.
2 """Pylab (matplotlib) support utilities.
3
3
4 Authors
4 Authors
5 -------
5 -------
6
6
7 * Fernando Perez.
7 * Fernando Perez.
8 * Brian Granger
8 * Brian Granger
9 """
9 """
10 from __future__ import print_function
10 from __future__ import print_function
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2009 The IPython Development Team
13 # Copyright (C) 2009 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import sys
23 import sys
24 from io import BytesIO
24 from io import BytesIO
25
25
26 from IPython.core.display import _pngxy
26 from IPython.core.display import _pngxy
27 from IPython.utils.decorators import flag_calls
27 from IPython.utils.decorators import flag_calls
28
28
29 # If user specifies a GUI, that dictates the backend, otherwise we read the
29 # If user specifies a GUI, that dictates the backend, otherwise we read the
30 # user's mpl default from the mpl rc structure
30 # user's mpl default from the mpl rc structure
31 backends = {'tk': 'TkAgg',
31 backends = {'tk': 'TkAgg',
32 'gtk': 'GTKAgg',
32 'gtk': 'GTKAgg',
33 'wx': 'WXAgg',
33 'wx': 'WXAgg',
34 'qt': 'Qt4Agg', # qt3 not supported
34 'qt': 'Qt4Agg', # qt3 not supported
35 'qt4': 'Qt4Agg',
35 'qt4': 'Qt4Agg',
36 'osx': 'MacOSX',
36 'osx': 'MacOSX',
37 'inline' : 'module://IPython.kernel.zmq.pylab.backend_inline'}
37 'inline' : 'module://IPython.kernel.zmq.pylab.backend_inline'}
38
38
39 # We also need a reverse backends2guis mapping that will properly choose which
39 # We also need a reverse backends2guis mapping that will properly choose which
40 # GUI support to activate based on the desired matplotlib backend. For the
40 # GUI support to activate based on the desired matplotlib backend. For the
41 # most part it's just a reverse of the above dict, but we also need to add a
41 # most part it's just a reverse of the above dict, but we also need to add a
42 # few others that map to the same GUI manually:
42 # few others that map to the same GUI manually:
43 backend2gui = dict(zip(backends.values(), backends.keys()))
43 backend2gui = dict(zip(backends.values(), backends.keys()))
44 # Our tests expect backend2gui to just return 'qt'
44 # Our tests expect backend2gui to just return 'qt'
45 backend2gui['Qt4Agg'] = 'qt'
45 backend2gui['Qt4Agg'] = 'qt'
46 # In the reverse mapping, there are a few extra valid matplotlib backends that
46 # In the reverse mapping, there are a few extra valid matplotlib backends that
47 # map to the same GUI support
47 # map to the same GUI support
48 backend2gui['GTK'] = backend2gui['GTKCairo'] = 'gtk'
48 backend2gui['GTK'] = backend2gui['GTKCairo'] = 'gtk'
49 backend2gui['WX'] = 'wx'
49 backend2gui['WX'] = 'wx'
50 backend2gui['CocoaAgg'] = 'osx'
50 backend2gui['CocoaAgg'] = 'osx'
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Matplotlib utilities
53 # Matplotlib utilities
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56
56
57 def getfigs(*fig_nums):
57 def getfigs(*fig_nums):
58 """Get a list of matplotlib figures by figure numbers.
58 """Get a list of matplotlib figures by figure numbers.
59
59
60 If no arguments are given, all available figures are returned. If the
60 If no arguments are given, all available figures are returned. If the
61 argument list contains references to invalid figures, a warning is printed
61 argument list contains references to invalid figures, a warning is printed
62 but the function continues pasting further figures.
62 but the function continues pasting further figures.
63
63
64 Parameters
64 Parameters
65 ----------
65 ----------
66 figs : tuple
66 figs : tuple
67 A tuple of ints giving the figure numbers of the figures to return.
67 A tuple of ints giving the figure numbers of the figures to return.
68 """
68 """
69 from matplotlib._pylab_helpers import Gcf
69 from matplotlib._pylab_helpers import Gcf
70 if not fig_nums:
70 if not fig_nums:
71 fig_managers = Gcf.get_all_fig_managers()
71 fig_managers = Gcf.get_all_fig_managers()
72 return [fm.canvas.figure for fm in fig_managers]
72 return [fm.canvas.figure for fm in fig_managers]
73 else:
73 else:
74 figs = []
74 figs = []
75 for num in fig_nums:
75 for num in fig_nums:
76 f = Gcf.figs.get(num)
76 f = Gcf.figs.get(num)
77 if f is None:
77 if f is None:
78 print(('Warning: figure %s not available.' % num))
78 print('Warning: figure %s not available.' % num)
79 else:
79 else:
80 figs.append(f.canvas.figure)
80 figs.append(f.canvas.figure)
81 return figs
81 return figs
82
82
83
83
84 def figsize(sizex, sizey):
84 def figsize(sizex, sizey):
85 """Set the default figure size to be [sizex, sizey].
85 """Set the default figure size to be [sizex, sizey].
86
86
87 This is just an easy to remember, convenience wrapper that sets::
87 This is just an easy to remember, convenience wrapper that sets::
88
88
89 matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
89 matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
90 """
90 """
91 import matplotlib
91 import matplotlib
92 matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
92 matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
93
93
94
94
95 def print_figure(fig, fmt='png'):
95 def print_figure(fig, fmt='png'):
96 """Convert a figure to svg or png for inline display."""
96 """Convert a figure to svg or png for inline display."""
97 from matplotlib import rcParams
97 from matplotlib import rcParams
98 # When there's an empty figure, we shouldn't return anything, otherwise we
98 # When there's an empty figure, we shouldn't return anything, otherwise we
99 # get big blank areas in the qt console.
99 # get big blank areas in the qt console.
100 if not fig.axes and not fig.lines:
100 if not fig.axes and not fig.lines:
101 return
101 return
102
102
103 fc = fig.get_facecolor()
103 fc = fig.get_facecolor()
104 ec = fig.get_edgecolor()
104 ec = fig.get_edgecolor()
105 bytes_io = BytesIO()
105 bytes_io = BytesIO()
106 dpi = rcParams['savefig.dpi']
106 dpi = rcParams['savefig.dpi']
107 if fmt == 'retina':
107 if fmt == 'retina':
108 dpi = dpi * 2
108 dpi = dpi * 2
109 fmt = 'png'
109 fmt = 'png'
110 fig.canvas.print_figure(bytes_io, format=fmt, bbox_inches='tight',
110 fig.canvas.print_figure(bytes_io, format=fmt, bbox_inches='tight',
111 facecolor=fc, edgecolor=ec, dpi=dpi)
111 facecolor=fc, edgecolor=ec, dpi=dpi)
112 data = bytes_io.getvalue()
112 data = bytes_io.getvalue()
113 return data
113 return data
114
114
115 def retina_figure(fig):
115 def retina_figure(fig):
116 """format a figure as a pixel-doubled (retina) PNG"""
116 """format a figure as a pixel-doubled (retina) PNG"""
117 pngdata = print_figure(fig, fmt='retina')
117 pngdata = print_figure(fig, fmt='retina')
118 w, h = _pngxy(pngdata)
118 w, h = _pngxy(pngdata)
119 metadata = dict(width=w//2, height=h//2)
119 metadata = dict(width=w//2, height=h//2)
120 return pngdata, metadata
120 return pngdata, metadata
121
121
122 # We need a little factory function here to create the closure where
122 # We need a little factory function here to create the closure where
123 # safe_execfile can live.
123 # safe_execfile can live.
124 def mpl_runner(safe_execfile):
124 def mpl_runner(safe_execfile):
125 """Factory to return a matplotlib-enabled runner for %run.
125 """Factory to return a matplotlib-enabled runner for %run.
126
126
127 Parameters
127 Parameters
128 ----------
128 ----------
129 safe_execfile : function
129 safe_execfile : function
130 This must be a function with the same interface as the
130 This must be a function with the same interface as the
131 :meth:`safe_execfile` method of IPython.
131 :meth:`safe_execfile` method of IPython.
132
132
133 Returns
133 Returns
134 -------
134 -------
135 A function suitable for use as the ``runner`` argument of the %run magic
135 A function suitable for use as the ``runner`` argument of the %run magic
136 function.
136 function.
137 """
137 """
138
138
139 def mpl_execfile(fname,*where,**kw):
139 def mpl_execfile(fname,*where,**kw):
140 """matplotlib-aware wrapper around safe_execfile.
140 """matplotlib-aware wrapper around safe_execfile.
141
141
142 Its interface is identical to that of the :func:`execfile` builtin.
142 Its interface is identical to that of the :func:`execfile` builtin.
143
143
144 This is ultimately a call to execfile(), but wrapped in safeties to
144 This is ultimately a call to execfile(), but wrapped in safeties to
145 properly handle interactive rendering."""
145 properly handle interactive rendering."""
146
146
147 import matplotlib
147 import matplotlib
148 import matplotlib.pylab as pylab
148 import matplotlib.pylab as pylab
149
149
150 #print '*** Matplotlib runner ***' # dbg
150 #print '*** Matplotlib runner ***' # dbg
151 # turn off rendering until end of script
151 # turn off rendering until end of script
152 is_interactive = matplotlib.rcParams['interactive']
152 is_interactive = matplotlib.rcParams['interactive']
153 matplotlib.interactive(False)
153 matplotlib.interactive(False)
154 safe_execfile(fname,*where,**kw)
154 safe_execfile(fname,*where,**kw)
155 matplotlib.interactive(is_interactive)
155 matplotlib.interactive(is_interactive)
156 # make rendering call now, if the user tried to do it
156 # make rendering call now, if the user tried to do it
157 if pylab.draw_if_interactive.called:
157 if pylab.draw_if_interactive.called:
158 pylab.draw()
158 pylab.draw()
159 pylab.draw_if_interactive.called = False
159 pylab.draw_if_interactive.called = False
160
160
161 return mpl_execfile
161 return mpl_execfile
162
162
163
163
164 def select_figure_format(shell, fmt):
164 def select_figure_format(shell, fmt):
165 """Select figure format for inline backend, can be 'png', 'retina', or 'svg'.
165 """Select figure format for inline backend, can be 'png', 'retina', or 'svg'.
166
166
167 Using this method ensures only one figure format is active at a time.
167 Using this method ensures only one figure format is active at a time.
168 """
168 """
169 from matplotlib.figure import Figure
169 from matplotlib.figure import Figure
170 from IPython.kernel.zmq.pylab import backend_inline
170 from IPython.kernel.zmq.pylab import backend_inline
171
171
172 svg_formatter = shell.display_formatter.formatters['image/svg+xml']
172 svg_formatter = shell.display_formatter.formatters['image/svg+xml']
173 png_formatter = shell.display_formatter.formatters['image/png']
173 png_formatter = shell.display_formatter.formatters['image/png']
174
174
175 if fmt == 'png':
175 if fmt == 'png':
176 svg_formatter.type_printers.pop(Figure, None)
176 svg_formatter.type_printers.pop(Figure, None)
177 png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png'))
177 png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png'))
178 elif fmt in ('png2x', 'retina'):
178 elif fmt in ('png2x', 'retina'):
179 svg_formatter.type_printers.pop(Figure, None)
179 svg_formatter.type_printers.pop(Figure, None)
180 png_formatter.for_type(Figure, retina_figure)
180 png_formatter.for_type(Figure, retina_figure)
181 elif fmt == 'svg':
181 elif fmt == 'svg':
182 png_formatter.type_printers.pop(Figure, None)
182 png_formatter.type_printers.pop(Figure, None)
183 svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg'))
183 svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg'))
184 else:
184 else:
185 raise ValueError("supported formats are: 'png', 'retina', 'svg', not %r" % fmt)
185 raise ValueError("supported formats are: 'png', 'retina', 'svg', not %r" % fmt)
186
186
187 # set the format to be used in the backend()
187 # set the format to be used in the backend()
188 backend_inline._figure_format = fmt
188 backend_inline._figure_format = fmt
189
189
190 #-----------------------------------------------------------------------------
190 #-----------------------------------------------------------------------------
191 # Code for initializing matplotlib and importing pylab
191 # Code for initializing matplotlib and importing pylab
192 #-----------------------------------------------------------------------------
192 #-----------------------------------------------------------------------------
193
193
194
194
195 def find_gui_and_backend(gui=None, gui_select=None):
195 def find_gui_and_backend(gui=None, gui_select=None):
196 """Given a gui string return the gui and mpl backend.
196 """Given a gui string return the gui and mpl backend.
197
197
198 Parameters
198 Parameters
199 ----------
199 ----------
200 gui : str
200 gui : str
201 Can be one of ('tk','gtk','wx','qt','qt4','inline').
201 Can be one of ('tk','gtk','wx','qt','qt4','inline').
202 gui_select : str
202 gui_select : str
203 Can be one of ('tk','gtk','wx','qt','qt4','inline').
203 Can be one of ('tk','gtk','wx','qt','qt4','inline').
204 This is any gui already selected by the shell.
204 This is any gui already selected by the shell.
205
205
206 Returns
206 Returns
207 -------
207 -------
208 A tuple of (gui, backend) where backend is one of ('TkAgg','GTKAgg',
208 A tuple of (gui, backend) where backend is one of ('TkAgg','GTKAgg',
209 'WXAgg','Qt4Agg','module://IPython.kernel.zmq.pylab.backend_inline').
209 'WXAgg','Qt4Agg','module://IPython.kernel.zmq.pylab.backend_inline').
210 """
210 """
211
211
212 import matplotlib
212 import matplotlib
213
213
214 if gui and gui != 'auto':
214 if gui and gui != 'auto':
215 # select backend based on requested gui
215 # select backend based on requested gui
216 backend = backends[gui]
216 backend = backends[gui]
217 else:
217 else:
218 # We need to read the backend from the original data structure, *not*
218 # We need to read the backend from the original data structure, *not*
219 # from mpl.rcParams, since a prior invocation of %matplotlib may have
219 # from mpl.rcParams, since a prior invocation of %matplotlib may have
220 # overwritten that.
220 # overwritten that.
221 # WARNING: this assumes matplotlib 1.1 or newer!!
221 # WARNING: this assumes matplotlib 1.1 or newer!!
222 backend = matplotlib.rcParamsOrig['backend']
222 backend = matplotlib.rcParamsOrig['backend']
223 # In this case, we need to find what the appropriate gui selection call
223 # In this case, we need to find what the appropriate gui selection call
224 # should be for IPython, so we can activate inputhook accordingly
224 # should be for IPython, so we can activate inputhook accordingly
225 gui = backend2gui.get(backend, None)
225 gui = backend2gui.get(backend, None)
226
226
227 # If we have already had a gui active, we need it and inline are the
227 # If we have already had a gui active, we need it and inline are the
228 # ones allowed.
228 # ones allowed.
229 if gui_select and gui != gui_select:
229 if gui_select and gui != gui_select:
230 gui = gui_select
230 gui = gui_select
231 backend = backends[gui]
231 backend = backends[gui]
232
232
233 return gui, backend
233 return gui, backend
234
234
235
235
236 def activate_matplotlib(backend):
236 def activate_matplotlib(backend):
237 """Activate the given backend and set interactive to True."""
237 """Activate the given backend and set interactive to True."""
238
238
239 import matplotlib
239 import matplotlib
240 matplotlib.interactive(True)
240 matplotlib.interactive(True)
241
241
242 # Matplotlib had a bug where even switch_backend could not force
242 # Matplotlib had a bug where even switch_backend could not force
243 # the rcParam to update. This needs to be set *before* the module
243 # the rcParam to update. This needs to be set *before* the module
244 # magic of switch_backend().
244 # magic of switch_backend().
245 matplotlib.rcParams['backend'] = backend
245 matplotlib.rcParams['backend'] = backend
246
246
247 import matplotlib.pyplot
247 import matplotlib.pyplot
248 matplotlib.pyplot.switch_backend(backend)
248 matplotlib.pyplot.switch_backend(backend)
249
249
250 # This must be imported last in the matplotlib series, after
250 # This must be imported last in the matplotlib series, after
251 # backend/interactivity choices have been made
251 # backend/interactivity choices have been made
252 import matplotlib.pylab as pylab
252 import matplotlib.pylab as pylab
253
253
254 pylab.show._needmain = False
254 pylab.show._needmain = False
255 # We need to detect at runtime whether show() is called by the user.
255 # We need to detect at runtime whether show() is called by the user.
256 # For this, we wrap it into a decorator which adds a 'called' flag.
256 # For this, we wrap it into a decorator which adds a 'called' flag.
257 pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)
257 pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)
258
258
259
259
260 def import_pylab(user_ns, import_all=True):
260 def import_pylab(user_ns, import_all=True):
261 """Populate the namespace with pylab-related values.
261 """Populate the namespace with pylab-related values.
262
262
263 Imports matplotlib, pylab, numpy, and everything from pylab and numpy.
263 Imports matplotlib, pylab, numpy, and everything from pylab and numpy.
264
264
265 Also imports a few names from IPython (figsize, display, getfigs)
265 Also imports a few names from IPython (figsize, display, getfigs)
266
266
267 """
267 """
268
268
269 # Import numpy as np/pyplot as plt are conventions we're trying to
269 # Import numpy as np/pyplot as plt are conventions we're trying to
270 # somewhat standardize on. Making them available to users by default
270 # somewhat standardize on. Making them available to users by default
271 # will greatly help this.
271 # will greatly help this.
272 s = ("import numpy\n"
272 s = ("import numpy\n"
273 "import matplotlib\n"
273 "import matplotlib\n"
274 "from matplotlib import pylab, mlab, pyplot\n"
274 "from matplotlib import pylab, mlab, pyplot\n"
275 "np = numpy\n"
275 "np = numpy\n"
276 "plt = pyplot\n"
276 "plt = pyplot\n"
277 )
277 )
278 exec(s, user_ns)
278 exec(s, user_ns)
279
279
280 if import_all:
280 if import_all:
281 s = ("from matplotlib.pylab import *\n"
281 s = ("from matplotlib.pylab import *\n"
282 "from numpy import *\n")
282 "from numpy import *\n")
283 exec(s, user_ns)
283 exec(s, user_ns)
284
284
285 # IPython symbols to add
285 # IPython symbols to add
286 user_ns['figsize'] = figsize
286 user_ns['figsize'] = figsize
287 from IPython.core.display import display
287 from IPython.core.display import display
288 # Add display and getfigs to the user's namespace
288 # Add display and getfigs to the user's namespace
289 user_ns['display'] = display
289 user_ns['display'] = display
290 user_ns['getfigs'] = getfigs
290 user_ns['getfigs'] = getfigs
291
291
292
292
293 def configure_inline_support(shell, backend):
293 def configure_inline_support(shell, backend):
294 """Configure an IPython shell object for matplotlib use.
294 """Configure an IPython shell object for matplotlib use.
295
295
296 Parameters
296 Parameters
297 ----------
297 ----------
298 shell : InteractiveShell instance
298 shell : InteractiveShell instance
299
299
300 backend : matplotlib backend
300 backend : matplotlib backend
301 """
301 """
302 # If using our svg payload backend, register the post-execution
302 # If using our svg payload backend, register the post-execution
303 # function that will pick up the results for display. This can only be
303 # function that will pick up the results for display. This can only be
304 # done with access to the real shell object.
304 # done with access to the real shell object.
305
305
306 # Note: if we can't load the inline backend, then there's no point
306 # Note: if we can't load the inline backend, then there's no point
307 # continuing (such as in terminal-only shells in environments without
307 # continuing (such as in terminal-only shells in environments without
308 # zeromq available).
308 # zeromq available).
309 try:
309 try:
310 from IPython.kernel.zmq.pylab.backend_inline import InlineBackend
310 from IPython.kernel.zmq.pylab.backend_inline import InlineBackend
311 except ImportError:
311 except ImportError:
312 return
312 return
313 from matplotlib import pyplot
313 from matplotlib import pyplot
314
314
315 cfg = InlineBackend.instance(parent=shell)
315 cfg = InlineBackend.instance(parent=shell)
316 cfg.shell = shell
316 cfg.shell = shell
317 if cfg not in shell.configurables:
317 if cfg not in shell.configurables:
318 shell.configurables.append(cfg)
318 shell.configurables.append(cfg)
319
319
320 if backend == backends['inline']:
320 if backend == backends['inline']:
321 from IPython.kernel.zmq.pylab.backend_inline import flush_figures
321 from IPython.kernel.zmq.pylab.backend_inline import flush_figures
322 shell.register_post_execute(flush_figures)
322 shell.register_post_execute(flush_figures)
323
323
324 # Save rcParams that will be overwrittern
324 # Save rcParams that will be overwrittern
325 shell._saved_rcParams = dict()
325 shell._saved_rcParams = dict()
326 for k in cfg.rc:
326 for k in cfg.rc:
327 shell._saved_rcParams[k] = pyplot.rcParams[k]
327 shell._saved_rcParams[k] = pyplot.rcParams[k]
328 # load inline_rc
328 # load inline_rc
329 pyplot.rcParams.update(cfg.rc)
329 pyplot.rcParams.update(cfg.rc)
330 else:
330 else:
331 from IPython.kernel.zmq.pylab.backend_inline import flush_figures
331 from IPython.kernel.zmq.pylab.backend_inline import flush_figures
332 if flush_figures in shell._post_execute:
332 if flush_figures in shell._post_execute:
333 shell._post_execute.pop(flush_figures)
333 shell._post_execute.pop(flush_figures)
334 if hasattr(shell, '_saved_rcParams'):
334 if hasattr(shell, '_saved_rcParams'):
335 pyplot.rcParams.update(shell._saved_rcParams)
335 pyplot.rcParams.update(shell._saved_rcParams)
336 del shell._saved_rcParams
336 del shell._saved_rcParams
337
337
338 # Setup the default figure format
338 # Setup the default figure format
339 select_figure_format(shell, cfg.figure_format)
339 select_figure_format(shell, cfg.figure_format)
340
340
@@ -1,410 +1,410 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 A mixin for :class:`~IPython.core.application.Application` classes that
3 A mixin for :class:`~IPython.core.application.Application` classes that
4 launch InteractiveShell instances, load extensions, etc.
4 launch InteractiveShell instances, load extensions, etc.
5
5
6 Authors
6 Authors
7 -------
7 -------
8
8
9 * Min Ragan-Kelley
9 * Min Ragan-Kelley
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2011 The IPython Development Team
13 # Copyright (C) 2008-2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 from __future__ import absolute_import
23 from __future__ import absolute_import
24 from __future__ import print_function
24 from __future__ import print_function
25
25
26 import glob
26 import glob
27 import os
27 import os
28 import sys
28 import sys
29
29
30 from IPython.config.application import boolean_flag
30 from IPython.config.application import boolean_flag
31 from IPython.config.configurable import Configurable
31 from IPython.config.configurable import Configurable
32 from IPython.config.loader import Config
32 from IPython.config.loader import Config
33 from IPython.core import pylabtools
33 from IPython.core import pylabtools
34 from IPython.utils import py3compat
34 from IPython.utils import py3compat
35 from IPython.utils.contexts import preserve_keys
35 from IPython.utils.contexts import preserve_keys
36 from IPython.utils.path import filefind
36 from IPython.utils.path import filefind
37 from IPython.utils.traitlets import (
37 from IPython.utils.traitlets import (
38 Unicode, Instance, List, Bool, CaselessStrEnum, Dict
38 Unicode, Instance, List, Bool, CaselessStrEnum, Dict
39 )
39 )
40 from IPython.lib.inputhook import guis
40 from IPython.lib.inputhook import guis
41
41
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43 # Aliases and Flags
43 # Aliases and Flags
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45
45
46 gui_keys = tuple(sorted([ key for key in guis if key is not None ]))
46 gui_keys = tuple(sorted([ key for key in guis if key is not None ]))
47
47
48 backend_keys = sorted(pylabtools.backends.keys())
48 backend_keys = sorted(pylabtools.backends.keys())
49 backend_keys.insert(0, 'auto')
49 backend_keys.insert(0, 'auto')
50
50
51 shell_flags = {}
51 shell_flags = {}
52
52
53 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
53 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
54 addflag('autoindent', 'InteractiveShell.autoindent',
54 addflag('autoindent', 'InteractiveShell.autoindent',
55 'Turn on autoindenting.', 'Turn off autoindenting.'
55 'Turn on autoindenting.', 'Turn off autoindenting.'
56 )
56 )
57 addflag('automagic', 'InteractiveShell.automagic',
57 addflag('automagic', 'InteractiveShell.automagic',
58 """Turn on the auto calling of magic commands. Type %%magic at the
58 """Turn on the auto calling of magic commands. Type %%magic at the
59 IPython prompt for more information.""",
59 IPython prompt for more information.""",
60 'Turn off the auto calling of magic commands.'
60 'Turn off the auto calling of magic commands.'
61 )
61 )
62 addflag('pdb', 'InteractiveShell.pdb',
62 addflag('pdb', 'InteractiveShell.pdb',
63 "Enable auto calling the pdb debugger after every exception.",
63 "Enable auto calling the pdb debugger after every exception.",
64 "Disable auto calling the pdb debugger after every exception."
64 "Disable auto calling the pdb debugger after every exception."
65 )
65 )
66 # pydb flag doesn't do any config, as core.debugger switches on import,
66 # pydb flag doesn't do any config, as core.debugger switches on import,
67 # which is before parsing. This just allows the flag to be passed.
67 # which is before parsing. This just allows the flag to be passed.
68 shell_flags.update(dict(
68 shell_flags.update(dict(
69 pydb = ({},
69 pydb = ({},
70 """Use the third party 'pydb' package as debugger, instead of pdb.
70 """Use the third party 'pydb' package as debugger, instead of pdb.
71 Requires that pydb is installed."""
71 Requires that pydb is installed."""
72 )
72 )
73 ))
73 ))
74 addflag('pprint', 'PlainTextFormatter.pprint',
74 addflag('pprint', 'PlainTextFormatter.pprint',
75 "Enable auto pretty printing of results.",
75 "Enable auto pretty printing of results.",
76 "Disable auto pretty printing of results."
76 "Disable auto pretty printing of results."
77 )
77 )
78 addflag('color-info', 'InteractiveShell.color_info',
78 addflag('color-info', 'InteractiveShell.color_info',
79 """IPython can display information about objects via a set of func-
79 """IPython can display information about objects via a set of func-
80 tions, and optionally can use colors for this, syntax highlighting
80 tions, and optionally can use colors for this, syntax highlighting
81 source code and various other elements. However, because this
81 source code and various other elements. However, because this
82 information is passed through a pager (like 'less') and many pagers get
82 information is passed through a pager (like 'less') and many pagers get
83 confused with color codes, this option is off by default. You can test
83 confused with color codes, this option is off by default. You can test
84 it and turn it on permanently in your ipython_config.py file if it
84 it and turn it on permanently in your ipython_config.py file if it
85 works for you. Test it and turn it on permanently if it works with
85 works for you. Test it and turn it on permanently if it works with
86 your system. The magic function %%color_info allows you to toggle this
86 your system. The magic function %%color_info allows you to toggle this
87 interactively for testing.""",
87 interactively for testing.""",
88 "Disable using colors for info related things."
88 "Disable using colors for info related things."
89 )
89 )
90 addflag('deep-reload', 'InteractiveShell.deep_reload',
90 addflag('deep-reload', 'InteractiveShell.deep_reload',
91 """Enable deep (recursive) reloading by default. IPython can use the
91 """Enable deep (recursive) reloading by default. IPython can use the
92 deep_reload module which reloads changes in modules recursively (it
92 deep_reload module which reloads changes in modules recursively (it
93 replaces the reload() function, so you don't need to change anything to
93 replaces the reload() function, so you don't need to change anything to
94 use it). deep_reload() forces a full reload of modules whose code may
94 use it). deep_reload() forces a full reload of modules whose code may
95 have changed, which the default reload() function does not. When
95 have changed, which the default reload() function does not. When
96 deep_reload is off, IPython will use the normal reload(), but
96 deep_reload is off, IPython will use the normal reload(), but
97 deep_reload will still be available as dreload(). This feature is off
97 deep_reload will still be available as dreload(). This feature is off
98 by default [which means that you have both normal reload() and
98 by default [which means that you have both normal reload() and
99 dreload()].""",
99 dreload()].""",
100 "Disable deep (recursive) reloading by default."
100 "Disable deep (recursive) reloading by default."
101 )
101 )
102 nosep_config = Config()
102 nosep_config = Config()
103 nosep_config.InteractiveShell.separate_in = ''
103 nosep_config.InteractiveShell.separate_in = ''
104 nosep_config.InteractiveShell.separate_out = ''
104 nosep_config.InteractiveShell.separate_out = ''
105 nosep_config.InteractiveShell.separate_out2 = ''
105 nosep_config.InteractiveShell.separate_out2 = ''
106
106
107 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
107 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
108 shell_flags['pylab'] = (
108 shell_flags['pylab'] = (
109 {'InteractiveShellApp' : {'pylab' : 'auto'}},
109 {'InteractiveShellApp' : {'pylab' : 'auto'}},
110 """Pre-load matplotlib and numpy for interactive use with
110 """Pre-load matplotlib and numpy for interactive use with
111 the default matplotlib backend."""
111 the default matplotlib backend."""
112 )
112 )
113 shell_flags['matplotlib'] = (
113 shell_flags['matplotlib'] = (
114 {'InteractiveShellApp' : {'matplotlib' : 'auto'}},
114 {'InteractiveShellApp' : {'matplotlib' : 'auto'}},
115 """Configure matplotlib for interactive use with
115 """Configure matplotlib for interactive use with
116 the default matplotlib backend."""
116 the default matplotlib backend."""
117 )
117 )
118
118
119 # it's possible we don't want short aliases for *all* of these:
119 # it's possible we don't want short aliases for *all* of these:
120 shell_aliases = dict(
120 shell_aliases = dict(
121 autocall='InteractiveShell.autocall',
121 autocall='InteractiveShell.autocall',
122 colors='InteractiveShell.colors',
122 colors='InteractiveShell.colors',
123 logfile='InteractiveShell.logfile',
123 logfile='InteractiveShell.logfile',
124 logappend='InteractiveShell.logappend',
124 logappend='InteractiveShell.logappend',
125 c='InteractiveShellApp.code_to_run',
125 c='InteractiveShellApp.code_to_run',
126 m='InteractiveShellApp.module_to_run',
126 m='InteractiveShellApp.module_to_run',
127 ext='InteractiveShellApp.extra_extension',
127 ext='InteractiveShellApp.extra_extension',
128 gui='InteractiveShellApp.gui',
128 gui='InteractiveShellApp.gui',
129 pylab='InteractiveShellApp.pylab',
129 pylab='InteractiveShellApp.pylab',
130 matplotlib='InteractiveShellApp.matplotlib',
130 matplotlib='InteractiveShellApp.matplotlib',
131 )
131 )
132 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
132 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
133
133
134 #-----------------------------------------------------------------------------
134 #-----------------------------------------------------------------------------
135 # Main classes and functions
135 # Main classes and functions
136 #-----------------------------------------------------------------------------
136 #-----------------------------------------------------------------------------
137
137
138 class InteractiveShellApp(Configurable):
138 class InteractiveShellApp(Configurable):
139 """A Mixin for applications that start InteractiveShell instances.
139 """A Mixin for applications that start InteractiveShell instances.
140
140
141 Provides configurables for loading extensions and executing files
141 Provides configurables for loading extensions and executing files
142 as part of configuring a Shell environment.
142 as part of configuring a Shell environment.
143
143
144 The following methods should be called by the :meth:`initialize` method
144 The following methods should be called by the :meth:`initialize` method
145 of the subclass:
145 of the subclass:
146
146
147 - :meth:`init_path`
147 - :meth:`init_path`
148 - :meth:`init_shell` (to be implemented by the subclass)
148 - :meth:`init_shell` (to be implemented by the subclass)
149 - :meth:`init_gui_pylab`
149 - :meth:`init_gui_pylab`
150 - :meth:`init_extensions`
150 - :meth:`init_extensions`
151 - :meth:`init_code`
151 - :meth:`init_code`
152 """
152 """
153 extensions = List(Unicode, config=True,
153 extensions = List(Unicode, config=True,
154 help="A list of dotted module names of IPython extensions to load."
154 help="A list of dotted module names of IPython extensions to load."
155 )
155 )
156 extra_extension = Unicode('', config=True,
156 extra_extension = Unicode('', config=True,
157 help="dotted module name of an IPython extension to load."
157 help="dotted module name of an IPython extension to load."
158 )
158 )
159 def _extra_extension_changed(self, name, old, new):
159 def _extra_extension_changed(self, name, old, new):
160 if new:
160 if new:
161 # add to self.extensions
161 # add to self.extensions
162 self.extensions.append(new)
162 self.extensions.append(new)
163
163
164 # Extensions that are always loaded (not configurable)
164 # Extensions that are always loaded (not configurable)
165 default_extensions = List(Unicode, [u'storemagic'], config=False)
165 default_extensions = List(Unicode, [u'storemagic'], config=False)
166
166
167 exec_files = List(Unicode, config=True,
167 exec_files = List(Unicode, config=True,
168 help="""List of files to run at IPython startup."""
168 help="""List of files to run at IPython startup."""
169 )
169 )
170 file_to_run = Unicode('', config=True,
170 file_to_run = Unicode('', config=True,
171 help="""A file to be run""")
171 help="""A file to be run""")
172
172
173 exec_lines = List(Unicode, config=True,
173 exec_lines = List(Unicode, config=True,
174 help="""lines of code to run at IPython startup."""
174 help="""lines of code to run at IPython startup."""
175 )
175 )
176 code_to_run = Unicode('', config=True,
176 code_to_run = Unicode('', config=True,
177 help="Execute the given command string."
177 help="Execute the given command string."
178 )
178 )
179 module_to_run = Unicode('', config=True,
179 module_to_run = Unicode('', config=True,
180 help="Run the module as a script."
180 help="Run the module as a script."
181 )
181 )
182 gui = CaselessStrEnum(gui_keys, config=True,
182 gui = CaselessStrEnum(gui_keys, config=True,
183 help="Enable GUI event loop integration with any of {0}.".format(gui_keys)
183 help="Enable GUI event loop integration with any of {0}.".format(gui_keys)
184 )
184 )
185 matplotlib = CaselessStrEnum(backend_keys,
185 matplotlib = CaselessStrEnum(backend_keys,
186 config=True,
186 config=True,
187 help="""Configure matplotlib for interactive use with
187 help="""Configure matplotlib for interactive use with
188 the default matplotlib backend."""
188 the default matplotlib backend."""
189 )
189 )
190 pylab = CaselessStrEnum(backend_keys,
190 pylab = CaselessStrEnum(backend_keys,
191 config=True,
191 config=True,
192 help="""Pre-load matplotlib and numpy for interactive use,
192 help="""Pre-load matplotlib and numpy for interactive use,
193 selecting a particular matplotlib backend and loop integration.
193 selecting a particular matplotlib backend and loop integration.
194 """
194 """
195 )
195 )
196 pylab_import_all = Bool(True, config=True,
196 pylab_import_all = Bool(True, config=True,
197 help="""If true, IPython will populate the user namespace with numpy, pylab, etc.
197 help="""If true, IPython will populate the user namespace with numpy, pylab, etc.
198 and an 'import *' is done from numpy and pylab, when using pylab mode.
198 and an 'import *' is done from numpy and pylab, when using pylab mode.
199
199
200 When False, pylab mode should not import any names into the user namespace.
200 When False, pylab mode should not import any names into the user namespace.
201 """
201 """
202 )
202 )
203 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
203 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
204
204
205 user_ns = Instance(dict, args=None, allow_none=True)
205 user_ns = Instance(dict, args=None, allow_none=True)
206 def _user_ns_changed(self, name, old, new):
206 def _user_ns_changed(self, name, old, new):
207 if self.shell is not None:
207 if self.shell is not None:
208 self.shell.user_ns = new
208 self.shell.user_ns = new
209 self.shell.init_user_ns()
209 self.shell.init_user_ns()
210
210
211 def init_path(self):
211 def init_path(self):
212 """Add current working directory, '', to sys.path"""
212 """Add current working directory, '', to sys.path"""
213 if sys.path[0] != '':
213 if sys.path[0] != '':
214 sys.path.insert(0, '')
214 sys.path.insert(0, '')
215
215
216 def init_shell(self):
216 def init_shell(self):
217 raise NotImplementedError("Override in subclasses")
217 raise NotImplementedError("Override in subclasses")
218
218
219 def init_gui_pylab(self):
219 def init_gui_pylab(self):
220 """Enable GUI event loop integration, taking pylab into account."""
220 """Enable GUI event loop integration, taking pylab into account."""
221 enable = False
221 enable = False
222 shell = self.shell
222 shell = self.shell
223 if self.pylab:
223 if self.pylab:
224 enable = lambda key: shell.enable_pylab(key, import_all=self.pylab_import_all)
224 enable = lambda key: shell.enable_pylab(key, import_all=self.pylab_import_all)
225 key = self.pylab
225 key = self.pylab
226 elif self.matplotlib:
226 elif self.matplotlib:
227 enable = shell.enable_matplotlib
227 enable = shell.enable_matplotlib
228 key = self.matplotlib
228 key = self.matplotlib
229 elif self.gui:
229 elif self.gui:
230 enable = shell.enable_gui
230 enable = shell.enable_gui
231 key = self.gui
231 key = self.gui
232
232
233 if not enable:
233 if not enable:
234 return
234 return
235
235
236 try:
236 try:
237 r = enable(key)
237 r = enable(key)
238 except ImportError:
238 except ImportError:
239 self.log.warn("Eventloop or matplotlib integration failed. Is matplotlib installed?")
239 self.log.warn("Eventloop or matplotlib integration failed. Is matplotlib installed?")
240 self.shell.showtraceback()
240 self.shell.showtraceback()
241 return
241 return
242 except Exception:
242 except Exception:
243 self.log.warn("GUI event loop or pylab initialization failed")
243 self.log.warn("GUI event loop or pylab initialization failed")
244 self.shell.showtraceback()
244 self.shell.showtraceback()
245 return
245 return
246
246
247 if isinstance(r, tuple):
247 if isinstance(r, tuple):
248 gui, backend = r[:2]
248 gui, backend = r[:2]
249 self.log.info("Enabling GUI event loop integration, "
249 self.log.info("Enabling GUI event loop integration, "
250 "eventloop=%s, matplotlib=%s", gui, backend)
250 "eventloop=%s, matplotlib=%s", gui, backend)
251 if key == "auto":
251 if key == "auto":
252 print(("Using matplotlib backend: %s" % backend))
252 print("Using matplotlib backend: %s" % backend)
253 else:
253 else:
254 gui = r
254 gui = r
255 self.log.info("Enabling GUI event loop integration, "
255 self.log.info("Enabling GUI event loop integration, "
256 "eventloop=%s", gui)
256 "eventloop=%s", gui)
257
257
258 def init_extensions(self):
258 def init_extensions(self):
259 """Load all IPython extensions in IPythonApp.extensions.
259 """Load all IPython extensions in IPythonApp.extensions.
260
260
261 This uses the :meth:`ExtensionManager.load_extensions` to load all
261 This uses the :meth:`ExtensionManager.load_extensions` to load all
262 the extensions listed in ``self.extensions``.
262 the extensions listed in ``self.extensions``.
263 """
263 """
264 try:
264 try:
265 self.log.debug("Loading IPython extensions...")
265 self.log.debug("Loading IPython extensions...")
266 extensions = self.default_extensions + self.extensions
266 extensions = self.default_extensions + self.extensions
267 for ext in extensions:
267 for ext in extensions:
268 try:
268 try:
269 self.log.info("Loading IPython extension: %s" % ext)
269 self.log.info("Loading IPython extension: %s" % ext)
270 self.shell.extension_manager.load_extension(ext)
270 self.shell.extension_manager.load_extension(ext)
271 except:
271 except:
272 self.log.warn("Error in loading extension: %s" % ext +
272 self.log.warn("Error in loading extension: %s" % ext +
273 "\nCheck your config files in %s" % self.profile_dir.location
273 "\nCheck your config files in %s" % self.profile_dir.location
274 )
274 )
275 self.shell.showtraceback()
275 self.shell.showtraceback()
276 except:
276 except:
277 self.log.warn("Unknown error in loading extensions:")
277 self.log.warn("Unknown error in loading extensions:")
278 self.shell.showtraceback()
278 self.shell.showtraceback()
279
279
280 def init_code(self):
280 def init_code(self):
281 """run the pre-flight code, specified via exec_lines"""
281 """run the pre-flight code, specified via exec_lines"""
282 self._run_startup_files()
282 self._run_startup_files()
283 self._run_exec_lines()
283 self._run_exec_lines()
284 self._run_exec_files()
284 self._run_exec_files()
285 self._run_cmd_line_code()
285 self._run_cmd_line_code()
286 self._run_module()
286 self._run_module()
287
287
288 # flush output, so itwon't be attached to the first cell
288 # flush output, so itwon't be attached to the first cell
289 sys.stdout.flush()
289 sys.stdout.flush()
290 sys.stderr.flush()
290 sys.stderr.flush()
291
291
292 # Hide variables defined here from %who etc.
292 # Hide variables defined here from %who etc.
293 self.shell.user_ns_hidden.update(self.shell.user_ns)
293 self.shell.user_ns_hidden.update(self.shell.user_ns)
294
294
295 def _run_exec_lines(self):
295 def _run_exec_lines(self):
296 """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
296 """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
297 if not self.exec_lines:
297 if not self.exec_lines:
298 return
298 return
299 try:
299 try:
300 self.log.debug("Running code from IPythonApp.exec_lines...")
300 self.log.debug("Running code from IPythonApp.exec_lines...")
301 for line in self.exec_lines:
301 for line in self.exec_lines:
302 try:
302 try:
303 self.log.info("Running code in user namespace: %s" %
303 self.log.info("Running code in user namespace: %s" %
304 line)
304 line)
305 self.shell.run_cell(line, store_history=False)
305 self.shell.run_cell(line, store_history=False)
306 except:
306 except:
307 self.log.warn("Error in executing line in user "
307 self.log.warn("Error in executing line in user "
308 "namespace: %s" % line)
308 "namespace: %s" % line)
309 self.shell.showtraceback()
309 self.shell.showtraceback()
310 except:
310 except:
311 self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
311 self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
312 self.shell.showtraceback()
312 self.shell.showtraceback()
313
313
314 def _exec_file(self, fname):
314 def _exec_file(self, fname):
315 try:
315 try:
316 full_filename = filefind(fname, [u'.', self.ipython_dir])
316 full_filename = filefind(fname, [u'.', self.ipython_dir])
317 except IOError as e:
317 except IOError as e:
318 self.log.warn("File not found: %r"%fname)
318 self.log.warn("File not found: %r"%fname)
319 return
319 return
320 # Make sure that the running script gets a proper sys.argv as if it
320 # Make sure that the running script gets a proper sys.argv as if it
321 # were run from a system shell.
321 # were run from a system shell.
322 save_argv = sys.argv
322 save_argv = sys.argv
323 sys.argv = [full_filename] + self.extra_args[1:]
323 sys.argv = [full_filename] + self.extra_args[1:]
324 # protect sys.argv from potential unicode strings on Python 2:
324 # protect sys.argv from potential unicode strings on Python 2:
325 if not py3compat.PY3:
325 if not py3compat.PY3:
326 sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
326 sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
327 try:
327 try:
328 if os.path.isfile(full_filename):
328 if os.path.isfile(full_filename):
329 self.log.info("Running file in user namespace: %s" %
329 self.log.info("Running file in user namespace: %s" %
330 full_filename)
330 full_filename)
331 # Ensure that __file__ is always defined to match Python
331 # Ensure that __file__ is always defined to match Python
332 # behavior.
332 # behavior.
333 with preserve_keys(self.shell.user_ns, '__file__'):
333 with preserve_keys(self.shell.user_ns, '__file__'):
334 self.shell.user_ns['__file__'] = fname
334 self.shell.user_ns['__file__'] = fname
335 if full_filename.endswith('.ipy'):
335 if full_filename.endswith('.ipy'):
336 self.shell.safe_execfile_ipy(full_filename)
336 self.shell.safe_execfile_ipy(full_filename)
337 else:
337 else:
338 # default to python, even without extension
338 # default to python, even without extension
339 self.shell.safe_execfile(full_filename,
339 self.shell.safe_execfile(full_filename,
340 self.shell.user_ns)
340 self.shell.user_ns)
341 finally:
341 finally:
342 sys.argv = save_argv
342 sys.argv = save_argv
343
343
344 def _run_startup_files(self):
344 def _run_startup_files(self):
345 """Run files from profile startup directory"""
345 """Run files from profile startup directory"""
346 startup_dir = self.profile_dir.startup_dir
346 startup_dir = self.profile_dir.startup_dir
347 startup_files = []
347 startup_files = []
348 if os.environ.get('PYTHONSTARTUP', False):
348 if os.environ.get('PYTHONSTARTUP', False):
349 startup_files.append(os.environ['PYTHONSTARTUP'])
349 startup_files.append(os.environ['PYTHONSTARTUP'])
350 startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
350 startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
351 startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
351 startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
352 if not startup_files:
352 if not startup_files:
353 return
353 return
354
354
355 self.log.debug("Running startup files from %s...", startup_dir)
355 self.log.debug("Running startup files from %s...", startup_dir)
356 try:
356 try:
357 for fname in sorted(startup_files):
357 for fname in sorted(startup_files):
358 self._exec_file(fname)
358 self._exec_file(fname)
359 except:
359 except:
360 self.log.warn("Unknown error in handling startup files:")
360 self.log.warn("Unknown error in handling startup files:")
361 self.shell.showtraceback()
361 self.shell.showtraceback()
362
362
363 def _run_exec_files(self):
363 def _run_exec_files(self):
364 """Run files from IPythonApp.exec_files"""
364 """Run files from IPythonApp.exec_files"""
365 if not self.exec_files:
365 if not self.exec_files:
366 return
366 return
367
367
368 self.log.debug("Running files in IPythonApp.exec_files...")
368 self.log.debug("Running files in IPythonApp.exec_files...")
369 try:
369 try:
370 for fname in self.exec_files:
370 for fname in self.exec_files:
371 self._exec_file(fname)
371 self._exec_file(fname)
372 except:
372 except:
373 self.log.warn("Unknown error in handling IPythonApp.exec_files:")
373 self.log.warn("Unknown error in handling IPythonApp.exec_files:")
374 self.shell.showtraceback()
374 self.shell.showtraceback()
375
375
376 def _run_cmd_line_code(self):
376 def _run_cmd_line_code(self):
377 """Run code or file specified at the command-line"""
377 """Run code or file specified at the command-line"""
378 if self.code_to_run:
378 if self.code_to_run:
379 line = self.code_to_run
379 line = self.code_to_run
380 try:
380 try:
381 self.log.info("Running code given at command line (c=): %s" %
381 self.log.info("Running code given at command line (c=): %s" %
382 line)
382 line)
383 self.shell.run_cell(line, store_history=False)
383 self.shell.run_cell(line, store_history=False)
384 except:
384 except:
385 self.log.warn("Error in executing line in user namespace: %s" %
385 self.log.warn("Error in executing line in user namespace: %s" %
386 line)
386 line)
387 self.shell.showtraceback()
387 self.shell.showtraceback()
388
388
389 # Like Python itself, ignore the second if the first of these is present
389 # Like Python itself, ignore the second if the first of these is present
390 elif self.file_to_run:
390 elif self.file_to_run:
391 fname = self.file_to_run
391 fname = self.file_to_run
392 try:
392 try:
393 self._exec_file(fname)
393 self._exec_file(fname)
394 except:
394 except:
395 self.log.warn("Error in executing file in user namespace: %s" %
395 self.log.warn("Error in executing file in user namespace: %s" %
396 fname)
396 fname)
397 self.shell.showtraceback()
397 self.shell.showtraceback()
398
398
399 def _run_module(self):
399 def _run_module(self):
400 """Run module specified at the command-line."""
400 """Run module specified at the command-line."""
401 if self.module_to_run:
401 if self.module_to_run:
402 # Make sure that the module gets a proper sys.argv as if it were
402 # Make sure that the module gets a proper sys.argv as if it were
403 # run using `python -m`.
403 # run using `python -m`.
404 save_argv = sys.argv
404 save_argv = sys.argv
405 sys.argv = [sys.executable] + self.extra_args
405 sys.argv = [sys.executable] + self.extra_args
406 try:
406 try:
407 self.shell.safe_run_module(self.module_to_run,
407 self.shell.safe_run_module(self.module_to_run,
408 self.shell.user_ns)
408 self.shell.user_ns)
409 finally:
409 finally:
410 sys.argv = save_argv
410 sys.argv = save_argv
@@ -1,508 +1,507 b''
1 """IPython extension to reload modules before executing user code.
1 """IPython extension to reload modules before executing user code.
2
2
3 ``autoreload`` reloads modules automatically before entering the execution of
3 ``autoreload`` reloads modules automatically before entering the execution of
4 code typed at the IPython prompt.
4 code typed at the IPython prompt.
5
5
6 This makes for example the following workflow possible:
6 This makes for example the following workflow possible:
7
7
8 .. sourcecode:: ipython
8 .. sourcecode:: ipython
9
9
10 In [1]: %load_ext autoreload
10 In [1]: %load_ext autoreload
11
11
12 In [2]: %autoreload 2
12 In [2]: %autoreload 2
13
13
14 In [3]: from foo import some_function
14 In [3]: from foo import some_function
15
15
16 In [4]: some_function()
16 In [4]: some_function()
17 Out[4]: 42
17 Out[4]: 42
18
18
19 In [5]: # open foo.py in an editor and change some_function to return 43
19 In [5]: # open foo.py in an editor and change some_function to return 43
20
20
21 In [6]: some_function()
21 In [6]: some_function()
22 Out[6]: 43
22 Out[6]: 43
23
23
24 The module was reloaded without reloading it explicitly, and the object
24 The module was reloaded without reloading it explicitly, and the object
25 imported with ``from foo import ...`` was also updated.
25 imported with ``from foo import ...`` was also updated.
26
26
27 Usage
27 Usage
28 =====
28 =====
29
29
30 The following magic commands are provided:
30 The following magic commands are provided:
31
31
32 ``%autoreload``
32 ``%autoreload``
33
33
34 Reload all modules (except those excluded by ``%aimport``)
34 Reload all modules (except those excluded by ``%aimport``)
35 automatically now.
35 automatically now.
36
36
37 ``%autoreload 0``
37 ``%autoreload 0``
38
38
39 Disable automatic reloading.
39 Disable automatic reloading.
40
40
41 ``%autoreload 1``
41 ``%autoreload 1``
42
42
43 Reload all modules imported with ``%aimport`` every time before
43 Reload all modules imported with ``%aimport`` every time before
44 executing the Python code typed.
44 executing the Python code typed.
45
45
46 ``%autoreload 2``
46 ``%autoreload 2``
47
47
48 Reload all modules (except those excluded by ``%aimport``) every
48 Reload all modules (except those excluded by ``%aimport``) every
49 time before executing the Python code typed.
49 time before executing the Python code typed.
50
50
51 ``%aimport``
51 ``%aimport``
52
52
53 List modules which are to be automatically imported or not to be imported.
53 List modules which are to be automatically imported or not to be imported.
54
54
55 ``%aimport foo``
55 ``%aimport foo``
56
56
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58
58
59 ``%aimport -foo``
59 ``%aimport -foo``
60
60
61 Mark module 'foo' to not be autoreloaded.
61 Mark module 'foo' to not be autoreloaded.
62
62
63 Caveats
63 Caveats
64 =======
64 =======
65
65
66 Reloading Python modules in a reliable way is in general difficult,
66 Reloading Python modules in a reliable way is in general difficult,
67 and unexpected things may occur. ``%autoreload`` tries to work around
67 and unexpected things may occur. ``%autoreload`` tries to work around
68 common pitfalls by replacing function code objects and parts of
68 common pitfalls by replacing function code objects and parts of
69 classes previously in the module with new versions. This makes the
69 classes previously in the module with new versions. This makes the
70 following things to work:
70 following things to work:
71
71
72 - Functions and classes imported via 'from xxx import foo' are upgraded
72 - Functions and classes imported via 'from xxx import foo' are upgraded
73 to new versions when 'xxx' is reloaded.
73 to new versions when 'xxx' is reloaded.
74
74
75 - Methods and properties of classes are upgraded on reload, so that
75 - Methods and properties of classes are upgraded on reload, so that
76 calling 'c.foo()' on an object 'c' created before the reload causes
76 calling 'c.foo()' on an object 'c' created before the reload causes
77 the new code for 'foo' to be executed.
77 the new code for 'foo' to be executed.
78
78
79 Some of the known remaining caveats are:
79 Some of the known remaining caveats are:
80
80
81 - Replacing code objects does not always succeed: changing a @property
81 - Replacing code objects does not always succeed: changing a @property
82 in a class to an ordinary method or a method to a member variable
82 in a class to an ordinary method or a method to a member variable
83 can cause problems (but in old objects only).
83 can cause problems (but in old objects only).
84
84
85 - Functions that are removed (eg. via monkey-patching) from a module
85 - Functions that are removed (eg. via monkey-patching) from a module
86 before it is reloaded are not upgraded.
86 before it is reloaded are not upgraded.
87
87
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
89 """
89 """
90 from __future__ import print_function
90 from __future__ import print_function
91
91
92 skip_doctest = True
92 skip_doctest = True
93
93
94 #-----------------------------------------------------------------------------
94 #-----------------------------------------------------------------------------
95 # Copyright (C) 2000 Thomas Heller
95 # Copyright (C) 2000 Thomas Heller
96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
97 # Copyright (C) 2012 The IPython Development Team
97 # Copyright (C) 2012 The IPython Development Team
98 #
98 #
99 # Distributed under the terms of the BSD License. The full license is in
99 # Distributed under the terms of the BSD License. The full license is in
100 # the file COPYING, distributed as part of this software.
100 # the file COPYING, distributed as part of this software.
101 #-----------------------------------------------------------------------------
101 #-----------------------------------------------------------------------------
102 #
102 #
103 # This IPython module is written by Pauli Virtanen, based on the autoreload
103 # This IPython module is written by Pauli Virtanen, based on the autoreload
104 # code by Thomas Heller.
104 # code by Thomas Heller.
105
105
106 #-----------------------------------------------------------------------------
106 #-----------------------------------------------------------------------------
107 # Imports
107 # Imports
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109
109
110 import os
110 import os
111 import sys
111 import sys
112 import traceback
112 import traceback
113 import types
113 import types
114 import weakref
114 import weakref
115
115
116 try:
116 try:
117 # Reload is not defined by default in Python3.
117 # Reload is not defined by default in Python3.
118 reload
118 reload
119 except NameError:
119 except NameError:
120 from imp import reload
120 from imp import reload
121
121
122 from IPython.utils import openpy
122 from IPython.utils import openpy
123 from IPython.utils.py3compat import PY3
123 from IPython.utils.py3compat import PY3
124
124
125 #------------------------------------------------------------------------------
125 #------------------------------------------------------------------------------
126 # Autoreload functionality
126 # Autoreload functionality
127 #------------------------------------------------------------------------------
127 #------------------------------------------------------------------------------
128
128
129 class ModuleReloader(object):
129 class ModuleReloader(object):
130 enabled = False
130 enabled = False
131 """Whether this reloader is enabled"""
131 """Whether this reloader is enabled"""
132
132
133 failed = {}
133 failed = {}
134 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
134 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
135
135
136 modules = {}
136 modules = {}
137 """Modules specially marked as autoreloadable."""
137 """Modules specially marked as autoreloadable."""
138
138
139 skip_modules = {}
139 skip_modules = {}
140 """Modules specially marked as not autoreloadable."""
140 """Modules specially marked as not autoreloadable."""
141
141
142 check_all = True
142 check_all = True
143 """Autoreload all modules, not just those listed in 'modules'"""
143 """Autoreload all modules, not just those listed in 'modules'"""
144
144
145 old_objects = {}
145 old_objects = {}
146 """(module-name, name) -> weakref, for replacing old code objects"""
146 """(module-name, name) -> weakref, for replacing old code objects"""
147
147
148 def mark_module_skipped(self, module_name):
148 def mark_module_skipped(self, module_name):
149 """Skip reloading the named module in the future"""
149 """Skip reloading the named module in the future"""
150 try:
150 try:
151 del self.modules[module_name]
151 del self.modules[module_name]
152 except KeyError:
152 except KeyError:
153 pass
153 pass
154 self.skip_modules[module_name] = True
154 self.skip_modules[module_name] = True
155
155
156 def mark_module_reloadable(self, module_name):
156 def mark_module_reloadable(self, module_name):
157 """Reload the named module in the future (if it is imported)"""
157 """Reload the named module in the future (if it is imported)"""
158 try:
158 try:
159 del self.skip_modules[module_name]
159 del self.skip_modules[module_name]
160 except KeyError:
160 except KeyError:
161 pass
161 pass
162 self.modules[module_name] = True
162 self.modules[module_name] = True
163
163
164 def aimport_module(self, module_name):
164 def aimport_module(self, module_name):
165 """Import a module, and mark it reloadable
165 """Import a module, and mark it reloadable
166
166
167 Returns
167 Returns
168 -------
168 -------
169 top_module : module
169 top_module : module
170 The imported module if it is top-level, or the top-level
170 The imported module if it is top-level, or the top-level
171 top_name : module
171 top_name : module
172 Name of top_module
172 Name of top_module
173
173
174 """
174 """
175 self.mark_module_reloadable(module_name)
175 self.mark_module_reloadable(module_name)
176
176
177 __import__(module_name)
177 __import__(module_name)
178 top_name = module_name.split('.')[0]
178 top_name = module_name.split('.')[0]
179 top_module = sys.modules[top_name]
179 top_module = sys.modules[top_name]
180 return top_module, top_name
180 return top_module, top_name
181
181
182 def check(self, check_all=False):
182 def check(self, check_all=False):
183 """Check whether some modules need to be reloaded."""
183 """Check whether some modules need to be reloaded."""
184
184
185 if not self.enabled and not check_all:
185 if not self.enabled and not check_all:
186 return
186 return
187
187
188 if check_all or self.check_all:
188 if check_all or self.check_all:
189 modules = list(sys.modules.keys())
189 modules = list(sys.modules.keys())
190 else:
190 else:
191 modules = list(self.modules.keys())
191 modules = list(self.modules.keys())
192
192
193 for modname in modules:
193 for modname in modules:
194 m = sys.modules.get(modname, None)
194 m = sys.modules.get(modname, None)
195
195
196 if modname in self.skip_modules:
196 if modname in self.skip_modules:
197 continue
197 continue
198
198
199 if not hasattr(m, '__file__'):
199 if not hasattr(m, '__file__'):
200 continue
200 continue
201
201
202 if m.__name__ == '__main__':
202 if m.__name__ == '__main__':
203 # we cannot reload(__main__)
203 # we cannot reload(__main__)
204 continue
204 continue
205
205
206 filename = m.__file__
206 filename = m.__file__
207 path, ext = os.path.splitext(filename)
207 path, ext = os.path.splitext(filename)
208
208
209 if ext.lower() == '.py':
209 if ext.lower() == '.py':
210 pyc_filename = openpy.cache_from_source(filename)
210 pyc_filename = openpy.cache_from_source(filename)
211 py_filename = filename
211 py_filename = filename
212 else:
212 else:
213 pyc_filename = filename
213 pyc_filename = filename
214 try:
214 try:
215 py_filename = openpy.source_from_cache(filename)
215 py_filename = openpy.source_from_cache(filename)
216 except ValueError:
216 except ValueError:
217 continue
217 continue
218
218
219 try:
219 try:
220 pymtime = os.stat(py_filename).st_mtime
220 pymtime = os.stat(py_filename).st_mtime
221 if pymtime <= os.stat(pyc_filename).st_mtime:
221 if pymtime <= os.stat(pyc_filename).st_mtime:
222 continue
222 continue
223 if self.failed.get(py_filename, None) == pymtime:
223 if self.failed.get(py_filename, None) == pymtime:
224 continue
224 continue
225 except OSError:
225 except OSError:
226 continue
226 continue
227
227
228 try:
228 try:
229 superreload(m, reload, self.old_objects)
229 superreload(m, reload, self.old_objects)
230 if py_filename in self.failed:
230 if py_filename in self.failed:
231 del self.failed[py_filename]
231 del self.failed[py_filename]
232 except:
232 except:
233 print("[autoreload of %s failed: %s]" % (
233 print("[autoreload of %s failed: %s]" % (
234 modname, traceback.format_exc(1)), file=sys.stderr)
234 modname, traceback.format_exc(1)), file=sys.stderr)
235 self.failed[py_filename] = pymtime
235 self.failed[py_filename] = pymtime
236
236
237 #------------------------------------------------------------------------------
237 #------------------------------------------------------------------------------
238 # superreload
238 # superreload
239 #------------------------------------------------------------------------------
239 #------------------------------------------------------------------------------
240
240
241 if PY3:
241 if PY3:
242 func_attrs = ['__code__', '__defaults__', '__doc__',
242 func_attrs = ['__code__', '__defaults__', '__doc__',
243 '__closure__', '__globals__', '__dict__']
243 '__closure__', '__globals__', '__dict__']
244 else:
244 else:
245 func_attrs = ['func_code', 'func_defaults', 'func_doc',
245 func_attrs = ['func_code', 'func_defaults', 'func_doc',
246 'func_closure', 'func_globals', 'func_dict']
246 'func_closure', 'func_globals', 'func_dict']
247
247
248
248
249 def update_function(old, new):
249 def update_function(old, new):
250 """Upgrade the code object of a function"""
250 """Upgrade the code object of a function"""
251 for name in func_attrs:
251 for name in func_attrs:
252 try:
252 try:
253 setattr(old, name, getattr(new, name))
253 setattr(old, name, getattr(new, name))
254 except (AttributeError, TypeError):
254 except (AttributeError, TypeError):
255 pass
255 pass
256
256
257
257
258 def update_class(old, new):
258 def update_class(old, new):
259 """Replace stuff in the __dict__ of a class, and upgrade
259 """Replace stuff in the __dict__ of a class, and upgrade
260 method code objects"""
260 method code objects"""
261 for key in list(old.__dict__.keys()):
261 for key in list(old.__dict__.keys()):
262 old_obj = getattr(old, key)
262 old_obj = getattr(old, key)
263
263
264 try:
264 try:
265 new_obj = getattr(new, key)
265 new_obj = getattr(new, key)
266 except AttributeError:
266 except AttributeError:
267 # obsolete attribute: remove it
267 # obsolete attribute: remove it
268 try:
268 try:
269 delattr(old, key)
269 delattr(old, key)
270 except (AttributeError, TypeError):
270 except (AttributeError, TypeError):
271 pass
271 pass
272 continue
272 continue
273
273
274 if update_generic(old_obj, new_obj): continue
274 if update_generic(old_obj, new_obj): continue
275
275
276 try:
276 try:
277 setattr(old, key, getattr(new, key))
277 setattr(old, key, getattr(new, key))
278 except (AttributeError, TypeError):
278 except (AttributeError, TypeError):
279 pass # skip non-writable attributes
279 pass # skip non-writable attributes
280
280
281
281
282 def update_property(old, new):
282 def update_property(old, new):
283 """Replace get/set/del functions of a property"""
283 """Replace get/set/del functions of a property"""
284 update_generic(old.fdel, new.fdel)
284 update_generic(old.fdel, new.fdel)
285 update_generic(old.fget, new.fget)
285 update_generic(old.fget, new.fget)
286 update_generic(old.fset, new.fset)
286 update_generic(old.fset, new.fset)
287
287
288
288
289 def isinstance2(a, b, typ):
289 def isinstance2(a, b, typ):
290 return isinstance(a, typ) and isinstance(b, typ)
290 return isinstance(a, typ) and isinstance(b, typ)
291
291
292
292
293 UPDATE_RULES = [
293 UPDATE_RULES = [
294 (lambda a, b: isinstance2(a, b, type),
294 (lambda a, b: isinstance2(a, b, type),
295 update_class),
295 update_class),
296 (lambda a, b: isinstance2(a, b, types.FunctionType),
296 (lambda a, b: isinstance2(a, b, types.FunctionType),
297 update_function),
297 update_function),
298 (lambda a, b: isinstance2(a, b, property),
298 (lambda a, b: isinstance2(a, b, property),
299 update_property),
299 update_property),
300 ]
300 ]
301
301
302
302
303 if PY3:
303 if PY3:
304 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
304 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
305 lambda a, b: update_function(a.__func__, b.__func__)),
305 lambda a, b: update_function(a.__func__, b.__func__)),
306 ])
306 ])
307 else:
307 else:
308 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
308 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
309 update_class),
309 update_class),
310 (lambda a, b: isinstance2(a, b, types.MethodType),
310 (lambda a, b: isinstance2(a, b, types.MethodType),
311 lambda a, b: update_function(a.__func__, b.__func__)),
311 lambda a, b: update_function(a.__func__, b.__func__)),
312 ])
312 ])
313
313
314
314
315 def update_generic(a, b):
315 def update_generic(a, b):
316 for type_check, update in UPDATE_RULES:
316 for type_check, update in UPDATE_RULES:
317 if type_check(a, b):
317 if type_check(a, b):
318 update(a, b)
318 update(a, b)
319 return True
319 return True
320 return False
320 return False
321
321
322
322
323 class StrongRef(object):
323 class StrongRef(object):
324 def __init__(self, obj):
324 def __init__(self, obj):
325 self.obj = obj
325 self.obj = obj
326 def __call__(self):
326 def __call__(self):
327 return self.obj
327 return self.obj
328
328
329
329
330 def superreload(module, reload=reload, old_objects={}):
330 def superreload(module, reload=reload, old_objects={}):
331 """Enhanced version of the builtin reload function.
331 """Enhanced version of the builtin reload function.
332
332
333 superreload remembers objects previously in the module, and
333 superreload remembers objects previously in the module, and
334
334
335 - upgrades the class dictionary of every old class in the module
335 - upgrades the class dictionary of every old class in the module
336 - upgrades the code object of every old function and method
336 - upgrades the code object of every old function and method
337 - clears the module's namespace before reloading
337 - clears the module's namespace before reloading
338
338
339 """
339 """
340
340
341 # collect old objects in the module
341 # collect old objects in the module
342 for name, obj in list(module.__dict__.items()):
342 for name, obj in list(module.__dict__.items()):
343 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
343 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
344 continue
344 continue
345 key = (module.__name__, name)
345 key = (module.__name__, name)
346 try:
346 try:
347 old_objects.setdefault(key, []).append(weakref.ref(obj))
347 old_objects.setdefault(key, []).append(weakref.ref(obj))
348 except TypeError:
348 except TypeError:
349 # weakref doesn't work for all types;
349 # weakref doesn't work for all types;
350 # create strong references for 'important' cases
350 # create strong references for 'important' cases
351 if not PY3 and isinstance(obj, types.ClassType):
351 if not PY3 and isinstance(obj, types.ClassType):
352 old_objects.setdefault(key, []).append(StrongRef(obj))
352 old_objects.setdefault(key, []).append(StrongRef(obj))
353
353
354 # reload module
354 # reload module
355 try:
355 try:
356 # clear namespace first from old cruft
356 # clear namespace first from old cruft
357 old_dict = module.__dict__.copy()
357 old_dict = module.__dict__.copy()
358 old_name = module.__name__
358 old_name = module.__name__
359 module.__dict__.clear()
359 module.__dict__.clear()
360 module.__dict__['__name__'] = old_name
360 module.__dict__['__name__'] = old_name
361 module.__dict__['__loader__'] = old_dict['__loader__']
361 module.__dict__['__loader__'] = old_dict['__loader__']
362 except (TypeError, AttributeError, KeyError):
362 except (TypeError, AttributeError, KeyError):
363 pass
363 pass
364
364
365 try:
365 try:
366 module = reload(module)
366 module = reload(module)
367 except:
367 except:
368 # restore module dictionary on failed reload
368 # restore module dictionary on failed reload
369 module.__dict__.update(old_dict)
369 module.__dict__.update(old_dict)
370 raise
370 raise
371
371
372 # iterate over all objects and update functions & classes
372 # iterate over all objects and update functions & classes
373 for name, new_obj in list(module.__dict__.items()):
373 for name, new_obj in list(module.__dict__.items()):
374 key = (module.__name__, name)
374 key = (module.__name__, name)
375 if key not in old_objects: continue
375 if key not in old_objects: continue
376
376
377 new_refs = []
377 new_refs = []
378 for old_ref in old_objects[key]:
378 for old_ref in old_objects[key]:
379 old_obj = old_ref()
379 old_obj = old_ref()
380 if old_obj is None: continue
380 if old_obj is None: continue
381 new_refs.append(old_ref)
381 new_refs.append(old_ref)
382 update_generic(old_obj, new_obj)
382 update_generic(old_obj, new_obj)
383
383
384 if new_refs:
384 if new_refs:
385 old_objects[key] = new_refs
385 old_objects[key] = new_refs
386 else:
386 else:
387 del old_objects[key]
387 del old_objects[key]
388
388
389 return module
389 return module
390
390
391 #------------------------------------------------------------------------------
391 #------------------------------------------------------------------------------
392 # IPython connectivity
392 # IPython connectivity
393 #------------------------------------------------------------------------------
393 #------------------------------------------------------------------------------
394
394
395 from IPython.core.hooks import TryNext
395 from IPython.core.hooks import TryNext
396 from IPython.core.magic import Magics, magics_class, line_magic
396 from IPython.core.magic import Magics, magics_class, line_magic
397
397
398 @magics_class
398 @magics_class
399 class AutoreloadMagics(Magics):
399 class AutoreloadMagics(Magics):
400 def __init__(self, *a, **kw):
400 def __init__(self, *a, **kw):
401 super(AutoreloadMagics, self).__init__(*a, **kw)
401 super(AutoreloadMagics, self).__init__(*a, **kw)
402 self._reloader = ModuleReloader()
402 self._reloader = ModuleReloader()
403 self._reloader.check_all = False
403 self._reloader.check_all = False
404
404
405 @line_magic
405 @line_magic
406 def autoreload(self, parameter_s=''):
406 def autoreload(self, parameter_s=''):
407 r"""%autoreload => Reload modules automatically
407 r"""%autoreload => Reload modules automatically
408
408
409 %autoreload
409 %autoreload
410 Reload all modules (except those excluded by %aimport) automatically
410 Reload all modules (except those excluded by %aimport) automatically
411 now.
411 now.
412
412
413 %autoreload 0
413 %autoreload 0
414 Disable automatic reloading.
414 Disable automatic reloading.
415
415
416 %autoreload 1
416 %autoreload 1
417 Reload all modules imported with %aimport every time before executing
417 Reload all modules imported with %aimport every time before executing
418 the Python code typed.
418 the Python code typed.
419
419
420 %autoreload 2
420 %autoreload 2
421 Reload all modules (except those excluded by %aimport) every time
421 Reload all modules (except those excluded by %aimport) every time
422 before executing the Python code typed.
422 before executing the Python code typed.
423
423
424 Reloading Python modules in a reliable way is in general
424 Reloading Python modules in a reliable way is in general
425 difficult, and unexpected things may occur. %autoreload tries to
425 difficult, and unexpected things may occur. %autoreload tries to
426 work around common pitfalls by replacing function code objects and
426 work around common pitfalls by replacing function code objects and
427 parts of classes previously in the module with new versions. This
427 parts of classes previously in the module with new versions. This
428 makes the following things to work:
428 makes the following things to work:
429
429
430 - Functions and classes imported via 'from xxx import foo' are upgraded
430 - Functions and classes imported via 'from xxx import foo' are upgraded
431 to new versions when 'xxx' is reloaded.
431 to new versions when 'xxx' is reloaded.
432
432
433 - Methods and properties of classes are upgraded on reload, so that
433 - Methods and properties of classes are upgraded on reload, so that
434 calling 'c.foo()' on an object 'c' created before the reload causes
434 calling 'c.foo()' on an object 'c' created before the reload causes
435 the new code for 'foo' to be executed.
435 the new code for 'foo' to be executed.
436
436
437 Some of the known remaining caveats are:
437 Some of the known remaining caveats are:
438
438
439 - Replacing code objects does not always succeed: changing a @property
439 - Replacing code objects does not always succeed: changing a @property
440 in a class to an ordinary method or a method to a member variable
440 in a class to an ordinary method or a method to a member variable
441 can cause problems (but in old objects only).
441 can cause problems (but in old objects only).
442
442
443 - Functions that are removed (eg. via monkey-patching) from a module
443 - Functions that are removed (eg. via monkey-patching) from a module
444 before it is reloaded are not upgraded.
444 before it is reloaded are not upgraded.
445
445
446 - C extension modules cannot be reloaded, and so cannot be
446 - C extension modules cannot be reloaded, and so cannot be
447 autoreloaded.
447 autoreloaded.
448
448
449 """
449 """
450 if parameter_s == '':
450 if parameter_s == '':
451 self._reloader.check(True)
451 self._reloader.check(True)
452 elif parameter_s == '0':
452 elif parameter_s == '0':
453 self._reloader.enabled = False
453 self._reloader.enabled = False
454 elif parameter_s == '1':
454 elif parameter_s == '1':
455 self._reloader.check_all = False
455 self._reloader.check_all = False
456 self._reloader.enabled = True
456 self._reloader.enabled = True
457 elif parameter_s == '2':
457 elif parameter_s == '2':
458 self._reloader.check_all = True
458 self._reloader.check_all = True
459 self._reloader.enabled = True
459 self._reloader.enabled = True
460
460
461 @line_magic
461 @line_magic
462 def aimport(self, parameter_s='', stream=None):
462 def aimport(self, parameter_s='', stream=None):
463 """%aimport => Import modules for automatic reloading.
463 """%aimport => Import modules for automatic reloading.
464
464
465 %aimport
465 %aimport
466 List modules to automatically import and not to import.
466 List modules to automatically import and not to import.
467
467
468 %aimport foo
468 %aimport foo
469 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
469 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
470
470
471 %aimport -foo
471 %aimport -foo
472 Mark module 'foo' to not be autoreloaded for %autoreload 1
472 Mark module 'foo' to not be autoreloaded for %autoreload 1
473 """
473 """
474 modname = parameter_s
474 modname = parameter_s
475 if not modname:
475 if not modname:
476 to_reload = sorted(self._reloader.modules.keys())
476 to_reload = sorted(self._reloader.modules.keys())
477 to_skip = sorted(self._reloader.skip_modules.keys())
477 to_skip = sorted(self._reloader.skip_modules.keys())
478 to_skip.sort()
479 if stream is None:
478 if stream is None:
480 stream = sys.stdout
479 stream = sys.stdout
481 if self._reloader.check_all:
480 if self._reloader.check_all:
482 stream.write("Modules to reload:\nall-except-skipped\n")
481 stream.write("Modules to reload:\nall-except-skipped\n")
483 else:
482 else:
484 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
483 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
485 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
484 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
486 elif modname.startswith('-'):
485 elif modname.startswith('-'):
487 modname = modname[1:]
486 modname = modname[1:]
488 self._reloader.mark_module_skipped(modname)
487 self._reloader.mark_module_skipped(modname)
489 else:
488 else:
490 top_module, top_name = self._reloader.aimport_module(modname)
489 top_module, top_name = self._reloader.aimport_module(modname)
491
490
492 # Inject module to user namespace
491 # Inject module to user namespace
493 self.shell.push({top_name: top_module})
492 self.shell.push({top_name: top_module})
494
493
495 def pre_run_code_hook(self, ip):
494 def pre_run_code_hook(self, ip):
496 if not self._reloader.enabled:
495 if not self._reloader.enabled:
497 raise TryNext
496 raise TryNext
498 try:
497 try:
499 self._reloader.check()
498 self._reloader.check()
500 except:
499 except:
501 pass
500 pass
502
501
503
502
504 def load_ipython_extension(ip):
503 def load_ipython_extension(ip):
505 """Load the extension in IPython."""
504 """Load the extension in IPython."""
506 auto_reload = AutoreloadMagics(ip)
505 auto_reload = AutoreloadMagics(ip)
507 ip.register_magics(auto_reload)
506 ip.register_magics(auto_reload)
508 ip.set_hook('pre_run_code_hook', auto_reload.pre_run_code_hook)
507 ip.set_hook('pre_run_code_hook', auto_reload.pre_run_code_hook)
@@ -1,695 +1,695 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 ======
3 ======
4 Rmagic
4 Rmagic
5 ======
5 ======
6
6
7 Magic command interface for interactive work with R via rpy2
7 Magic command interface for interactive work with R via rpy2
8
8
9 .. note::
9 .. note::
10
10
11 The ``rpy2`` package needs to be installed separately. It
11 The ``rpy2`` package needs to be installed separately. It
12 can be obtained using ``easy_install`` or ``pip``.
12 can be obtained using ``easy_install`` or ``pip``.
13
13
14 You will also need a working copy of R.
14 You will also need a working copy of R.
15
15
16 Usage
16 Usage
17 =====
17 =====
18
18
19 To enable the magics below, execute ``%load_ext rmagic``.
19 To enable the magics below, execute ``%load_ext rmagic``.
20
20
21 ``%R``
21 ``%R``
22
22
23 {R_DOC}
23 {R_DOC}
24
24
25 ``%Rpush``
25 ``%Rpush``
26
26
27 {RPUSH_DOC}
27 {RPUSH_DOC}
28
28
29 ``%Rpull``
29 ``%Rpull``
30
30
31 {RPULL_DOC}
31 {RPULL_DOC}
32
32
33 ``%Rget``
33 ``%Rget``
34
34
35 {RGET_DOC}
35 {RGET_DOC}
36
36
37 """
37 """
38 from __future__ import print_function
38 from __future__ import print_function
39
39
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41 # Copyright (C) 2012 The IPython Development Team
41 # Copyright (C) 2012 The IPython Development Team
42 #
42 #
43 # Distributed under the terms of the BSD License. The full license is in
43 # Distributed under the terms of the BSD License. The full license is in
44 # the file COPYING, distributed as part of this software.
44 # the file COPYING, distributed as part of this software.
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46
46
47 import sys
47 import sys
48 import tempfile
48 import tempfile
49 from glob import glob
49 from glob import glob
50 from shutil import rmtree
50 from shutil import rmtree
51
51
52 # numpy and rpy2 imports
52 # numpy and rpy2 imports
53
53
54 import numpy as np
54 import numpy as np
55
55
56 import rpy2.rinterface as ri
56 import rpy2.rinterface as ri
57 import rpy2.robjects as ro
57 import rpy2.robjects as ro
58 try:
58 try:
59 from rpy2.robjects import pandas2ri
59 from rpy2.robjects import pandas2ri
60 pandas2ri.activate()
60 pandas2ri.activate()
61 except ImportError:
61 except ImportError:
62 pandas2ri = None
62 pandas2ri = None
63 from rpy2.robjects import numpy2ri
63 from rpy2.robjects import numpy2ri
64 numpy2ri.activate()
64 numpy2ri.activate()
65
65
66 # IPython imports
66 # IPython imports
67
67
68 from IPython.core.displaypub import publish_display_data
68 from IPython.core.displaypub import publish_display_data
69 from IPython.core.magic import (Magics, magics_class, line_magic,
69 from IPython.core.magic import (Magics, magics_class, line_magic,
70 line_cell_magic, needs_local_scope)
70 line_cell_magic, needs_local_scope)
71 from IPython.testing.skipdoctest import skip_doctest
71 from IPython.testing.skipdoctest import skip_doctest
72 from IPython.core.magic_arguments import (
72 from IPython.core.magic_arguments import (
73 argument, magic_arguments, parse_argstring
73 argument, magic_arguments, parse_argstring
74 )
74 )
75 from IPython.external.simplegeneric import generic
75 from IPython.external.simplegeneric import generic
76 from IPython.utils.py3compat import (str_to_unicode, unicode_to_str, PY3,
76 from IPython.utils.py3compat import (str_to_unicode, unicode_to_str, PY3,
77 unicode_type)
77 unicode_type)
78
78
79 class RInterpreterError(ri.RRuntimeError):
79 class RInterpreterError(ri.RRuntimeError):
80 """An error when running R code in a %%R magic cell."""
80 """An error when running R code in a %%R magic cell."""
81 def __init__(self, line, err, stdout):
81 def __init__(self, line, err, stdout):
82 self.line = line
82 self.line = line
83 self.err = err.rstrip()
83 self.err = err.rstrip()
84 self.stdout = stdout.rstrip()
84 self.stdout = stdout.rstrip()
85
85
86 def __unicode__(self):
86 def __unicode__(self):
87 s = 'Failed to parse and evaluate line %r.\nR error message: %r' % \
87 s = 'Failed to parse and evaluate line %r.\nR error message: %r' % \
88 (self.line, self.err)
88 (self.line, self.err)
89 if self.stdout and (self.stdout != self.err):
89 if self.stdout and (self.stdout != self.err):
90 s += '\nR stdout:\n' + self.stdout
90 s += '\nR stdout:\n' + self.stdout
91 return s
91 return s
92
92
93 if PY3:
93 if PY3:
94 __str__ = __unicode__
94 __str__ = __unicode__
95 else:
95 else:
96 def __str__(self):
96 def __str__(self):
97 return unicode_to_str(unicode(self), 'utf-8')
97 return unicode_to_str(unicode(self), 'utf-8')
98
98
99 def Rconverter(Robj, dataframe=False):
99 def Rconverter(Robj, dataframe=False):
100 """
100 """
101 Convert an object in R's namespace to one suitable
101 Convert an object in R's namespace to one suitable
102 for ipython's namespace.
102 for ipython's namespace.
103
103
104 For a data.frame, it tries to return a structured array.
104 For a data.frame, it tries to return a structured array.
105 It first checks for colnames, then names.
105 It first checks for colnames, then names.
106 If all are NULL, it returns np.asarray(Robj), else
106 If all are NULL, it returns np.asarray(Robj), else
107 it tries to construct a recarray
107 it tries to construct a recarray
108
108
109 Parameters
109 Parameters
110 ----------
110 ----------
111
111
112 Robj: an R object returned from rpy2
112 Robj: an R object returned from rpy2
113 """
113 """
114 is_data_frame = ro.r('is.data.frame')
114 is_data_frame = ro.r('is.data.frame')
115 colnames = ro.r('colnames')
115 colnames = ro.r('colnames')
116 rownames = ro.r('rownames') # with pandas, these could be used for the index
116 rownames = ro.r('rownames') # with pandas, these could be used for the index
117 names = ro.r('names')
117 names = ro.r('names')
118
118
119 if dataframe:
119 if dataframe:
120 as_data_frame = ro.r('as.data.frame')
120 as_data_frame = ro.r('as.data.frame')
121 cols = colnames(Robj)
121 cols = colnames(Robj)
122 _names = names(Robj)
122 _names = names(Robj)
123 if cols != ri.NULL:
123 if cols != ri.NULL:
124 Robj = as_data_frame(Robj)
124 Robj = as_data_frame(Robj)
125 names = tuple(np.array(cols))
125 names = tuple(np.array(cols))
126 elif _names != ri.NULL:
126 elif _names != ri.NULL:
127 names = tuple(np.array(_names))
127 names = tuple(np.array(_names))
128 else: # failed to find names
128 else: # failed to find names
129 return np.asarray(Robj)
129 return np.asarray(Robj)
130 Robj = np.rec.fromarrays(Robj, names = names)
130 Robj = np.rec.fromarrays(Robj, names = names)
131 return np.asarray(Robj)
131 return np.asarray(Robj)
132
132
133 @generic
133 @generic
134 def pyconverter(pyobj):
134 def pyconverter(pyobj):
135 """Convert Python objects to R objects. Add types using the decorator:
135 """Convert Python objects to R objects. Add types using the decorator:
136
136
137 @pyconverter.when_type
137 @pyconverter.when_type
138 """
138 """
139 return pyobj
139 return pyobj
140
140
141 # The default conversion for lists seems to make them a nested list. That has
141 # The default conversion for lists seems to make them a nested list. That has
142 # some advantages, but is rarely convenient, so for interactive use, we convert
142 # some advantages, but is rarely convenient, so for interactive use, we convert
143 # lists to a numpy array, which becomes an R vector.
143 # lists to a numpy array, which becomes an R vector.
144 @pyconverter.when_type(list)
144 @pyconverter.when_type(list)
145 def pyconverter_list(pyobj):
145 def pyconverter_list(pyobj):
146 return np.asarray(pyobj)
146 return np.asarray(pyobj)
147
147
148 if pandas2ri is None:
148 if pandas2ri is None:
149 # pandas2ri was new in rpy2 2.3.3, so for now we'll fallback to pandas'
149 # pandas2ri was new in rpy2 2.3.3, so for now we'll fallback to pandas'
150 # conversion function.
150 # conversion function.
151 try:
151 try:
152 from pandas import DataFrame
152 from pandas import DataFrame
153 from pandas.rpy.common import convert_to_r_dataframe
153 from pandas.rpy.common import convert_to_r_dataframe
154 @pyconverter.when_type(DataFrame)
154 @pyconverter.when_type(DataFrame)
155 def pyconverter_dataframe(pyobj):
155 def pyconverter_dataframe(pyobj):
156 return convert_to_r_dataframe(pyobj, strings_as_factors=True)
156 return convert_to_r_dataframe(pyobj, strings_as_factors=True)
157 except ImportError:
157 except ImportError:
158 pass
158 pass
159
159
160 @magics_class
160 @magics_class
161 class RMagics(Magics):
161 class RMagics(Magics):
162 """A set of magics useful for interactive work with R via rpy2.
162 """A set of magics useful for interactive work with R via rpy2.
163 """
163 """
164
164
165 def __init__(self, shell, Rconverter=Rconverter,
165 def __init__(self, shell, Rconverter=Rconverter,
166 pyconverter=pyconverter,
166 pyconverter=pyconverter,
167 cache_display_data=False):
167 cache_display_data=False):
168 """
168 """
169 Parameters
169 Parameters
170 ----------
170 ----------
171
171
172 shell : IPython shell
172 shell : IPython shell
173
173
174 Rconverter : callable
174 Rconverter : callable
175 To be called on values taken from R before putting them in the
175 To be called on values taken from R before putting them in the
176 IPython namespace.
176 IPython namespace.
177
177
178 pyconverter : callable
178 pyconverter : callable
179 To be called on values in ipython namespace before
179 To be called on values in ipython namespace before
180 assigning to variables in rpy2.
180 assigning to variables in rpy2.
181
181
182 cache_display_data : bool
182 cache_display_data : bool
183 If True, the published results of the final call to R are
183 If True, the published results of the final call to R are
184 cached in the variable 'display_cache'.
184 cached in the variable 'display_cache'.
185
185
186 """
186 """
187 super(RMagics, self).__init__(shell)
187 super(RMagics, self).__init__(shell)
188 self.cache_display_data = cache_display_data
188 self.cache_display_data = cache_display_data
189
189
190 self.r = ro.R()
190 self.r = ro.R()
191
191
192 self.Rstdout_cache = []
192 self.Rstdout_cache = []
193 self.pyconverter = pyconverter
193 self.pyconverter = pyconverter
194 self.Rconverter = Rconverter
194 self.Rconverter = Rconverter
195
195
196 def eval(self, line):
196 def eval(self, line):
197 '''
197 '''
198 Parse and evaluate a line of R code with rpy2.
198 Parse and evaluate a line of R code with rpy2.
199 Returns the output to R's stdout() connection,
199 Returns the output to R's stdout() connection,
200 the value generated by evaluating the code, and a
200 the value generated by evaluating the code, and a
201 boolean indicating whether the return value would be
201 boolean indicating whether the return value would be
202 visible if the line of code were evaluated in an R REPL.
202 visible if the line of code were evaluated in an R REPL.
203
203
204 R Code evaluation and visibility determination are
204 R Code evaluation and visibility determination are
205 done via an R call of the form withVisible({<code>})
205 done via an R call of the form withVisible({<code>})
206
206
207 '''
207 '''
208 old_writeconsole = ri.get_writeconsole()
208 old_writeconsole = ri.get_writeconsole()
209 ri.set_writeconsole(self.write_console)
209 ri.set_writeconsole(self.write_console)
210 try:
210 try:
211 res = ro.r("withVisible({%s})" % line)
211 res = ro.r("withVisible({%s})" % line)
212 value = res[0] #value (R object)
212 value = res[0] #value (R object)
213 visible = ro.conversion.ri2py(res[1])[0] #visible (boolean)
213 visible = ro.conversion.ri2py(res[1])[0] #visible (boolean)
214 except (ri.RRuntimeError, ValueError) as exception:
214 except (ri.RRuntimeError, ValueError) as exception:
215 warning_or_other_msg = self.flush() # otherwise next return seems to have copy of error
215 warning_or_other_msg = self.flush() # otherwise next return seems to have copy of error
216 raise RInterpreterError(line, str_to_unicode(str(exception)), warning_or_other_msg)
216 raise RInterpreterError(line, str_to_unicode(str(exception)), warning_or_other_msg)
217 text_output = self.flush()
217 text_output = self.flush()
218 ri.set_writeconsole(old_writeconsole)
218 ri.set_writeconsole(old_writeconsole)
219 return text_output, value, visible
219 return text_output, value, visible
220
220
221 def write_console(self, output):
221 def write_console(self, output):
222 '''
222 '''
223 A hook to capture R's stdout in a cache.
223 A hook to capture R's stdout in a cache.
224 '''
224 '''
225 self.Rstdout_cache.append(output)
225 self.Rstdout_cache.append(output)
226
226
227 def flush(self):
227 def flush(self):
228 '''
228 '''
229 Flush R's stdout cache to a string, returning the string.
229 Flush R's stdout cache to a string, returning the string.
230 '''
230 '''
231 value = ''.join([str_to_unicode(s, 'utf-8') for s in self.Rstdout_cache])
231 value = ''.join([str_to_unicode(s, 'utf-8') for s in self.Rstdout_cache])
232 self.Rstdout_cache = []
232 self.Rstdout_cache = []
233 return value
233 return value
234
234
235 @skip_doctest
235 @skip_doctest
236 @needs_local_scope
236 @needs_local_scope
237 @line_magic
237 @line_magic
238 def Rpush(self, line, local_ns=None):
238 def Rpush(self, line, local_ns=None):
239 '''
239 '''
240 A line-level magic for R that pushes
240 A line-level magic for R that pushes
241 variables from python to rpy2. The line should be made up
241 variables from python to rpy2. The line should be made up
242 of whitespace separated variable names in the IPython
242 of whitespace separated variable names in the IPython
243 namespace::
243 namespace::
244
244
245 In [7]: import numpy as np
245 In [7]: import numpy as np
246
246
247 In [8]: X = np.array([4.5,6.3,7.9])
247 In [8]: X = np.array([4.5,6.3,7.9])
248
248
249 In [9]: X.mean()
249 In [9]: X.mean()
250 Out[9]: 6.2333333333333343
250 Out[9]: 6.2333333333333343
251
251
252 In [10]: %Rpush X
252 In [10]: %Rpush X
253
253
254 In [11]: %R mean(X)
254 In [11]: %R mean(X)
255 Out[11]: array([ 6.23333333])
255 Out[11]: array([ 6.23333333])
256
256
257 '''
257 '''
258 if local_ns is None:
258 if local_ns is None:
259 local_ns = {}
259 local_ns = {}
260
260
261 inputs = line.split(' ')
261 inputs = line.split(' ')
262 for input in inputs:
262 for input in inputs:
263 try:
263 try:
264 val = local_ns[input]
264 val = local_ns[input]
265 except KeyError:
265 except KeyError:
266 try:
266 try:
267 val = self.shell.user_ns[input]
267 val = self.shell.user_ns[input]
268 except KeyError:
268 except KeyError:
269 # reraise the KeyError as a NameError so that it looks like
269 # reraise the KeyError as a NameError so that it looks like
270 # the standard python behavior when you use an unnamed
270 # the standard python behavior when you use an unnamed
271 # variable
271 # variable
272 raise NameError("name '%s' is not defined" % input)
272 raise NameError("name '%s' is not defined" % input)
273
273
274 self.r.assign(input, self.pyconverter(val))
274 self.r.assign(input, self.pyconverter(val))
275
275
276 @skip_doctest
276 @skip_doctest
277 @magic_arguments()
277 @magic_arguments()
278 @argument(
278 @argument(
279 '-d', '--as_dataframe', action='store_true',
279 '-d', '--as_dataframe', action='store_true',
280 default=False,
280 default=False,
281 help='Convert objects to data.frames before returning to ipython.'
281 help='Convert objects to data.frames before returning to ipython.'
282 )
282 )
283 @argument(
283 @argument(
284 'outputs',
284 'outputs',
285 nargs='*',
285 nargs='*',
286 )
286 )
287 @line_magic
287 @line_magic
288 def Rpull(self, line):
288 def Rpull(self, line):
289 '''
289 '''
290 A line-level magic for R that pulls
290 A line-level magic for R that pulls
291 variables from python to rpy2::
291 variables from python to rpy2::
292
292
293 In [18]: _ = %R x = c(3,4,6.7); y = c(4,6,7); z = c('a',3,4)
293 In [18]: _ = %R x = c(3,4,6.7); y = c(4,6,7); z = c('a',3,4)
294
294
295 In [19]: %Rpull x y z
295 In [19]: %Rpull x y z
296
296
297 In [20]: x
297 In [20]: x
298 Out[20]: array([ 3. , 4. , 6.7])
298 Out[20]: array([ 3. , 4. , 6.7])
299
299
300 In [21]: y
300 In [21]: y
301 Out[21]: array([ 4., 6., 7.])
301 Out[21]: array([ 4., 6., 7.])
302
302
303 In [22]: z
303 In [22]: z
304 Out[22]:
304 Out[22]:
305 array(['a', '3', '4'],
305 array(['a', '3', '4'],
306 dtype='|S1')
306 dtype='|S1')
307
307
308
308
309 If --as_dataframe, then each object is returned as a structured array
309 If --as_dataframe, then each object is returned as a structured array
310 after first passed through "as.data.frame" in R before
310 after first passed through "as.data.frame" in R before
311 being calling self.Rconverter.
311 being calling self.Rconverter.
312 This is useful when a structured array is desired as output, or
312 This is useful when a structured array is desired as output, or
313 when the object in R has mixed data types.
313 when the object in R has mixed data types.
314 See the %%R docstring for more examples.
314 See the %%R docstring for more examples.
315
315
316 Notes
316 Notes
317 -----
317 -----
318
318
319 Beware that R names can have '.' so this is not fool proof.
319 Beware that R names can have '.' so this is not fool proof.
320 To avoid this, don't name your R objects with '.'s...
320 To avoid this, don't name your R objects with '.'s...
321
321
322 '''
322 '''
323 args = parse_argstring(self.Rpull, line)
323 args = parse_argstring(self.Rpull, line)
324 outputs = args.outputs
324 outputs = args.outputs
325 for output in outputs:
325 for output in outputs:
326 self.shell.push({output:self.Rconverter(self.r(output),dataframe=args.as_dataframe)})
326 self.shell.push({output:self.Rconverter(self.r(output),dataframe=args.as_dataframe)})
327
327
328 @skip_doctest
328 @skip_doctest
329 @magic_arguments()
329 @magic_arguments()
330 @argument(
330 @argument(
331 '-d', '--as_dataframe', action='store_true',
331 '-d', '--as_dataframe', action='store_true',
332 default=False,
332 default=False,
333 help='Convert objects to data.frames before returning to ipython.'
333 help='Convert objects to data.frames before returning to ipython.'
334 )
334 )
335 @argument(
335 @argument(
336 'output',
336 'output',
337 nargs=1,
337 nargs=1,
338 type=str,
338 type=str,
339 )
339 )
340 @line_magic
340 @line_magic
341 def Rget(self, line):
341 def Rget(self, line):
342 '''
342 '''
343 Return an object from rpy2, possibly as a structured array (if possible).
343 Return an object from rpy2, possibly as a structured array (if possible).
344 Similar to Rpull except only one argument is accepted and the value is
344 Similar to Rpull except only one argument is accepted and the value is
345 returned rather than pushed to self.shell.user_ns::
345 returned rather than pushed to self.shell.user_ns::
346
346
347 In [3]: dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')]
347 In [3]: dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')]
348
348
349 In [4]: datapy = np.array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5, 'e')], dtype=dtype)
349 In [4]: datapy = np.array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5, 'e')], dtype=dtype)
350
350
351 In [5]: %R -i datapy
351 In [5]: %R -i datapy
352
352
353 In [6]: %Rget datapy
353 In [6]: %Rget datapy
354 Out[6]:
354 Out[6]:
355 array([['1', '2', '3', '4'],
355 array([['1', '2', '3', '4'],
356 ['2', '3', '2', '5'],
356 ['2', '3', '2', '5'],
357 ['a', 'b', 'c', 'e']],
357 ['a', 'b', 'c', 'e']],
358 dtype='|S1')
358 dtype='|S1')
359
359
360 In [7]: %Rget -d datapy
360 In [7]: %Rget -d datapy
361 Out[7]:
361 Out[7]:
362 array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5.0, 'e')],
362 array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5.0, 'e')],
363 dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')])
363 dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')])
364
364
365 '''
365 '''
366 args = parse_argstring(self.Rget, line)
366 args = parse_argstring(self.Rget, line)
367 output = args.output
367 output = args.output
368 return self.Rconverter(self.r(output[0]),dataframe=args.as_dataframe)
368 return self.Rconverter(self.r(output[0]),dataframe=args.as_dataframe)
369
369
370
370
371 @skip_doctest
371 @skip_doctest
372 @magic_arguments()
372 @magic_arguments()
373 @argument(
373 @argument(
374 '-i', '--input', action='append',
374 '-i', '--input', action='append',
375 help='Names of input variable from shell.user_ns to be assigned to R variables of the same names after calling self.pyconverter. Multiple names can be passed separated only by commas with no whitespace.'
375 help='Names of input variable from shell.user_ns to be assigned to R variables of the same names after calling self.pyconverter. Multiple names can be passed separated only by commas with no whitespace.'
376 )
376 )
377 @argument(
377 @argument(
378 '-o', '--output', action='append',
378 '-o', '--output', action='append',
379 help='Names of variables to be pushed from rpy2 to shell.user_ns after executing cell body and applying self.Rconverter. Multiple names can be passed separated only by commas with no whitespace.'
379 help='Names of variables to be pushed from rpy2 to shell.user_ns after executing cell body and applying self.Rconverter. Multiple names can be passed separated only by commas with no whitespace.'
380 )
380 )
381 @argument(
381 @argument(
382 '-w', '--width', type=int,
382 '-w', '--width', type=int,
383 help='Width of png plotting device sent as an argument to *png* in R.'
383 help='Width of png plotting device sent as an argument to *png* in R.'
384 )
384 )
385 @argument(
385 @argument(
386 '-h', '--height', type=int,
386 '-h', '--height', type=int,
387 help='Height of png plotting device sent as an argument to *png* in R.'
387 help='Height of png plotting device sent as an argument to *png* in R.'
388 )
388 )
389
389
390 @argument(
390 @argument(
391 '-d', '--dataframe', action='append',
391 '-d', '--dataframe', action='append',
392 help='Convert these objects to data.frames and return as structured arrays.'
392 help='Convert these objects to data.frames and return as structured arrays.'
393 )
393 )
394 @argument(
394 @argument(
395 '-u', '--units', type=unicode_type, choices=["px", "in", "cm", "mm"],
395 '-u', '--units', type=unicode_type, choices=["px", "in", "cm", "mm"],
396 help='Units of png plotting device sent as an argument to *png* in R. One of ["px", "in", "cm", "mm"].'
396 help='Units of png plotting device sent as an argument to *png* in R. One of ["px", "in", "cm", "mm"].'
397 )
397 )
398 @argument(
398 @argument(
399 '-r', '--res', type=int,
399 '-r', '--res', type=int,
400 help='Resolution of png plotting device sent as an argument to *png* in R. Defaults to 72 if *units* is one of ["in", "cm", "mm"].'
400 help='Resolution of png plotting device sent as an argument to *png* in R. Defaults to 72 if *units* is one of ["in", "cm", "mm"].'
401 )
401 )
402 @argument(
402 @argument(
403 '-p', '--pointsize', type=int,
403 '-p', '--pointsize', type=int,
404 help='Pointsize of png plotting device sent as an argument to *png* in R.'
404 help='Pointsize of png plotting device sent as an argument to *png* in R.'
405 )
405 )
406 @argument(
406 @argument(
407 '-b', '--bg',
407 '-b', '--bg',
408 help='Background of png plotting device sent as an argument to *png* in R.'
408 help='Background of png plotting device sent as an argument to *png* in R.'
409 )
409 )
410 @argument(
410 @argument(
411 '-n', '--noreturn',
411 '-n', '--noreturn',
412 help='Force the magic to not return anything.',
412 help='Force the magic to not return anything.',
413 action='store_true',
413 action='store_true',
414 default=False
414 default=False
415 )
415 )
416 @argument(
416 @argument(
417 'code',
417 'code',
418 nargs='*',
418 nargs='*',
419 )
419 )
420 @needs_local_scope
420 @needs_local_scope
421 @line_cell_magic
421 @line_cell_magic
422 def R(self, line, cell=None, local_ns=None):
422 def R(self, line, cell=None, local_ns=None):
423 '''
423 '''
424 Execute code in R, and pull some of the results back into the Python namespace.
424 Execute code in R, and pull some of the results back into the Python namespace.
425
425
426 In line mode, this will evaluate an expression and convert the returned value to a Python object.
426 In line mode, this will evaluate an expression and convert the returned value to a Python object.
427 The return value is determined by rpy2's behaviour of returning the result of evaluating the
427 The return value is determined by rpy2's behaviour of returning the result of evaluating the
428 final line.
428 final line.
429
429
430 Multiple R lines can be executed by joining them with semicolons::
430 Multiple R lines can be executed by joining them with semicolons::
431
431
432 In [9]: %R X=c(1,4,5,7); sd(X); mean(X)
432 In [9]: %R X=c(1,4,5,7); sd(X); mean(X)
433 Out[9]: array([ 4.25])
433 Out[9]: array([ 4.25])
434
434
435 In cell mode, this will run a block of R code. The resulting value
435 In cell mode, this will run a block of R code. The resulting value
436 is printed if it would printed be when evaluating the same code
436 is printed if it would printed be when evaluating the same code
437 within a standard R REPL.
437 within a standard R REPL.
438
438
439 Nothing is returned to python by default in cell mode::
439 Nothing is returned to python by default in cell mode::
440
440
441 In [10]: %%R
441 In [10]: %%R
442 ....: Y = c(2,4,3,9)
442 ....: Y = c(2,4,3,9)
443 ....: summary(lm(Y~X))
443 ....: summary(lm(Y~X))
444
444
445 Call:
445 Call:
446 lm(formula = Y ~ X)
446 lm(formula = Y ~ X)
447
447
448 Residuals:
448 Residuals:
449 1 2 3 4
449 1 2 3 4
450 0.88 -0.24 -2.28 1.64
450 0.88 -0.24 -2.28 1.64
451
451
452 Coefficients:
452 Coefficients:
453 Estimate Std. Error t value Pr(>|t|)
453 Estimate Std. Error t value Pr(>|t|)
454 (Intercept) 0.0800 2.3000 0.035 0.975
454 (Intercept) 0.0800 2.3000 0.035 0.975
455 X 1.0400 0.4822 2.157 0.164
455 X 1.0400 0.4822 2.157 0.164
456
456
457 Residual standard error: 2.088 on 2 degrees of freedom
457 Residual standard error: 2.088 on 2 degrees of freedom
458 Multiple R-squared: 0.6993,Adjusted R-squared: 0.549
458 Multiple R-squared: 0.6993,Adjusted R-squared: 0.549
459 F-statistic: 4.651 on 1 and 2 DF, p-value: 0.1638
459 F-statistic: 4.651 on 1 and 2 DF, p-value: 0.1638
460
460
461 In the notebook, plots are published as the output of the cell::
461 In the notebook, plots are published as the output of the cell::
462
462
463 %R plot(X, Y)
463 %R plot(X, Y)
464
464
465 will create a scatter plot of X bs Y.
465 will create a scatter plot of X bs Y.
466
466
467 If cell is not None and line has some R code, it is prepended to
467 If cell is not None and line has some R code, it is prepended to
468 the R code in cell.
468 the R code in cell.
469
469
470 Objects can be passed back and forth between rpy2 and python via the -i -o flags in line::
470 Objects can be passed back and forth between rpy2 and python via the -i -o flags in line::
471
471
472 In [14]: Z = np.array([1,4,5,10])
472 In [14]: Z = np.array([1,4,5,10])
473
473
474 In [15]: %R -i Z mean(Z)
474 In [15]: %R -i Z mean(Z)
475 Out[15]: array([ 5.])
475 Out[15]: array([ 5.])
476
476
477
477
478 In [16]: %R -o W W=Z*mean(Z)
478 In [16]: %R -o W W=Z*mean(Z)
479 Out[16]: array([ 5., 20., 25., 50.])
479 Out[16]: array([ 5., 20., 25., 50.])
480
480
481 In [17]: W
481 In [17]: W
482 Out[17]: array([ 5., 20., 25., 50.])
482 Out[17]: array([ 5., 20., 25., 50.])
483
483
484 The return value is determined by these rules:
484 The return value is determined by these rules:
485
485
486 * If the cell is not None, the magic returns None.
486 * If the cell is not None, the magic returns None.
487
487
488 * If the cell evaluates as False, the resulting value is returned
488 * If the cell evaluates as False, the resulting value is returned
489 unless the final line prints something to the console, in
489 unless the final line prints something to the console, in
490 which case None is returned.
490 which case None is returned.
491
491
492 * If the final line results in a NULL value when evaluated
492 * If the final line results in a NULL value when evaluated
493 by rpy2, then None is returned.
493 by rpy2, then None is returned.
494
494
495 * No attempt is made to convert the final value to a structured array.
495 * No attempt is made to convert the final value to a structured array.
496 Use the --dataframe flag or %Rget to push / return a structured array.
496 Use the --dataframe flag or %Rget to push / return a structured array.
497
497
498 * If the -n flag is present, there is no return value.
498 * If the -n flag is present, there is no return value.
499
499
500 * A trailing ';' will also result in no return value as the last
500 * A trailing ';' will also result in no return value as the last
501 value in the line is an empty string.
501 value in the line is an empty string.
502
502
503 The --dataframe argument will attempt to return structured arrays.
503 The --dataframe argument will attempt to return structured arrays.
504 This is useful for dataframes with
504 This is useful for dataframes with
505 mixed data types. Note also that for a data.frame,
505 mixed data types. Note also that for a data.frame,
506 if it is returned as an ndarray, it is transposed::
506 if it is returned as an ndarray, it is transposed::
507
507
508 In [18]: dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')]
508 In [18]: dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')]
509
509
510 In [19]: datapy = np.array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5, 'e')], dtype=dtype)
510 In [19]: datapy = np.array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5, 'e')], dtype=dtype)
511
511
512 In [20]: %%R -o datar
512 In [20]: %%R -o datar
513 datar = datapy
513 datar = datapy
514 ....:
514 ....:
515
515
516 In [21]: datar
516 In [21]: datar
517 Out[21]:
517 Out[21]:
518 array([['1', '2', '3', '4'],
518 array([['1', '2', '3', '4'],
519 ['2', '3', '2', '5'],
519 ['2', '3', '2', '5'],
520 ['a', 'b', 'c', 'e']],
520 ['a', 'b', 'c', 'e']],
521 dtype='|S1')
521 dtype='|S1')
522
522
523 In [22]: %%R -d datar
523 In [22]: %%R -d datar
524 datar = datapy
524 datar = datapy
525 ....:
525 ....:
526
526
527 In [23]: datar
527 In [23]: datar
528 Out[23]:
528 Out[23]:
529 array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5.0, 'e')],
529 array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5.0, 'e')],
530 dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')])
530 dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')])
531
531
532 The --dataframe argument first tries colnames, then names.
532 The --dataframe argument first tries colnames, then names.
533 If both are NULL, it returns an ndarray (i.e. unstructured)::
533 If both are NULL, it returns an ndarray (i.e. unstructured)::
534
534
535 In [1]: %R mydata=c(4,6,8.3); NULL
535 In [1]: %R mydata=c(4,6,8.3); NULL
536
536
537 In [2]: %R -d mydata
537 In [2]: %R -d mydata
538
538
539 In [3]: mydata
539 In [3]: mydata
540 Out[3]: array([ 4. , 6. , 8.3])
540 Out[3]: array([ 4. , 6. , 8.3])
541
541
542 In [4]: %R names(mydata) = c('a','b','c'); NULL
542 In [4]: %R names(mydata) = c('a','b','c'); NULL
543
543
544 In [5]: %R -d mydata
544 In [5]: %R -d mydata
545
545
546 In [6]: mydata
546 In [6]: mydata
547 Out[6]:
547 Out[6]:
548 array((4.0, 6.0, 8.3),
548 array((4.0, 6.0, 8.3),
549 dtype=[('a', '<f8'), ('b', '<f8'), ('c', '<f8')])
549 dtype=[('a', '<f8'), ('b', '<f8'), ('c', '<f8')])
550
550
551 In [7]: %R -o mydata
551 In [7]: %R -o mydata
552
552
553 In [8]: mydata
553 In [8]: mydata
554 Out[8]: array([ 4. , 6. , 8.3])
554 Out[8]: array([ 4. , 6. , 8.3])
555
555
556 '''
556 '''
557
557
558 args = parse_argstring(self.R, line)
558 args = parse_argstring(self.R, line)
559
559
560 # arguments 'code' in line are prepended to
560 # arguments 'code' in line are prepended to
561 # the cell lines
561 # the cell lines
562
562
563 if cell is None:
563 if cell is None:
564 code = ''
564 code = ''
565 return_output = True
565 return_output = True
566 line_mode = True
566 line_mode = True
567 else:
567 else:
568 code = cell
568 code = cell
569 return_output = False
569 return_output = False
570 line_mode = False
570 line_mode = False
571
571
572 code = ' '.join(args.code) + code
572 code = ' '.join(args.code) + code
573
573
574 # if there is no local namespace then default to an empty dict
574 # if there is no local namespace then default to an empty dict
575 if local_ns is None:
575 if local_ns is None:
576 local_ns = {}
576 local_ns = {}
577
577
578 if args.input:
578 if args.input:
579 for input in ','.join(args.input).split(','):
579 for input in ','.join(args.input).split(','):
580 try:
580 try:
581 val = local_ns[input]
581 val = local_ns[input]
582 except KeyError:
582 except KeyError:
583 try:
583 try:
584 val = self.shell.user_ns[input]
584 val = self.shell.user_ns[input]
585 except KeyError:
585 except KeyError:
586 raise NameError("name '%s' is not defined" % input)
586 raise NameError("name '%s' is not defined" % input)
587 self.r.assign(input, self.pyconverter(val))
587 self.r.assign(input, self.pyconverter(val))
588
588
589 if getattr(args, 'units') is not None:
589 if getattr(args, 'units') is not None:
590 if args.units != "px" and getattr(args, 'res') is None:
590 if args.units != "px" and getattr(args, 'res') is None:
591 args.res = 72
591 args.res = 72
592 args.units = '"%s"' % args.units
592 args.units = '"%s"' % args.units
593
593
594 png_argdict = dict([(n, getattr(args, n)) for n in ['units', 'res', 'height', 'width', 'bg', 'pointsize']])
594 png_argdict = dict([(n, getattr(args, n)) for n in ['units', 'res', 'height', 'width', 'bg', 'pointsize']])
595 png_args = ','.join(['%s=%s' % (o,v) for o, v in png_argdict.items() if v is not None])
595 png_args = ','.join(['%s=%s' % (o,v) for o, v in png_argdict.items() if v is not None])
596 # execute the R code in a temporary directory
596 # execute the R code in a temporary directory
597
597
598 tmpd = tempfile.mkdtemp()
598 tmpd = tempfile.mkdtemp()
599 self.r('png("%s/Rplots%%03d.png",%s)' % (tmpd.replace('\\', '/'), png_args))
599 self.r('png("%s/Rplots%%03d.png",%s)' % (tmpd.replace('\\', '/'), png_args))
600
600
601 text_output = ''
601 text_output = ''
602 try:
602 try:
603 if line_mode:
603 if line_mode:
604 for line in code.split(';'):
604 for line in code.split(';'):
605 text_result, result, visible = self.eval(line)
605 text_result, result, visible = self.eval(line)
606 text_output += text_result
606 text_output += text_result
607 if text_result:
607 if text_result:
608 # the last line printed something to the console so we won't return it
608 # the last line printed something to the console so we won't return it
609 return_output = False
609 return_output = False
610 else:
610 else:
611 text_result, result, visible = self.eval(code)
611 text_result, result, visible = self.eval(code)
612 text_output += text_result
612 text_output += text_result
613 if visible:
613 if visible:
614 old_writeconsole = ri.get_writeconsole()
614 old_writeconsole = ri.get_writeconsole()
615 ri.set_writeconsole(self.write_console)
615 ri.set_writeconsole(self.write_console)
616 ro.r.show(result)
616 ro.r.show(result)
617 text_output += self.flush()
617 text_output += self.flush()
618 ri.set_writeconsole(old_writeconsole)
618 ri.set_writeconsole(old_writeconsole)
619
619
620 except RInterpreterError as e:
620 except RInterpreterError as e:
621 print((e.stdout))
621 print(e.stdout)
622 if not e.stdout.endswith(e.err):
622 if not e.stdout.endswith(e.err):
623 print((e.err))
623 print(e.err)
624 rmtree(tmpd)
624 rmtree(tmpd)
625 return
625 return
626
626
627 self.r('dev.off()')
627 self.r('dev.off()')
628
628
629 # read out all the saved .png files
629 # read out all the saved .png files
630
630
631 images = [open(imgfile, 'rb').read() for imgfile in glob("%s/Rplots*png" % tmpd)]
631 images = [open(imgfile, 'rb').read() for imgfile in glob("%s/Rplots*png" % tmpd)]
632
632
633 # now publish the images
633 # now publish the images
634 # mimicking IPython/zmq/pylab/backend_inline.py
634 # mimicking IPython/zmq/pylab/backend_inline.py
635 fmt = 'png'
635 fmt = 'png'
636 mimetypes = { 'png' : 'image/png', 'svg' : 'image/svg+xml' }
636 mimetypes = { 'png' : 'image/png', 'svg' : 'image/svg+xml' }
637 mime = mimetypes[fmt]
637 mime = mimetypes[fmt]
638
638
639 # publish the printed R objects, if any
639 # publish the printed R objects, if any
640
640
641 display_data = []
641 display_data = []
642 if text_output:
642 if text_output:
643 display_data.append(('RMagic.R', {'text/plain':text_output}))
643 display_data.append(('RMagic.R', {'text/plain':text_output}))
644
644
645 # flush text streams before sending figures, helps a little with output
645 # flush text streams before sending figures, helps a little with output
646 for image in images:
646 for image in images:
647 # synchronization in the console (though it's a bandaid, not a real sln)
647 # synchronization in the console (though it's a bandaid, not a real sln)
648 sys.stdout.flush(); sys.stderr.flush()
648 sys.stdout.flush(); sys.stderr.flush()
649 display_data.append(('RMagic.R', {mime: image}))
649 display_data.append(('RMagic.R', {mime: image}))
650
650
651 # kill the temporary directory
651 # kill the temporary directory
652 rmtree(tmpd)
652 rmtree(tmpd)
653
653
654 # try to turn every output into a numpy array
654 # try to turn every output into a numpy array
655 # this means that output are assumed to be castable
655 # this means that output are assumed to be castable
656 # as numpy arrays
656 # as numpy arrays
657
657
658 if args.output:
658 if args.output:
659 for output in ','.join(args.output).split(','):
659 for output in ','.join(args.output).split(','):
660 self.shell.push({output:self.Rconverter(self.r(output), dataframe=False)})
660 self.shell.push({output:self.Rconverter(self.r(output), dataframe=False)})
661
661
662 if args.dataframe:
662 if args.dataframe:
663 for output in ','.join(args.dataframe).split(','):
663 for output in ','.join(args.dataframe).split(','):
664 self.shell.push({output:self.Rconverter(self.r(output), dataframe=True)})
664 self.shell.push({output:self.Rconverter(self.r(output), dataframe=True)})
665
665
666 for tag, disp_d in display_data:
666 for tag, disp_d in display_data:
667 publish_display_data(tag, disp_d)
667 publish_display_data(tag, disp_d)
668
668
669 # this will keep a reference to the display_data
669 # this will keep a reference to the display_data
670 # which might be useful to other objects who happen to use
670 # which might be useful to other objects who happen to use
671 # this method
671 # this method
672
672
673 if self.cache_display_data:
673 if self.cache_display_data:
674 self.display_cache = display_data
674 self.display_cache = display_data
675
675
676 # if in line mode and return_output, return the result as an ndarray
676 # if in line mode and return_output, return the result as an ndarray
677 if return_output and not args.noreturn:
677 if return_output and not args.noreturn:
678 if result != ri.NULL:
678 if result != ri.NULL:
679 return self.Rconverter(result, dataframe=False)
679 return self.Rconverter(result, dataframe=False)
680
680
681 __doc__ = __doc__.format(
681 __doc__ = __doc__.format(
682 R_DOC = ' '*8 + RMagics.R.__doc__,
682 R_DOC = ' '*8 + RMagics.R.__doc__,
683 RPUSH_DOC = ' '*8 + RMagics.Rpush.__doc__,
683 RPUSH_DOC = ' '*8 + RMagics.Rpush.__doc__,
684 RPULL_DOC = ' '*8 + RMagics.Rpull.__doc__,
684 RPULL_DOC = ' '*8 + RMagics.Rpull.__doc__,
685 RGET_DOC = ' '*8 + RMagics.Rget.__doc__
685 RGET_DOC = ' '*8 + RMagics.Rget.__doc__
686 )
686 )
687
687
688
688
689 def load_ipython_extension(ip):
689 def load_ipython_extension(ip):
690 """Load the extension in IPython."""
690 """Load the extension in IPython."""
691 ip.register_magics(RMagics)
691 ip.register_magics(RMagics)
692 # Initialising rpy2 interferes with readline. Since, at this point, we've
692 # Initialising rpy2 interferes with readline. Since, at this point, we've
693 # probably just loaded rpy2, we reset the delimiters. See issue gh-2759.
693 # probably just loaded rpy2, we reset the delimiters. See issue gh-2759.
694 if ip.has_readline:
694 if ip.has_readline:
695 ip.readline.set_completer_delims(ip.readline_delims)
695 ip.readline.set_completer_delims(ip.readline_delims)
@@ -1,234 +1,234 b''
1 #!/usr/bin/python
1 #!/usr/bin/python
2 """Utility function for installing MathJax javascript library into
2 """Utility function for installing MathJax javascript library into
3 your IPython nbextensions directory, for offline use.
3 your IPython nbextensions directory, for offline use.
4
4
5 Authors:
5 Authors:
6
6
7 * Min RK
7 * Min RK
8 * Mark Sienkiewicz
8 * Mark Sienkiewicz
9 * Matthias Bussonnier
9 * Matthias Bussonnier
10
10
11 To download and install MathJax:
11 To download and install MathJax:
12
12
13 From Python:
13 From Python:
14
14
15 >>> from IPython.external.mathjax import install_mathjax
15 >>> from IPython.external.mathjax import install_mathjax
16 >>> install_mathjax()
16 >>> install_mathjax()
17
17
18 From the command line:
18 From the command line:
19
19
20 $ python -m IPython.external.mathjax
20 $ python -m IPython.external.mathjax
21
21
22 To a specific location:
22 To a specific location:
23
23
24 $ python -m IPython.external.mathjax -i /usr/share/
24 $ python -m IPython.external.mathjax -i /usr/share/
25
25
26 will install mathjax to /usr/share/mathjax
26 will install mathjax to /usr/share/mathjax
27
27
28 To install MathJax from a file you have already downloaded:
28 To install MathJax from a file you have already downloaded:
29
29
30 $ python -m IPython.external.mathjax mathjax-xxx.tar.gz
30 $ python -m IPython.external.mathjax mathjax-xxx.tar.gz
31 $ python -m IPython.external.mathjax mathjax-xxx.zip
31 $ python -m IPython.external.mathjax mathjax-xxx.zip
32
32
33 It will not install MathJax if it is already there. Use -r to
33 It will not install MathJax if it is already there. Use -r to
34 replace the existing copy of MathJax.
34 replace the existing copy of MathJax.
35
35
36 To find the directory where IPython would like MathJax installed:
36 To find the directory where IPython would like MathJax installed:
37
37
38 $ python -m IPython.external.mathjax -d
38 $ python -m IPython.external.mathjax -d
39
39
40 """
40 """
41 from __future__ import print_function
41 from __future__ import print_function
42
42
43
43
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45 # Copyright (C) 2011 The IPython Development Team
45 # Copyright (C) 2011 The IPython Development Team
46 #
46 #
47 # Distributed under the terms of the BSD License. The full license is in
47 # Distributed under the terms of the BSD License. The full license is in
48 # the file COPYING, distributed as part of this software.
48 # the file COPYING, distributed as part of this software.
49 #-----------------------------------------------------------------------------
49 #-----------------------------------------------------------------------------
50
50
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Imports
53 # Imports
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56 import argparse
56 import argparse
57 import os
57 import os
58 import shutil
58 import shutil
59 import sys
59 import sys
60 import tarfile
60 import tarfile
61 import urllib2
61 import urllib2
62 import zipfile
62 import zipfile
63
63
64 from IPython.utils.path import get_ipython_dir
64 from IPython.utils.path import get_ipython_dir
65
65
66 #-----------------------------------------------------------------------------
66 #-----------------------------------------------------------------------------
67 #
67 #
68 #-----------------------------------------------------------------------------
68 #-----------------------------------------------------------------------------
69
69
70 # Where mathjax will be installed
70 # Where mathjax will be installed
71
71
72 nbextensions = os.path.join(get_ipython_dir(), 'nbextensions')
72 nbextensions = os.path.join(get_ipython_dir(), 'nbextensions')
73 default_dest = os.path.join(nbextensions, 'mathjax')
73 default_dest = os.path.join(nbextensions, 'mathjax')
74
74
75 # Test for access to install mathjax
75 # Test for access to install mathjax
76
76
77 def prepare_dest(dest, replace=False):
77 def prepare_dest(dest, replace=False):
78 """prepare the destination folder for mathjax install
78 """prepare the destination folder for mathjax install
79
79
80 Returns False if mathjax appears to already be installed and there is nothing to do,
80 Returns False if mathjax appears to already be installed and there is nothing to do,
81 True otherwise.
81 True otherwise.
82 """
82 """
83
83
84 parent = os.path.abspath(os.path.join(dest, os.path.pardir))
84 parent = os.path.abspath(os.path.join(dest, os.path.pardir))
85 if not os.path.exists(parent):
85 if not os.path.exists(parent):
86 os.makedirs(parent)
86 os.makedirs(parent)
87
87
88 if os.path.exists(dest):
88 if os.path.exists(dest):
89 if replace:
89 if replace:
90 print("removing existing MathJax at %s" % dest)
90 print("removing existing MathJax at %s" % dest)
91 shutil.rmtree(dest)
91 shutil.rmtree(dest)
92 return True
92 return True
93 else:
93 else:
94 mathjax_js = os.path.join(dest, 'MathJax.js')
94 mathjax_js = os.path.join(dest, 'MathJax.js')
95 if not os.path.exists(mathjax_js):
95 if not os.path.exists(mathjax_js):
96 raise IOError("%s exists, but does not contain MathJax.js" % dest)
96 raise IOError("%s exists, but does not contain MathJax.js" % dest)
97 print("%s already exists" % mathjax_js)
97 print("%s already exists" % mathjax_js)
98 return False
98 return False
99 else:
99 else:
100 return True
100 return True
101
101
102
102
103 def extract_tar(fd, dest):
103 def extract_tar(fd, dest):
104 """extract a tarball from filelike `fd` to destination `dest`"""
104 """extract a tarball from filelike `fd` to destination `dest`"""
105 # use 'r|gz' stream mode, because socket file-like objects can't seek:
105 # use 'r|gz' stream mode, because socket file-like objects can't seek:
106 tar = tarfile.open(fileobj=fd, mode='r|gz')
106 tar = tarfile.open(fileobj=fd, mode='r|gz')
107
107
108 # The first entry in the archive is the top-level dir
108 # The first entry in the archive is the top-level dir
109 topdir = tar.firstmember.path
109 topdir = tar.firstmember.path
110
110
111 # extract the archive (contains a single directory) to the destination directory
111 # extract the archive (contains a single directory) to the destination directory
112 parent = os.path.abspath(os.path.join(dest, os.path.pardir))
112 parent = os.path.abspath(os.path.join(dest, os.path.pardir))
113 tar.extractall(parent)
113 tar.extractall(parent)
114
114
115 # it will be mathjax-MathJax-<sha>, rename to just mathjax
115 # it will be mathjax-MathJax-<sha>, rename to just mathjax
116 os.rename(os.path.join(parent, topdir), dest)
116 os.rename(os.path.join(parent, topdir), dest)
117
117
118
118
119 def extract_zip(fd, dest):
119 def extract_zip(fd, dest):
120 """extract a zip file from filelike `fd` to destination `dest`"""
120 """extract a zip file from filelike `fd` to destination `dest`"""
121 z = zipfile.ZipFile(fd, 'r')
121 z = zipfile.ZipFile(fd, 'r')
122
122
123 # The first entry in the archive is the top-level dir
123 # The first entry in the archive is the top-level dir
124 topdir = z.namelist()[0]
124 topdir = z.namelist()[0]
125
125
126 # extract the archive (contains a single directory) to the static/ directory
126 # extract the archive (contains a single directory) to the static/ directory
127 parent = os.path.abspath(os.path.join(dest, os.path.pardir))
127 parent = os.path.abspath(os.path.join(dest, os.path.pardir))
128 z.extractall(parent)
128 z.extractall(parent)
129
129
130 # it will be mathjax-MathJax-<sha>, rename to just mathjax
130 # it will be mathjax-MathJax-<sha>, rename to just mathjax
131 d = os.path.join(parent, topdir)
131 d = os.path.join(parent, topdir)
132 os.rename(os.path.join(parent, topdir), dest)
132 os.rename(os.path.join(parent, topdir), dest)
133
133
134
134
135 def install_mathjax(tag='v2.2', dest=default_dest, replace=False, file=None, extractor=extract_tar):
135 def install_mathjax(tag='v2.2', dest=default_dest, replace=False, file=None, extractor=extract_tar):
136 """Download and/or install MathJax for offline use.
136 """Download and/or install MathJax for offline use.
137
137
138 This will install mathjax to the nbextensions dir in your IPYTHONDIR.
138 This will install mathjax to the nbextensions dir in your IPYTHONDIR.
139
139
140 MathJax is a ~15MB download, and ~150MB installed.
140 MathJax is a ~15MB download, and ~150MB installed.
141
141
142 Parameters
142 Parameters
143 ----------
143 ----------
144
144
145 replace : bool [False]
145 replace : bool [False]
146 Whether to remove and replace an existing install.
146 Whether to remove and replace an existing install.
147 dest : str [IPYTHONDIR/nbextensions/mathjax]
147 dest : str [IPYTHONDIR/nbextensions/mathjax]
148 Where to install mathjax
148 Where to install mathjax
149 tag : str ['v2.2']
149 tag : str ['v2.2']
150 Which tag to download. Default is 'v2.2', the current stable release,
150 Which tag to download. Default is 'v2.2', the current stable release,
151 but alternatives include 'v1.1a' and 'master'.
151 but alternatives include 'v1.1a' and 'master'.
152 file : file like object [ defualt to content of https://github.com/mathjax/MathJax/tarball/#{tag}]
152 file : file like object [ defualt to content of https://github.com/mathjax/MathJax/tarball/#{tag}]
153 File handle from which to untar/unzip/... mathjax
153 File handle from which to untar/unzip/... mathjax
154 extractor : function
154 extractor : function
155 Method to use to untar/unzip/... `file`
155 Method to use to untar/unzip/... `file`
156 """
156 """
157 try:
157 try:
158 anything_to_do = prepare_dest(dest, replace)
158 anything_to_do = prepare_dest(dest, replace)
159 except OSError as e:
159 except OSError as e:
160 print(("ERROR %s, require write access to %s" % (e, dest)))
160 print("ERROR %s, require write access to %s" % (e, dest))
161 return 1
161 return 1
162 else:
162 else:
163 if not anything_to_do:
163 if not anything_to_do:
164 return 0
164 return 0
165
165
166 if file is None:
166 if file is None:
167 # download mathjax
167 # download mathjax
168 mathjax_url = "https://github.com/mathjax/MathJax/archive/%s.tar.gz" %tag
168 mathjax_url = "https://github.com/mathjax/MathJax/archive/%s.tar.gz" %tag
169 print("Downloading mathjax source from %s" % mathjax_url)
169 print("Downloading mathjax source from %s" % mathjax_url)
170 response = urllib2.urlopen(mathjax_url)
170 response = urllib2.urlopen(mathjax_url)
171 file = response.fp
171 file = response.fp
172
172
173 print("Extracting to %s" % dest)
173 print("Extracting to %s" % dest)
174 extractor(file, dest)
174 extractor(file, dest)
175 return 0
175 return 0
176
176
177
177
178 def main():
178 def main():
179 parser = argparse.ArgumentParser(
179 parser = argparse.ArgumentParser(
180 description="""Install mathjax from internet or local archive""",
180 description="""Install mathjax from internet or local archive""",
181 )
181 )
182
182
183 parser.add_argument(
183 parser.add_argument(
184 '-i',
184 '-i',
185 '--install-dir',
185 '--install-dir',
186 default=nbextensions,
186 default=nbextensions,
187 help='custom installation directory. Mathjax will be installed in here/mathjax')
187 help='custom installation directory. Mathjax will be installed in here/mathjax')
188
188
189 parser.add_argument(
189 parser.add_argument(
190 '-d',
190 '-d',
191 '--print-dest',
191 '--print-dest',
192 action='store_true',
192 action='store_true',
193 help='print where mathjax would be installed and exit')
193 help='print where mathjax would be installed and exit')
194 parser.add_argument(
194 parser.add_argument(
195 '-r',
195 '-r',
196 '--replace',
196 '--replace',
197 action='store_true',
197 action='store_true',
198 help='Whether to replace current mathjax if it already exists')
198 help='Whether to replace current mathjax if it already exists')
199 parser.add_argument('filename',
199 parser.add_argument('filename',
200 help="the local tar/zip-ball filename containing mathjax",
200 help="the local tar/zip-ball filename containing mathjax",
201 nargs='?',
201 nargs='?',
202 metavar='filename')
202 metavar='filename')
203
203
204 pargs = parser.parse_args()
204 pargs = parser.parse_args()
205
205
206 dest = os.path.join(pargs.install_dir, 'mathjax')
206 dest = os.path.join(pargs.install_dir, 'mathjax')
207
207
208 if pargs.print_dest:
208 if pargs.print_dest:
209 print(dest)
209 print(dest)
210 return
210 return
211
211
212 # remove/replace existing mathjax?
212 # remove/replace existing mathjax?
213 replace = pargs.replace
213 replace = pargs.replace
214
214
215 # do it
215 # do it
216 if pargs.filename:
216 if pargs.filename:
217 fname = pargs.filename
217 fname = pargs.filename
218
218
219 # automatically detect zip/tar - could do something based
219 # automatically detect zip/tar - could do something based
220 # on file content, but really not cost-effective here.
220 # on file content, but really not cost-effective here.
221 if fname.endswith('.zip'):
221 if fname.endswith('.zip'):
222 extractor = extract_zip
222 extractor = extract_zip
223 else :
223 else :
224 extractor = extract_tar
224 extractor = extract_tar
225 # do it
225 # do it
226 return install_mathjax(file=open(fname, "rb"), replace=replace, extractor=extractor, dest=dest)
226 return install_mathjax(file=open(fname, "rb"), replace=replace, extractor=extractor, dest=dest)
227 else:
227 else:
228 return install_mathjax(replace=replace, dest=dest)
228 return install_mathjax(replace=replace, dest=dest)
229
229
230
230
231 if __name__ == '__main__' :
231 if __name__ == '__main__' :
232 sys.exit(main())
232 sys.exit(main())
233
233
234 __all__ = ['install_mathjax', 'main', 'default_dest']
234 __all__ = ['install_mathjax', 'main', 'default_dest']
@@ -1,583 +1,583 b''
1 """Module for interactive demos using IPython.
1 """Module for interactive demos using IPython.
2
2
3 This module implements a few classes for running Python scripts interactively
3 This module implements a few classes for running Python scripts interactively
4 in IPython for demonstrations. With very simple markup (a few tags in
4 in IPython for demonstrations. With very simple markup (a few tags in
5 comments), you can control points where the script stops executing and returns
5 comments), you can control points where the script stops executing and returns
6 control to IPython.
6 control to IPython.
7
7
8
8
9 Provided classes
9 Provided classes
10 ----------------
10 ----------------
11
11
12 The classes are (see their docstrings for further details):
12 The classes are (see their docstrings for further details):
13
13
14 - Demo: pure python demos
14 - Demo: pure python demos
15
15
16 - IPythonDemo: demos with input to be processed by IPython as if it had been
16 - IPythonDemo: demos with input to be processed by IPython as if it had been
17 typed interactively (so magics work, as well as any other special syntax you
17 typed interactively (so magics work, as well as any other special syntax you
18 may have added via input prefilters).
18 may have added via input prefilters).
19
19
20 - LineDemo: single-line version of the Demo class. These demos are executed
20 - LineDemo: single-line version of the Demo class. These demos are executed
21 one line at a time, and require no markup.
21 one line at a time, and require no markup.
22
22
23 - IPythonLineDemo: IPython version of the LineDemo class (the demo is
23 - IPythonLineDemo: IPython version of the LineDemo class (the demo is
24 executed a line at a time, but processed via IPython).
24 executed a line at a time, but processed via IPython).
25
25
26 - ClearMixin: mixin to make Demo classes with less visual clutter. It
26 - ClearMixin: mixin to make Demo classes with less visual clutter. It
27 declares an empty marquee and a pre_cmd that clears the screen before each
27 declares an empty marquee and a pre_cmd that clears the screen before each
28 block (see Subclassing below).
28 block (see Subclassing below).
29
29
30 - ClearDemo, ClearIPDemo: mixin-enabled versions of the Demo and IPythonDemo
30 - ClearDemo, ClearIPDemo: mixin-enabled versions of the Demo and IPythonDemo
31 classes.
31 classes.
32
32
33 Inheritance diagram:
33 Inheritance diagram:
34
34
35 .. inheritance-diagram:: IPython.lib.demo
35 .. inheritance-diagram:: IPython.lib.demo
36 :parts: 3
36 :parts: 3
37
37
38 Subclassing
38 Subclassing
39 -----------
39 -----------
40
40
41 The classes here all include a few methods meant to make customization by
41 The classes here all include a few methods meant to make customization by
42 subclassing more convenient. Their docstrings below have some more details:
42 subclassing more convenient. Their docstrings below have some more details:
43
43
44 - marquee(): generates a marquee to provide visible on-screen markers at each
44 - marquee(): generates a marquee to provide visible on-screen markers at each
45 block start and end.
45 block start and end.
46
46
47 - pre_cmd(): run right before the execution of each block.
47 - pre_cmd(): run right before the execution of each block.
48
48
49 - post_cmd(): run right after the execution of each block. If the block
49 - post_cmd(): run right after the execution of each block. If the block
50 raises an exception, this is NOT called.
50 raises an exception, this is NOT called.
51
51
52
52
53 Operation
53 Operation
54 ---------
54 ---------
55
55
56 The file is run in its own empty namespace (though you can pass it a string of
56 The file is run in its own empty namespace (though you can pass it a string of
57 arguments as if in a command line environment, and it will see those as
57 arguments as if in a command line environment, and it will see those as
58 sys.argv). But at each stop, the global IPython namespace is updated with the
58 sys.argv). But at each stop, the global IPython namespace is updated with the
59 current internal demo namespace, so you can work interactively with the data
59 current internal demo namespace, so you can work interactively with the data
60 accumulated so far.
60 accumulated so far.
61
61
62 By default, each block of code is printed (with syntax highlighting) before
62 By default, each block of code is printed (with syntax highlighting) before
63 executing it and you have to confirm execution. This is intended to show the
63 executing it and you have to confirm execution. This is intended to show the
64 code to an audience first so you can discuss it, and only proceed with
64 code to an audience first so you can discuss it, and only proceed with
65 execution once you agree. There are a few tags which allow you to modify this
65 execution once you agree. There are a few tags which allow you to modify this
66 behavior.
66 behavior.
67
67
68 The supported tags are:
68 The supported tags are:
69
69
70 # <demo> stop
70 # <demo> stop
71
71
72 Defines block boundaries, the points where IPython stops execution of the
72 Defines block boundaries, the points where IPython stops execution of the
73 file and returns to the interactive prompt.
73 file and returns to the interactive prompt.
74
74
75 You can optionally mark the stop tag with extra dashes before and after the
75 You can optionally mark the stop tag with extra dashes before and after the
76 word 'stop', to help visually distinguish the blocks in a text editor:
76 word 'stop', to help visually distinguish the blocks in a text editor:
77
77
78 # <demo> --- stop ---
78 # <demo> --- stop ---
79
79
80
80
81 # <demo> silent
81 # <demo> silent
82
82
83 Make a block execute silently (and hence automatically). Typically used in
83 Make a block execute silently (and hence automatically). Typically used in
84 cases where you have some boilerplate or initialization code which you need
84 cases where you have some boilerplate or initialization code which you need
85 executed but do not want to be seen in the demo.
85 executed but do not want to be seen in the demo.
86
86
87 # <demo> auto
87 # <demo> auto
88
88
89 Make a block execute automatically, but still being printed. Useful for
89 Make a block execute automatically, but still being printed. Useful for
90 simple code which does not warrant discussion, since it avoids the extra
90 simple code which does not warrant discussion, since it avoids the extra
91 manual confirmation.
91 manual confirmation.
92
92
93 # <demo> auto_all
93 # <demo> auto_all
94
94
95 This tag can _only_ be in the first block, and if given it overrides the
95 This tag can _only_ be in the first block, and if given it overrides the
96 individual auto tags to make the whole demo fully automatic (no block asks
96 individual auto tags to make the whole demo fully automatic (no block asks
97 for confirmation). It can also be given at creation time (or the attribute
97 for confirmation). It can also be given at creation time (or the attribute
98 set later) to override what's in the file.
98 set later) to override what's in the file.
99
99
100 While _any_ python file can be run as a Demo instance, if there are no stop
100 While _any_ python file can be run as a Demo instance, if there are no stop
101 tags the whole file will run in a single block (no different that calling
101 tags the whole file will run in a single block (no different that calling
102 first %pycat and then %run). The minimal markup to make this useful is to
102 first %pycat and then %run). The minimal markup to make this useful is to
103 place a set of stop tags; the other tags are only there to let you fine-tune
103 place a set of stop tags; the other tags are only there to let you fine-tune
104 the execution.
104 the execution.
105
105
106 This is probably best explained with the simple example file below. You can
106 This is probably best explained with the simple example file below. You can
107 copy this into a file named ex_demo.py, and try running it via::
107 copy this into a file named ex_demo.py, and try running it via::
108
108
109 from IPython.demo import Demo
109 from IPython.demo import Demo
110 d = Demo('ex_demo.py')
110 d = Demo('ex_demo.py')
111 d()
111 d()
112
112
113 Each time you call the demo object, it runs the next block. The demo object
113 Each time you call the demo object, it runs the next block. The demo object
114 has a few useful methods for navigation, like again(), edit(), jump(), seek()
114 has a few useful methods for navigation, like again(), edit(), jump(), seek()
115 and back(). It can be reset for a new run via reset() or reloaded from disk
115 and back(). It can be reset for a new run via reset() or reloaded from disk
116 (in case you've edited the source) via reload(). See their docstrings below.
116 (in case you've edited the source) via reload(). See their docstrings below.
117
117
118 Note: To make this simpler to explore, a file called "demo-exercizer.py" has
118 Note: To make this simpler to explore, a file called "demo-exercizer.py" has
119 been added to the "docs/examples/core" directory. Just cd to this directory in
119 been added to the "docs/examples/core" directory. Just cd to this directory in
120 an IPython session, and type::
120 an IPython session, and type::
121
121
122 %run demo-exercizer.py
122 %run demo-exercizer.py
123
123
124 and then follow the directions.
124 and then follow the directions.
125
125
126 Example
126 Example
127 -------
127 -------
128
128
129 The following is a very simple example of a valid demo file.
129 The following is a very simple example of a valid demo file.
130
130
131 ::
131 ::
132
132
133 #################### EXAMPLE DEMO <ex_demo.py> ###############################
133 #################### EXAMPLE DEMO <ex_demo.py> ###############################
134 '''A simple interactive demo to illustrate the use of IPython's Demo class.'''
134 '''A simple interactive demo to illustrate the use of IPython's Demo class.'''
135
135
136 print 'Hello, welcome to an interactive IPython demo.'
136 print 'Hello, welcome to an interactive IPython demo.'
137
137
138 # The mark below defines a block boundary, which is a point where IPython will
138 # The mark below defines a block boundary, which is a point where IPython will
139 # stop execution and return to the interactive prompt. The dashes are actually
139 # stop execution and return to the interactive prompt. The dashes are actually
140 # optional and used only as a visual aid to clearly separate blocks while
140 # optional and used only as a visual aid to clearly separate blocks while
141 # editing the demo code.
141 # editing the demo code.
142 # <demo> stop
142 # <demo> stop
143
143
144 x = 1
144 x = 1
145 y = 2
145 y = 2
146
146
147 # <demo> stop
147 # <demo> stop
148
148
149 # the mark below makes this block as silent
149 # the mark below makes this block as silent
150 # <demo> silent
150 # <demo> silent
151
151
152 print 'This is a silent block, which gets executed but not printed.'
152 print 'This is a silent block, which gets executed but not printed.'
153
153
154 # <demo> stop
154 # <demo> stop
155 # <demo> auto
155 # <demo> auto
156 print 'This is an automatic block.'
156 print 'This is an automatic block.'
157 print 'It is executed without asking for confirmation, but printed.'
157 print 'It is executed without asking for confirmation, but printed.'
158 z = x+y
158 z = x+y
159
159
160 print 'z=',x
160 print 'z=',x
161
161
162 # <demo> stop
162 # <demo> stop
163 # This is just another normal block.
163 # This is just another normal block.
164 print 'z is now:', z
164 print 'z is now:', z
165
165
166 print 'bye!'
166 print 'bye!'
167 ################### END EXAMPLE DEMO <ex_demo.py> ############################
167 ################### END EXAMPLE DEMO <ex_demo.py> ############################
168 """
168 """
169
169
170 from __future__ import unicode_literals
170 from __future__ import unicode_literals
171
171
172 #*****************************************************************************
172 #*****************************************************************************
173 # Copyright (C) 2005-2006 Fernando Perez. <Fernando.Perez@colorado.edu>
173 # Copyright (C) 2005-2006 Fernando Perez. <Fernando.Perez@colorado.edu>
174 #
174 #
175 # Distributed under the terms of the BSD License. The full license is in
175 # Distributed under the terms of the BSD License. The full license is in
176 # the file COPYING, distributed as part of this software.
176 # the file COPYING, distributed as part of this software.
177 #
177 #
178 #*****************************************************************************
178 #*****************************************************************************
179 from __future__ import print_function
179 from __future__ import print_function
180
180
181 import os
181 import os
182 import re
182 import re
183 import shlex
183 import shlex
184 import sys
184 import sys
185
185
186 from IPython.utils import io
186 from IPython.utils import io
187 from IPython.utils.text import marquee
187 from IPython.utils.text import marquee
188 from IPython.utils import openpy
188 from IPython.utils import openpy
189 from IPython.utils import py3compat
189 from IPython.utils import py3compat
190 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
190 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
191
191
192 class DemoError(Exception): pass
192 class DemoError(Exception): pass
193
193
194 def re_mark(mark):
194 def re_mark(mark):
195 return re.compile(r'^\s*#\s+<demo>\s+%s\s*$' % mark,re.MULTILINE)
195 return re.compile(r'^\s*#\s+<demo>\s+%s\s*$' % mark,re.MULTILINE)
196
196
197 class Demo(object):
197 class Demo(object):
198
198
199 re_stop = re_mark('-*\s?stop\s?-*')
199 re_stop = re_mark('-*\s?stop\s?-*')
200 re_silent = re_mark('silent')
200 re_silent = re_mark('silent')
201 re_auto = re_mark('auto')
201 re_auto = re_mark('auto')
202 re_auto_all = re_mark('auto_all')
202 re_auto_all = re_mark('auto_all')
203
203
204 def __init__(self,src,title='',arg_str='',auto_all=None):
204 def __init__(self,src,title='',arg_str='',auto_all=None):
205 """Make a new demo object. To run the demo, simply call the object.
205 """Make a new demo object. To run the demo, simply call the object.
206
206
207 See the module docstring for full details and an example (you can use
207 See the module docstring for full details and an example (you can use
208 IPython.Demo? in IPython to see it).
208 IPython.Demo? in IPython to see it).
209
209
210 Inputs:
210 Inputs:
211
211
212 - src is either a file, or file-like object, or a
212 - src is either a file, or file-like object, or a
213 string that can be resolved to a filename.
213 string that can be resolved to a filename.
214
214
215 Optional inputs:
215 Optional inputs:
216
216
217 - title: a string to use as the demo name. Of most use when the demo
217 - title: a string to use as the demo name. Of most use when the demo
218 you are making comes from an object that has no filename, or if you
218 you are making comes from an object that has no filename, or if you
219 want an alternate denotation distinct from the filename.
219 want an alternate denotation distinct from the filename.
220
220
221 - arg_str(''): a string of arguments, internally converted to a list
221 - arg_str(''): a string of arguments, internally converted to a list
222 just like sys.argv, so the demo script can see a similar
222 just like sys.argv, so the demo script can see a similar
223 environment.
223 environment.
224
224
225 - auto_all(None): global flag to run all blocks automatically without
225 - auto_all(None): global flag to run all blocks automatically without
226 confirmation. This attribute overrides the block-level tags and
226 confirmation. This attribute overrides the block-level tags and
227 applies to the whole demo. It is an attribute of the object, and
227 applies to the whole demo. It is an attribute of the object, and
228 can be changed at runtime simply by reassigning it to a boolean
228 can be changed at runtime simply by reassigning it to a boolean
229 value.
229 value.
230 """
230 """
231 if hasattr(src, "read"):
231 if hasattr(src, "read"):
232 # It seems to be a file or a file-like object
232 # It seems to be a file or a file-like object
233 self.fname = "from a file-like object"
233 self.fname = "from a file-like object"
234 if title == '':
234 if title == '':
235 self.title = "from a file-like object"
235 self.title = "from a file-like object"
236 else:
236 else:
237 self.title = title
237 self.title = title
238 else:
238 else:
239 # Assume it's a string or something that can be converted to one
239 # Assume it's a string or something that can be converted to one
240 self.fname = src
240 self.fname = src
241 if title == '':
241 if title == '':
242 (filepath, filename) = os.path.split(src)
242 (filepath, filename) = os.path.split(src)
243 self.title = filename
243 self.title = filename
244 else:
244 else:
245 self.title = title
245 self.title = title
246 self.sys_argv = [src] + shlex.split(arg_str)
246 self.sys_argv = [src] + shlex.split(arg_str)
247 self.auto_all = auto_all
247 self.auto_all = auto_all
248 self.src = src
248 self.src = src
249
249
250 # get a few things from ipython. While it's a bit ugly design-wise,
250 # get a few things from ipython. While it's a bit ugly design-wise,
251 # it ensures that things like color scheme and the like are always in
251 # it ensures that things like color scheme and the like are always in
252 # sync with the ipython mode being used. This class is only meant to
252 # sync with the ipython mode being used. This class is only meant to
253 # be used inside ipython anyways, so it's OK.
253 # be used inside ipython anyways, so it's OK.
254 ip = get_ipython() # this is in builtins whenever IPython is running
254 ip = get_ipython() # this is in builtins whenever IPython is running
255 self.ip_ns = ip.user_ns
255 self.ip_ns = ip.user_ns
256 self.ip_colorize = ip.pycolorize
256 self.ip_colorize = ip.pycolorize
257 self.ip_showtb = ip.showtraceback
257 self.ip_showtb = ip.showtraceback
258 self.ip_run_cell = ip.run_cell
258 self.ip_run_cell = ip.run_cell
259 self.shell = ip
259 self.shell = ip
260
260
261 # load user data and initialize data structures
261 # load user data and initialize data structures
262 self.reload()
262 self.reload()
263
263
264 def fload(self):
264 def fload(self):
265 """Load file object."""
265 """Load file object."""
266 # read data and parse into blocks
266 # read data and parse into blocks
267 if hasattr(self, 'fobj') and self.fobj is not None:
267 if hasattr(self, 'fobj') and self.fobj is not None:
268 self.fobj.close()
268 self.fobj.close()
269 if hasattr(self.src, "read"):
269 if hasattr(self.src, "read"):
270 # It seems to be a file or a file-like object
270 # It seems to be a file or a file-like object
271 self.fobj = self.src
271 self.fobj = self.src
272 else:
272 else:
273 # Assume it's a string or something that can be converted to one
273 # Assume it's a string or something that can be converted to one
274 self.fobj = openpy.open(self.fname)
274 self.fobj = openpy.open(self.fname)
275
275
276 def reload(self):
276 def reload(self):
277 """Reload source from disk and initialize state."""
277 """Reload source from disk and initialize state."""
278 self.fload()
278 self.fload()
279
279
280 self.src = "".join(openpy.strip_encoding_cookie(self.fobj))
280 self.src = "".join(openpy.strip_encoding_cookie(self.fobj))
281 src_b = [b.strip() for b in self.re_stop.split(self.src) if b]
281 src_b = [b.strip() for b in self.re_stop.split(self.src) if b]
282 self._silent = [bool(self.re_silent.findall(b)) for b in src_b]
282 self._silent = [bool(self.re_silent.findall(b)) for b in src_b]
283 self._auto = [bool(self.re_auto.findall(b)) for b in src_b]
283 self._auto = [bool(self.re_auto.findall(b)) for b in src_b]
284
284
285 # if auto_all is not given (def. None), we read it from the file
285 # if auto_all is not given (def. None), we read it from the file
286 if self.auto_all is None:
286 if self.auto_all is None:
287 self.auto_all = bool(self.re_auto_all.findall(src_b[0]))
287 self.auto_all = bool(self.re_auto_all.findall(src_b[0]))
288 else:
288 else:
289 self.auto_all = bool(self.auto_all)
289 self.auto_all = bool(self.auto_all)
290
290
291 # Clean the sources from all markup so it doesn't get displayed when
291 # Clean the sources from all markup so it doesn't get displayed when
292 # running the demo
292 # running the demo
293 src_blocks = []
293 src_blocks = []
294 auto_strip = lambda s: self.re_auto.sub('',s)
294 auto_strip = lambda s: self.re_auto.sub('',s)
295 for i,b in enumerate(src_b):
295 for i,b in enumerate(src_b):
296 if self._auto[i]:
296 if self._auto[i]:
297 src_blocks.append(auto_strip(b))
297 src_blocks.append(auto_strip(b))
298 else:
298 else:
299 src_blocks.append(b)
299 src_blocks.append(b)
300 # remove the auto_all marker
300 # remove the auto_all marker
301 src_blocks[0] = self.re_auto_all.sub('',src_blocks[0])
301 src_blocks[0] = self.re_auto_all.sub('',src_blocks[0])
302
302
303 self.nblocks = len(src_blocks)
303 self.nblocks = len(src_blocks)
304 self.src_blocks = src_blocks
304 self.src_blocks = src_blocks
305
305
306 # also build syntax-highlighted source
306 # also build syntax-highlighted source
307 self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
307 self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
308
308
309 # ensure clean namespace and seek offset
309 # ensure clean namespace and seek offset
310 self.reset()
310 self.reset()
311
311
312 def reset(self):
312 def reset(self):
313 """Reset the namespace and seek pointer to restart the demo"""
313 """Reset the namespace and seek pointer to restart the demo"""
314 self.user_ns = {}
314 self.user_ns = {}
315 self.finished = False
315 self.finished = False
316 self.block_index = 0
316 self.block_index = 0
317
317
318 def _validate_index(self,index):
318 def _validate_index(self,index):
319 if index<0 or index>=self.nblocks:
319 if index<0 or index>=self.nblocks:
320 raise ValueError('invalid block index %s' % index)
320 raise ValueError('invalid block index %s' % index)
321
321
322 def _get_index(self,index):
322 def _get_index(self,index):
323 """Get the current block index, validating and checking status.
323 """Get the current block index, validating and checking status.
324
324
325 Returns None if the demo is finished"""
325 Returns None if the demo is finished"""
326
326
327 if index is None:
327 if index is None:
328 if self.finished:
328 if self.finished:
329 print('Demo finished. Use <demo_name>.reset() if you want to rerun it.', file=io.stdout)
329 print('Demo finished. Use <demo_name>.reset() if you want to rerun it.', file=io.stdout)
330 return None
330 return None
331 index = self.block_index
331 index = self.block_index
332 else:
332 else:
333 self._validate_index(index)
333 self._validate_index(index)
334 return index
334 return index
335
335
336 def seek(self,index):
336 def seek(self,index):
337 """Move the current seek pointer to the given block.
337 """Move the current seek pointer to the given block.
338
338
339 You can use negative indices to seek from the end, with identical
339 You can use negative indices to seek from the end, with identical
340 semantics to those of Python lists."""
340 semantics to those of Python lists."""
341 if index<0:
341 if index<0:
342 index = self.nblocks + index
342 index = self.nblocks + index
343 self._validate_index(index)
343 self._validate_index(index)
344 self.block_index = index
344 self.block_index = index
345 self.finished = False
345 self.finished = False
346
346
347 def back(self,num=1):
347 def back(self,num=1):
348 """Move the seek pointer back num blocks (default is 1)."""
348 """Move the seek pointer back num blocks (default is 1)."""
349 self.seek(self.block_index-num)
349 self.seek(self.block_index-num)
350
350
351 def jump(self,num=1):
351 def jump(self,num=1):
352 """Jump a given number of blocks relative to the current one.
352 """Jump a given number of blocks relative to the current one.
353
353
354 The offset can be positive or negative, defaults to 1."""
354 The offset can be positive or negative, defaults to 1."""
355 self.seek(self.block_index+num)
355 self.seek(self.block_index+num)
356
356
357 def again(self):
357 def again(self):
358 """Move the seek pointer back one block and re-execute."""
358 """Move the seek pointer back one block and re-execute."""
359 self.back(1)
359 self.back(1)
360 self()
360 self()
361
361
362 def edit(self,index=None):
362 def edit(self,index=None):
363 """Edit a block.
363 """Edit a block.
364
364
365 If no number is given, use the last block executed.
365 If no number is given, use the last block executed.
366
366
367 This edits the in-memory copy of the demo, it does NOT modify the
367 This edits the in-memory copy of the demo, it does NOT modify the
368 original source file. If you want to do that, simply open the file in
368 original source file. If you want to do that, simply open the file in
369 an editor and use reload() when you make changes to the file. This
369 an editor and use reload() when you make changes to the file. This
370 method is meant to let you change a block during a demonstration for
370 method is meant to let you change a block during a demonstration for
371 explanatory purposes, without damaging your original script."""
371 explanatory purposes, without damaging your original script."""
372
372
373 index = self._get_index(index)
373 index = self._get_index(index)
374 if index is None:
374 if index is None:
375 return
375 return
376 # decrease the index by one (unless we're at the very beginning), so
376 # decrease the index by one (unless we're at the very beginning), so
377 # that the default demo.edit() call opens up the sblock we've last run
377 # that the default demo.edit() call opens up the sblock we've last run
378 if index>0:
378 if index>0:
379 index -= 1
379 index -= 1
380
380
381 filename = self.shell.mktempfile(self.src_blocks[index])
381 filename = self.shell.mktempfile(self.src_blocks[index])
382 self.shell.hooks.editor(filename,1)
382 self.shell.hooks.editor(filename,1)
383 with open(filename, 'r') as f:
383 with open(filename, 'r') as f:
384 new_block = f.read()
384 new_block = f.read()
385 # update the source and colored block
385 # update the source and colored block
386 self.src_blocks[index] = new_block
386 self.src_blocks[index] = new_block
387 self.src_blocks_colored[index] = self.ip_colorize(new_block)
387 self.src_blocks_colored[index] = self.ip_colorize(new_block)
388 self.block_index = index
388 self.block_index = index
389 # call to run with the newly edited index
389 # call to run with the newly edited index
390 self()
390 self()
391
391
392 def show(self,index=None):
392 def show(self,index=None):
393 """Show a single block on screen"""
393 """Show a single block on screen"""
394
394
395 index = self._get_index(index)
395 index = self._get_index(index)
396 if index is None:
396 if index is None:
397 return
397 return
398
398
399 print(self.marquee('<%s> block # %s (%s remaining)' %
399 print(self.marquee('<%s> block # %s (%s remaining)' %
400 (self.title,index,self.nblocks-index-1)), file=io.stdout)
400 (self.title,index,self.nblocks-index-1)), file=io.stdout)
401 print((self.src_blocks_colored[index]), file=io.stdout)
401 print(self.src_blocks_colored[index], file=io.stdout)
402 sys.stdout.flush()
402 sys.stdout.flush()
403
403
404 def show_all(self):
404 def show_all(self):
405 """Show entire demo on screen, block by block"""
405 """Show entire demo on screen, block by block"""
406
406
407 fname = self.title
407 fname = self.title
408 title = self.title
408 title = self.title
409 nblocks = self.nblocks
409 nblocks = self.nblocks
410 silent = self._silent
410 silent = self._silent
411 marquee = self.marquee
411 marquee = self.marquee
412 for index,block in enumerate(self.src_blocks_colored):
412 for index,block in enumerate(self.src_blocks_colored):
413 if silent[index]:
413 if silent[index]:
414 print(marquee('<%s> SILENT block # %s (%s remaining)' %
414 print(marquee('<%s> SILENT block # %s (%s remaining)' %
415 (title,index,nblocks-index-1)), file=io.stdout)
415 (title,index,nblocks-index-1)), file=io.stdout)
416 else:
416 else:
417 print(marquee('<%s> block # %s (%s remaining)' %
417 print(marquee('<%s> block # %s (%s remaining)' %
418 (title,index,nblocks-index-1)), file=io.stdout)
418 (title,index,nblocks-index-1)), file=io.stdout)
419 print(block, end=' ', file=io.stdout)
419 print(block, end=' ', file=io.stdout)
420 sys.stdout.flush()
420 sys.stdout.flush()
421
421
422 def run_cell(self,source):
422 def run_cell(self,source):
423 """Execute a string with one or more lines of code"""
423 """Execute a string with one or more lines of code"""
424
424
425 exec(source, self.user_ns)
425 exec(source, self.user_ns)
426
426
427 def __call__(self,index=None):
427 def __call__(self,index=None):
428 """run a block of the demo.
428 """run a block of the demo.
429
429
430 If index is given, it should be an integer >=1 and <= nblocks. This
430 If index is given, it should be an integer >=1 and <= nblocks. This
431 means that the calling convention is one off from typical Python
431 means that the calling convention is one off from typical Python
432 lists. The reason for the inconsistency is that the demo always
432 lists. The reason for the inconsistency is that the demo always
433 prints 'Block n/N, and N is the total, so it would be very odd to use
433 prints 'Block n/N, and N is the total, so it would be very odd to use
434 zero-indexing here."""
434 zero-indexing here."""
435
435
436 index = self._get_index(index)
436 index = self._get_index(index)
437 if index is None:
437 if index is None:
438 return
438 return
439 try:
439 try:
440 marquee = self.marquee
440 marquee = self.marquee
441 next_block = self.src_blocks[index]
441 next_block = self.src_blocks[index]
442 self.block_index += 1
442 self.block_index += 1
443 if self._silent[index]:
443 if self._silent[index]:
444 print(marquee('Executing silent block # %s (%s remaining)' %
444 print(marquee('Executing silent block # %s (%s remaining)' %
445 (index,self.nblocks-index-1)), file=io.stdout)
445 (index,self.nblocks-index-1)), file=io.stdout)
446 else:
446 else:
447 self.pre_cmd()
447 self.pre_cmd()
448 self.show(index)
448 self.show(index)
449 if self.auto_all or self._auto[index]:
449 if self.auto_all or self._auto[index]:
450 print(marquee('output:'), file=io.stdout)
450 print(marquee('output:'), file=io.stdout)
451 else:
451 else:
452 print(marquee('Press <q> to quit, <Enter> to execute...'), end=' ', file=io.stdout)
452 print(marquee('Press <q> to quit, <Enter> to execute...'), end=' ', file=io.stdout)
453 ans = py3compat.input().strip()
453 ans = py3compat.input().strip()
454 if ans:
454 if ans:
455 print(marquee('Block NOT executed'), file=io.stdout)
455 print(marquee('Block NOT executed'), file=io.stdout)
456 return
456 return
457 try:
457 try:
458 save_argv = sys.argv
458 save_argv = sys.argv
459 sys.argv = self.sys_argv
459 sys.argv = self.sys_argv
460 self.run_cell(next_block)
460 self.run_cell(next_block)
461 self.post_cmd()
461 self.post_cmd()
462 finally:
462 finally:
463 sys.argv = save_argv
463 sys.argv = save_argv
464
464
465 except:
465 except:
466 self.ip_showtb(filename=self.fname)
466 self.ip_showtb(filename=self.fname)
467 else:
467 else:
468 self.ip_ns.update(self.user_ns)
468 self.ip_ns.update(self.user_ns)
469
469
470 if self.block_index == self.nblocks:
470 if self.block_index == self.nblocks:
471 mq1 = self.marquee('END OF DEMO')
471 mq1 = self.marquee('END OF DEMO')
472 if mq1:
472 if mq1:
473 # avoid spurious print >>io.stdout,s if empty marquees are used
473 # avoid spurious print >>io.stdout,s if empty marquees are used
474 print(file=io.stdout)
474 print(file=io.stdout)
475 print(mq1, file=io.stdout)
475 print(mq1, file=io.stdout)
476 print(self.marquee('Use <demo_name>.reset() if you want to rerun it.'), file=io.stdout)
476 print(self.marquee('Use <demo_name>.reset() if you want to rerun it.'), file=io.stdout)
477 self.finished = True
477 self.finished = True
478
478
479 # These methods are meant to be overridden by subclasses who may wish to
479 # These methods are meant to be overridden by subclasses who may wish to
480 # customize the behavior of of their demos.
480 # customize the behavior of of their demos.
481 def marquee(self,txt='',width=78,mark='*'):
481 def marquee(self,txt='',width=78,mark='*'):
482 """Return the input string centered in a 'marquee'."""
482 """Return the input string centered in a 'marquee'."""
483 return marquee(txt,width,mark)
483 return marquee(txt,width,mark)
484
484
485 def pre_cmd(self):
485 def pre_cmd(self):
486 """Method called before executing each block."""
486 """Method called before executing each block."""
487 pass
487 pass
488
488
489 def post_cmd(self):
489 def post_cmd(self):
490 """Method called after executing each block."""
490 """Method called after executing each block."""
491 pass
491 pass
492
492
493
493
494 class IPythonDemo(Demo):
494 class IPythonDemo(Demo):
495 """Class for interactive demos with IPython's input processing applied.
495 """Class for interactive demos with IPython's input processing applied.
496
496
497 This subclasses Demo, but instead of executing each block by the Python
497 This subclasses Demo, but instead of executing each block by the Python
498 interpreter (via exec), it actually calls IPython on it, so that any input
498 interpreter (via exec), it actually calls IPython on it, so that any input
499 filters which may be in place are applied to the input block.
499 filters which may be in place are applied to the input block.
500
500
501 If you have an interactive environment which exposes special input
501 If you have an interactive environment which exposes special input
502 processing, you can use this class instead to write demo scripts which
502 processing, you can use this class instead to write demo scripts which
503 operate exactly as if you had typed them interactively. The default Demo
503 operate exactly as if you had typed them interactively. The default Demo
504 class requires the input to be valid, pure Python code.
504 class requires the input to be valid, pure Python code.
505 """
505 """
506
506
507 def run_cell(self,source):
507 def run_cell(self,source):
508 """Execute a string with one or more lines of code"""
508 """Execute a string with one or more lines of code"""
509
509
510 self.shell.run_cell(source)
510 self.shell.run_cell(source)
511
511
512 class LineDemo(Demo):
512 class LineDemo(Demo):
513 """Demo where each line is executed as a separate block.
513 """Demo where each line is executed as a separate block.
514
514
515 The input script should be valid Python code.
515 The input script should be valid Python code.
516
516
517 This class doesn't require any markup at all, and it's meant for simple
517 This class doesn't require any markup at all, and it's meant for simple
518 scripts (with no nesting or any kind of indentation) which consist of
518 scripts (with no nesting or any kind of indentation) which consist of
519 multiple lines of input to be executed, one at a time, as if they had been
519 multiple lines of input to be executed, one at a time, as if they had been
520 typed in the interactive prompt.
520 typed in the interactive prompt.
521
521
522 Note: the input can not have *any* indentation, which means that only
522 Note: the input can not have *any* indentation, which means that only
523 single-lines of input are accepted, not even function definitions are
523 single-lines of input are accepted, not even function definitions are
524 valid."""
524 valid."""
525
525
526 def reload(self):
526 def reload(self):
527 """Reload source from disk and initialize state."""
527 """Reload source from disk and initialize state."""
528 # read data and parse into blocks
528 # read data and parse into blocks
529 self.fload()
529 self.fload()
530 lines = self.fobj.readlines()
530 lines = self.fobj.readlines()
531 src_b = [l for l in lines if l.strip()]
531 src_b = [l for l in lines if l.strip()]
532 nblocks = len(src_b)
532 nblocks = len(src_b)
533 self.src = ''.join(lines)
533 self.src = ''.join(lines)
534 self._silent = [False]*nblocks
534 self._silent = [False]*nblocks
535 self._auto = [True]*nblocks
535 self._auto = [True]*nblocks
536 self.auto_all = True
536 self.auto_all = True
537 self.nblocks = nblocks
537 self.nblocks = nblocks
538 self.src_blocks = src_b
538 self.src_blocks = src_b
539
539
540 # also build syntax-highlighted source
540 # also build syntax-highlighted source
541 self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
541 self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
542
542
543 # ensure clean namespace and seek offset
543 # ensure clean namespace and seek offset
544 self.reset()
544 self.reset()
545
545
546
546
547 class IPythonLineDemo(IPythonDemo,LineDemo):
547 class IPythonLineDemo(IPythonDemo,LineDemo):
548 """Variant of the LineDemo class whose input is processed by IPython."""
548 """Variant of the LineDemo class whose input is processed by IPython."""
549 pass
549 pass
550
550
551
551
552 class ClearMixin(object):
552 class ClearMixin(object):
553 """Use this mixin to make Demo classes with less visual clutter.
553 """Use this mixin to make Demo classes with less visual clutter.
554
554
555 Demos using this mixin will clear the screen before every block and use
555 Demos using this mixin will clear the screen before every block and use
556 blank marquees.
556 blank marquees.
557
557
558 Note that in order for the methods defined here to actually override those
558 Note that in order for the methods defined here to actually override those
559 of the classes it's mixed with, it must go /first/ in the inheritance
559 of the classes it's mixed with, it must go /first/ in the inheritance
560 tree. For example:
560 tree. For example:
561
561
562 class ClearIPDemo(ClearMixin,IPythonDemo): pass
562 class ClearIPDemo(ClearMixin,IPythonDemo): pass
563
563
564 will provide an IPythonDemo class with the mixin's features.
564 will provide an IPythonDemo class with the mixin's features.
565 """
565 """
566
566
567 def marquee(self,txt='',width=78,mark='*'):
567 def marquee(self,txt='',width=78,mark='*'):
568 """Blank marquee that returns '' no matter what the input."""
568 """Blank marquee that returns '' no matter what the input."""
569 return ''
569 return ''
570
570
571 def pre_cmd(self):
571 def pre_cmd(self):
572 """Method called before executing each block.
572 """Method called before executing each block.
573
573
574 This one simply clears the screen."""
574 This one simply clears the screen."""
575 from IPython.utils.terminal import term_clear
575 from IPython.utils.terminal import term_clear
576 term_clear()
576 term_clear()
577
577
578 class ClearDemo(ClearMixin,Demo):
578 class ClearDemo(ClearMixin,Demo):
579 pass
579 pass
580
580
581
581
582 class ClearIPDemo(ClearMixin,IPythonDemo):
582 class ClearIPDemo(ClearMixin,IPythonDemo):
583 pass
583 pass
@@ -1,112 +1,112 b''
1 """PostProcessor for serving reveal.js HTML slideshows."""
1 """PostProcessor for serving reveal.js HTML slideshows."""
2 from __future__ import print_function
2 from __future__ import print_function
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 #Copyright (c) 2013, the IPython Development Team.
4 #Copyright (c) 2013, the IPython Development Team.
5 #
5 #
6 #Distributed under the terms of the Modified BSD License.
6 #Distributed under the terms of the Modified BSD License.
7 #
7 #
8 #The full license is in the file COPYING.txt, distributed with this software.
8 #The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 import os
15 import os
16 import webbrowser
16 import webbrowser
17
17
18 from tornado import web, ioloop, httpserver
18 from tornado import web, ioloop, httpserver
19 from tornado.httpclient import AsyncHTTPClient
19 from tornado.httpclient import AsyncHTTPClient
20
20
21 from IPython.utils.traitlets import Bool, Unicode, Int
21 from IPython.utils.traitlets import Bool, Unicode, Int
22
22
23 from .base import PostProcessorBase
23 from .base import PostProcessorBase
24
24
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Classes
26 # Classes
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28
28
29 class ProxyHandler(web.RequestHandler):
29 class ProxyHandler(web.RequestHandler):
30 """handler the proxies requests from a local prefix to a CDN"""
30 """handler the proxies requests from a local prefix to a CDN"""
31 @web.asynchronous
31 @web.asynchronous
32 def get(self, prefix, url):
32 def get(self, prefix, url):
33 """proxy a request to a CDN"""
33 """proxy a request to a CDN"""
34 proxy_url = "/".join([self.settings['cdn'], url])
34 proxy_url = "/".join([self.settings['cdn'], url])
35 client = self.settings['client']
35 client = self.settings['client']
36 client.fetch(proxy_url, callback=self.finish_get)
36 client.fetch(proxy_url, callback=self.finish_get)
37
37
38 def finish_get(self, response):
38 def finish_get(self, response):
39 """finish the request"""
39 """finish the request"""
40 # copy potentially relevant headers
40 # copy potentially relevant headers
41 for header in ["Content-Type", "Cache-Control", "Date", "Last-Modified", "Expires"]:
41 for header in ["Content-Type", "Cache-Control", "Date", "Last-Modified", "Expires"]:
42 if header in response.headers:
42 if header in response.headers:
43 self.set_header(header, response.headers[header])
43 self.set_header(header, response.headers[header])
44 self.finish(response.body)
44 self.finish(response.body)
45
45
46 class ServePostProcessor(PostProcessorBase):
46 class ServePostProcessor(PostProcessorBase):
47 """Post processor designed to serve files
47 """Post processor designed to serve files
48
48
49 Proxies reveal.js requests to a CDN if no local reveal.js is present
49 Proxies reveal.js requests to a CDN if no local reveal.js is present
50 """
50 """
51
51
52
52
53 open_in_browser = Bool(True, config=True,
53 open_in_browser = Bool(True, config=True,
54 help="""Should the browser be opened automatically?"""
54 help="""Should the browser be opened automatically?"""
55 )
55 )
56 reveal_cdn = Unicode("https://cdn.jsdelivr.net/reveal.js/2.4.0", config=True,
56 reveal_cdn = Unicode("https://cdn.jsdelivr.net/reveal.js/2.4.0", config=True,
57 help="""URL for reveal.js CDN."""
57 help="""URL for reveal.js CDN."""
58 )
58 )
59 reveal_prefix = Unicode("reveal.js", config=True, help="URL prefix for reveal.js")
59 reveal_prefix = Unicode("reveal.js", config=True, help="URL prefix for reveal.js")
60 ip = Unicode("127.0.0.1", config=True, help="The IP address to listen on.")
60 ip = Unicode("127.0.0.1", config=True, help="The IP address to listen on.")
61 port = Int(8000, config=True, help="port for the server to listen on.")
61 port = Int(8000, config=True, help="port for the server to listen on.")
62
62
63 def postprocess(self, input):
63 def postprocess(self, input):
64 """Serve the build directory with a webserver."""
64 """Serve the build directory with a webserver."""
65 dirname, filename = os.path.split(input)
65 dirname, filename = os.path.split(input)
66 handlers = [
66 handlers = [
67 (r"/(.+)", web.StaticFileHandler, {'path' : dirname}),
67 (r"/(.+)", web.StaticFileHandler, {'path' : dirname}),
68 (r"/", web.RedirectHandler, {"url": "/%s" % filename})
68 (r"/", web.RedirectHandler, {"url": "/%s" % filename})
69 ]
69 ]
70
70
71 if ('://' in self.reveal_prefix or self.reveal_prefix.startswith("//")):
71 if ('://' in self.reveal_prefix or self.reveal_prefix.startswith("//")):
72 # reveal specifically from CDN, nothing to do
72 # reveal specifically from CDN, nothing to do
73 pass
73 pass
74 elif os.path.isdir(os.path.join(dirname, self.reveal_prefix)):
74 elif os.path.isdir(os.path.join(dirname, self.reveal_prefix)):
75 # reveal prefix exists
75 # reveal prefix exists
76 self.log.info("Serving local %s", self.reveal_prefix)
76 self.log.info("Serving local %s", self.reveal_prefix)
77 else:
77 else:
78 self.log.info("Redirecting %s requests to %s", self.reveal_prefix, self.reveal_cdn)
78 self.log.info("Redirecting %s requests to %s", self.reveal_prefix, self.reveal_cdn)
79 handlers.insert(0, (r"/(%s)/(.*)" % self.reveal_prefix, ProxyHandler))
79 handlers.insert(0, (r"/(%s)/(.*)" % self.reveal_prefix, ProxyHandler))
80
80
81 app = web.Application(handlers,
81 app = web.Application(handlers,
82 cdn=self.reveal_cdn,
82 cdn=self.reveal_cdn,
83 client=AsyncHTTPClient(),
83 client=AsyncHTTPClient(),
84 )
84 )
85 # hook up tornado logging to our logger
85 # hook up tornado logging to our logger
86 try:
86 try:
87 from tornado import log
87 from tornado import log
88 log.app_log = self.log
88 log.app_log = self.log
89 except ImportError:
89 except ImportError:
90 # old tornado (<= 3), ignore
90 # old tornado (<= 3), ignore
91 pass
91 pass
92
92
93 http_server = httpserver.HTTPServer(app)
93 http_server = httpserver.HTTPServer(app)
94 http_server.listen(self.port, address=self.ip)
94 http_server.listen(self.port, address=self.ip)
95 url = "http://%s:%i/%s" % (self.ip, self.port, filename)
95 url = "http://%s:%i/%s" % (self.ip, self.port, filename)
96 print(("Serving your slides at %s" % url))
96 print("Serving your slides at %s" % url)
97 print("Use Control-C to stop this server")
97 print("Use Control-C to stop this server")
98 if self.open_in_browser:
98 if self.open_in_browser:
99 webbrowser.open(url, new=2)
99 webbrowser.open(url, new=2)
100 try:
100 try:
101 ioloop.IOLoop.instance().start()
101 ioloop.IOLoop.instance().start()
102 except KeyboardInterrupt:
102 except KeyboardInterrupt:
103 print("\nInterrupted")
103 print("\nInterrupted")
104
104
105 def main(path):
105 def main(path):
106 """allow running this module to serve the slides"""
106 """allow running this module to serve the slides"""
107 server = ServePostProcessor()
107 server = ServePostProcessor()
108 server(path)
108 server(path)
109
109
110 if __name__ == '__main__':
110 if __name__ == '__main__':
111 import sys
111 import sys
112 main(sys.argv[1])
112 main(sys.argv[1])
@@ -1,43 +1,43 b''
1 """
1 """
2 Contains debug writer.
2 Contains debug writer.
3 """
3 """
4 from __future__ import print_function
4 from __future__ import print_function
5 #-----------------------------------------------------------------------------
5 #-----------------------------------------------------------------------------
6 #Copyright (c) 2013, the IPython Development Team.
6 #Copyright (c) 2013, the IPython Development Team.
7 #
7 #
8 #Distributed under the terms of the Modified BSD License.
8 #Distributed under the terms of the Modified BSD License.
9 #
9 #
10 #The full license is in the file COPYING.txt, distributed with this software.
10 #The full license is in the file COPYING.txt, distributed with this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 from .base import WriterBase
17 from .base import WriterBase
18 from pprint import pprint
18 from pprint import pprint
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Classes
21 # Classes
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 class DebugWriter(WriterBase):
24 class DebugWriter(WriterBase):
25 """Consumes output from nbconvert export...() methods and writes usefull
25 """Consumes output from nbconvert export...() methods and writes usefull
26 debugging information to the stdout. The information includes a list of
26 debugging information to the stdout. The information includes a list of
27 resources that were extracted from the notebook(s) during export."""
27 resources that were extracted from the notebook(s) during export."""
28
28
29
29
30 def write(self, output, resources, notebook_name='notebook', **kw):
30 def write(self, output, resources, notebook_name='notebook', **kw):
31 """
31 """
32 Consume and write Jinja output.
32 Consume and write Jinja output.
33
33
34 See base for more...
34 See base for more...
35 """
35 """
36
36
37 if isinstance(resources['outputs'], dict):
37 if isinstance(resources['outputs'], dict):
38 print(("outputs extracted from %s" % notebook_name))
38 print("outputs extracted from %s" % notebook_name)
39 print(('-' * 80))
39 print('-' * 80)
40 pprint(resources['outputs'], indent=2, width=70)
40 pprint(resources['outputs'], indent=2, width=70)
41 else:
41 else:
42 print(("no outputs extracted from %s" % notebook_name))
42 print("no outputs extracted from %s" % notebook_name)
43 print(('=' * 80))
43 print('=' * 80)
@@ -1,1855 +1,1855 b''
1 """A semi-synchronous Client for the ZMQ cluster
1 """A semi-synchronous Client for the ZMQ cluster
2
2
3 Authors:
3 Authors:
4
4
5 * MinRK
5 * MinRK
6 """
6 """
7 from __future__ import print_function
7 from __future__ import print_function
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2010-2011 The IPython Development Team
9 # Copyright (C) 2010-2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 import os
19 import os
20 import json
20 import json
21 import sys
21 import sys
22 from threading import Thread, Event
22 from threading import Thread, Event
23 import time
23 import time
24 import warnings
24 import warnings
25 from datetime import datetime
25 from datetime import datetime
26 from getpass import getpass
26 from getpass import getpass
27 from pprint import pprint
27 from pprint import pprint
28
28
29 pjoin = os.path.join
29 pjoin = os.path.join
30
30
31 import zmq
31 import zmq
32 # from zmq.eventloop import ioloop, zmqstream
32 # from zmq.eventloop import ioloop, zmqstream
33
33
34 from IPython.config.configurable import MultipleInstanceError
34 from IPython.config.configurable import MultipleInstanceError
35 from IPython.core.application import BaseIPythonApplication
35 from IPython.core.application import BaseIPythonApplication
36 from IPython.core.profiledir import ProfileDir, ProfileDirError
36 from IPython.core.profiledir import ProfileDir, ProfileDirError
37
37
38 from IPython.utils.capture import RichOutput
38 from IPython.utils.capture import RichOutput
39 from IPython.utils.coloransi import TermColors
39 from IPython.utils.coloransi import TermColors
40 from IPython.utils.jsonutil import rekey
40 from IPython.utils.jsonutil import rekey
41 from IPython.utils.localinterfaces import localhost, is_local_ip
41 from IPython.utils.localinterfaces import localhost, is_local_ip
42 from IPython.utils.path import get_ipython_dir
42 from IPython.utils.path import get_ipython_dir
43 from IPython.utils.py3compat import cast_bytes, string_types, xrange, iteritems
43 from IPython.utils.py3compat import cast_bytes, string_types, xrange, iteritems
44 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
44 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
45 Dict, List, Bool, Set, Any)
45 Dict, List, Bool, Set, Any)
46 from IPython.external.decorator import decorator
46 from IPython.external.decorator import decorator
47 from IPython.external.ssh import tunnel
47 from IPython.external.ssh import tunnel
48
48
49 from IPython.parallel import Reference
49 from IPython.parallel import Reference
50 from IPython.parallel import error
50 from IPython.parallel import error
51 from IPython.parallel import util
51 from IPython.parallel import util
52
52
53 from IPython.kernel.zmq.session import Session, Message
53 from IPython.kernel.zmq.session import Session, Message
54 from IPython.kernel.zmq import serialize
54 from IPython.kernel.zmq import serialize
55
55
56 from .asyncresult import AsyncResult, AsyncHubResult
56 from .asyncresult import AsyncResult, AsyncHubResult
57 from .view import DirectView, LoadBalancedView
57 from .view import DirectView, LoadBalancedView
58
58
59 #--------------------------------------------------------------------------
59 #--------------------------------------------------------------------------
60 # Decorators for Client methods
60 # Decorators for Client methods
61 #--------------------------------------------------------------------------
61 #--------------------------------------------------------------------------
62
62
63 @decorator
63 @decorator
64 def spin_first(f, self, *args, **kwargs):
64 def spin_first(f, self, *args, **kwargs):
65 """Call spin() to sync state prior to calling the method."""
65 """Call spin() to sync state prior to calling the method."""
66 self.spin()
66 self.spin()
67 return f(self, *args, **kwargs)
67 return f(self, *args, **kwargs)
68
68
69
69
70 #--------------------------------------------------------------------------
70 #--------------------------------------------------------------------------
71 # Classes
71 # Classes
72 #--------------------------------------------------------------------------
72 #--------------------------------------------------------------------------
73
73
74
74
75 class ExecuteReply(RichOutput):
75 class ExecuteReply(RichOutput):
76 """wrapper for finished Execute results"""
76 """wrapper for finished Execute results"""
77 def __init__(self, msg_id, content, metadata):
77 def __init__(self, msg_id, content, metadata):
78 self.msg_id = msg_id
78 self.msg_id = msg_id
79 self._content = content
79 self._content = content
80 self.execution_count = content['execution_count']
80 self.execution_count = content['execution_count']
81 self.metadata = metadata
81 self.metadata = metadata
82
82
83 # RichOutput overrides
83 # RichOutput overrides
84
84
85 @property
85 @property
86 def source(self):
86 def source(self):
87 pyout = self.metadata['pyout']
87 pyout = self.metadata['pyout']
88 if pyout:
88 if pyout:
89 return pyout.get('source', '')
89 return pyout.get('source', '')
90
90
91 @property
91 @property
92 def data(self):
92 def data(self):
93 pyout = self.metadata['pyout']
93 pyout = self.metadata['pyout']
94 if pyout:
94 if pyout:
95 return pyout.get('data', {})
95 return pyout.get('data', {})
96
96
97 @property
97 @property
98 def _metadata(self):
98 def _metadata(self):
99 pyout = self.metadata['pyout']
99 pyout = self.metadata['pyout']
100 if pyout:
100 if pyout:
101 return pyout.get('metadata', {})
101 return pyout.get('metadata', {})
102
102
103 def display(self):
103 def display(self):
104 from IPython.display import publish_display_data
104 from IPython.display import publish_display_data
105 publish_display_data(self.source, self.data, self.metadata)
105 publish_display_data(self.source, self.data, self.metadata)
106
106
107 def _repr_mime_(self, mime):
107 def _repr_mime_(self, mime):
108 if mime not in self.data:
108 if mime not in self.data:
109 return
109 return
110 data = self.data[mime]
110 data = self.data[mime]
111 if mime in self._metadata:
111 if mime in self._metadata:
112 return data, self._metadata[mime]
112 return data, self._metadata[mime]
113 else:
113 else:
114 return data
114 return data
115
115
116 def __getitem__(self, key):
116 def __getitem__(self, key):
117 return self.metadata[key]
117 return self.metadata[key]
118
118
119 def __getattr__(self, key):
119 def __getattr__(self, key):
120 if key not in self.metadata:
120 if key not in self.metadata:
121 raise AttributeError(key)
121 raise AttributeError(key)
122 return self.metadata[key]
122 return self.metadata[key]
123
123
124 def __repr__(self):
124 def __repr__(self):
125 pyout = self.metadata['pyout'] or {'data':{}}
125 pyout = self.metadata['pyout'] or {'data':{}}
126 text_out = pyout['data'].get('text/plain', '')
126 text_out = pyout['data'].get('text/plain', '')
127 if len(text_out) > 32:
127 if len(text_out) > 32:
128 text_out = text_out[:29] + '...'
128 text_out = text_out[:29] + '...'
129
129
130 return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
130 return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
131
131
132 def _repr_pretty_(self, p, cycle):
132 def _repr_pretty_(self, p, cycle):
133 pyout = self.metadata['pyout'] or {'data':{}}
133 pyout = self.metadata['pyout'] or {'data':{}}
134 text_out = pyout['data'].get('text/plain', '')
134 text_out = pyout['data'].get('text/plain', '')
135
135
136 if not text_out:
136 if not text_out:
137 return
137 return
138
138
139 try:
139 try:
140 ip = get_ipython()
140 ip = get_ipython()
141 except NameError:
141 except NameError:
142 colors = "NoColor"
142 colors = "NoColor"
143 else:
143 else:
144 colors = ip.colors
144 colors = ip.colors
145
145
146 if colors == "NoColor":
146 if colors == "NoColor":
147 out = normal = ""
147 out = normal = ""
148 else:
148 else:
149 out = TermColors.Red
149 out = TermColors.Red
150 normal = TermColors.Normal
150 normal = TermColors.Normal
151
151
152 if '\n' in text_out and not text_out.startswith('\n'):
152 if '\n' in text_out and not text_out.startswith('\n'):
153 # add newline for multiline reprs
153 # add newline for multiline reprs
154 text_out = '\n' + text_out
154 text_out = '\n' + text_out
155
155
156 p.text(
156 p.text(
157 out + u'Out[%i:%i]: ' % (
157 out + u'Out[%i:%i]: ' % (
158 self.metadata['engine_id'], self.execution_count
158 self.metadata['engine_id'], self.execution_count
159 ) + normal + text_out
159 ) + normal + text_out
160 )
160 )
161
161
162
162
163 class Metadata(dict):
163 class Metadata(dict):
164 """Subclass of dict for initializing metadata values.
164 """Subclass of dict for initializing metadata values.
165
165
166 Attribute access works on keys.
166 Attribute access works on keys.
167
167
168 These objects have a strict set of keys - errors will raise if you try
168 These objects have a strict set of keys - errors will raise if you try
169 to add new keys.
169 to add new keys.
170 """
170 """
171 def __init__(self, *args, **kwargs):
171 def __init__(self, *args, **kwargs):
172 dict.__init__(self)
172 dict.__init__(self)
173 md = {'msg_id' : None,
173 md = {'msg_id' : None,
174 'submitted' : None,
174 'submitted' : None,
175 'started' : None,
175 'started' : None,
176 'completed' : None,
176 'completed' : None,
177 'received' : None,
177 'received' : None,
178 'engine_uuid' : None,
178 'engine_uuid' : None,
179 'engine_id' : None,
179 'engine_id' : None,
180 'follow' : None,
180 'follow' : None,
181 'after' : None,
181 'after' : None,
182 'status' : None,
182 'status' : None,
183
183
184 'pyin' : None,
184 'pyin' : None,
185 'pyout' : None,
185 'pyout' : None,
186 'pyerr' : None,
186 'pyerr' : None,
187 'stdout' : '',
187 'stdout' : '',
188 'stderr' : '',
188 'stderr' : '',
189 'outputs' : [],
189 'outputs' : [],
190 'data': {},
190 'data': {},
191 'outputs_ready' : False,
191 'outputs_ready' : False,
192 }
192 }
193 self.update(md)
193 self.update(md)
194 self.update(dict(*args, **kwargs))
194 self.update(dict(*args, **kwargs))
195
195
196 def __getattr__(self, key):
196 def __getattr__(self, key):
197 """getattr aliased to getitem"""
197 """getattr aliased to getitem"""
198 if key in self:
198 if key in self:
199 return self[key]
199 return self[key]
200 else:
200 else:
201 raise AttributeError(key)
201 raise AttributeError(key)
202
202
203 def __setattr__(self, key, value):
203 def __setattr__(self, key, value):
204 """setattr aliased to setitem, with strict"""
204 """setattr aliased to setitem, with strict"""
205 if key in self:
205 if key in self:
206 self[key] = value
206 self[key] = value
207 else:
207 else:
208 raise AttributeError(key)
208 raise AttributeError(key)
209
209
210 def __setitem__(self, key, value):
210 def __setitem__(self, key, value):
211 """strict static key enforcement"""
211 """strict static key enforcement"""
212 if key in self:
212 if key in self:
213 dict.__setitem__(self, key, value)
213 dict.__setitem__(self, key, value)
214 else:
214 else:
215 raise KeyError(key)
215 raise KeyError(key)
216
216
217
217
218 class Client(HasTraits):
218 class Client(HasTraits):
219 """A semi-synchronous client to the IPython ZMQ cluster
219 """A semi-synchronous client to the IPython ZMQ cluster
220
220
221 Parameters
221 Parameters
222 ----------
222 ----------
223
223
224 url_file : str/unicode; path to ipcontroller-client.json
224 url_file : str/unicode; path to ipcontroller-client.json
225 This JSON file should contain all the information needed to connect to a cluster,
225 This JSON file should contain all the information needed to connect to a cluster,
226 and is likely the only argument needed.
226 and is likely the only argument needed.
227 Connection information for the Hub's registration. If a json connector
227 Connection information for the Hub's registration. If a json connector
228 file is given, then likely no further configuration is necessary.
228 file is given, then likely no further configuration is necessary.
229 [Default: use profile]
229 [Default: use profile]
230 profile : bytes
230 profile : bytes
231 The name of the Cluster profile to be used to find connector information.
231 The name of the Cluster profile to be used to find connector information.
232 If run from an IPython application, the default profile will be the same
232 If run from an IPython application, the default profile will be the same
233 as the running application, otherwise it will be 'default'.
233 as the running application, otherwise it will be 'default'.
234 cluster_id : str
234 cluster_id : str
235 String id to added to runtime files, to prevent name collisions when using
235 String id to added to runtime files, to prevent name collisions when using
236 multiple clusters with a single profile simultaneously.
236 multiple clusters with a single profile simultaneously.
237 When set, will look for files named like: 'ipcontroller-<cluster_id>-client.json'
237 When set, will look for files named like: 'ipcontroller-<cluster_id>-client.json'
238 Since this is text inserted into filenames, typical recommendations apply:
238 Since this is text inserted into filenames, typical recommendations apply:
239 Simple character strings are ideal, and spaces are not recommended (but
239 Simple character strings are ideal, and spaces are not recommended (but
240 should generally work)
240 should generally work)
241 context : zmq.Context
241 context : zmq.Context
242 Pass an existing zmq.Context instance, otherwise the client will create its own.
242 Pass an existing zmq.Context instance, otherwise the client will create its own.
243 debug : bool
243 debug : bool
244 flag for lots of message printing for debug purposes
244 flag for lots of message printing for debug purposes
245 timeout : int/float
245 timeout : int/float
246 time (in seconds) to wait for connection replies from the Hub
246 time (in seconds) to wait for connection replies from the Hub
247 [Default: 10]
247 [Default: 10]
248
248
249 #-------------- session related args ----------------
249 #-------------- session related args ----------------
250
250
251 config : Config object
251 config : Config object
252 If specified, this will be relayed to the Session for configuration
252 If specified, this will be relayed to the Session for configuration
253 username : str
253 username : str
254 set username for the session object
254 set username for the session object
255
255
256 #-------------- ssh related args ----------------
256 #-------------- ssh related args ----------------
257 # These are args for configuring the ssh tunnel to be used
257 # These are args for configuring the ssh tunnel to be used
258 # credentials are used to forward connections over ssh to the Controller
258 # credentials are used to forward connections over ssh to the Controller
259 # Note that the ip given in `addr` needs to be relative to sshserver
259 # Note that the ip given in `addr` needs to be relative to sshserver
260 # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
260 # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
261 # and set sshserver as the same machine the Controller is on. However,
261 # and set sshserver as the same machine the Controller is on. However,
262 # the only requirement is that sshserver is able to see the Controller
262 # the only requirement is that sshserver is able to see the Controller
263 # (i.e. is within the same trusted network).
263 # (i.e. is within the same trusted network).
264
264
265 sshserver : str
265 sshserver : str
266 A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port'
266 A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port'
267 If keyfile or password is specified, and this is not, it will default to
267 If keyfile or password is specified, and this is not, it will default to
268 the ip given in addr.
268 the ip given in addr.
269 sshkey : str; path to ssh private key file
269 sshkey : str; path to ssh private key file
270 This specifies a key to be used in ssh login, default None.
270 This specifies a key to be used in ssh login, default None.
271 Regular default ssh keys will be used without specifying this argument.
271 Regular default ssh keys will be used without specifying this argument.
272 password : str
272 password : str
273 Your ssh password to sshserver. Note that if this is left None,
273 Your ssh password to sshserver. Note that if this is left None,
274 you will be prompted for it if passwordless key based login is unavailable.
274 you will be prompted for it if passwordless key based login is unavailable.
275 paramiko : bool
275 paramiko : bool
276 flag for whether to use paramiko instead of shell ssh for tunneling.
276 flag for whether to use paramiko instead of shell ssh for tunneling.
277 [default: True on win32, False else]
277 [default: True on win32, False else]
278
278
279
279
280 Attributes
280 Attributes
281 ----------
281 ----------
282
282
283 ids : list of int engine IDs
283 ids : list of int engine IDs
284 requesting the ids attribute always synchronizes
284 requesting the ids attribute always synchronizes
285 the registration state. To request ids without synchronization,
285 the registration state. To request ids without synchronization,
286 use semi-private _ids attributes.
286 use semi-private _ids attributes.
287
287
288 history : list of msg_ids
288 history : list of msg_ids
289 a list of msg_ids, keeping track of all the execution
289 a list of msg_ids, keeping track of all the execution
290 messages you have submitted in order.
290 messages you have submitted in order.
291
291
292 outstanding : set of msg_ids
292 outstanding : set of msg_ids
293 a set of msg_ids that have been submitted, but whose
293 a set of msg_ids that have been submitted, but whose
294 results have not yet been received.
294 results have not yet been received.
295
295
296 results : dict
296 results : dict
297 a dict of all our results, keyed by msg_id
297 a dict of all our results, keyed by msg_id
298
298
299 block : bool
299 block : bool
300 determines default behavior when block not specified
300 determines default behavior when block not specified
301 in execution methods
301 in execution methods
302
302
303 Methods
303 Methods
304 -------
304 -------
305
305
306 spin
306 spin
307 flushes incoming results and registration state changes
307 flushes incoming results and registration state changes
308 control methods spin, and requesting `ids` also ensures up to date
308 control methods spin, and requesting `ids` also ensures up to date
309
309
310 wait
310 wait
311 wait on one or more msg_ids
311 wait on one or more msg_ids
312
312
313 execution methods
313 execution methods
314 apply
314 apply
315 legacy: execute, run
315 legacy: execute, run
316
316
317 data movement
317 data movement
318 push, pull, scatter, gather
318 push, pull, scatter, gather
319
319
320 query methods
320 query methods
321 queue_status, get_result, purge, result_status
321 queue_status, get_result, purge, result_status
322
322
323 control methods
323 control methods
324 abort, shutdown
324 abort, shutdown
325
325
326 """
326 """
327
327
328
328
329 block = Bool(False)
329 block = Bool(False)
330 outstanding = Set()
330 outstanding = Set()
331 results = Instance('collections.defaultdict', (dict,))
331 results = Instance('collections.defaultdict', (dict,))
332 metadata = Instance('collections.defaultdict', (Metadata,))
332 metadata = Instance('collections.defaultdict', (Metadata,))
333 history = List()
333 history = List()
334 debug = Bool(False)
334 debug = Bool(False)
335 _spin_thread = Any()
335 _spin_thread = Any()
336 _stop_spinning = Any()
336 _stop_spinning = Any()
337
337
338 profile=Unicode()
338 profile=Unicode()
339 def _profile_default(self):
339 def _profile_default(self):
340 if BaseIPythonApplication.initialized():
340 if BaseIPythonApplication.initialized():
341 # an IPython app *might* be running, try to get its profile
341 # an IPython app *might* be running, try to get its profile
342 try:
342 try:
343 return BaseIPythonApplication.instance().profile
343 return BaseIPythonApplication.instance().profile
344 except (AttributeError, MultipleInstanceError):
344 except (AttributeError, MultipleInstanceError):
345 # could be a *different* subclass of config.Application,
345 # could be a *different* subclass of config.Application,
346 # which would raise one of these two errors.
346 # which would raise one of these two errors.
347 return u'default'
347 return u'default'
348 else:
348 else:
349 return u'default'
349 return u'default'
350
350
351
351
352 _outstanding_dict = Instance('collections.defaultdict', (set,))
352 _outstanding_dict = Instance('collections.defaultdict', (set,))
353 _ids = List()
353 _ids = List()
354 _connected=Bool(False)
354 _connected=Bool(False)
355 _ssh=Bool(False)
355 _ssh=Bool(False)
356 _context = Instance('zmq.Context')
356 _context = Instance('zmq.Context')
357 _config = Dict()
357 _config = Dict()
358 _engines=Instance(util.ReverseDict, (), {})
358 _engines=Instance(util.ReverseDict, (), {})
359 # _hub_socket=Instance('zmq.Socket')
359 # _hub_socket=Instance('zmq.Socket')
360 _query_socket=Instance('zmq.Socket')
360 _query_socket=Instance('zmq.Socket')
361 _control_socket=Instance('zmq.Socket')
361 _control_socket=Instance('zmq.Socket')
362 _iopub_socket=Instance('zmq.Socket')
362 _iopub_socket=Instance('zmq.Socket')
363 _notification_socket=Instance('zmq.Socket')
363 _notification_socket=Instance('zmq.Socket')
364 _mux_socket=Instance('zmq.Socket')
364 _mux_socket=Instance('zmq.Socket')
365 _task_socket=Instance('zmq.Socket')
365 _task_socket=Instance('zmq.Socket')
366 _task_scheme=Unicode()
366 _task_scheme=Unicode()
367 _closed = False
367 _closed = False
368 _ignored_control_replies=Integer(0)
368 _ignored_control_replies=Integer(0)
369 _ignored_hub_replies=Integer(0)
369 _ignored_hub_replies=Integer(0)
370
370
371 def __new__(self, *args, **kw):
371 def __new__(self, *args, **kw):
372 # don't raise on positional args
372 # don't raise on positional args
373 return HasTraits.__new__(self, **kw)
373 return HasTraits.__new__(self, **kw)
374
374
375 def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None,
375 def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None,
376 context=None, debug=False,
376 context=None, debug=False,
377 sshserver=None, sshkey=None, password=None, paramiko=None,
377 sshserver=None, sshkey=None, password=None, paramiko=None,
378 timeout=10, cluster_id=None, **extra_args
378 timeout=10, cluster_id=None, **extra_args
379 ):
379 ):
380 if profile:
380 if profile:
381 super(Client, self).__init__(debug=debug, profile=profile)
381 super(Client, self).__init__(debug=debug, profile=profile)
382 else:
382 else:
383 super(Client, self).__init__(debug=debug)
383 super(Client, self).__init__(debug=debug)
384 if context is None:
384 if context is None:
385 context = zmq.Context.instance()
385 context = zmq.Context.instance()
386 self._context = context
386 self._context = context
387 self._stop_spinning = Event()
387 self._stop_spinning = Event()
388
388
389 if 'url_or_file' in extra_args:
389 if 'url_or_file' in extra_args:
390 url_file = extra_args['url_or_file']
390 url_file = extra_args['url_or_file']
391 warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning)
391 warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning)
392
392
393 if url_file and util.is_url(url_file):
393 if url_file and util.is_url(url_file):
394 raise ValueError("single urls cannot be specified, url-files must be used.")
394 raise ValueError("single urls cannot be specified, url-files must be used.")
395
395
396 self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
396 self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
397
397
398 if self._cd is not None:
398 if self._cd is not None:
399 if url_file is None:
399 if url_file is None:
400 if not cluster_id:
400 if not cluster_id:
401 client_json = 'ipcontroller-client.json'
401 client_json = 'ipcontroller-client.json'
402 else:
402 else:
403 client_json = 'ipcontroller-%s-client.json' % cluster_id
403 client_json = 'ipcontroller-%s-client.json' % cluster_id
404 url_file = pjoin(self._cd.security_dir, client_json)
404 url_file = pjoin(self._cd.security_dir, client_json)
405 if url_file is None:
405 if url_file is None:
406 raise ValueError(
406 raise ValueError(
407 "I can't find enough information to connect to a hub!"
407 "I can't find enough information to connect to a hub!"
408 " Please specify at least one of url_file or profile."
408 " Please specify at least one of url_file or profile."
409 )
409 )
410
410
411 with open(url_file) as f:
411 with open(url_file) as f:
412 cfg = json.load(f)
412 cfg = json.load(f)
413
413
414 self._task_scheme = cfg['task_scheme']
414 self._task_scheme = cfg['task_scheme']
415
415
416 # sync defaults from args, json:
416 # sync defaults from args, json:
417 if sshserver:
417 if sshserver:
418 cfg['ssh'] = sshserver
418 cfg['ssh'] = sshserver
419
419
420 location = cfg.setdefault('location', None)
420 location = cfg.setdefault('location', None)
421
421
422 proto,addr = cfg['interface'].split('://')
422 proto,addr = cfg['interface'].split('://')
423 addr = util.disambiguate_ip_address(addr, location)
423 addr = util.disambiguate_ip_address(addr, location)
424 cfg['interface'] = "%s://%s" % (proto, addr)
424 cfg['interface'] = "%s://%s" % (proto, addr)
425
425
426 # turn interface,port into full urls:
426 # turn interface,port into full urls:
427 for key in ('control', 'task', 'mux', 'iopub', 'notification', 'registration'):
427 for key in ('control', 'task', 'mux', 'iopub', 'notification', 'registration'):
428 cfg[key] = cfg['interface'] + ':%i' % cfg[key]
428 cfg[key] = cfg['interface'] + ':%i' % cfg[key]
429
429
430 url = cfg['registration']
430 url = cfg['registration']
431
431
432 if location is not None and addr == localhost():
432 if location is not None and addr == localhost():
433 # location specified, and connection is expected to be local
433 # location specified, and connection is expected to be local
434 if not is_local_ip(location) and not sshserver:
434 if not is_local_ip(location) and not sshserver:
435 # load ssh from JSON *only* if the controller is not on
435 # load ssh from JSON *only* if the controller is not on
436 # this machine
436 # this machine
437 sshserver=cfg['ssh']
437 sshserver=cfg['ssh']
438 if not is_local_ip(location) and not sshserver:
438 if not is_local_ip(location) and not sshserver:
439 # warn if no ssh specified, but SSH is probably needed
439 # warn if no ssh specified, but SSH is probably needed
440 # This is only a warning, because the most likely cause
440 # This is only a warning, because the most likely cause
441 # is a local Controller on a laptop whose IP is dynamic
441 # is a local Controller on a laptop whose IP is dynamic
442 warnings.warn("""
442 warnings.warn("""
443 Controller appears to be listening on localhost, but not on this machine.
443 Controller appears to be listening on localhost, but not on this machine.
444 If this is true, you should specify Client(...,sshserver='you@%s')
444 If this is true, you should specify Client(...,sshserver='you@%s')
445 or instruct your controller to listen on an external IP."""%location,
445 or instruct your controller to listen on an external IP."""%location,
446 RuntimeWarning)
446 RuntimeWarning)
447 elif not sshserver:
447 elif not sshserver:
448 # otherwise sync with cfg
448 # otherwise sync with cfg
449 sshserver = cfg['ssh']
449 sshserver = cfg['ssh']
450
450
451 self._config = cfg
451 self._config = cfg
452
452
453 self._ssh = bool(sshserver or sshkey or password)
453 self._ssh = bool(sshserver or sshkey or password)
454 if self._ssh and sshserver is None:
454 if self._ssh and sshserver is None:
455 # default to ssh via localhost
455 # default to ssh via localhost
456 sshserver = addr
456 sshserver = addr
457 if self._ssh and password is None:
457 if self._ssh and password is None:
458 if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
458 if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
459 password=False
459 password=False
460 else:
460 else:
461 password = getpass("SSH Password for %s: "%sshserver)
461 password = getpass("SSH Password for %s: "%sshserver)
462 ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
462 ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
463
463
464 # configure and construct the session
464 # configure and construct the session
465 try:
465 try:
466 extra_args['packer'] = cfg['pack']
466 extra_args['packer'] = cfg['pack']
467 extra_args['unpacker'] = cfg['unpack']
467 extra_args['unpacker'] = cfg['unpack']
468 extra_args['key'] = cast_bytes(cfg['key'])
468 extra_args['key'] = cast_bytes(cfg['key'])
469 extra_args['signature_scheme'] = cfg['signature_scheme']
469 extra_args['signature_scheme'] = cfg['signature_scheme']
470 except KeyError as exc:
470 except KeyError as exc:
471 msg = '\n'.join([
471 msg = '\n'.join([
472 "Connection file is invalid (missing '{}'), possibly from an old version of IPython.",
472 "Connection file is invalid (missing '{}'), possibly from an old version of IPython.",
473 "If you are reusing connection files, remove them and start ipcontroller again."
473 "If you are reusing connection files, remove them and start ipcontroller again."
474 ])
474 ])
475 raise ValueError(msg.format(exc.message))
475 raise ValueError(msg.format(exc.message))
476
476
477 self.session = Session(**extra_args)
477 self.session = Session(**extra_args)
478
478
479 self._query_socket = self._context.socket(zmq.DEALER)
479 self._query_socket = self._context.socket(zmq.DEALER)
480
480
481 if self._ssh:
481 if self._ssh:
482 tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs)
482 tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs)
483 else:
483 else:
484 self._query_socket.connect(cfg['registration'])
484 self._query_socket.connect(cfg['registration'])
485
485
486 self.session.debug = self.debug
486 self.session.debug = self.debug
487
487
488 self._notification_handlers = {'registration_notification' : self._register_engine,
488 self._notification_handlers = {'registration_notification' : self._register_engine,
489 'unregistration_notification' : self._unregister_engine,
489 'unregistration_notification' : self._unregister_engine,
490 'shutdown_notification' : lambda msg: self.close(),
490 'shutdown_notification' : lambda msg: self.close(),
491 }
491 }
492 self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
492 self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
493 'apply_reply' : self._handle_apply_reply}
493 'apply_reply' : self._handle_apply_reply}
494
494
495 try:
495 try:
496 self._connect(sshserver, ssh_kwargs, timeout)
496 self._connect(sshserver, ssh_kwargs, timeout)
497 except:
497 except:
498 self.close(linger=0)
498 self.close(linger=0)
499 raise
499 raise
500
500
501 # last step: setup magics, if we are in IPython:
501 # last step: setup magics, if we are in IPython:
502
502
503 try:
503 try:
504 ip = get_ipython()
504 ip = get_ipython()
505 except NameError:
505 except NameError:
506 return
506 return
507 else:
507 else:
508 if 'px' not in ip.magics_manager.magics:
508 if 'px' not in ip.magics_manager.magics:
509 # in IPython but we are the first Client.
509 # in IPython but we are the first Client.
510 # activate a default view for parallel magics.
510 # activate a default view for parallel magics.
511 self.activate()
511 self.activate()
512
512
513 def __del__(self):
513 def __del__(self):
514 """cleanup sockets, but _not_ context."""
514 """cleanup sockets, but _not_ context."""
515 self.close()
515 self.close()
516
516
517 def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
517 def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
518 if ipython_dir is None:
518 if ipython_dir is None:
519 ipython_dir = get_ipython_dir()
519 ipython_dir = get_ipython_dir()
520 if profile_dir is not None:
520 if profile_dir is not None:
521 try:
521 try:
522 self._cd = ProfileDir.find_profile_dir(profile_dir)
522 self._cd = ProfileDir.find_profile_dir(profile_dir)
523 return
523 return
524 except ProfileDirError:
524 except ProfileDirError:
525 pass
525 pass
526 elif profile is not None:
526 elif profile is not None:
527 try:
527 try:
528 self._cd = ProfileDir.find_profile_dir_by_name(
528 self._cd = ProfileDir.find_profile_dir_by_name(
529 ipython_dir, profile)
529 ipython_dir, profile)
530 return
530 return
531 except ProfileDirError:
531 except ProfileDirError:
532 pass
532 pass
533 self._cd = None
533 self._cd = None
534
534
535 def _update_engines(self, engines):
535 def _update_engines(self, engines):
536 """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
536 """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
537 for k,v in iteritems(engines):
537 for k,v in iteritems(engines):
538 eid = int(k)
538 eid = int(k)
539 if eid not in self._engines:
539 if eid not in self._engines:
540 self._ids.append(eid)
540 self._ids.append(eid)
541 self._engines[eid] = v
541 self._engines[eid] = v
542 self._ids = sorted(self._ids)
542 self._ids = sorted(self._ids)
543 if sorted(self._engines.keys()) != list(range(len(self._engines))) and \
543 if sorted(self._engines.keys()) != list(range(len(self._engines))) and \
544 self._task_scheme == 'pure' and self._task_socket:
544 self._task_scheme == 'pure' and self._task_socket:
545 self._stop_scheduling_tasks()
545 self._stop_scheduling_tasks()
546
546
547 def _stop_scheduling_tasks(self):
547 def _stop_scheduling_tasks(self):
548 """Stop scheduling tasks because an engine has been unregistered
548 """Stop scheduling tasks because an engine has been unregistered
549 from a pure ZMQ scheduler.
549 from a pure ZMQ scheduler.
550 """
550 """
551 self._task_socket.close()
551 self._task_socket.close()
552 self._task_socket = None
552 self._task_socket = None
553 msg = "An engine has been unregistered, and we are using pure " +\
553 msg = "An engine has been unregistered, and we are using pure " +\
554 "ZMQ task scheduling. Task farming will be disabled."
554 "ZMQ task scheduling. Task farming will be disabled."
555 if self.outstanding:
555 if self.outstanding:
556 msg += " If you were running tasks when this happened, " +\
556 msg += " If you were running tasks when this happened, " +\
557 "some `outstanding` msg_ids may never resolve."
557 "some `outstanding` msg_ids may never resolve."
558 warnings.warn(msg, RuntimeWarning)
558 warnings.warn(msg, RuntimeWarning)
559
559
560 def _build_targets(self, targets):
560 def _build_targets(self, targets):
561 """Turn valid target IDs or 'all' into two lists:
561 """Turn valid target IDs or 'all' into two lists:
562 (int_ids, uuids).
562 (int_ids, uuids).
563 """
563 """
564 if not self._ids:
564 if not self._ids:
565 # flush notification socket if no engines yet, just in case
565 # flush notification socket if no engines yet, just in case
566 if not self.ids:
566 if not self.ids:
567 raise error.NoEnginesRegistered("Can't build targets without any engines")
567 raise error.NoEnginesRegistered("Can't build targets without any engines")
568
568
569 if targets is None:
569 if targets is None:
570 targets = self._ids
570 targets = self._ids
571 elif isinstance(targets, string_types):
571 elif isinstance(targets, string_types):
572 if targets.lower() == 'all':
572 if targets.lower() == 'all':
573 targets = self._ids
573 targets = self._ids
574 else:
574 else:
575 raise TypeError("%r not valid str target, must be 'all'"%(targets))
575 raise TypeError("%r not valid str target, must be 'all'"%(targets))
576 elif isinstance(targets, int):
576 elif isinstance(targets, int):
577 if targets < 0:
577 if targets < 0:
578 targets = self.ids[targets]
578 targets = self.ids[targets]
579 if targets not in self._ids:
579 if targets not in self._ids:
580 raise IndexError("No such engine: %i"%targets)
580 raise IndexError("No such engine: %i"%targets)
581 targets = [targets]
581 targets = [targets]
582
582
583 if isinstance(targets, slice):
583 if isinstance(targets, slice):
584 indices = list(range(len(self._ids))[targets])
584 indices = list(range(len(self._ids))[targets])
585 ids = self.ids
585 ids = self.ids
586 targets = [ ids[i] for i in indices ]
586 targets = [ ids[i] for i in indices ]
587
587
588 if not isinstance(targets, (tuple, list, xrange)):
588 if not isinstance(targets, (tuple, list, xrange)):
589 raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
589 raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
590
590
591 return [cast_bytes(self._engines[t]) for t in targets], list(targets)
591 return [cast_bytes(self._engines[t]) for t in targets], list(targets)
592
592
593 def _connect(self, sshserver, ssh_kwargs, timeout):
593 def _connect(self, sshserver, ssh_kwargs, timeout):
594 """setup all our socket connections to the cluster. This is called from
594 """setup all our socket connections to the cluster. This is called from
595 __init__."""
595 __init__."""
596
596
597 # Maybe allow reconnecting?
597 # Maybe allow reconnecting?
598 if self._connected:
598 if self._connected:
599 return
599 return
600 self._connected=True
600 self._connected=True
601
601
602 def connect_socket(s, url):
602 def connect_socket(s, url):
603 if self._ssh:
603 if self._ssh:
604 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
604 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
605 else:
605 else:
606 return s.connect(url)
606 return s.connect(url)
607
607
608 self.session.send(self._query_socket, 'connection_request')
608 self.session.send(self._query_socket, 'connection_request')
609 # use Poller because zmq.select has wrong units in pyzmq 2.1.7
609 # use Poller because zmq.select has wrong units in pyzmq 2.1.7
610 poller = zmq.Poller()
610 poller = zmq.Poller()
611 poller.register(self._query_socket, zmq.POLLIN)
611 poller.register(self._query_socket, zmq.POLLIN)
612 # poll expects milliseconds, timeout is seconds
612 # poll expects milliseconds, timeout is seconds
613 evts = poller.poll(timeout*1000)
613 evts = poller.poll(timeout*1000)
614 if not evts:
614 if not evts:
615 raise error.TimeoutError("Hub connection request timed out")
615 raise error.TimeoutError("Hub connection request timed out")
616 idents,msg = self.session.recv(self._query_socket,mode=0)
616 idents,msg = self.session.recv(self._query_socket,mode=0)
617 if self.debug:
617 if self.debug:
618 pprint(msg)
618 pprint(msg)
619 content = msg['content']
619 content = msg['content']
620 # self._config['registration'] = dict(content)
620 # self._config['registration'] = dict(content)
621 cfg = self._config
621 cfg = self._config
622 if content['status'] == 'ok':
622 if content['status'] == 'ok':
623 self._mux_socket = self._context.socket(zmq.DEALER)
623 self._mux_socket = self._context.socket(zmq.DEALER)
624 connect_socket(self._mux_socket, cfg['mux'])
624 connect_socket(self._mux_socket, cfg['mux'])
625
625
626 self._task_socket = self._context.socket(zmq.DEALER)
626 self._task_socket = self._context.socket(zmq.DEALER)
627 connect_socket(self._task_socket, cfg['task'])
627 connect_socket(self._task_socket, cfg['task'])
628
628
629 self._notification_socket = self._context.socket(zmq.SUB)
629 self._notification_socket = self._context.socket(zmq.SUB)
630 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
630 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
631 connect_socket(self._notification_socket, cfg['notification'])
631 connect_socket(self._notification_socket, cfg['notification'])
632
632
633 self._control_socket = self._context.socket(zmq.DEALER)
633 self._control_socket = self._context.socket(zmq.DEALER)
634 connect_socket(self._control_socket, cfg['control'])
634 connect_socket(self._control_socket, cfg['control'])
635
635
636 self._iopub_socket = self._context.socket(zmq.SUB)
636 self._iopub_socket = self._context.socket(zmq.SUB)
637 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
637 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
638 connect_socket(self._iopub_socket, cfg['iopub'])
638 connect_socket(self._iopub_socket, cfg['iopub'])
639
639
640 self._update_engines(dict(content['engines']))
640 self._update_engines(dict(content['engines']))
641 else:
641 else:
642 self._connected = False
642 self._connected = False
643 raise Exception("Failed to connect!")
643 raise Exception("Failed to connect!")
644
644
645 #--------------------------------------------------------------------------
645 #--------------------------------------------------------------------------
646 # handlers and callbacks for incoming messages
646 # handlers and callbacks for incoming messages
647 #--------------------------------------------------------------------------
647 #--------------------------------------------------------------------------
648
648
649 def _unwrap_exception(self, content):
649 def _unwrap_exception(self, content):
650 """unwrap exception, and remap engine_id to int."""
650 """unwrap exception, and remap engine_id to int."""
651 e = error.unwrap_exception(content)
651 e = error.unwrap_exception(content)
652 # print e.traceback
652 # print e.traceback
653 if e.engine_info:
653 if e.engine_info:
654 e_uuid = e.engine_info['engine_uuid']
654 e_uuid = e.engine_info['engine_uuid']
655 eid = self._engines[e_uuid]
655 eid = self._engines[e_uuid]
656 e.engine_info['engine_id'] = eid
656 e.engine_info['engine_id'] = eid
657 return e
657 return e
658
658
659 def _extract_metadata(self, msg):
659 def _extract_metadata(self, msg):
660 header = msg['header']
660 header = msg['header']
661 parent = msg['parent_header']
661 parent = msg['parent_header']
662 msg_meta = msg['metadata']
662 msg_meta = msg['metadata']
663 content = msg['content']
663 content = msg['content']
664 md = {'msg_id' : parent['msg_id'],
664 md = {'msg_id' : parent['msg_id'],
665 'received' : datetime.now(),
665 'received' : datetime.now(),
666 'engine_uuid' : msg_meta.get('engine', None),
666 'engine_uuid' : msg_meta.get('engine', None),
667 'follow' : msg_meta.get('follow', []),
667 'follow' : msg_meta.get('follow', []),
668 'after' : msg_meta.get('after', []),
668 'after' : msg_meta.get('after', []),
669 'status' : content['status'],
669 'status' : content['status'],
670 }
670 }
671
671
672 if md['engine_uuid'] is not None:
672 if md['engine_uuid'] is not None:
673 md['engine_id'] = self._engines.get(md['engine_uuid'], None)
673 md['engine_id'] = self._engines.get(md['engine_uuid'], None)
674
674
675 if 'date' in parent:
675 if 'date' in parent:
676 md['submitted'] = parent['date']
676 md['submitted'] = parent['date']
677 if 'started' in msg_meta:
677 if 'started' in msg_meta:
678 md['started'] = msg_meta['started']
678 md['started'] = msg_meta['started']
679 if 'date' in header:
679 if 'date' in header:
680 md['completed'] = header['date']
680 md['completed'] = header['date']
681 return md
681 return md
682
682
683 def _register_engine(self, msg):
683 def _register_engine(self, msg):
684 """Register a new engine, and update our connection info."""
684 """Register a new engine, and update our connection info."""
685 content = msg['content']
685 content = msg['content']
686 eid = content['id']
686 eid = content['id']
687 d = {eid : content['uuid']}
687 d = {eid : content['uuid']}
688 self._update_engines(d)
688 self._update_engines(d)
689
689
690 def _unregister_engine(self, msg):
690 def _unregister_engine(self, msg):
691 """Unregister an engine that has died."""
691 """Unregister an engine that has died."""
692 content = msg['content']
692 content = msg['content']
693 eid = int(content['id'])
693 eid = int(content['id'])
694 if eid in self._ids:
694 if eid in self._ids:
695 self._ids.remove(eid)
695 self._ids.remove(eid)
696 uuid = self._engines.pop(eid)
696 uuid = self._engines.pop(eid)
697
697
698 self._handle_stranded_msgs(eid, uuid)
698 self._handle_stranded_msgs(eid, uuid)
699
699
700 if self._task_socket and self._task_scheme == 'pure':
700 if self._task_socket and self._task_scheme == 'pure':
701 self._stop_scheduling_tasks()
701 self._stop_scheduling_tasks()
702
702
703 def _handle_stranded_msgs(self, eid, uuid):
703 def _handle_stranded_msgs(self, eid, uuid):
704 """Handle messages known to be on an engine when the engine unregisters.
704 """Handle messages known to be on an engine when the engine unregisters.
705
705
706 It is possible that this will fire prematurely - that is, an engine will
706 It is possible that this will fire prematurely - that is, an engine will
707 go down after completing a result, and the client will be notified
707 go down after completing a result, and the client will be notified
708 of the unregistration and later receive the successful result.
708 of the unregistration and later receive the successful result.
709 """
709 """
710
710
711 outstanding = self._outstanding_dict[uuid]
711 outstanding = self._outstanding_dict[uuid]
712
712
713 for msg_id in list(outstanding):
713 for msg_id in list(outstanding):
714 if msg_id in self.results:
714 if msg_id in self.results:
715 # we already
715 # we already
716 continue
716 continue
717 try:
717 try:
718 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
718 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
719 except:
719 except:
720 content = error.wrap_exception()
720 content = error.wrap_exception()
721 # build a fake message:
721 # build a fake message:
722 msg = self.session.msg('apply_reply', content=content)
722 msg = self.session.msg('apply_reply', content=content)
723 msg['parent_header']['msg_id'] = msg_id
723 msg['parent_header']['msg_id'] = msg_id
724 msg['metadata']['engine'] = uuid
724 msg['metadata']['engine'] = uuid
725 self._handle_apply_reply(msg)
725 self._handle_apply_reply(msg)
726
726
727 def _handle_execute_reply(self, msg):
727 def _handle_execute_reply(self, msg):
728 """Save the reply to an execute_request into our results.
728 """Save the reply to an execute_request into our results.
729
729
730 execute messages are never actually used. apply is used instead.
730 execute messages are never actually used. apply is used instead.
731 """
731 """
732
732
733 parent = msg['parent_header']
733 parent = msg['parent_header']
734 msg_id = parent['msg_id']
734 msg_id = parent['msg_id']
735 if msg_id not in self.outstanding:
735 if msg_id not in self.outstanding:
736 if msg_id in self.history:
736 if msg_id in self.history:
737 print(("got stale result: %s"%msg_id))
737 print("got stale result: %s"%msg_id)
738 else:
738 else:
739 print(("got unknown result: %s"%msg_id))
739 print("got unknown result: %s"%msg_id)
740 else:
740 else:
741 self.outstanding.remove(msg_id)
741 self.outstanding.remove(msg_id)
742
742
743 content = msg['content']
743 content = msg['content']
744 header = msg['header']
744 header = msg['header']
745
745
746 # construct metadata:
746 # construct metadata:
747 md = self.metadata[msg_id]
747 md = self.metadata[msg_id]
748 md.update(self._extract_metadata(msg))
748 md.update(self._extract_metadata(msg))
749 # is this redundant?
749 # is this redundant?
750 self.metadata[msg_id] = md
750 self.metadata[msg_id] = md
751
751
752 e_outstanding = self._outstanding_dict[md['engine_uuid']]
752 e_outstanding = self._outstanding_dict[md['engine_uuid']]
753 if msg_id in e_outstanding:
753 if msg_id in e_outstanding:
754 e_outstanding.remove(msg_id)
754 e_outstanding.remove(msg_id)
755
755
756 # construct result:
756 # construct result:
757 if content['status'] == 'ok':
757 if content['status'] == 'ok':
758 self.results[msg_id] = ExecuteReply(msg_id, content, md)
758 self.results[msg_id] = ExecuteReply(msg_id, content, md)
759 elif content['status'] == 'aborted':
759 elif content['status'] == 'aborted':
760 self.results[msg_id] = error.TaskAborted(msg_id)
760 self.results[msg_id] = error.TaskAborted(msg_id)
761 elif content['status'] == 'resubmitted':
761 elif content['status'] == 'resubmitted':
762 # TODO: handle resubmission
762 # TODO: handle resubmission
763 pass
763 pass
764 else:
764 else:
765 self.results[msg_id] = self._unwrap_exception(content)
765 self.results[msg_id] = self._unwrap_exception(content)
766
766
767 def _handle_apply_reply(self, msg):
767 def _handle_apply_reply(self, msg):
768 """Save the reply to an apply_request into our results."""
768 """Save the reply to an apply_request into our results."""
769 parent = msg['parent_header']
769 parent = msg['parent_header']
770 msg_id = parent['msg_id']
770 msg_id = parent['msg_id']
771 if msg_id not in self.outstanding:
771 if msg_id not in self.outstanding:
772 if msg_id in self.history:
772 if msg_id in self.history:
773 print(("got stale result: %s"%msg_id))
773 print("got stale result: %s"%msg_id)
774 print(self.results[msg_id])
774 print(self.results[msg_id])
775 print(msg)
775 print(msg)
776 else:
776 else:
777 print(("got unknown result: %s"%msg_id))
777 print("got unknown result: %s"%msg_id)
778 else:
778 else:
779 self.outstanding.remove(msg_id)
779 self.outstanding.remove(msg_id)
780 content = msg['content']
780 content = msg['content']
781 header = msg['header']
781 header = msg['header']
782
782
783 # construct metadata:
783 # construct metadata:
784 md = self.metadata[msg_id]
784 md = self.metadata[msg_id]
785 md.update(self._extract_metadata(msg))
785 md.update(self._extract_metadata(msg))
786 # is this redundant?
786 # is this redundant?
787 self.metadata[msg_id] = md
787 self.metadata[msg_id] = md
788
788
789 e_outstanding = self._outstanding_dict[md['engine_uuid']]
789 e_outstanding = self._outstanding_dict[md['engine_uuid']]
790 if msg_id in e_outstanding:
790 if msg_id in e_outstanding:
791 e_outstanding.remove(msg_id)
791 e_outstanding.remove(msg_id)
792
792
793 # construct result:
793 # construct result:
794 if content['status'] == 'ok':
794 if content['status'] == 'ok':
795 self.results[msg_id] = serialize.unserialize_object(msg['buffers'])[0]
795 self.results[msg_id] = serialize.unserialize_object(msg['buffers'])[0]
796 elif content['status'] == 'aborted':
796 elif content['status'] == 'aborted':
797 self.results[msg_id] = error.TaskAborted(msg_id)
797 self.results[msg_id] = error.TaskAborted(msg_id)
798 elif content['status'] == 'resubmitted':
798 elif content['status'] == 'resubmitted':
799 # TODO: handle resubmission
799 # TODO: handle resubmission
800 pass
800 pass
801 else:
801 else:
802 self.results[msg_id] = self._unwrap_exception(content)
802 self.results[msg_id] = self._unwrap_exception(content)
803
803
804 def _flush_notifications(self):
804 def _flush_notifications(self):
805 """Flush notifications of engine registrations waiting
805 """Flush notifications of engine registrations waiting
806 in ZMQ queue."""
806 in ZMQ queue."""
807 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
807 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
808 while msg is not None:
808 while msg is not None:
809 if self.debug:
809 if self.debug:
810 pprint(msg)
810 pprint(msg)
811 msg_type = msg['header']['msg_type']
811 msg_type = msg['header']['msg_type']
812 handler = self._notification_handlers.get(msg_type, None)
812 handler = self._notification_handlers.get(msg_type, None)
813 if handler is None:
813 if handler is None:
814 raise Exception("Unhandled message type: %s" % msg_type)
814 raise Exception("Unhandled message type: %s" % msg_type)
815 else:
815 else:
816 handler(msg)
816 handler(msg)
817 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
817 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
818
818
819 def _flush_results(self, sock):
819 def _flush_results(self, sock):
820 """Flush task or queue results waiting in ZMQ queue."""
820 """Flush task or queue results waiting in ZMQ queue."""
821 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
821 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
822 while msg is not None:
822 while msg is not None:
823 if self.debug:
823 if self.debug:
824 pprint(msg)
824 pprint(msg)
825 msg_type = msg['header']['msg_type']
825 msg_type = msg['header']['msg_type']
826 handler = self._queue_handlers.get(msg_type, None)
826 handler = self._queue_handlers.get(msg_type, None)
827 if handler is None:
827 if handler is None:
828 raise Exception("Unhandled message type: %s" % msg_type)
828 raise Exception("Unhandled message type: %s" % msg_type)
829 else:
829 else:
830 handler(msg)
830 handler(msg)
831 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
831 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
832
832
833 def _flush_control(self, sock):
833 def _flush_control(self, sock):
834 """Flush replies from the control channel waiting
834 """Flush replies from the control channel waiting
835 in the ZMQ queue.
835 in the ZMQ queue.
836
836
837 Currently: ignore them."""
837 Currently: ignore them."""
838 if self._ignored_control_replies <= 0:
838 if self._ignored_control_replies <= 0:
839 return
839 return
840 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
840 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
841 while msg is not None:
841 while msg is not None:
842 self._ignored_control_replies -= 1
842 self._ignored_control_replies -= 1
843 if self.debug:
843 if self.debug:
844 pprint(msg)
844 pprint(msg)
845 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
845 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
846
846
847 def _flush_ignored_control(self):
847 def _flush_ignored_control(self):
848 """flush ignored control replies"""
848 """flush ignored control replies"""
849 while self._ignored_control_replies > 0:
849 while self._ignored_control_replies > 0:
850 self.session.recv(self._control_socket)
850 self.session.recv(self._control_socket)
851 self._ignored_control_replies -= 1
851 self._ignored_control_replies -= 1
852
852
853 def _flush_ignored_hub_replies(self):
853 def _flush_ignored_hub_replies(self):
854 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
854 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
855 while msg is not None:
855 while msg is not None:
856 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
856 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
857
857
858 def _flush_iopub(self, sock):
858 def _flush_iopub(self, sock):
859 """Flush replies from the iopub channel waiting
859 """Flush replies from the iopub channel waiting
860 in the ZMQ queue.
860 in the ZMQ queue.
861 """
861 """
862 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
862 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
863 while msg is not None:
863 while msg is not None:
864 if self.debug:
864 if self.debug:
865 pprint(msg)
865 pprint(msg)
866 parent = msg['parent_header']
866 parent = msg['parent_header']
867 # ignore IOPub messages with no parent.
867 # ignore IOPub messages with no parent.
868 # Caused by print statements or warnings from before the first execution.
868 # Caused by print statements or warnings from before the first execution.
869 if not parent:
869 if not parent:
870 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
870 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
871 continue
871 continue
872 msg_id = parent['msg_id']
872 msg_id = parent['msg_id']
873 content = msg['content']
873 content = msg['content']
874 header = msg['header']
874 header = msg['header']
875 msg_type = msg['header']['msg_type']
875 msg_type = msg['header']['msg_type']
876
876
877 # init metadata:
877 # init metadata:
878 md = self.metadata[msg_id]
878 md = self.metadata[msg_id]
879
879
880 if msg_type == 'stream':
880 if msg_type == 'stream':
881 name = content['name']
881 name = content['name']
882 s = md[name] or ''
882 s = md[name] or ''
883 md[name] = s + content['data']
883 md[name] = s + content['data']
884 elif msg_type == 'pyerr':
884 elif msg_type == 'pyerr':
885 md.update({'pyerr' : self._unwrap_exception(content)})
885 md.update({'pyerr' : self._unwrap_exception(content)})
886 elif msg_type == 'pyin':
886 elif msg_type == 'pyin':
887 md.update({'pyin' : content['code']})
887 md.update({'pyin' : content['code']})
888 elif msg_type == 'display_data':
888 elif msg_type == 'display_data':
889 md['outputs'].append(content)
889 md['outputs'].append(content)
890 elif msg_type == 'pyout':
890 elif msg_type == 'pyout':
891 md['pyout'] = content
891 md['pyout'] = content
892 elif msg_type == 'data_message':
892 elif msg_type == 'data_message':
893 data, remainder = serialize.unserialize_object(msg['buffers'])
893 data, remainder = serialize.unserialize_object(msg['buffers'])
894 md['data'].update(data)
894 md['data'].update(data)
895 elif msg_type == 'status':
895 elif msg_type == 'status':
896 # idle message comes after all outputs
896 # idle message comes after all outputs
897 if content['execution_state'] == 'idle':
897 if content['execution_state'] == 'idle':
898 md['outputs_ready'] = True
898 md['outputs_ready'] = True
899 else:
899 else:
900 # unhandled msg_type (status, etc.)
900 # unhandled msg_type (status, etc.)
901 pass
901 pass
902
902
903 # reduntant?
903 # reduntant?
904 self.metadata[msg_id] = md
904 self.metadata[msg_id] = md
905
905
906 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
906 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
907
907
908 #--------------------------------------------------------------------------
908 #--------------------------------------------------------------------------
909 # len, getitem
909 # len, getitem
910 #--------------------------------------------------------------------------
910 #--------------------------------------------------------------------------
911
911
912 def __len__(self):
912 def __len__(self):
913 """len(client) returns # of engines."""
913 """len(client) returns # of engines."""
914 return len(self.ids)
914 return len(self.ids)
915
915
916 def __getitem__(self, key):
916 def __getitem__(self, key):
917 """index access returns DirectView multiplexer objects
917 """index access returns DirectView multiplexer objects
918
918
919 Must be int, slice, or list/tuple/xrange of ints"""
919 Must be int, slice, or list/tuple/xrange of ints"""
920 if not isinstance(key, (int, slice, tuple, list, xrange)):
920 if not isinstance(key, (int, slice, tuple, list, xrange)):
921 raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
921 raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
922 else:
922 else:
923 return self.direct_view(key)
923 return self.direct_view(key)
924
924
925 #--------------------------------------------------------------------------
925 #--------------------------------------------------------------------------
926 # Begin public methods
926 # Begin public methods
927 #--------------------------------------------------------------------------
927 #--------------------------------------------------------------------------
928
928
929 @property
929 @property
930 def ids(self):
930 def ids(self):
931 """Always up-to-date ids property."""
931 """Always up-to-date ids property."""
932 self._flush_notifications()
932 self._flush_notifications()
933 # always copy:
933 # always copy:
934 return list(self._ids)
934 return list(self._ids)
935
935
936 def activate(self, targets='all', suffix=''):
936 def activate(self, targets='all', suffix=''):
937 """Create a DirectView and register it with IPython magics
937 """Create a DirectView and register it with IPython magics
938
938
939 Defines the magics `%px, %autopx, %pxresult, %%px`
939 Defines the magics `%px, %autopx, %pxresult, %%px`
940
940
941 Parameters
941 Parameters
942 ----------
942 ----------
943
943
944 targets: int, list of ints, or 'all'
944 targets: int, list of ints, or 'all'
945 The engines on which the view's magics will run
945 The engines on which the view's magics will run
946 suffix: str [default: '']
946 suffix: str [default: '']
947 The suffix, if any, for the magics. This allows you to have
947 The suffix, if any, for the magics. This allows you to have
948 multiple views associated with parallel magics at the same time.
948 multiple views associated with parallel magics at the same time.
949
949
950 e.g. ``rc.activate(targets=0, suffix='0')`` will give you
950 e.g. ``rc.activate(targets=0, suffix='0')`` will give you
951 the magics ``%px0``, ``%pxresult0``, etc. for running magics just
951 the magics ``%px0``, ``%pxresult0``, etc. for running magics just
952 on engine 0.
952 on engine 0.
953 """
953 """
954 view = self.direct_view(targets)
954 view = self.direct_view(targets)
955 view.block = True
955 view.block = True
956 view.activate(suffix)
956 view.activate(suffix)
957 return view
957 return view
958
958
959 def close(self, linger=None):
959 def close(self, linger=None):
960 """Close my zmq Sockets
960 """Close my zmq Sockets
961
961
962 If `linger`, set the zmq LINGER socket option,
962 If `linger`, set the zmq LINGER socket option,
963 which allows discarding of messages.
963 which allows discarding of messages.
964 """
964 """
965 if self._closed:
965 if self._closed:
966 return
966 return
967 self.stop_spin_thread()
967 self.stop_spin_thread()
968 snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
968 snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
969 for name in snames:
969 for name in snames:
970 socket = getattr(self, name)
970 socket = getattr(self, name)
971 if socket is not None and not socket.closed:
971 if socket is not None and not socket.closed:
972 if linger is not None:
972 if linger is not None:
973 socket.close(linger=linger)
973 socket.close(linger=linger)
974 else:
974 else:
975 socket.close()
975 socket.close()
976 self._closed = True
976 self._closed = True
977
977
978 def _spin_every(self, interval=1):
978 def _spin_every(self, interval=1):
979 """target func for use in spin_thread"""
979 """target func for use in spin_thread"""
980 while True:
980 while True:
981 if self._stop_spinning.is_set():
981 if self._stop_spinning.is_set():
982 return
982 return
983 time.sleep(interval)
983 time.sleep(interval)
984 self.spin()
984 self.spin()
985
985
986 def spin_thread(self, interval=1):
986 def spin_thread(self, interval=1):
987 """call Client.spin() in a background thread on some regular interval
987 """call Client.spin() in a background thread on some regular interval
988
988
989 This helps ensure that messages don't pile up too much in the zmq queue
989 This helps ensure that messages don't pile up too much in the zmq queue
990 while you are working on other things, or just leaving an idle terminal.
990 while you are working on other things, or just leaving an idle terminal.
991
991
992 It also helps limit potential padding of the `received` timestamp
992 It also helps limit potential padding of the `received` timestamp
993 on AsyncResult objects, used for timings.
993 on AsyncResult objects, used for timings.
994
994
995 Parameters
995 Parameters
996 ----------
996 ----------
997
997
998 interval : float, optional
998 interval : float, optional
999 The interval on which to spin the client in the background thread
999 The interval on which to spin the client in the background thread
1000 (simply passed to time.sleep).
1000 (simply passed to time.sleep).
1001
1001
1002 Notes
1002 Notes
1003 -----
1003 -----
1004
1004
1005 For precision timing, you may want to use this method to put a bound
1005 For precision timing, you may want to use this method to put a bound
1006 on the jitter (in seconds) in `received` timestamps used
1006 on the jitter (in seconds) in `received` timestamps used
1007 in AsyncResult.wall_time.
1007 in AsyncResult.wall_time.
1008
1008
1009 """
1009 """
1010 if self._spin_thread is not None:
1010 if self._spin_thread is not None:
1011 self.stop_spin_thread()
1011 self.stop_spin_thread()
1012 self._stop_spinning.clear()
1012 self._stop_spinning.clear()
1013 self._spin_thread = Thread(target=self._spin_every, args=(interval,))
1013 self._spin_thread = Thread(target=self._spin_every, args=(interval,))
1014 self._spin_thread.daemon = True
1014 self._spin_thread.daemon = True
1015 self._spin_thread.start()
1015 self._spin_thread.start()
1016
1016
1017 def stop_spin_thread(self):
1017 def stop_spin_thread(self):
1018 """stop background spin_thread, if any"""
1018 """stop background spin_thread, if any"""
1019 if self._spin_thread is not None:
1019 if self._spin_thread is not None:
1020 self._stop_spinning.set()
1020 self._stop_spinning.set()
1021 self._spin_thread.join()
1021 self._spin_thread.join()
1022 self._spin_thread = None
1022 self._spin_thread = None
1023
1023
1024 def spin(self):
1024 def spin(self):
1025 """Flush any registration notifications and execution results
1025 """Flush any registration notifications and execution results
1026 waiting in the ZMQ queue.
1026 waiting in the ZMQ queue.
1027 """
1027 """
1028 if self._notification_socket:
1028 if self._notification_socket:
1029 self._flush_notifications()
1029 self._flush_notifications()
1030 if self._iopub_socket:
1030 if self._iopub_socket:
1031 self._flush_iopub(self._iopub_socket)
1031 self._flush_iopub(self._iopub_socket)
1032 if self._mux_socket:
1032 if self._mux_socket:
1033 self._flush_results(self._mux_socket)
1033 self._flush_results(self._mux_socket)
1034 if self._task_socket:
1034 if self._task_socket:
1035 self._flush_results(self._task_socket)
1035 self._flush_results(self._task_socket)
1036 if self._control_socket:
1036 if self._control_socket:
1037 self._flush_control(self._control_socket)
1037 self._flush_control(self._control_socket)
1038 if self._query_socket:
1038 if self._query_socket:
1039 self._flush_ignored_hub_replies()
1039 self._flush_ignored_hub_replies()
1040
1040
1041 def wait(self, jobs=None, timeout=-1):
1041 def wait(self, jobs=None, timeout=-1):
1042 """waits on one or more `jobs`, for up to `timeout` seconds.
1042 """waits on one or more `jobs`, for up to `timeout` seconds.
1043
1043
1044 Parameters
1044 Parameters
1045 ----------
1045 ----------
1046
1046
1047 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
1047 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
1048 ints are indices to self.history
1048 ints are indices to self.history
1049 strs are msg_ids
1049 strs are msg_ids
1050 default: wait on all outstanding messages
1050 default: wait on all outstanding messages
1051 timeout : float
1051 timeout : float
1052 a time in seconds, after which to give up.
1052 a time in seconds, after which to give up.
1053 default is -1, which means no timeout
1053 default is -1, which means no timeout
1054
1054
1055 Returns
1055 Returns
1056 -------
1056 -------
1057
1057
1058 True : when all msg_ids are done
1058 True : when all msg_ids are done
1059 False : timeout reached, some msg_ids still outstanding
1059 False : timeout reached, some msg_ids still outstanding
1060 """
1060 """
1061 tic = time.time()
1061 tic = time.time()
1062 if jobs is None:
1062 if jobs is None:
1063 theids = self.outstanding
1063 theids = self.outstanding
1064 else:
1064 else:
1065 if isinstance(jobs, string_types + (int, AsyncResult)):
1065 if isinstance(jobs, string_types + (int, AsyncResult)):
1066 jobs = [jobs]
1066 jobs = [jobs]
1067 theids = set()
1067 theids = set()
1068 for job in jobs:
1068 for job in jobs:
1069 if isinstance(job, int):
1069 if isinstance(job, int):
1070 # index access
1070 # index access
1071 job = self.history[job]
1071 job = self.history[job]
1072 elif isinstance(job, AsyncResult):
1072 elif isinstance(job, AsyncResult):
1073 theids.update(job.msg_ids)
1073 theids.update(job.msg_ids)
1074 continue
1074 continue
1075 theids.add(job)
1075 theids.add(job)
1076 if not theids.intersection(self.outstanding):
1076 if not theids.intersection(self.outstanding):
1077 return True
1077 return True
1078 self.spin()
1078 self.spin()
1079 while theids.intersection(self.outstanding):
1079 while theids.intersection(self.outstanding):
1080 if timeout >= 0 and ( time.time()-tic ) > timeout:
1080 if timeout >= 0 and ( time.time()-tic ) > timeout:
1081 break
1081 break
1082 time.sleep(1e-3)
1082 time.sleep(1e-3)
1083 self.spin()
1083 self.spin()
1084 return len(theids.intersection(self.outstanding)) == 0
1084 return len(theids.intersection(self.outstanding)) == 0
1085
1085
1086 #--------------------------------------------------------------------------
1086 #--------------------------------------------------------------------------
1087 # Control methods
1087 # Control methods
1088 #--------------------------------------------------------------------------
1088 #--------------------------------------------------------------------------
1089
1089
1090 @spin_first
1090 @spin_first
1091 def clear(self, targets=None, block=None):
1091 def clear(self, targets=None, block=None):
1092 """Clear the namespace in target(s)."""
1092 """Clear the namespace in target(s)."""
1093 block = self.block if block is None else block
1093 block = self.block if block is None else block
1094 targets = self._build_targets(targets)[0]
1094 targets = self._build_targets(targets)[0]
1095 for t in targets:
1095 for t in targets:
1096 self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
1096 self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
1097 error = False
1097 error = False
1098 if block:
1098 if block:
1099 self._flush_ignored_control()
1099 self._flush_ignored_control()
1100 for i in range(len(targets)):
1100 for i in range(len(targets)):
1101 idents,msg = self.session.recv(self._control_socket,0)
1101 idents,msg = self.session.recv(self._control_socket,0)
1102 if self.debug:
1102 if self.debug:
1103 pprint(msg)
1103 pprint(msg)
1104 if msg['content']['status'] != 'ok':
1104 if msg['content']['status'] != 'ok':
1105 error = self._unwrap_exception(msg['content'])
1105 error = self._unwrap_exception(msg['content'])
1106 else:
1106 else:
1107 self._ignored_control_replies += len(targets)
1107 self._ignored_control_replies += len(targets)
1108 if error:
1108 if error:
1109 raise error
1109 raise error
1110
1110
1111
1111
1112 @spin_first
1112 @spin_first
1113 def abort(self, jobs=None, targets=None, block=None):
1113 def abort(self, jobs=None, targets=None, block=None):
1114 """Abort specific jobs from the execution queues of target(s).
1114 """Abort specific jobs from the execution queues of target(s).
1115
1115
1116 This is a mechanism to prevent jobs that have already been submitted
1116 This is a mechanism to prevent jobs that have already been submitted
1117 from executing.
1117 from executing.
1118
1118
1119 Parameters
1119 Parameters
1120 ----------
1120 ----------
1121
1121
1122 jobs : msg_id, list of msg_ids, or AsyncResult
1122 jobs : msg_id, list of msg_ids, or AsyncResult
1123 The jobs to be aborted
1123 The jobs to be aborted
1124
1124
1125 If unspecified/None: abort all outstanding jobs.
1125 If unspecified/None: abort all outstanding jobs.
1126
1126
1127 """
1127 """
1128 block = self.block if block is None else block
1128 block = self.block if block is None else block
1129 jobs = jobs if jobs is not None else list(self.outstanding)
1129 jobs = jobs if jobs is not None else list(self.outstanding)
1130 targets = self._build_targets(targets)[0]
1130 targets = self._build_targets(targets)[0]
1131
1131
1132 msg_ids = []
1132 msg_ids = []
1133 if isinstance(jobs, string_types + (AsyncResult,)):
1133 if isinstance(jobs, string_types + (AsyncResult,)):
1134 jobs = [jobs]
1134 jobs = [jobs]
1135 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1135 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1136 if bad_ids:
1136 if bad_ids:
1137 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1137 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1138 for j in jobs:
1138 for j in jobs:
1139 if isinstance(j, AsyncResult):
1139 if isinstance(j, AsyncResult):
1140 msg_ids.extend(j.msg_ids)
1140 msg_ids.extend(j.msg_ids)
1141 else:
1141 else:
1142 msg_ids.append(j)
1142 msg_ids.append(j)
1143 content = dict(msg_ids=msg_ids)
1143 content = dict(msg_ids=msg_ids)
1144 for t in targets:
1144 for t in targets:
1145 self.session.send(self._control_socket, 'abort_request',
1145 self.session.send(self._control_socket, 'abort_request',
1146 content=content, ident=t)
1146 content=content, ident=t)
1147 error = False
1147 error = False
1148 if block:
1148 if block:
1149 self._flush_ignored_control()
1149 self._flush_ignored_control()
1150 for i in range(len(targets)):
1150 for i in range(len(targets)):
1151 idents,msg = self.session.recv(self._control_socket,0)
1151 idents,msg = self.session.recv(self._control_socket,0)
1152 if self.debug:
1152 if self.debug:
1153 pprint(msg)
1153 pprint(msg)
1154 if msg['content']['status'] != 'ok':
1154 if msg['content']['status'] != 'ok':
1155 error = self._unwrap_exception(msg['content'])
1155 error = self._unwrap_exception(msg['content'])
1156 else:
1156 else:
1157 self._ignored_control_replies += len(targets)
1157 self._ignored_control_replies += len(targets)
1158 if error:
1158 if error:
1159 raise error
1159 raise error
1160
1160
1161 @spin_first
1161 @spin_first
1162 def shutdown(self, targets='all', restart=False, hub=False, block=None):
1162 def shutdown(self, targets='all', restart=False, hub=False, block=None):
1163 """Terminates one or more engine processes, optionally including the hub.
1163 """Terminates one or more engine processes, optionally including the hub.
1164
1164
1165 Parameters
1165 Parameters
1166 ----------
1166 ----------
1167
1167
1168 targets: list of ints or 'all' [default: all]
1168 targets: list of ints or 'all' [default: all]
1169 Which engines to shutdown.
1169 Which engines to shutdown.
1170 hub: bool [default: False]
1170 hub: bool [default: False]
1171 Whether to include the Hub. hub=True implies targets='all'.
1171 Whether to include the Hub. hub=True implies targets='all'.
1172 block: bool [default: self.block]
1172 block: bool [default: self.block]
1173 Whether to wait for clean shutdown replies or not.
1173 Whether to wait for clean shutdown replies or not.
1174 restart: bool [default: False]
1174 restart: bool [default: False]
1175 NOT IMPLEMENTED
1175 NOT IMPLEMENTED
1176 whether to restart engines after shutting them down.
1176 whether to restart engines after shutting them down.
1177 """
1177 """
1178 from IPython.parallel.error import NoEnginesRegistered
1178 from IPython.parallel.error import NoEnginesRegistered
1179 if restart:
1179 if restart:
1180 raise NotImplementedError("Engine restart is not yet implemented")
1180 raise NotImplementedError("Engine restart is not yet implemented")
1181
1181
1182 block = self.block if block is None else block
1182 block = self.block if block is None else block
1183 if hub:
1183 if hub:
1184 targets = 'all'
1184 targets = 'all'
1185 try:
1185 try:
1186 targets = self._build_targets(targets)[0]
1186 targets = self._build_targets(targets)[0]
1187 except NoEnginesRegistered:
1187 except NoEnginesRegistered:
1188 targets = []
1188 targets = []
1189 for t in targets:
1189 for t in targets:
1190 self.session.send(self._control_socket, 'shutdown_request',
1190 self.session.send(self._control_socket, 'shutdown_request',
1191 content={'restart':restart},ident=t)
1191 content={'restart':restart},ident=t)
1192 error = False
1192 error = False
1193 if block or hub:
1193 if block or hub:
1194 self._flush_ignored_control()
1194 self._flush_ignored_control()
1195 for i in range(len(targets)):
1195 for i in range(len(targets)):
1196 idents,msg = self.session.recv(self._control_socket, 0)
1196 idents,msg = self.session.recv(self._control_socket, 0)
1197 if self.debug:
1197 if self.debug:
1198 pprint(msg)
1198 pprint(msg)
1199 if msg['content']['status'] != 'ok':
1199 if msg['content']['status'] != 'ok':
1200 error = self._unwrap_exception(msg['content'])
1200 error = self._unwrap_exception(msg['content'])
1201 else:
1201 else:
1202 self._ignored_control_replies += len(targets)
1202 self._ignored_control_replies += len(targets)
1203
1203
1204 if hub:
1204 if hub:
1205 time.sleep(0.25)
1205 time.sleep(0.25)
1206 self.session.send(self._query_socket, 'shutdown_request')
1206 self.session.send(self._query_socket, 'shutdown_request')
1207 idents,msg = self.session.recv(self._query_socket, 0)
1207 idents,msg = self.session.recv(self._query_socket, 0)
1208 if self.debug:
1208 if self.debug:
1209 pprint(msg)
1209 pprint(msg)
1210 if msg['content']['status'] != 'ok':
1210 if msg['content']['status'] != 'ok':
1211 error = self._unwrap_exception(msg['content'])
1211 error = self._unwrap_exception(msg['content'])
1212
1212
1213 if error:
1213 if error:
1214 raise error
1214 raise error
1215
1215
1216 #--------------------------------------------------------------------------
1216 #--------------------------------------------------------------------------
1217 # Execution related methods
1217 # Execution related methods
1218 #--------------------------------------------------------------------------
1218 #--------------------------------------------------------------------------
1219
1219
1220 def _maybe_raise(self, result):
1220 def _maybe_raise(self, result):
1221 """wrapper for maybe raising an exception if apply failed."""
1221 """wrapper for maybe raising an exception if apply failed."""
1222 if isinstance(result, error.RemoteError):
1222 if isinstance(result, error.RemoteError):
1223 raise result
1223 raise result
1224
1224
1225 return result
1225 return result
1226
1226
1227 def send_apply_request(self, socket, f, args=None, kwargs=None, metadata=None, track=False,
1227 def send_apply_request(self, socket, f, args=None, kwargs=None, metadata=None, track=False,
1228 ident=None):
1228 ident=None):
1229 """construct and send an apply message via a socket.
1229 """construct and send an apply message via a socket.
1230
1230
1231 This is the principal method with which all engine execution is performed by views.
1231 This is the principal method with which all engine execution is performed by views.
1232 """
1232 """
1233
1233
1234 if self._closed:
1234 if self._closed:
1235 raise RuntimeError("Client cannot be used after its sockets have been closed")
1235 raise RuntimeError("Client cannot be used after its sockets have been closed")
1236
1236
1237 # defaults:
1237 # defaults:
1238 args = args if args is not None else []
1238 args = args if args is not None else []
1239 kwargs = kwargs if kwargs is not None else {}
1239 kwargs = kwargs if kwargs is not None else {}
1240 metadata = metadata if metadata is not None else {}
1240 metadata = metadata if metadata is not None else {}
1241
1241
1242 # validate arguments
1242 # validate arguments
1243 if not callable(f) and not isinstance(f, Reference):
1243 if not callable(f) and not isinstance(f, Reference):
1244 raise TypeError("f must be callable, not %s"%type(f))
1244 raise TypeError("f must be callable, not %s"%type(f))
1245 if not isinstance(args, (tuple, list)):
1245 if not isinstance(args, (tuple, list)):
1246 raise TypeError("args must be tuple or list, not %s"%type(args))
1246 raise TypeError("args must be tuple or list, not %s"%type(args))
1247 if not isinstance(kwargs, dict):
1247 if not isinstance(kwargs, dict):
1248 raise TypeError("kwargs must be dict, not %s"%type(kwargs))
1248 raise TypeError("kwargs must be dict, not %s"%type(kwargs))
1249 if not isinstance(metadata, dict):
1249 if not isinstance(metadata, dict):
1250 raise TypeError("metadata must be dict, not %s"%type(metadata))
1250 raise TypeError("metadata must be dict, not %s"%type(metadata))
1251
1251
1252 bufs = serialize.pack_apply_message(f, args, kwargs,
1252 bufs = serialize.pack_apply_message(f, args, kwargs,
1253 buffer_threshold=self.session.buffer_threshold,
1253 buffer_threshold=self.session.buffer_threshold,
1254 item_threshold=self.session.item_threshold,
1254 item_threshold=self.session.item_threshold,
1255 )
1255 )
1256
1256
1257 msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
1257 msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
1258 metadata=metadata, track=track)
1258 metadata=metadata, track=track)
1259
1259
1260 msg_id = msg['header']['msg_id']
1260 msg_id = msg['header']['msg_id']
1261 self.outstanding.add(msg_id)
1261 self.outstanding.add(msg_id)
1262 if ident:
1262 if ident:
1263 # possibly routed to a specific engine
1263 # possibly routed to a specific engine
1264 if isinstance(ident, list):
1264 if isinstance(ident, list):
1265 ident = ident[-1]
1265 ident = ident[-1]
1266 if ident in self._engines.values():
1266 if ident in self._engines.values():
1267 # save for later, in case of engine death
1267 # save for later, in case of engine death
1268 self._outstanding_dict[ident].add(msg_id)
1268 self._outstanding_dict[ident].add(msg_id)
1269 self.history.append(msg_id)
1269 self.history.append(msg_id)
1270 self.metadata[msg_id]['submitted'] = datetime.now()
1270 self.metadata[msg_id]['submitted'] = datetime.now()
1271
1271
1272 return msg
1272 return msg
1273
1273
1274 def send_execute_request(self, socket, code, silent=True, metadata=None, ident=None):
1274 def send_execute_request(self, socket, code, silent=True, metadata=None, ident=None):
1275 """construct and send an execute request via a socket.
1275 """construct and send an execute request via a socket.
1276
1276
1277 """
1277 """
1278
1278
1279 if self._closed:
1279 if self._closed:
1280 raise RuntimeError("Client cannot be used after its sockets have been closed")
1280 raise RuntimeError("Client cannot be used after its sockets have been closed")
1281
1281
1282 # defaults:
1282 # defaults:
1283 metadata = metadata if metadata is not None else {}
1283 metadata = metadata if metadata is not None else {}
1284
1284
1285 # validate arguments
1285 # validate arguments
1286 if not isinstance(code, string_types):
1286 if not isinstance(code, string_types):
1287 raise TypeError("code must be text, not %s" % type(code))
1287 raise TypeError("code must be text, not %s" % type(code))
1288 if not isinstance(metadata, dict):
1288 if not isinstance(metadata, dict):
1289 raise TypeError("metadata must be dict, not %s" % type(metadata))
1289 raise TypeError("metadata must be dict, not %s" % type(metadata))
1290
1290
1291 content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})
1291 content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})
1292
1292
1293
1293
1294 msg = self.session.send(socket, "execute_request", content=content, ident=ident,
1294 msg = self.session.send(socket, "execute_request", content=content, ident=ident,
1295 metadata=metadata)
1295 metadata=metadata)
1296
1296
1297 msg_id = msg['header']['msg_id']
1297 msg_id = msg['header']['msg_id']
1298 self.outstanding.add(msg_id)
1298 self.outstanding.add(msg_id)
1299 if ident:
1299 if ident:
1300 # possibly routed to a specific engine
1300 # possibly routed to a specific engine
1301 if isinstance(ident, list):
1301 if isinstance(ident, list):
1302 ident = ident[-1]
1302 ident = ident[-1]
1303 if ident in self._engines.values():
1303 if ident in self._engines.values():
1304 # save for later, in case of engine death
1304 # save for later, in case of engine death
1305 self._outstanding_dict[ident].add(msg_id)
1305 self._outstanding_dict[ident].add(msg_id)
1306 self.history.append(msg_id)
1306 self.history.append(msg_id)
1307 self.metadata[msg_id]['submitted'] = datetime.now()
1307 self.metadata[msg_id]['submitted'] = datetime.now()
1308
1308
1309 return msg
1309 return msg
1310
1310
1311 #--------------------------------------------------------------------------
1311 #--------------------------------------------------------------------------
1312 # construct a View object
1312 # construct a View object
1313 #--------------------------------------------------------------------------
1313 #--------------------------------------------------------------------------
1314
1314
1315 def load_balanced_view(self, targets=None):
1315 def load_balanced_view(self, targets=None):
1316 """construct a DirectView object.
1316 """construct a DirectView object.
1317
1317
1318 If no arguments are specified, create a LoadBalancedView
1318 If no arguments are specified, create a LoadBalancedView
1319 using all engines.
1319 using all engines.
1320
1320
1321 Parameters
1321 Parameters
1322 ----------
1322 ----------
1323
1323
1324 targets: list,slice,int,etc. [default: use all engines]
1324 targets: list,slice,int,etc. [default: use all engines]
1325 The subset of engines across which to load-balance
1325 The subset of engines across which to load-balance
1326 """
1326 """
1327 if targets == 'all':
1327 if targets == 'all':
1328 targets = None
1328 targets = None
1329 if targets is not None:
1329 if targets is not None:
1330 targets = self._build_targets(targets)[1]
1330 targets = self._build_targets(targets)[1]
1331 return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
1331 return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
1332
1332
1333 def direct_view(self, targets='all'):
1333 def direct_view(self, targets='all'):
1334 """construct a DirectView object.
1334 """construct a DirectView object.
1335
1335
1336 If no targets are specified, create a DirectView using all engines.
1336 If no targets are specified, create a DirectView using all engines.
1337
1337
1338 rc.direct_view('all') is distinguished from rc[:] in that 'all' will
1338 rc.direct_view('all') is distinguished from rc[:] in that 'all' will
1339 evaluate the target engines at each execution, whereas rc[:] will connect to
1339 evaluate the target engines at each execution, whereas rc[:] will connect to
1340 all *current* engines, and that list will not change.
1340 all *current* engines, and that list will not change.
1341
1341
1342 That is, 'all' will always use all engines, whereas rc[:] will not use
1342 That is, 'all' will always use all engines, whereas rc[:] will not use
1343 engines added after the DirectView is constructed.
1343 engines added after the DirectView is constructed.
1344
1344
1345 Parameters
1345 Parameters
1346 ----------
1346 ----------
1347
1347
1348 targets: list,slice,int,etc. [default: use all engines]
1348 targets: list,slice,int,etc. [default: use all engines]
1349 The engines to use for the View
1349 The engines to use for the View
1350 """
1350 """
1351 single = isinstance(targets, int)
1351 single = isinstance(targets, int)
1352 # allow 'all' to be lazily evaluated at each execution
1352 # allow 'all' to be lazily evaluated at each execution
1353 if targets != 'all':
1353 if targets != 'all':
1354 targets = self._build_targets(targets)[1]
1354 targets = self._build_targets(targets)[1]
1355 if single:
1355 if single:
1356 targets = targets[0]
1356 targets = targets[0]
1357 return DirectView(client=self, socket=self._mux_socket, targets=targets)
1357 return DirectView(client=self, socket=self._mux_socket, targets=targets)
1358
1358
1359 #--------------------------------------------------------------------------
1359 #--------------------------------------------------------------------------
1360 # Query methods
1360 # Query methods
1361 #--------------------------------------------------------------------------
1361 #--------------------------------------------------------------------------
1362
1362
1363 @spin_first
1363 @spin_first
1364 def get_result(self, indices_or_msg_ids=None, block=None):
1364 def get_result(self, indices_or_msg_ids=None, block=None):
1365 """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
1365 """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
1366
1366
1367 If the client already has the results, no request to the Hub will be made.
1367 If the client already has the results, no request to the Hub will be made.
1368
1368
1369 This is a convenient way to construct AsyncResult objects, which are wrappers
1369 This is a convenient way to construct AsyncResult objects, which are wrappers
1370 that include metadata about execution, and allow for awaiting results that
1370 that include metadata about execution, and allow for awaiting results that
1371 were not submitted by this Client.
1371 were not submitted by this Client.
1372
1372
1373 It can also be a convenient way to retrieve the metadata associated with
1373 It can also be a convenient way to retrieve the metadata associated with
1374 blocking execution, since it always retrieves
1374 blocking execution, since it always retrieves
1375
1375
1376 Examples
1376 Examples
1377 --------
1377 --------
1378 ::
1378 ::
1379
1379
1380 In [10]: r = client.apply()
1380 In [10]: r = client.apply()
1381
1381
1382 Parameters
1382 Parameters
1383 ----------
1383 ----------
1384
1384
1385 indices_or_msg_ids : integer history index, str msg_id, or list of either
1385 indices_or_msg_ids : integer history index, str msg_id, or list of either
1386 The indices or msg_ids of indices to be retrieved
1386 The indices or msg_ids of indices to be retrieved
1387
1387
1388 block : bool
1388 block : bool
1389 Whether to wait for the result to be done
1389 Whether to wait for the result to be done
1390
1390
1391 Returns
1391 Returns
1392 -------
1392 -------
1393
1393
1394 AsyncResult
1394 AsyncResult
1395 A single AsyncResult object will always be returned.
1395 A single AsyncResult object will always be returned.
1396
1396
1397 AsyncHubResult
1397 AsyncHubResult
1398 A subclass of AsyncResult that retrieves results from the Hub
1398 A subclass of AsyncResult that retrieves results from the Hub
1399
1399
1400 """
1400 """
1401 block = self.block if block is None else block
1401 block = self.block if block is None else block
1402 if indices_or_msg_ids is None:
1402 if indices_or_msg_ids is None:
1403 indices_or_msg_ids = -1
1403 indices_or_msg_ids = -1
1404
1404
1405 single_result = False
1405 single_result = False
1406 if not isinstance(indices_or_msg_ids, (list,tuple)):
1406 if not isinstance(indices_or_msg_ids, (list,tuple)):
1407 indices_or_msg_ids = [indices_or_msg_ids]
1407 indices_or_msg_ids = [indices_or_msg_ids]
1408 single_result = True
1408 single_result = True
1409
1409
1410 theids = []
1410 theids = []
1411 for id in indices_or_msg_ids:
1411 for id in indices_or_msg_ids:
1412 if isinstance(id, int):
1412 if isinstance(id, int):
1413 id = self.history[id]
1413 id = self.history[id]
1414 if not isinstance(id, string_types):
1414 if not isinstance(id, string_types):
1415 raise TypeError("indices must be str or int, not %r"%id)
1415 raise TypeError("indices must be str or int, not %r"%id)
1416 theids.append(id)
1416 theids.append(id)
1417
1417
1418 local_ids = [msg_id for msg_id in theids if (msg_id in self.outstanding or msg_id in self.results)]
1418 local_ids = [msg_id for msg_id in theids if (msg_id in self.outstanding or msg_id in self.results)]
1419 remote_ids = [msg_id for msg_id in theids if msg_id not in local_ids]
1419 remote_ids = [msg_id for msg_id in theids if msg_id not in local_ids]
1420
1420
1421 # given single msg_id initially, get_result shot get the result itself,
1421 # given single msg_id initially, get_result shot get the result itself,
1422 # not a length-one list
1422 # not a length-one list
1423 if single_result:
1423 if single_result:
1424 theids = theids[0]
1424 theids = theids[0]
1425
1425
1426 if remote_ids:
1426 if remote_ids:
1427 ar = AsyncHubResult(self, msg_ids=theids)
1427 ar = AsyncHubResult(self, msg_ids=theids)
1428 else:
1428 else:
1429 ar = AsyncResult(self, msg_ids=theids)
1429 ar = AsyncResult(self, msg_ids=theids)
1430
1430
1431 if block:
1431 if block:
1432 ar.wait()
1432 ar.wait()
1433
1433
1434 return ar
1434 return ar
1435
1435
1436 @spin_first
1436 @spin_first
1437 def resubmit(self, indices_or_msg_ids=None, metadata=None, block=None):
1437 def resubmit(self, indices_or_msg_ids=None, metadata=None, block=None):
1438 """Resubmit one or more tasks.
1438 """Resubmit one or more tasks.
1439
1439
1440 in-flight tasks may not be resubmitted.
1440 in-flight tasks may not be resubmitted.
1441
1441
1442 Parameters
1442 Parameters
1443 ----------
1443 ----------
1444
1444
1445 indices_or_msg_ids : integer history index, str msg_id, or list of either
1445 indices_or_msg_ids : integer history index, str msg_id, or list of either
1446 The indices or msg_ids of indices to be retrieved
1446 The indices or msg_ids of indices to be retrieved
1447
1447
1448 block : bool
1448 block : bool
1449 Whether to wait for the result to be done
1449 Whether to wait for the result to be done
1450
1450
1451 Returns
1451 Returns
1452 -------
1452 -------
1453
1453
1454 AsyncHubResult
1454 AsyncHubResult
1455 A subclass of AsyncResult that retrieves results from the Hub
1455 A subclass of AsyncResult that retrieves results from the Hub
1456
1456
1457 """
1457 """
1458 block = self.block if block is None else block
1458 block = self.block if block is None else block
1459 if indices_or_msg_ids is None:
1459 if indices_or_msg_ids is None:
1460 indices_or_msg_ids = -1
1460 indices_or_msg_ids = -1
1461
1461
1462 if not isinstance(indices_or_msg_ids, (list,tuple)):
1462 if not isinstance(indices_or_msg_ids, (list,tuple)):
1463 indices_or_msg_ids = [indices_or_msg_ids]
1463 indices_or_msg_ids = [indices_or_msg_ids]
1464
1464
1465 theids = []
1465 theids = []
1466 for id in indices_or_msg_ids:
1466 for id in indices_or_msg_ids:
1467 if isinstance(id, int):
1467 if isinstance(id, int):
1468 id = self.history[id]
1468 id = self.history[id]
1469 if not isinstance(id, string_types):
1469 if not isinstance(id, string_types):
1470 raise TypeError("indices must be str or int, not %r"%id)
1470 raise TypeError("indices must be str or int, not %r"%id)
1471 theids.append(id)
1471 theids.append(id)
1472
1472
1473 content = dict(msg_ids = theids)
1473 content = dict(msg_ids = theids)
1474
1474
1475 self.session.send(self._query_socket, 'resubmit_request', content)
1475 self.session.send(self._query_socket, 'resubmit_request', content)
1476
1476
1477 zmq.select([self._query_socket], [], [])
1477 zmq.select([self._query_socket], [], [])
1478 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1478 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1479 if self.debug:
1479 if self.debug:
1480 pprint(msg)
1480 pprint(msg)
1481 content = msg['content']
1481 content = msg['content']
1482 if content['status'] != 'ok':
1482 if content['status'] != 'ok':
1483 raise self._unwrap_exception(content)
1483 raise self._unwrap_exception(content)
1484 mapping = content['resubmitted']
1484 mapping = content['resubmitted']
1485 new_ids = [ mapping[msg_id] for msg_id in theids ]
1485 new_ids = [ mapping[msg_id] for msg_id in theids ]
1486
1486
1487 ar = AsyncHubResult(self, msg_ids=new_ids)
1487 ar = AsyncHubResult(self, msg_ids=new_ids)
1488
1488
1489 if block:
1489 if block:
1490 ar.wait()
1490 ar.wait()
1491
1491
1492 return ar
1492 return ar
1493
1493
1494 @spin_first
1494 @spin_first
1495 def result_status(self, msg_ids, status_only=True):
1495 def result_status(self, msg_ids, status_only=True):
1496 """Check on the status of the result(s) of the apply request with `msg_ids`.
1496 """Check on the status of the result(s) of the apply request with `msg_ids`.
1497
1497
1498 If status_only is False, then the actual results will be retrieved, else
1498 If status_only is False, then the actual results will be retrieved, else
1499 only the status of the results will be checked.
1499 only the status of the results will be checked.
1500
1500
1501 Parameters
1501 Parameters
1502 ----------
1502 ----------
1503
1503
1504 msg_ids : list of msg_ids
1504 msg_ids : list of msg_ids
1505 if int:
1505 if int:
1506 Passed as index to self.history for convenience.
1506 Passed as index to self.history for convenience.
1507 status_only : bool (default: True)
1507 status_only : bool (default: True)
1508 if False:
1508 if False:
1509 Retrieve the actual results of completed tasks.
1509 Retrieve the actual results of completed tasks.
1510
1510
1511 Returns
1511 Returns
1512 -------
1512 -------
1513
1513
1514 results : dict
1514 results : dict
1515 There will always be the keys 'pending' and 'completed', which will
1515 There will always be the keys 'pending' and 'completed', which will
1516 be lists of msg_ids that are incomplete or complete. If `status_only`
1516 be lists of msg_ids that are incomplete or complete. If `status_only`
1517 is False, then completed results will be keyed by their `msg_id`.
1517 is False, then completed results will be keyed by their `msg_id`.
1518 """
1518 """
1519 if not isinstance(msg_ids, (list,tuple)):
1519 if not isinstance(msg_ids, (list,tuple)):
1520 msg_ids = [msg_ids]
1520 msg_ids = [msg_ids]
1521
1521
1522 theids = []
1522 theids = []
1523 for msg_id in msg_ids:
1523 for msg_id in msg_ids:
1524 if isinstance(msg_id, int):
1524 if isinstance(msg_id, int):
1525 msg_id = self.history[msg_id]
1525 msg_id = self.history[msg_id]
1526 if not isinstance(msg_id, string_types):
1526 if not isinstance(msg_id, string_types):
1527 raise TypeError("msg_ids must be str, not %r"%msg_id)
1527 raise TypeError("msg_ids must be str, not %r"%msg_id)
1528 theids.append(msg_id)
1528 theids.append(msg_id)
1529
1529
1530 completed = []
1530 completed = []
1531 local_results = {}
1531 local_results = {}
1532
1532
1533 # comment this block out to temporarily disable local shortcut:
1533 # comment this block out to temporarily disable local shortcut:
1534 for msg_id in theids:
1534 for msg_id in theids:
1535 if msg_id in self.results:
1535 if msg_id in self.results:
1536 completed.append(msg_id)
1536 completed.append(msg_id)
1537 local_results[msg_id] = self.results[msg_id]
1537 local_results[msg_id] = self.results[msg_id]
1538 theids.remove(msg_id)
1538 theids.remove(msg_id)
1539
1539
1540 if theids: # some not locally cached
1540 if theids: # some not locally cached
1541 content = dict(msg_ids=theids, status_only=status_only)
1541 content = dict(msg_ids=theids, status_only=status_only)
1542 msg = self.session.send(self._query_socket, "result_request", content=content)
1542 msg = self.session.send(self._query_socket, "result_request", content=content)
1543 zmq.select([self._query_socket], [], [])
1543 zmq.select([self._query_socket], [], [])
1544 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1544 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1545 if self.debug:
1545 if self.debug:
1546 pprint(msg)
1546 pprint(msg)
1547 content = msg['content']
1547 content = msg['content']
1548 if content['status'] != 'ok':
1548 if content['status'] != 'ok':
1549 raise self._unwrap_exception(content)
1549 raise self._unwrap_exception(content)
1550 buffers = msg['buffers']
1550 buffers = msg['buffers']
1551 else:
1551 else:
1552 content = dict(completed=[],pending=[])
1552 content = dict(completed=[],pending=[])
1553
1553
1554 content['completed'].extend(completed)
1554 content['completed'].extend(completed)
1555
1555
1556 if status_only:
1556 if status_only:
1557 return content
1557 return content
1558
1558
1559 failures = []
1559 failures = []
1560 # load cached results into result:
1560 # load cached results into result:
1561 content.update(local_results)
1561 content.update(local_results)
1562
1562
1563 # update cache with results:
1563 # update cache with results:
1564 for msg_id in sorted(theids):
1564 for msg_id in sorted(theids):
1565 if msg_id in content['completed']:
1565 if msg_id in content['completed']:
1566 rec = content[msg_id]
1566 rec = content[msg_id]
1567 parent = rec['header']
1567 parent = rec['header']
1568 header = rec['result_header']
1568 header = rec['result_header']
1569 rcontent = rec['result_content']
1569 rcontent = rec['result_content']
1570 iodict = rec['io']
1570 iodict = rec['io']
1571 if isinstance(rcontent, str):
1571 if isinstance(rcontent, str):
1572 rcontent = self.session.unpack(rcontent)
1572 rcontent = self.session.unpack(rcontent)
1573
1573
1574 md = self.metadata[msg_id]
1574 md = self.metadata[msg_id]
1575 md_msg = dict(
1575 md_msg = dict(
1576 content=rcontent,
1576 content=rcontent,
1577 parent_header=parent,
1577 parent_header=parent,
1578 header=header,
1578 header=header,
1579 metadata=rec['result_metadata'],
1579 metadata=rec['result_metadata'],
1580 )
1580 )
1581 md.update(self._extract_metadata(md_msg))
1581 md.update(self._extract_metadata(md_msg))
1582 if rec.get('received'):
1582 if rec.get('received'):
1583 md['received'] = rec['received']
1583 md['received'] = rec['received']
1584 md.update(iodict)
1584 md.update(iodict)
1585
1585
1586 if rcontent['status'] == 'ok':
1586 if rcontent['status'] == 'ok':
1587 if header['msg_type'] == 'apply_reply':
1587 if header['msg_type'] == 'apply_reply':
1588 res,buffers = serialize.unserialize_object(buffers)
1588 res,buffers = serialize.unserialize_object(buffers)
1589 elif header['msg_type'] == 'execute_reply':
1589 elif header['msg_type'] == 'execute_reply':
1590 res = ExecuteReply(msg_id, rcontent, md)
1590 res = ExecuteReply(msg_id, rcontent, md)
1591 else:
1591 else:
1592 raise KeyError("unhandled msg type: %r" % header['msg_type'])
1592 raise KeyError("unhandled msg type: %r" % header['msg_type'])
1593 else:
1593 else:
1594 res = self._unwrap_exception(rcontent)
1594 res = self._unwrap_exception(rcontent)
1595 failures.append(res)
1595 failures.append(res)
1596
1596
1597 self.results[msg_id] = res
1597 self.results[msg_id] = res
1598 content[msg_id] = res
1598 content[msg_id] = res
1599
1599
1600 if len(theids) == 1 and failures:
1600 if len(theids) == 1 and failures:
1601 raise failures[0]
1601 raise failures[0]
1602
1602
1603 error.collect_exceptions(failures, "result_status")
1603 error.collect_exceptions(failures, "result_status")
1604 return content
1604 return content
1605
1605
1606 @spin_first
1606 @spin_first
1607 def queue_status(self, targets='all', verbose=False):
1607 def queue_status(self, targets='all', verbose=False):
1608 """Fetch the status of engine queues.
1608 """Fetch the status of engine queues.
1609
1609
1610 Parameters
1610 Parameters
1611 ----------
1611 ----------
1612
1612
1613 targets : int/str/list of ints/strs
1613 targets : int/str/list of ints/strs
1614 the engines whose states are to be queried.
1614 the engines whose states are to be queried.
1615 default : all
1615 default : all
1616 verbose : bool
1616 verbose : bool
1617 Whether to return lengths only, or lists of ids for each element
1617 Whether to return lengths only, or lists of ids for each element
1618 """
1618 """
1619 if targets == 'all':
1619 if targets == 'all':
1620 # allow 'all' to be evaluated on the engine
1620 # allow 'all' to be evaluated on the engine
1621 engine_ids = None
1621 engine_ids = None
1622 else:
1622 else:
1623 engine_ids = self._build_targets(targets)[1]
1623 engine_ids = self._build_targets(targets)[1]
1624 content = dict(targets=engine_ids, verbose=verbose)
1624 content = dict(targets=engine_ids, verbose=verbose)
1625 self.session.send(self._query_socket, "queue_request", content=content)
1625 self.session.send(self._query_socket, "queue_request", content=content)
1626 idents,msg = self.session.recv(self._query_socket, 0)
1626 idents,msg = self.session.recv(self._query_socket, 0)
1627 if self.debug:
1627 if self.debug:
1628 pprint(msg)
1628 pprint(msg)
1629 content = msg['content']
1629 content = msg['content']
1630 status = content.pop('status')
1630 status = content.pop('status')
1631 if status != 'ok':
1631 if status != 'ok':
1632 raise self._unwrap_exception(content)
1632 raise self._unwrap_exception(content)
1633 content = rekey(content)
1633 content = rekey(content)
1634 if isinstance(targets, int):
1634 if isinstance(targets, int):
1635 return content[targets]
1635 return content[targets]
1636 else:
1636 else:
1637 return content
1637 return content
1638
1638
1639 def _build_msgids_from_target(self, targets=None):
1639 def _build_msgids_from_target(self, targets=None):
1640 """Build a list of msg_ids from the list of engine targets"""
1640 """Build a list of msg_ids from the list of engine targets"""
1641 if not targets: # needed as _build_targets otherwise uses all engines
1641 if not targets: # needed as _build_targets otherwise uses all engines
1642 return []
1642 return []
1643 target_ids = self._build_targets(targets)[0]
1643 target_ids = self._build_targets(targets)[0]
1644 return [md_id for md_id in self.metadata if self.metadata[md_id]["engine_uuid"] in target_ids]
1644 return [md_id for md_id in self.metadata if self.metadata[md_id]["engine_uuid"] in target_ids]
1645
1645
1646 def _build_msgids_from_jobs(self, jobs=None):
1646 def _build_msgids_from_jobs(self, jobs=None):
1647 """Build a list of msg_ids from "jobs" """
1647 """Build a list of msg_ids from "jobs" """
1648 if not jobs:
1648 if not jobs:
1649 return []
1649 return []
1650 msg_ids = []
1650 msg_ids = []
1651 if isinstance(jobs, string_types + (AsyncResult,)):
1651 if isinstance(jobs, string_types + (AsyncResult,)):
1652 jobs = [jobs]
1652 jobs = [jobs]
1653 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1653 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1654 if bad_ids:
1654 if bad_ids:
1655 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1655 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1656 for j in jobs:
1656 for j in jobs:
1657 if isinstance(j, AsyncResult):
1657 if isinstance(j, AsyncResult):
1658 msg_ids.extend(j.msg_ids)
1658 msg_ids.extend(j.msg_ids)
1659 else:
1659 else:
1660 msg_ids.append(j)
1660 msg_ids.append(j)
1661 return msg_ids
1661 return msg_ids
1662
1662
1663 def purge_local_results(self, jobs=[], targets=[]):
1663 def purge_local_results(self, jobs=[], targets=[]):
1664 """Clears the client caches of results and frees such memory.
1664 """Clears the client caches of results and frees such memory.
1665
1665
1666 Individual results can be purged by msg_id, or the entire
1666 Individual results can be purged by msg_id, or the entire
1667 history of specific targets can be purged.
1667 history of specific targets can be purged.
1668
1668
1669 Use `purge_local_results('all')` to scrub everything from the Clients's db.
1669 Use `purge_local_results('all')` to scrub everything from the Clients's db.
1670
1670
1671 The client must have no outstanding tasks before purging the caches.
1671 The client must have no outstanding tasks before purging the caches.
1672 Raises `AssertionError` if there are still outstanding tasks.
1672 Raises `AssertionError` if there are still outstanding tasks.
1673
1673
1674 After this call all `AsyncResults` are invalid and should be discarded.
1674 After this call all `AsyncResults` are invalid and should be discarded.
1675
1675
1676 If you must "reget" the results, you can still do so by using
1676 If you must "reget" the results, you can still do so by using
1677 `client.get_result(msg_id)` or `client.get_result(asyncresult)`. This will
1677 `client.get_result(msg_id)` or `client.get_result(asyncresult)`. This will
1678 redownload the results from the hub if they are still available
1678 redownload the results from the hub if they are still available
1679 (i.e `client.purge_hub_results(...)` has not been called.
1679 (i.e `client.purge_hub_results(...)` has not been called.
1680
1680
1681 Parameters
1681 Parameters
1682 ----------
1682 ----------
1683
1683
1684 jobs : str or list of str or AsyncResult objects
1684 jobs : str or list of str or AsyncResult objects
1685 the msg_ids whose results should be purged.
1685 the msg_ids whose results should be purged.
1686 targets : int/str/list of ints/strs
1686 targets : int/str/list of ints/strs
1687 The targets, by int_id, whose entire results are to be purged.
1687 The targets, by int_id, whose entire results are to be purged.
1688
1688
1689 default : None
1689 default : None
1690 """
1690 """
1691 assert not self.outstanding, "Can't purge a client with outstanding tasks!"
1691 assert not self.outstanding, "Can't purge a client with outstanding tasks!"
1692
1692
1693 if not targets and not jobs:
1693 if not targets and not jobs:
1694 raise ValueError("Must specify at least one of `targets` and `jobs`")
1694 raise ValueError("Must specify at least one of `targets` and `jobs`")
1695
1695
1696 if jobs == 'all':
1696 if jobs == 'all':
1697 self.results.clear()
1697 self.results.clear()
1698 self.metadata.clear()
1698 self.metadata.clear()
1699 return
1699 return
1700 else:
1700 else:
1701 msg_ids = []
1701 msg_ids = []
1702 msg_ids.extend(self._build_msgids_from_target(targets))
1702 msg_ids.extend(self._build_msgids_from_target(targets))
1703 msg_ids.extend(self._build_msgids_from_jobs(jobs))
1703 msg_ids.extend(self._build_msgids_from_jobs(jobs))
1704 for mid in msg_ids:
1704 for mid in msg_ids:
1705 self.results.pop(mid)
1705 self.results.pop(mid)
1706 self.metadata.pop(mid)
1706 self.metadata.pop(mid)
1707
1707
1708
1708
1709 @spin_first
1709 @spin_first
1710 def purge_hub_results(self, jobs=[], targets=[]):
1710 def purge_hub_results(self, jobs=[], targets=[]):
1711 """Tell the Hub to forget results.
1711 """Tell the Hub to forget results.
1712
1712
1713 Individual results can be purged by msg_id, or the entire
1713 Individual results can be purged by msg_id, or the entire
1714 history of specific targets can be purged.
1714 history of specific targets can be purged.
1715
1715
1716 Use `purge_results('all')` to scrub everything from the Hub's db.
1716 Use `purge_results('all')` to scrub everything from the Hub's db.
1717
1717
1718 Parameters
1718 Parameters
1719 ----------
1719 ----------
1720
1720
1721 jobs : str or list of str or AsyncResult objects
1721 jobs : str or list of str or AsyncResult objects
1722 the msg_ids whose results should be forgotten.
1722 the msg_ids whose results should be forgotten.
1723 targets : int/str/list of ints/strs
1723 targets : int/str/list of ints/strs
1724 The targets, by int_id, whose entire history is to be purged.
1724 The targets, by int_id, whose entire history is to be purged.
1725
1725
1726 default : None
1726 default : None
1727 """
1727 """
1728 if not targets and not jobs:
1728 if not targets and not jobs:
1729 raise ValueError("Must specify at least one of `targets` and `jobs`")
1729 raise ValueError("Must specify at least one of `targets` and `jobs`")
1730 if targets:
1730 if targets:
1731 targets = self._build_targets(targets)[1]
1731 targets = self._build_targets(targets)[1]
1732
1732
1733 # construct msg_ids from jobs
1733 # construct msg_ids from jobs
1734 if jobs == 'all':
1734 if jobs == 'all':
1735 msg_ids = jobs
1735 msg_ids = jobs
1736 else:
1736 else:
1737 msg_ids = self._build_msgids_from_jobs(jobs)
1737 msg_ids = self._build_msgids_from_jobs(jobs)
1738
1738
1739 content = dict(engine_ids=targets, msg_ids=msg_ids)
1739 content = dict(engine_ids=targets, msg_ids=msg_ids)
1740 self.session.send(self._query_socket, "purge_request", content=content)
1740 self.session.send(self._query_socket, "purge_request", content=content)
1741 idents, msg = self.session.recv(self._query_socket, 0)
1741 idents, msg = self.session.recv(self._query_socket, 0)
1742 if self.debug:
1742 if self.debug:
1743 pprint(msg)
1743 pprint(msg)
1744 content = msg['content']
1744 content = msg['content']
1745 if content['status'] != 'ok':
1745 if content['status'] != 'ok':
1746 raise self._unwrap_exception(content)
1746 raise self._unwrap_exception(content)
1747
1747
1748 def purge_results(self, jobs=[], targets=[]):
1748 def purge_results(self, jobs=[], targets=[]):
1749 """Clears the cached results from both the hub and the local client
1749 """Clears the cached results from both the hub and the local client
1750
1750
1751 Individual results can be purged by msg_id, or the entire
1751 Individual results can be purged by msg_id, or the entire
1752 history of specific targets can be purged.
1752 history of specific targets can be purged.
1753
1753
1754 Use `purge_results('all')` to scrub every cached result from both the Hub's and
1754 Use `purge_results('all')` to scrub every cached result from both the Hub's and
1755 the Client's db.
1755 the Client's db.
1756
1756
1757 Equivalent to calling both `purge_hub_results()` and `purge_client_results()` with
1757 Equivalent to calling both `purge_hub_results()` and `purge_client_results()` with
1758 the same arguments.
1758 the same arguments.
1759
1759
1760 Parameters
1760 Parameters
1761 ----------
1761 ----------
1762
1762
1763 jobs : str or list of str or AsyncResult objects
1763 jobs : str or list of str or AsyncResult objects
1764 the msg_ids whose results should be forgotten.
1764 the msg_ids whose results should be forgotten.
1765 targets : int/str/list of ints/strs
1765 targets : int/str/list of ints/strs
1766 The targets, by int_id, whose entire history is to be purged.
1766 The targets, by int_id, whose entire history is to be purged.
1767
1767
1768 default : None
1768 default : None
1769 """
1769 """
1770 self.purge_local_results(jobs=jobs, targets=targets)
1770 self.purge_local_results(jobs=jobs, targets=targets)
1771 self.purge_hub_results(jobs=jobs, targets=targets)
1771 self.purge_hub_results(jobs=jobs, targets=targets)
1772
1772
1773 def purge_everything(self):
1773 def purge_everything(self):
1774 """Clears all content from previous Tasks from both the hub and the local client
1774 """Clears all content from previous Tasks from both the hub and the local client
1775
1775
1776 In addition to calling `purge_results("all")` it also deletes the history and
1776 In addition to calling `purge_results("all")` it also deletes the history and
1777 other bookkeeping lists.
1777 other bookkeeping lists.
1778 """
1778 """
1779 self.purge_results("all")
1779 self.purge_results("all")
1780 self.history = []
1780 self.history = []
1781 self.session.digest_history.clear()
1781 self.session.digest_history.clear()
1782
1782
1783 @spin_first
1783 @spin_first
1784 def hub_history(self):
1784 def hub_history(self):
1785 """Get the Hub's history
1785 """Get the Hub's history
1786
1786
1787 Just like the Client, the Hub has a history, which is a list of msg_ids.
1787 Just like the Client, the Hub has a history, which is a list of msg_ids.
1788 This will contain the history of all clients, and, depending on configuration,
1788 This will contain the history of all clients, and, depending on configuration,
1789 may contain history across multiple cluster sessions.
1789 may contain history across multiple cluster sessions.
1790
1790
1791 Any msg_id returned here is a valid argument to `get_result`.
1791 Any msg_id returned here is a valid argument to `get_result`.
1792
1792
1793 Returns
1793 Returns
1794 -------
1794 -------
1795
1795
1796 msg_ids : list of strs
1796 msg_ids : list of strs
1797 list of all msg_ids, ordered by task submission time.
1797 list of all msg_ids, ordered by task submission time.
1798 """
1798 """
1799
1799
1800 self.session.send(self._query_socket, "history_request", content={})
1800 self.session.send(self._query_socket, "history_request", content={})
1801 idents, msg = self.session.recv(self._query_socket, 0)
1801 idents, msg = self.session.recv(self._query_socket, 0)
1802
1802
1803 if self.debug:
1803 if self.debug:
1804 pprint(msg)
1804 pprint(msg)
1805 content = msg['content']
1805 content = msg['content']
1806 if content['status'] != 'ok':
1806 if content['status'] != 'ok':
1807 raise self._unwrap_exception(content)
1807 raise self._unwrap_exception(content)
1808 else:
1808 else:
1809 return content['history']
1809 return content['history']
1810
1810
1811 @spin_first
1811 @spin_first
1812 def db_query(self, query, keys=None):
1812 def db_query(self, query, keys=None):
1813 """Query the Hub's TaskRecord database
1813 """Query the Hub's TaskRecord database
1814
1814
1815 This will return a list of task record dicts that match `query`
1815 This will return a list of task record dicts that match `query`
1816
1816
1817 Parameters
1817 Parameters
1818 ----------
1818 ----------
1819
1819
1820 query : mongodb query dict
1820 query : mongodb query dict
1821 The search dict. See mongodb query docs for details.
1821 The search dict. See mongodb query docs for details.
1822 keys : list of strs [optional]
1822 keys : list of strs [optional]
1823 The subset of keys to be returned. The default is to fetch everything but buffers.
1823 The subset of keys to be returned. The default is to fetch everything but buffers.
1824 'msg_id' will *always* be included.
1824 'msg_id' will *always* be included.
1825 """
1825 """
1826 if isinstance(keys, string_types):
1826 if isinstance(keys, string_types):
1827 keys = [keys]
1827 keys = [keys]
1828 content = dict(query=query, keys=keys)
1828 content = dict(query=query, keys=keys)
1829 self.session.send(self._query_socket, "db_request", content=content)
1829 self.session.send(self._query_socket, "db_request", content=content)
1830 idents, msg = self.session.recv(self._query_socket, 0)
1830 idents, msg = self.session.recv(self._query_socket, 0)
1831 if self.debug:
1831 if self.debug:
1832 pprint(msg)
1832 pprint(msg)
1833 content = msg['content']
1833 content = msg['content']
1834 if content['status'] != 'ok':
1834 if content['status'] != 'ok':
1835 raise self._unwrap_exception(content)
1835 raise self._unwrap_exception(content)
1836
1836
1837 records = content['records']
1837 records = content['records']
1838
1838
1839 buffer_lens = content['buffer_lens']
1839 buffer_lens = content['buffer_lens']
1840 result_buffer_lens = content['result_buffer_lens']
1840 result_buffer_lens = content['result_buffer_lens']
1841 buffers = msg['buffers']
1841 buffers = msg['buffers']
1842 has_bufs = buffer_lens is not None
1842 has_bufs = buffer_lens is not None
1843 has_rbufs = result_buffer_lens is not None
1843 has_rbufs = result_buffer_lens is not None
1844 for i,rec in enumerate(records):
1844 for i,rec in enumerate(records):
1845 # relink buffers
1845 # relink buffers
1846 if has_bufs:
1846 if has_bufs:
1847 blen = buffer_lens[i]
1847 blen = buffer_lens[i]
1848 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
1848 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
1849 if has_rbufs:
1849 if has_rbufs:
1850 blen = result_buffer_lens[i]
1850 blen = result_buffer_lens[i]
1851 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
1851 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
1852
1852
1853 return records
1853 return records
1854
1854
1855 __all__ = [ 'Client' ]
1855 __all__ = [ 'Client' ]
General Comments 0
You need to be logged in to leave comments. Login now