Show More
The requested changes are too big and content was truncated. Show full diff
@@ -0,0 +1,5 b'' | |||||
|
1 | This is the IPython directory. | |||
|
2 | ||||
|
3 | For more information on configuring IPython, do: | |||
|
4 | ||||
|
5 | ipython config -h |
@@ -0,0 +1,219 b'' | |||||
|
1 | # encoding: utf-8 | |||
|
2 | """ | |||
|
3 | An application for managing IPython profiles. | |||
|
4 | ||||
|
5 | To be invoked as the `ipython profile` subcommand. | |||
|
6 | ||||
|
7 | Authors: | |||
|
8 | ||||
|
9 | * Min RK | |||
|
10 | ||||
|
11 | """ | |||
|
12 | ||||
|
13 | #----------------------------------------------------------------------------- | |||
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |||
|
15 | # | |||
|
16 | # Distributed under the terms of the BSD License. The full license is in | |||
|
17 | # the file COPYING, distributed as part of this software. | |||
|
18 | #----------------------------------------------------------------------------- | |||
|
19 | ||||
|
20 | #----------------------------------------------------------------------------- | |||
|
21 | # Imports | |||
|
22 | #----------------------------------------------------------------------------- | |||
|
23 | ||||
|
24 | import logging | |||
|
25 | import os | |||
|
26 | ||||
|
27 | from IPython.config.application import Application, boolean_flag | |||
|
28 | from IPython.core.application import ( | |||
|
29 | BaseIPythonApplication, base_flags, base_aliases | |||
|
30 | ) | |||
|
31 | from IPython.core.profiledir import ProfileDir | |||
|
32 | from IPython.utils.path import get_ipython_dir | |||
|
33 | from IPython.utils.traitlets import Unicode, Bool, Dict | |||
|
34 | ||||
|
35 | #----------------------------------------------------------------------------- | |||
|
36 | # Constants | |||
|
37 | #----------------------------------------------------------------------------- | |||
|
38 | ||||
|
39 | create_help = """Create an IPython profile by name | |||
|
40 | ||||
|
41 | Create an ipython profile directory by its name or | |||
|
42 | profile directory path. Profile directories contain | |||
|
43 | configuration, log and security related files and are named | |||
|
44 | using the convention 'profile_<name>'. By default they are | |||
|
45 | located in your ipython directory. Once created, you will | |||
|
46 | can edit the configuration files in the profile | |||
|
47 | directory to configure IPython. Most users will create a | |||
|
48 | profile directory by name, | |||
|
49 | `ipython profile create myprofile`, which will put the directory | |||
|
50 | in `<ipython_dir>/profile_myprofile`. | |||
|
51 | """ | |||
|
52 | list_help = """List available IPython profiles | |||
|
53 | ||||
|
54 | List all available profiles, by profile location, that can | |||
|
55 | be found in the current working directly or in the ipython | |||
|
56 | directory. Profile directories are named using the convention | |||
|
57 | 'profile_<profile>'. | |||
|
58 | """ | |||
|
59 | profile_help = """Manage IPython profiles | |||
|
60 | ||||
|
61 | Profile directories contain | |||
|
62 | configuration, log and security related files and are named | |||
|
63 | using the convention 'profile_<name>'. By default they are | |||
|
64 | located in your ipython directory. You can create profiles | |||
|
65 | with `ipython profile create <name>`, or see the profiles you | |||
|
66 | already have with `ipython profile list` | |||
|
67 | ||||
|
68 | To get started configuring IPython, simply do: | |||
|
69 | ||||
|
70 | $> ipython profile create | |||
|
71 | ||||
|
72 | and IPython will create the default profile in <ipython_dir>/profile_default, | |||
|
73 | where you can edit ipython_config.py to start configuring IPython. | |||
|
74 | ||||
|
75 | """ | |||
|
76 | ||||
|
77 | #----------------------------------------------------------------------------- | |||
|
78 | # Profile Application Class (for `ipython profile` subcommand) | |||
|
79 | #----------------------------------------------------------------------------- | |||
|
80 | ||||
|
81 | ||||
|
82 | ||||
|
83 | class ProfileList(Application): | |||
|
84 | name = u'ipython-profile' | |||
|
85 | description = list_help | |||
|
86 | ||||
|
87 | aliases = Dict(dict( | |||
|
88 | ipython_dir = 'ProfileList.ipython_dir', | |||
|
89 | log_level = 'Application.log_level', | |||
|
90 | )) | |||
|
91 | flags = Dict(dict( | |||
|
92 | debug = ({'Application' : {'log_level' : 0}}, | |||
|
93 | "Set log_level to 0, maximizing log output." | |||
|
94 | ) | |||
|
95 | )) | |||
|
96 | ipython_dir = Unicode(get_ipython_dir(), config=True, | |||
|
97 | help=""" | |||
|
98 | The name of the IPython directory. This directory is used for logging | |||
|
99 | configuration (through profiles), history storage, etc. The default | |||
|
100 | is usually $HOME/.ipython. This options can also be specified through | |||
|
101 | the environment variable IPYTHON_DIR. | |||
|
102 | """ | |||
|
103 | ) | |||
|
104 | ||||
|
105 | def list_profile_dirs(self): | |||
|
106 | # Find the search paths | |||
|
107 | paths = [os.getcwdu(), self.ipython_dir] | |||
|
108 | ||||
|
109 | self.log.warn('Searching for IPython profiles in paths: %r' % paths) | |||
|
110 | for path in paths: | |||
|
111 | files = os.listdir(path) | |||
|
112 | for f in files: | |||
|
113 | full_path = os.path.join(path, f) | |||
|
114 | if os.path.isdir(full_path) and f.startswith('profile_'): | |||
|
115 | profile = f.split('_',1)[-1] | |||
|
116 | start_cmd = 'ipython profile=%s' % profile | |||
|
117 | print start_cmd + " ==> " + full_path | |||
|
118 | ||||
|
119 | def start(self): | |||
|
120 | self.list_profile_dirs() | |||
|
121 | ||||
|
122 | ||||
|
123 | create_flags = {} | |||
|
124 | create_flags.update(base_flags) | |||
|
125 | create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite', | |||
|
126 | "reset config files to defaults", "leave existing config files")) | |||
|
127 | create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel', | |||
|
128 | "Include parallel computing config files", | |||
|
129 | "Don't include parallel computing config files")) | |||
|
130 | ||||
|
131 | class ProfileCreate(BaseIPythonApplication): | |||
|
132 | name = u'ipython-profile' | |||
|
133 | description = create_help | |||
|
134 | auto_create = Bool(True, config=False) | |||
|
135 | ||||
|
136 | def _copy_config_files_default(self): | |||
|
137 | return True | |||
|
138 | ||||
|
139 | parallel = Bool(False, config=True, | |||
|
140 | help="whether to include parallel computing config files") | |||
|
141 | def _parallel_changed(self, name, old, new): | |||
|
142 | parallel_files = [ 'ipcontroller_config.py', | |||
|
143 | 'ipengine_config.py', | |||
|
144 | 'ipcluster_config.py' | |||
|
145 | ] | |||
|
146 | if new: | |||
|
147 | for cf in parallel_files: | |||
|
148 | self.config_files.append(cf) | |||
|
149 | else: | |||
|
150 | for cf in parallel_files: | |||
|
151 | if cf in self.config_files: | |||
|
152 | self.config_files.remove(cf) | |||
|
153 | ||||
|
154 | def parse_command_line(self, argv): | |||
|
155 | super(ProfileCreate, self).parse_command_line(argv) | |||
|
156 | # accept positional arg as profile name | |||
|
157 | if self.extra_args: | |||
|
158 | self.profile = self.extra_args[0] | |||
|
159 | ||||
|
160 | flags = Dict(create_flags) | |||
|
161 | ||||
|
162 | aliases = Dict(dict(profile='BaseIPythonApplication.profile')) | |||
|
163 | ||||
|
164 | classes = [ProfileDir] | |||
|
165 | ||||
|
166 | def init_config_files(self): | |||
|
167 | super(ProfileCreate, self).init_config_files() | |||
|
168 | # use local imports, since these classes may import from here | |||
|
169 | from IPython.frontend.terminal.ipapp import TerminalIPythonApp | |||
|
170 | apps = [TerminalIPythonApp] | |||
|
171 | try: | |||
|
172 | from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp | |||
|
173 | except ImportError: | |||
|
174 | pass | |||
|
175 | else: | |||
|
176 | apps.append(IPythonQtConsoleApp) | |||
|
177 | if self.parallel: | |||
|
178 | from IPython.parallel.apps.ipcontrollerapp import IPControllerApp | |||
|
179 | from IPython.parallel.apps.ipengineapp import IPEngineApp | |||
|
180 | from IPython.parallel.apps.ipclusterapp import IPClusterStart | |||
|
181 | from IPython.parallel.apps.iploggerapp import IPLoggerApp | |||
|
182 | apps.extend([ | |||
|
183 | IPControllerApp, | |||
|
184 | IPEngineApp, | |||
|
185 | IPClusterStart, | |||
|
186 | IPLoggerApp, | |||
|
187 | ]) | |||
|
188 | for App in apps: | |||
|
189 | app = App() | |||
|
190 | app.config.update(self.config) | |||
|
191 | app.log = self.log | |||
|
192 | app.overwrite = self.overwrite | |||
|
193 | app.copy_config_files=True | |||
|
194 | app.profile = self.profile | |||
|
195 | app.init_profile_dir() | |||
|
196 | app.init_config_files() | |||
|
197 | ||||
|
198 | def stage_default_config_file(self): | |||
|
199 | pass | |||
|
200 | ||||
|
201 | class ProfileApp(Application): | |||
|
202 | name = u'ipython-profile' | |||
|
203 | description = profile_help | |||
|
204 | ||||
|
205 | subcommands = Dict(dict( | |||
|
206 | create = (ProfileCreate, "Create a new profile dir with default config files"), | |||
|
207 | list = (ProfileList, "List existing profiles") | |||
|
208 | )) | |||
|
209 | ||||
|
210 | def start(self): | |||
|
211 | if self.subapp is None: | |||
|
212 | print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()) | |||
|
213 | ||||
|
214 | self.print_description() | |||
|
215 | self.print_subcommands() | |||
|
216 | self.exit(1) | |||
|
217 | else: | |||
|
218 | return self.subapp.start() | |||
|
219 |
@@ -0,0 +1,206 b'' | |||||
|
1 | # encoding: utf-8 | |||
|
2 | """ | |||
|
3 | An object for managing IPython profile directories. | |||
|
4 | ||||
|
5 | Authors: | |||
|
6 | ||||
|
7 | * Brian Granger | |||
|
8 | * Fernando Perez | |||
|
9 | * Min RK | |||
|
10 | ||||
|
11 | """ | |||
|
12 | ||||
|
13 | #----------------------------------------------------------------------------- | |||
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |||
|
15 | # | |||
|
16 | # Distributed under the terms of the BSD License. The full license is in | |||
|
17 | # the file COPYING, distributed as part of this software. | |||
|
18 | #----------------------------------------------------------------------------- | |||
|
19 | ||||
|
20 | #----------------------------------------------------------------------------- | |||
|
21 | # Imports | |||
|
22 | #----------------------------------------------------------------------------- | |||
|
23 | ||||
|
24 | import os | |||
|
25 | import shutil | |||
|
26 | import sys | |||
|
27 | ||||
|
28 | from IPython.config.configurable import Configurable | |||
|
29 | from IPython.config.loader import Config | |||
|
30 | from IPython.utils.path import get_ipython_package_dir, expand_path | |||
|
31 | from IPython.utils.traitlets import List, Unicode, Bool | |||
|
32 | ||||
|
33 | #----------------------------------------------------------------------------- | |||
|
34 | # Classes and functions | |||
|
35 | #----------------------------------------------------------------------------- | |||
|
36 | ||||
|
37 | ||||
|
38 | #----------------------------------------------------------------------------- | |||
|
39 | # Module errors | |||
|
40 | #----------------------------------------------------------------------------- | |||
|
41 | ||||
|
42 | class ProfileDirError(Exception): | |||
|
43 | pass | |||
|
44 | ||||
|
45 | ||||
|
46 | #----------------------------------------------------------------------------- | |||
|
47 | # Class for managing profile directories | |||
|
48 | #----------------------------------------------------------------------------- | |||
|
49 | ||||
|
50 | class ProfileDir(Configurable): | |||
|
51 | """An object to manage the profile directory and its resources. | |||
|
52 | ||||
|
53 | The profile directory is used by all IPython applications, to manage | |||
|
54 | configuration, logging and security. | |||
|
55 | ||||
|
56 | This object knows how to find, create and manage these directories. This | |||
|
57 | should be used by any code that wants to handle profiles. | |||
|
58 | """ | |||
|
59 | ||||
|
60 | security_dir_name = Unicode('security') | |||
|
61 | log_dir_name = Unicode('log') | |||
|
62 | pid_dir_name = Unicode('pid') | |||
|
63 | security_dir = Unicode(u'') | |||
|
64 | log_dir = Unicode(u'') | |||
|
65 | pid_dir = Unicode(u'') | |||
|
66 | ||||
|
67 | location = Unicode(u'', config=True, | |||
|
68 | help="""Set the profile location directly. This overrides the logic used by the | |||
|
69 | `profile` option.""", | |||
|
70 | ) | |||
|
71 | ||||
|
72 | _location_isset = Bool(False) # flag for detecting multiply set location | |||
|
73 | ||||
|
74 | def _location_changed(self, name, old, new): | |||
|
75 | if self._location_isset: | |||
|
76 | raise RuntimeError("Cannot set profile location more than once.") | |||
|
77 | self._location_isset = True | |||
|
78 | if not os.path.isdir(new): | |||
|
79 | os.makedirs(new) | |||
|
80 | ||||
|
81 | # ensure config files exist: | |||
|
82 | self.security_dir = os.path.join(new, self.security_dir_name) | |||
|
83 | self.log_dir = os.path.join(new, self.log_dir_name) | |||
|
84 | self.pid_dir = os.path.join(new, self.pid_dir_name) | |||
|
85 | self.check_dirs() | |||
|
86 | ||||
|
87 | def _log_dir_changed(self, name, old, new): | |||
|
88 | self.check_log_dir() | |||
|
89 | ||||
|
90 | def check_log_dir(self): | |||
|
91 | if not os.path.isdir(self.log_dir): | |||
|
92 | os.mkdir(self.log_dir) | |||
|
93 | ||||
|
94 | def _security_dir_changed(self, name, old, new): | |||
|
95 | self.check_security_dir() | |||
|
96 | ||||
|
97 | def check_security_dir(self): | |||
|
98 | if not os.path.isdir(self.security_dir): | |||
|
99 | os.mkdir(self.security_dir, 0700) | |||
|
100 | else: | |||
|
101 | os.chmod(self.security_dir, 0700) | |||
|
102 | ||||
|
103 | def _pid_dir_changed(self, name, old, new): | |||
|
104 | self.check_pid_dir() | |||
|
105 | ||||
|
106 | def check_pid_dir(self): | |||
|
107 | if not os.path.isdir(self.pid_dir): | |||
|
108 | os.mkdir(self.pid_dir, 0700) | |||
|
109 | else: | |||
|
110 | os.chmod(self.pid_dir, 0700) | |||
|
111 | ||||
|
112 | def check_dirs(self): | |||
|
113 | self.check_security_dir() | |||
|
114 | self.check_log_dir() | |||
|
115 | self.check_pid_dir() | |||
|
116 | ||||
|
117 | def copy_config_file(self, config_file, path=None, overwrite=False): | |||
|
118 | """Copy a default config file into the active profile directory. | |||
|
119 | ||||
|
120 | Default configuration files are kept in :mod:`IPython.config.default`. | |||
|
121 | This function moves these from that location to the working profile | |||
|
122 | directory. | |||
|
123 | """ | |||
|
124 | dst = os.path.join(self.location, config_file) | |||
|
125 | if os.path.isfile(dst) and not overwrite: | |||
|
126 | return | |||
|
127 | if path is None: | |||
|
128 | path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default') | |||
|
129 | src = os.path.join(path, config_file) | |||
|
130 | shutil.copy(src, dst) | |||
|
131 | ||||
|
132 | @classmethod | |||
|
133 | def create_profile_dir(cls, profile_dir, config=None): | |||
|
134 | """Create a new profile directory given a full path. | |||
|
135 | ||||
|
136 | Parameters | |||
|
137 | ---------- | |||
|
138 | profile_dir : str | |||
|
139 | The full path to the profile directory. If it does exist, it will | |||
|
140 | be used. If not, it will be created. | |||
|
141 | """ | |||
|
142 | return cls(location=profile_dir, config=config) | |||
|
143 | ||||
|
144 | @classmethod | |||
|
145 | def create_profile_dir_by_name(cls, path, name=u'default', config=None): | |||
|
146 | """Create a profile dir by profile name and path. | |||
|
147 | ||||
|
148 | Parameters | |||
|
149 | ---------- | |||
|
150 | path : unicode | |||
|
151 | The path (directory) to put the profile directory in. | |||
|
152 | name : unicode | |||
|
153 | The name of the profile. The name of the profile directory will | |||
|
154 | be "profile_<profile>". | |||
|
155 | """ | |||
|
156 | if not os.path.isdir(path): | |||
|
157 | raise ProfileDirError('Directory not found: %s' % path) | |||
|
158 | profile_dir = os.path.join(path, u'profile_' + name) | |||
|
159 | return cls(location=profile_dir, config=config) | |||
|
160 | ||||
|
161 | @classmethod | |||
|
162 | def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): | |||
|
163 | """Find an existing profile dir by profile name, return its ProfileDir. | |||
|
164 | ||||
|
165 | This searches through a sequence of paths for a profile dir. If it | |||
|
166 | is not found, a :class:`ProfileDirError` exception will be raised. | |||
|
167 | ||||
|
168 | The search path algorithm is: | |||
|
169 | 1. ``os.getcwd()`` | |||
|
170 | 2. ``ipython_dir`` | |||
|
171 | ||||
|
172 | Parameters | |||
|
173 | ---------- | |||
|
174 | ipython_dir : unicode or str | |||
|
175 | The IPython directory to use. | |||
|
176 | name : unicode or str | |||
|
177 | The name of the profile. The name of the profile directory | |||
|
178 | will be "profile_<profile>". | |||
|
179 | """ | |||
|
180 | dirname = u'profile_' + name | |||
|
181 | paths = [os.getcwdu(), ipython_dir] | |||
|
182 | for p in paths: | |||
|
183 | profile_dir = os.path.join(p, dirname) | |||
|
184 | if os.path.isdir(profile_dir): | |||
|
185 | return cls(location=profile_dir, config=config) | |||
|
186 | else: | |||
|
187 | raise ProfileDirError('Profile directory not found in paths: %s' % dirname) | |||
|
188 | ||||
|
189 | @classmethod | |||
|
190 | def find_profile_dir(cls, profile_dir, config=None): | |||
|
191 | """Find/create a profile dir and return its ProfileDir. | |||
|
192 | ||||
|
193 | This will create the profile directory if it doesn't exist. | |||
|
194 | ||||
|
195 | Parameters | |||
|
196 | ---------- | |||
|
197 | profile_dir : unicode or str | |||
|
198 | The path of the profile directory. This is expanded using | |||
|
199 | :func:`IPython.utils.genutils.expand_path`. | |||
|
200 | """ | |||
|
201 | profile_dir = expand_path(profile_dir) | |||
|
202 | if not os.path.isdir(profile_dir): | |||
|
203 | raise ProfileDirError('Profile directory not found: %s' % profile_dir) | |||
|
204 | return cls(location=profile_dir, config=config) | |||
|
205 | ||||
|
206 |
@@ -0,0 +1,244 b'' | |||||
|
1 | #!/usr/bin/env python | |||
|
2 | # encoding: utf-8 | |||
|
3 | """ | |||
|
4 | A mixin for :class:`~IPython.core.application.Application` classes that | |||
|
5 | launch InteractiveShell instances, load extensions, etc. | |||
|
6 | ||||
|
7 | Authors | |||
|
8 | ------- | |||
|
9 | ||||
|
10 | * Min Ragan-Kelley | |||
|
11 | """ | |||
|
12 | ||||
|
13 | #----------------------------------------------------------------------------- | |||
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |||
|
15 | # | |||
|
16 | # Distributed under the terms of the BSD License. The full license is in | |||
|
17 | # the file COPYING, distributed as part of this software. | |||
|
18 | #----------------------------------------------------------------------------- | |||
|
19 | ||||
|
20 | #----------------------------------------------------------------------------- | |||
|
21 | # Imports | |||
|
22 | #----------------------------------------------------------------------------- | |||
|
23 | ||||
|
24 | from __future__ import absolute_import | |||
|
25 | ||||
|
26 | import os | |||
|
27 | import sys | |||
|
28 | ||||
|
29 | from IPython.config.application import boolean_flag | |||
|
30 | from IPython.config.configurable import Configurable | |||
|
31 | from IPython.config.loader import Config | |||
|
32 | from IPython.utils.path import filefind | |||
|
33 | from IPython.utils.traitlets import Unicode, Instance, List | |||
|
34 | ||||
|
35 | #----------------------------------------------------------------------------- | |||
|
36 | # Aliases and Flags | |||
|
37 | #----------------------------------------------------------------------------- | |||
|
38 | ||||
|
39 | shell_flags = {} | |||
|
40 | ||||
|
41 | addflag = lambda *args: shell_flags.update(boolean_flag(*args)) | |||
|
42 | addflag('autoindent', 'InteractiveShell.autoindent', | |||
|
43 | 'Turn on autoindenting.', 'Turn off autoindenting.' | |||
|
44 | ) | |||
|
45 | addflag('automagic', 'InteractiveShell.automagic', | |||
|
46 | """Turn on the auto calling of magic commands. Type %%magic at the | |||
|
47 | IPython prompt for more information.""", | |||
|
48 | 'Turn off the auto calling of magic commands.' | |||
|
49 | ) | |||
|
50 | addflag('pdb', 'InteractiveShell.pdb', | |||
|
51 | "Enable auto calling the pdb debugger after every exception.", | |||
|
52 | "Disable auto calling the pdb debugger after every exception." | |||
|
53 | ) | |||
|
54 | addflag('pprint', 'PlainTextFormatter.pprint', | |||
|
55 | "Enable auto pretty printing of results.", | |||
|
56 | "Disable auto auto pretty printing of results." | |||
|
57 | ) | |||
|
58 | addflag('color-info', 'InteractiveShell.color_info', | |||
|
59 | """IPython can display information about objects via a set of func- | |||
|
60 | tions, and optionally can use colors for this, syntax highlighting | |||
|
61 | source code and various other elements. However, because this | |||
|
62 | information is passed through a pager (like 'less') and many pagers get | |||
|
63 | confused with color codes, this option is off by default. You can test | |||
|
64 | it and turn it on permanently in your ipython_config.py file if it | |||
|
65 | works for you. Test it and turn it on permanently if it works with | |||
|
66 | your system. The magic function %%color_info allows you to toggle this | |||
|
67 | interactively for testing.""", | |||
|
68 | "Disable using colors for info related things." | |||
|
69 | ) | |||
|
70 | addflag('deep-reload', 'InteractiveShell.deep_reload', | |||
|
71 | """Enable deep (recursive) reloading by default. IPython can use the | |||
|
72 | deep_reload module which reloads changes in modules recursively (it | |||
|
73 | replaces the reload() function, so you don't need to change anything to | |||
|
74 | use it). deep_reload() forces a full reload of modules whose code may | |||
|
75 | have changed, which the default reload() function does not. When | |||
|
76 | deep_reload is off, IPython will use the normal reload(), but | |||
|
77 | deep_reload will still be available as dreload(). This feature is off | |||
|
78 | by default [which means that you have both normal reload() and | |||
|
79 | dreload()].""", | |||
|
80 | "Disable deep (recursive) reloading by default." | |||
|
81 | ) | |||
|
82 | nosep_config = Config() | |||
|
83 | nosep_config.InteractiveShell.separate_in = '' | |||
|
84 | nosep_config.InteractiveShell.separate_out = '' | |||
|
85 | nosep_config.InteractiveShell.separate_out2 = '' | |||
|
86 | ||||
|
87 | shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.") | |||
|
88 | ||||
|
89 | ||||
|
90 | # it's possible we don't want short aliases for *all* of these: | |||
|
91 | shell_aliases = dict( | |||
|
92 | autocall='InteractiveShell.autocall', | |||
|
93 | cache_size='InteractiveShell.cache_size', | |||
|
94 | colors='InteractiveShell.colors', | |||
|
95 | logfile='InteractiveShell.logfile', | |||
|
96 | log_append='InteractiveShell.logappend', | |||
|
97 | c='InteractiveShellApp.code_to_run', | |||
|
98 | ext='InteractiveShellApp.extra_extension', | |||
|
99 | ) | |||
|
100 | ||||
|
101 | #----------------------------------------------------------------------------- | |||
|
102 | # Main classes and functions | |||
|
103 | #----------------------------------------------------------------------------- | |||
|
104 | ||||
|
105 | class InteractiveShellApp(Configurable): | |||
|
106 | """A Mixin for applications that start InteractiveShell instances. | |||
|
107 | ||||
|
108 | Provides configurables for loading extensions and executing files | |||
|
109 | as part of configuring a Shell environment. | |||
|
110 | ||||
|
111 | Provides init_extensions() and init_code() methods, to be called | |||
|
112 | after init_shell(), which must be implemented by subclasses. | |||
|
113 | """ | |||
|
114 | extensions = List(Unicode, config=True, | |||
|
115 | help="A list of dotted module names of IPython extensions to load." | |||
|
116 | ) | |||
|
117 | extra_extension = Unicode('', config=True, | |||
|
118 | help="dotted module name of an IPython extension to load." | |||
|
119 | ) | |||
|
120 | def _extra_extension_changed(self, name, old, new): | |||
|
121 | if new: | |||
|
122 | # add to self.extensions | |||
|
123 | self.extensions.append(new) | |||
|
124 | ||||
|
125 | exec_files = List(Unicode, config=True, | |||
|
126 | help="""List of files to run at IPython startup.""" | |||
|
127 | ) | |||
|
128 | file_to_run = Unicode('', config=True, | |||
|
129 | help="""A file to be run""") | |||
|
130 | ||||
|
131 | exec_lines = List(Unicode, config=True, | |||
|
132 | help="""lines of code to run at IPython startup.""" | |||
|
133 | ) | |||
|
134 | code_to_run = Unicode('', config=True, | |||
|
135 | help="Execute the given command string." | |||
|
136 | ) | |||
|
137 | shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') | |||
|
138 | ||||
|
139 | def init_shell(self): | |||
|
140 | raise NotImplementedError("Override in subclasses") | |||
|
141 | ||||
|
142 | def init_extensions(self): | |||
|
143 | """Load all IPython extensions in IPythonApp.extensions. | |||
|
144 | ||||
|
145 | This uses the :meth:`ExtensionManager.load_extensions` to load all | |||
|
146 | the extensions listed in ``self.extensions``. | |||
|
147 | """ | |||
|
148 | if not self.extensions: | |||
|
149 | return | |||
|
150 | try: | |||
|
151 | self.log.debug("Loading IPython extensions...") | |||
|
152 | extensions = self.extensions | |||
|
153 | for ext in extensions: | |||
|
154 | try: | |||
|
155 | self.log.info("Loading IPython extension: %s" % ext) | |||
|
156 | self.shell.extension_manager.load_extension(ext) | |||
|
157 | except: | |||
|
158 | self.log.warn("Error in loading extension: %s" % ext) | |||
|
159 | self.shell.showtraceback() | |||
|
160 | except: | |||
|
161 | self.log.warn("Unknown error in loading extensions:") | |||
|
162 | self.shell.showtraceback() | |||
|
163 | ||||
|
164 | def init_code(self): | |||
|
165 | """run the pre-flight code, specified via exec_lines""" | |||
|
166 | self._run_exec_lines() | |||
|
167 | self._run_exec_files() | |||
|
168 | self._run_cmd_line_code() | |||
|
169 | ||||
|
170 | def _run_exec_lines(self): | |||
|
171 | """Run lines of code in IPythonApp.exec_lines in the user's namespace.""" | |||
|
172 | if not self.exec_lines: | |||
|
173 | return | |||
|
174 | try: | |||
|
175 | self.log.debug("Running code from IPythonApp.exec_lines...") | |||
|
176 | for line in self.exec_lines: | |||
|
177 | try: | |||
|
178 | self.log.info("Running code in user namespace: %s" % | |||
|
179 | line) | |||
|
180 | self.shell.run_cell(line, store_history=False) | |||
|
181 | except: | |||
|
182 | self.log.warn("Error in executing line in user " | |||
|
183 | "namespace: %s" % line) | |||
|
184 | self.shell.showtraceback() | |||
|
185 | except: | |||
|
186 | self.log.warn("Unknown error in handling IPythonApp.exec_lines:") | |||
|
187 | self.shell.showtraceback() | |||
|
188 | ||||
|
189 | def _exec_file(self, fname): | |||
|
190 | full_filename = filefind(fname, [u'.', self.ipython_dir]) | |||
|
191 | if os.path.isfile(full_filename): | |||
|
192 | if full_filename.endswith(u'.py'): | |||
|
193 | self.log.info("Running file in user namespace: %s" % | |||
|
194 | full_filename) | |||
|
195 | # Ensure that __file__ is always defined to match Python behavior | |||
|
196 | self.shell.user_ns['__file__'] = fname | |||
|
197 | try: | |||
|
198 | self.shell.safe_execfile(full_filename, self.shell.user_ns) | |||
|
199 | finally: | |||
|
200 | del self.shell.user_ns['__file__'] | |||
|
201 | elif full_filename.endswith('.ipy'): | |||
|
202 | self.log.info("Running file in user namespace: %s" % | |||
|
203 | full_filename) | |||
|
204 | self.shell.safe_execfile_ipy(full_filename) | |||
|
205 | else: | |||
|
206 | self.log.warn("File does not have a .py or .ipy extension: <%s>" | |||
|
207 | % full_filename) | |||
|
208 | ||||
|
209 | def _run_exec_files(self): | |||
|
210 | """Run files from IPythonApp.exec_files""" | |||
|
211 | if not self.exec_files: | |||
|
212 | return | |||
|
213 | ||||
|
214 | self.log.debug("Running files in IPythonApp.exec_files...") | |||
|
215 | try: | |||
|
216 | for fname in self.exec_files: | |||
|
217 | self._exec_file(fname) | |||
|
218 | except: | |||
|
219 | self.log.warn("Unknown error in handling IPythonApp.exec_files:") | |||
|
220 | self.shell.showtraceback() | |||
|
221 | ||||
|
222 | def _run_cmd_line_code(self): | |||
|
223 | """Run code or file specified at the command-line""" | |||
|
224 | if self.code_to_run: | |||
|
225 | line = self.code_to_run | |||
|
226 | try: | |||
|
227 | self.log.info("Running code given at command line (c=): %s" % | |||
|
228 | line) | |||
|
229 | self.shell.run_cell(line, store_history=False) | |||
|
230 | except: | |||
|
231 | self.log.warn("Error in executing line in user namespace: %s" % | |||
|
232 | line) | |||
|
233 | self.shell.showtraceback() | |||
|
234 | ||||
|
235 | # Like Python itself, ignore the second if the first of these is present | |||
|
236 | elif self.file_to_run: | |||
|
237 | fname = self.file_to_run | |||
|
238 | try: | |||
|
239 | self._exec_file(fname) | |||
|
240 | except: | |||
|
241 | self.log.warn("Error in executing file in user namespace: %s" % | |||
|
242 | fname) | |||
|
243 | self.shell.showtraceback() | |||
|
244 |
@@ -0,0 +1,267 b'' | |||||
|
1 | #!/usr/bin/env python | |||
|
2 | # encoding: utf-8 | |||
|
3 | """ | |||
|
4 | The Base Application class for IPython.parallel apps | |||
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Brian Granger | |||
|
9 | * Min RK | |||
|
10 | ||||
|
11 | """ | |||
|
12 | ||||
|
13 | #----------------------------------------------------------------------------- | |||
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |||
|
15 | # | |||
|
16 | # Distributed under the terms of the BSD License. The full license is in | |||
|
17 | # the file COPYING, distributed as part of this software. | |||
|
18 | #----------------------------------------------------------------------------- | |||
|
19 | ||||
|
20 | #----------------------------------------------------------------------------- | |||
|
21 | # Imports | |||
|
22 | #----------------------------------------------------------------------------- | |||
|
23 | ||||
|
24 | from __future__ import with_statement | |||
|
25 | ||||
|
26 | import os | |||
|
27 | import logging | |||
|
28 | import re | |||
|
29 | import sys | |||
|
30 | ||||
|
31 | from subprocess import Popen, PIPE | |||
|
32 | ||||
|
33 | from IPython.core import release | |||
|
34 | from IPython.core.crashhandler import CrashHandler | |||
|
35 | from IPython.core.application import ( | |||
|
36 | BaseIPythonApplication, | |||
|
37 | base_aliases as base_ip_aliases, | |||
|
38 | base_flags as base_ip_flags | |||
|
39 | ) | |||
|
40 | from IPython.utils.path import expand_path | |||
|
41 | ||||
|
42 | from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List | |||
|
43 | ||||
|
44 | #----------------------------------------------------------------------------- | |||
|
45 | # Module errors | |||
|
46 | #----------------------------------------------------------------------------- | |||
|
47 | ||||
|
48 | class PIDFileError(Exception): | |||
|
49 | pass | |||
|
50 | ||||
|
51 | ||||
|
52 | #----------------------------------------------------------------------------- | |||
|
53 | # Crash handler for this application | |||
|
54 | #----------------------------------------------------------------------------- | |||
|
55 | ||||
|
56 | ||||
|
57 | _message_template = """\ | |||
|
58 | Oops, $self.app_name crashed. We do our best to make it stable, but... | |||
|
59 | ||||
|
60 | A crash report was automatically generated with the following information: | |||
|
61 | - A verbatim copy of the crash traceback. | |||
|
62 | - Data on your current $self.app_name configuration. | |||
|
63 | ||||
|
64 | It was left in the file named: | |||
|
65 | \t'$self.crash_report_fname' | |||
|
66 | If you can email this file to the developers, the information in it will help | |||
|
67 | them in understanding and correcting the problem. | |||
|
68 | ||||
|
69 | You can mail it to: $self.contact_name at $self.contact_email | |||
|
70 | with the subject '$self.app_name Crash Report'. | |||
|
71 | ||||
|
72 | If you want to do it now, the following command will work (under Unix): | |||
|
73 | mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname | |||
|
74 | ||||
|
75 | To ensure accurate tracking of this issue, please file a report about it at: | |||
|
76 | $self.bug_tracker | |||
|
77 | """ | |||
|
78 | ||||
|
79 | class ParallelCrashHandler(CrashHandler): | |||
|
80 | """sys.excepthook for IPython itself, leaves a detailed report on disk.""" | |||
|
81 | ||||
|
82 | message_template = _message_template | |||
|
83 | ||||
|
84 | def __init__(self, app): | |||
|
85 | contact_name = release.authors['Min'][0] | |||
|
86 | contact_email = release.authors['Min'][1] | |||
|
87 | bug_tracker = 'http://github.com/ipython/ipython/issues' | |||
|
88 | super(ParallelCrashHandler,self).__init__( | |||
|
89 | app, contact_name, contact_email, bug_tracker | |||
|
90 | ) | |||
|
91 | ||||
|
92 | ||||
|
93 | #----------------------------------------------------------------------------- | |||
|
94 | # Main application | |||
|
95 | #----------------------------------------------------------------------------- | |||
|
96 | base_aliases = {} | |||
|
97 | base_aliases.update(base_ip_aliases) | |||
|
98 | base_aliases.update({ | |||
|
99 | 'profile_dir' : 'ProfileDir.location', | |||
|
100 | 'log_level' : 'BaseParallelApplication.log_level', | |||
|
101 | 'work_dir' : 'BaseParallelApplication.work_dir', | |||
|
102 | 'log_to_file' : 'BaseParallelApplication.log_to_file', | |||
|
103 | 'clean_logs' : 'BaseParallelApplication.clean_logs', | |||
|
104 | 'log_url' : 'BaseParallelApplication.log_url', | |||
|
105 | }) | |||
|
106 | ||||
|
107 | base_flags = { | |||
|
108 | 'log-to-file' : ( | |||
|
109 | {'BaseParallelApplication' : {'log_to_file' : True}}, | |||
|
110 | "send log output to a file" | |||
|
111 | ) | |||
|
112 | } | |||
|
113 | base_flags.update(base_ip_flags) | |||
|
114 | ||||
|
115 | class BaseParallelApplication(BaseIPythonApplication): | |||
|
116 | """The base Application for IPython.parallel apps | |||
|
117 | ||||
|
118 | Principle extensions to BaseIPyythonApplication: | |||
|
119 | ||||
|
120 | * work_dir | |||
|
121 | * remote logging via pyzmq | |||
|
122 | * IOLoop instance | |||
|
123 | """ | |||
|
124 | ||||
|
125 | crash_handler_class = ParallelCrashHandler | |||
|
126 | ||||
|
127 | def _log_level_default(self): | |||
|
128 | # temporarily override default_log_level to INFO | |||
|
129 | return logging.INFO | |||
|
130 | ||||
|
131 | work_dir = Unicode(os.getcwdu(), config=True, | |||
|
132 | help='Set the working dir for the process.' | |||
|
133 | ) | |||
|
134 | def _work_dir_changed(self, name, old, new): | |||
|
135 | self.work_dir = unicode(expand_path(new)) | |||
|
136 | ||||
|
137 | log_to_file = Bool(config=True, | |||
|
138 | help="whether to log to a file") | |||
|
139 | ||||
|
140 | clean_logs = Bool(False, config=True, | |||
|
141 | help="whether to cleanup old logfiles before starting") | |||
|
142 | ||||
|
143 | log_url = Unicode('', config=True, | |||
|
144 | help="The ZMQ URL of the iplogger to aggregate logging.") | |||
|
145 | ||||
|
146 | def _config_files_default(self): | |||
|
147 | return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py'] | |||
|
148 | ||||
|
149 | loop = Instance('zmq.eventloop.ioloop.IOLoop') | |||
|
150 | def _loop_default(self): | |||
|
151 | from zmq.eventloop.ioloop import IOLoop | |||
|
152 | return IOLoop.instance() | |||
|
153 | ||||
|
154 | aliases = Dict(base_aliases) | |||
|
155 | flags = Dict(base_flags) | |||
|
156 | ||||
|
157 | def initialize(self, argv=None): | |||
|
158 | """initialize the app""" | |||
|
159 | super(BaseParallelApplication, self).initialize(argv) | |||
|
160 | self.to_work_dir() | |||
|
161 | self.reinit_logging() | |||
|
162 | ||||
|
163 | def to_work_dir(self): | |||
|
164 | wd = self.work_dir | |||
|
165 | if unicode(wd) != os.getcwdu(): | |||
|
166 | os.chdir(wd) | |||
|
167 | self.log.info("Changing to working dir: %s" % wd) | |||
|
168 | # This is the working dir by now. | |||
|
169 | sys.path.insert(0, '') | |||
|
170 | ||||
|
171 | def reinit_logging(self): | |||
|
172 | # Remove old log files | |||
|
173 | log_dir = self.profile_dir.log_dir | |||
|
174 | if self.clean_logs: | |||
|
175 | for f in os.listdir(log_dir): | |||
|
176 | if re.match(r'%s-\d+\.(log|err|out)'%self.name,f): | |||
|
177 | os.remove(os.path.join(log_dir, f)) | |||
|
178 | if self.log_to_file: | |||
|
179 | # Start logging to the new log file | |||
|
180 | log_filename = self.name + u'-' + str(os.getpid()) + u'.log' | |||
|
181 | logfile = os.path.join(log_dir, log_filename) | |||
|
182 | open_log_file = open(logfile, 'w') | |||
|
183 | else: | |||
|
184 | open_log_file = None | |||
|
185 | if open_log_file is not None: | |||
|
186 | self.log.removeHandler(self._log_handler) | |||
|
187 | self._log_handler = logging.StreamHandler(open_log_file) | |||
|
188 | self._log_formatter = logging.Formatter("[%(name)s] %(message)s") | |||
|
189 | self._log_handler.setFormatter(self._log_formatter) | |||
|
190 | self.log.addHandler(self._log_handler) | |||
|
191 | ||||
|
192 | def write_pid_file(self, overwrite=False): | |||
|
193 | """Create a .pid file in the pid_dir with my pid. | |||
|
194 | ||||
|
195 | This must be called after pre_construct, which sets `self.pid_dir`. | |||
|
196 | This raises :exc:`PIDFileError` if the pid file exists already. | |||
|
197 | """ | |||
|
198 | pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid') | |||
|
199 | if os.path.isfile(pid_file): | |||
|
200 | pid = self.get_pid_from_file() | |||
|
201 | if not overwrite: | |||
|
202 | raise PIDFileError( | |||
|
203 | 'The pid file [%s] already exists. \nThis could mean that this ' | |||
|
204 | 'server is already running with [pid=%s].' % (pid_file, pid) | |||
|
205 | ) | |||
|
206 | with open(pid_file, 'w') as f: | |||
|
207 | self.log.info("Creating pid file: %s" % pid_file) | |||
|
208 | f.write(repr(os.getpid())+'\n') | |||
|
209 | ||||
|
210 | def remove_pid_file(self): | |||
|
211 | """Remove the pid file. | |||
|
212 | ||||
|
213 | This should be called at shutdown by registering a callback with | |||
|
214 | :func:`reactor.addSystemEventTrigger`. This needs to return | |||
|
215 | ``None``. | |||
|
216 | """ | |||
|
217 | pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid') | |||
|
218 | if os.path.isfile(pid_file): | |||
|
219 | try: | |||
|
220 | self.log.info("Removing pid file: %s" % pid_file) | |||
|
221 | os.remove(pid_file) | |||
|
222 | except: | |||
|
223 | self.log.warn("Error removing the pid file: %s" % pid_file) | |||
|
224 | ||||
|
225 | def get_pid_from_file(self): | |||
|
226 | """Get the pid from the pid file. | |||
|
227 | ||||
|
228 | If the pid file doesn't exist a :exc:`PIDFileError` is raised. | |||
|
229 | """ | |||
|
230 | pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid') | |||
|
231 | if os.path.isfile(pid_file): | |||
|
232 | with open(pid_file, 'r') as f: | |||
|
233 | s = f.read().strip() | |||
|
234 | try: | |||
|
235 | pid = int(s) | |||
|
236 | except: | |||
|
237 | raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s)) | |||
|
238 | return pid | |||
|
239 | else: | |||
|
240 | raise PIDFileError('pid file not found: %s' % pid_file) | |||
|
241 | ||||
|
242 | def check_pid(self, pid): | |||
|
243 | if os.name == 'nt': | |||
|
244 | try: | |||
|
245 | import ctypes | |||
|
246 | # returns 0 if no such process (of ours) exists | |||
|
247 | # positive int otherwise | |||
|
248 | p = ctypes.windll.kernel32.OpenProcess(1,0,pid) | |||
|
249 | except Exception: | |||
|
250 | self.log.warn( | |||
|
251 | "Could not determine whether pid %i is running via `OpenProcess`. " | |||
|
252 | " Making the likely assumption that it is."%pid | |||
|
253 | ) | |||
|
254 | return True | |||
|
255 | return bool(p) | |||
|
256 | else: | |||
|
257 | try: | |||
|
258 | p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE) | |||
|
259 | output,_ = p.communicate() | |||
|
260 | except OSError: | |||
|
261 | self.log.warn( | |||
|
262 | "Could not determine whether pid %i is running via `ps x`. " | |||
|
263 | " Making the likely assumption that it is."%pid | |||
|
264 | ) | |||
|
265 | return True | |||
|
266 | pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE)) | |||
|
267 | return pid in pids |
@@ -0,0 +1,26 b'' | |||||
|
1 | """daemonize function from twisted.scripts._twistd_unix.""" | |||
|
2 | ||||
|
3 | #----------------------------------------------------------------------------- | |||
|
4 | # Copyright (c) Twisted Matrix Laboratories. | |||
|
5 | # See Twisted's LICENSE for details. | |||
|
6 | # http://twistedmatrix.com/ | |||
|
7 | #----------------------------------------------------------------------------- | |||
|
8 | ||||
|
9 | import os, errno | |||
|
10 | ||||
|
11 | def daemonize(): | |||
|
12 | # See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16 | |||
|
13 | if os.fork(): # launch child and... | |||
|
14 | os._exit(0) # kill off parent | |||
|
15 | os.setsid() | |||
|
16 | if os.fork(): # launch child and... | |||
|
17 | os._exit(0) # kill off parent again. | |||
|
18 | null = os.open('/dev/null', os.O_RDWR) | |||
|
19 | for i in range(3): | |||
|
20 | try: | |||
|
21 | os.dup2(null, i) | |||
|
22 | except OSError, e: | |||
|
23 | if e.errno != errno.EBADF: | |||
|
24 | raise | |||
|
25 | os.close(null) | |||
|
26 |
1 | NO CONTENT: new file 100644 |
|
NO CONTENT: new file 100644 | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: new file 100644 |
|
NO CONTENT: new file 100644 | ||
The requested commit or file is too big and content was truncated. Show full diff |
@@ -5,6 +5,7 b' A base class for a configurable application.' | |||||
5 | Authors: |
|
5 | Authors: | |
6 |
|
6 | |||
7 | * Brian Granger |
|
7 | * Brian Granger | |
|
8 | * Min RK | |||
8 | """ |
|
9 | """ | |
9 |
|
10 | |||
10 | #----------------------------------------------------------------------------- |
|
11 | #----------------------------------------------------------------------------- | |
@@ -18,22 +19,29 b' Authors:' | |||||
18 | # Imports |
|
19 | # Imports | |
19 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
20 |
|
21 | |||
21 | from copy import deepcopy |
|
|||
22 | import logging |
|
22 | import logging | |
|
23 | import os | |||
|
24 | import re | |||
23 | import sys |
|
25 | import sys | |
|
26 | from copy import deepcopy | |||
24 |
|
27 | |||
25 | from IPython.config.configurable import SingletonConfigurable |
|
28 | from IPython.config.configurable import SingletonConfigurable | |
26 | from IPython.config.loader import ( |
|
29 | from IPython.config.loader import ( | |
27 | KeyValueConfigLoader, PyFileConfigLoader, Config |
|
30 | KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError | |
28 | ) |
|
31 | ) | |
29 |
|
32 | |||
30 | from IPython.utils.traitlets import ( |
|
33 | from IPython.utils.traitlets import ( | |
31 | Unicode, List, Int, Enum, Dict |
|
34 | Unicode, List, Int, Enum, Dict, Instance | |
32 | ) |
|
35 | ) | |
33 |
from IPython.utils. |
|
36 | from IPython.utils.importstring import import_item | |
|
37 | from IPython.utils.text import indent, wrap_paragraphs, dedent | |||
|
38 | ||||
|
39 | #----------------------------------------------------------------------------- | |||
|
40 | # function for re-wrapping a helpstring | |||
|
41 | #----------------------------------------------------------------------------- | |||
34 |
|
42 | |||
35 | #----------------------------------------------------------------------------- |
|
43 | #----------------------------------------------------------------------------- | |
36 | # Descriptions for |
|
44 | # Descriptions for the various sections | |
37 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
38 |
|
46 | |||
39 | flag_description = """ |
|
47 | flag_description = """ | |
@@ -41,6 +49,8 b" Flags are command-line arguments passed as '--<flag>'." | |||||
41 | These take no parameters, unlike regular key-value arguments. |
|
49 | These take no parameters, unlike regular key-value arguments. | |
42 | They are typically used for setting boolean flags, or enabling |
|
50 | They are typically used for setting boolean flags, or enabling | |
43 | modes that involve setting multiple options together. |
|
51 | modes that involve setting multiple options together. | |
|
52 | ||||
|
53 | Flags *always* begin with '--', never just one '-'. | |||
44 | """.strip() # trim newlines of front and back |
|
54 | """.strip() # trim newlines of front and back | |
45 |
|
55 | |||
46 | alias_description = """ |
|
56 | alias_description = """ | |
@@ -61,12 +71,16 b' This line is evaluated in Python, so simple expressions are allowed, e.g.' | |||||
61 | #----------------------------------------------------------------------------- |
|
71 | #----------------------------------------------------------------------------- | |
62 |
|
72 | |||
63 |
|
73 | |||
|
74 | class ApplicationError(Exception): | |||
|
75 | pass | |||
|
76 | ||||
|
77 | ||||
64 | class Application(SingletonConfigurable): |
|
78 | class Application(SingletonConfigurable): | |
65 | """A singleton application with full configuration support.""" |
|
79 | """A singleton application with full configuration support.""" | |
66 |
|
80 | |||
67 | # The name of the application, will usually match the name of the command |
|
81 | # The name of the application, will usually match the name of the command | |
68 | # line application |
|
82 | # line application | |
69 |
|
|
83 | name = Unicode(u'application') | |
70 |
|
84 | |||
71 | # The description of the application that is printed at the beginning |
|
85 | # The description of the application that is printed at the beginning | |
72 | # of the help. |
|
86 | # of the help. | |
@@ -85,9 +99,16 b' class Application(SingletonConfigurable):' | |||||
85 | version = Unicode(u'0.0') |
|
99 | version = Unicode(u'0.0') | |
86 |
|
100 | |||
87 | # The log level for the application |
|
101 | # The log level for the application | |
88 | log_level = Enum((0,10,20,30,40,50), default_value=logging.WARN, |
|
102 | log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), | |
89 |
|
|
103 | default_value=logging.WARN, | |
90 | help="Set the log level (0,10,20,30,40,50).") |
|
104 | config=True, | |
|
105 | help="Set the log level by value or name.") | |||
|
106 | def _log_level_changed(self, name, old, new): | |||
|
107 | """Adjust the log level when log_level is set.""" | |||
|
108 | if isinstance(new, basestring): | |||
|
109 | new = getattr(logging, new) | |||
|
110 | self.log_level = new | |||
|
111 | self.log.setLevel(new) | |||
91 |
|
112 | |||
92 | # the alias map for configurables |
|
113 | # the alias map for configurables | |
93 | aliases = Dict(dict(log_level='Application.log_level')) |
|
114 | aliases = Dict(dict(log_level='Application.log_level')) | |
@@ -97,6 +118,25 b' class Application(SingletonConfigurable):' | |||||
97 | # this must be a dict of two-tuples, the first element being the Config/dict |
|
118 | # this must be a dict of two-tuples, the first element being the Config/dict | |
98 | # and the second being the help string for the flag |
|
119 | # and the second being the help string for the flag | |
99 | flags = Dict() |
|
120 | flags = Dict() | |
|
121 | def _flags_changed(self, name, old, new): | |||
|
122 | """ensure flags dict is valid""" | |||
|
123 | for key,value in new.iteritems(): | |||
|
124 | assert len(value) == 2, "Bad flag: %r:%s"%(key,value) | |||
|
125 | assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value) | |||
|
126 | assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value) | |||
|
127 | ||||
|
128 | ||||
|
129 | # subcommands for launching other applications | |||
|
130 | # if this is not empty, this will be a parent Application | |||
|
131 | # this must be a dict of two-tuples, | |||
|
132 | # the first element being the application class/import string | |||
|
133 | # and the second being the help string for the subcommand | |||
|
134 | subcommands = Dict() | |||
|
135 | # parse_command_line will initialize a subapp, if requested | |||
|
136 | subapp = Instance('IPython.config.application.Application', allow_none=True) | |||
|
137 | ||||
|
138 | # extra command-line arguments that don't set config values | |||
|
139 | extra_args = List(Unicode) | |||
100 |
|
140 | |||
101 |
|
141 | |||
102 | def __init__(self, **kwargs): |
|
142 | def __init__(self, **kwargs): | |
@@ -105,13 +145,13 b' class Application(SingletonConfigurable):' | |||||
105 | # options. |
|
145 | # options. | |
106 | self.classes.insert(0, self.__class__) |
|
146 | self.classes.insert(0, self.__class__) | |
107 |
|
147 | |||
108 | # ensure self.flags dict is valid |
|
|||
109 | for key,value in self.flags.iteritems(): |
|
|||
110 | assert len(value) == 2, "Bad flag: %r:%s"%(key,value) |
|
|||
111 | assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value) |
|
|||
112 | assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value) |
|
|||
113 | self.init_logging() |
|
148 | self.init_logging() | |
114 |
|
149 | |||
|
150 | def _config_changed(self, name, old, new): | |||
|
151 | SingletonConfigurable._config_changed(self, name, old, new) | |||
|
152 | self.log.debug('Config changed:') | |||
|
153 | self.log.debug(repr(new)) | |||
|
154 | ||||
115 | def init_logging(self): |
|
155 | def init_logging(self): | |
116 | """Start logging for this application. |
|
156 | """Start logging for this application. | |
117 |
|
157 | |||
@@ -125,69 +165,114 b' class Application(SingletonConfigurable):' | |||||
125 | self._log_formatter = logging.Formatter("[%(name)s] %(message)s") |
|
165 | self._log_formatter = logging.Formatter("[%(name)s] %(message)s") | |
126 | self._log_handler.setFormatter(self._log_formatter) |
|
166 | self._log_handler.setFormatter(self._log_formatter) | |
127 | self.log.addHandler(self._log_handler) |
|
167 | self.log.addHandler(self._log_handler) | |
128 |
|
168 | |||
129 | def _log_level_changed(self, name, old, new): |
|
169 | def initialize(self, argv=None): | |
130 | """Adjust the log level when log_level is set.""" |
|
170 | """Do the basic steps to configure me. | |
131 | self.log.setLevel(new) |
|
171 | ||
|
172 | Override in subclasses. | |||
|
173 | """ | |||
|
174 | self.parse_command_line(argv) | |||
|
175 | ||||
|
176 | ||||
|
177 | def start(self): | |||
|
178 | """Start the app mainloop. | |||
|
179 | ||||
|
180 | Override in subclasses. | |||
|
181 | """ | |||
|
182 | if self.subapp is not None: | |||
|
183 | return self.subapp.start() | |||
132 |
|
184 | |||
133 | def print_alias_help(self): |
|
185 | def print_alias_help(self): | |
134 |
""" |
|
186 | """Print the alias part of the help.""" | |
135 | if not self.aliases: |
|
187 | if not self.aliases: | |
136 | return |
|
188 | return | |
137 |
|
|
189 | ||
138 |
|
|
190 | lines = ['Aliases'] | |
139 | print "-------" |
|
191 | lines.append('-'*len(lines[0])) | |
140 | print self.alias_description |
|
192 | lines.append('') | |
141 |
|
193 | for p in wrap_paragraphs(self.alias_description): | ||
|
194 | lines.append(p) | |||
|
195 | lines.append('') | |||
142 |
|
196 | |||
143 | classdict = {} |
|
197 | classdict = {} | |
144 | for c in self.classes: |
|
198 | for cls in self.classes: | |
145 | classdict[c.__name__] = c |
|
199 | # include all parents (up to, but excluding Configurable) in available names | |
|
200 | for c in cls.mro()[:-3]: | |||
|
201 | classdict[c.__name__] = c | |||
146 |
|
202 | |||
147 | for alias, longname in self.aliases.iteritems(): |
|
203 | for alias, longname in self.aliases.iteritems(): | |
148 | classname, traitname = longname.split('.',1) |
|
204 | classname, traitname = longname.split('.',1) | |
149 | cls = classdict[classname] |
|
205 | cls = classdict[classname] | |
150 |
|
206 | |||
151 | trait = cls.class_traits(config=True)[traitname] |
|
207 | trait = cls.class_traits(config=True)[traitname] | |
152 |
help = trait |
|
208 | help = cls.class_get_trait_help(trait) | |
153 | print alias, "(%s)"%longname, ':', trait.__class__.__name__ |
|
209 | help = help.replace(longname, "%s (%s)"%(alias, longname), 1) | |
154 |
|
|
210 | lines.append(help) | |
155 | print indent(help) |
|
211 | lines.append('') | |
156 |
|
212 | print '\n'.join(lines) | ||
157 |
|
213 | |||
158 | def print_flag_help(self): |
|
214 | def print_flag_help(self): | |
159 |
""" |
|
215 | """Print the flag part of the help.""" | |
160 | if not self.flags: |
|
216 | if not self.flags: | |
161 | return |
|
217 | return | |
162 |
|
218 | |||
163 |
|
|
219 | lines = ['Flags'] | |
164 | print "-----" |
|
220 | lines.append('-'*len(lines[0])) | |
165 | print self.flag_description |
|
221 | lines.append('') | |
166 |
|
222 | for p in wrap_paragraphs(self.flag_description): | ||
|
223 | lines.append(p) | |||
|
224 | lines.append('') | |||
167 |
|
225 | |||
168 | for m, (cfg,help) in self.flags.iteritems(): |
|
226 | for m, (cfg,help) in self.flags.iteritems(): | |
169 |
|
|
227 | lines.append('--'+m) | |
170 |
|
|
228 | lines.append(indent(dedent(help.strip()))) | |
171 |
|
229 | lines.append('') | ||
|
230 | print '\n'.join(lines) | |||
172 |
|
231 | |||
173 |
def print_ |
|
232 | def print_subcommands(self): | |
174 | """Print the help for each Configurable class in self.classes.""" |
|
233 | """Print the subcommand part of the help.""" | |
|
234 | if not self.subcommands: | |||
|
235 | return | |||
|
236 | ||||
|
237 | lines = ["Subcommands"] | |||
|
238 | lines.append('-'*len(lines[0])) | |||
|
239 | for subc, (cls,help) in self.subcommands.iteritems(): | |||
|
240 | lines.append("%s : %s"%(subc, cls)) | |||
|
241 | if help: | |||
|
242 | lines.append(indent(dedent(help.strip()))) | |||
|
243 | lines.append('') | |||
|
244 | print '\n'.join(lines) | |||
|
245 | ||||
|
246 | def print_help(self, classes=False): | |||
|
247 | """Print the help for each Configurable class in self.classes. | |||
|
248 | ||||
|
249 | If classes=False (the default), only flags and aliases are printed. | |||
|
250 | """ | |||
|
251 | self.print_subcommands() | |||
175 | self.print_flag_help() |
|
252 | self.print_flag_help() | |
176 | self.print_alias_help() |
|
253 | self.print_alias_help() | |
177 | if self.classes: |
|
|||
178 | print "Class parameters" |
|
|||
179 | print "----------------" |
|
|||
180 | print self.keyvalue_description |
|
|||
181 |
|
||||
182 |
|
254 | |||
183 |
f |
|
255 | if classes: | |
184 |
|
|
256 | if self.classes: | |
|
257 | print "Class parameters" | |||
|
258 | print "----------------" | |||
|
259 | ||||
|
260 | for p in wrap_paragraphs(self.keyvalue_description): | |||
|
261 | print p | |||
|
262 | ||||
|
263 | ||||
|
264 | for cls in self.classes: | |||
|
265 | cls.class_print_help() | |||
|
266 | ||||
|
267 | else: | |||
|
268 | print "To see all available configurables, use `--help-all`" | |||
185 |
|
269 | |||
186 |
|
270 | |||
187 | def print_description(self): |
|
271 | def print_description(self): | |
188 | """Print the application description.""" |
|
272 | """Print the application description.""" | |
189 |
|
|
273 | for p in wrap_paragraphs(self.description): | |
190 |
|
274 | print p | ||
|
275 | ||||
191 |
|
276 | |||
192 | def print_version(self): |
|
277 | def print_version(self): | |
193 | """Print the version string.""" |
|
278 | """Print the version string.""" | |
@@ -201,30 +286,108 b' class Application(SingletonConfigurable):' | |||||
201 | newconfig._merge(config) |
|
286 | newconfig._merge(config) | |
202 | # Save the combined config as self.config, which triggers the traits |
|
287 | # Save the combined config as self.config, which triggers the traits | |
203 | # events. |
|
288 | # events. | |
204 | self.config = config |
|
289 | self.config = newconfig | |
205 |
|
290 | |||
|
291 | def initialize_subcommand(self, subc, argv=None): | |||
|
292 | """Initialize a subcommand with argv.""" | |||
|
293 | subapp,help = self.subcommands.get(subc) | |||
|
294 | ||||
|
295 | if isinstance(subapp, basestring): | |||
|
296 | subapp = import_item(subapp) | |||
|
297 | ||||
|
298 | # clear existing instances | |||
|
299 | self.__class__.clear_instance() | |||
|
300 | # instantiate | |||
|
301 | self.subapp = subapp.instance() | |||
|
302 | # and initialize subapp | |||
|
303 | self.subapp.initialize(argv) | |||
|
304 | ||||
206 | def parse_command_line(self, argv=None): |
|
305 | def parse_command_line(self, argv=None): | |
207 | """Parse the command line arguments.""" |
|
306 | """Parse the command line arguments.""" | |
208 | argv = sys.argv[1:] if argv is None else argv |
|
307 | argv = sys.argv[1:] if argv is None else argv | |
209 |
|
308 | |||
210 | if '-h' in argv or '--help' in argv: |
|
309 | if self.subcommands and len(argv) > 0: | |
|
310 | # we have subcommands, and one may have been specified | |||
|
311 | subc, subargv = argv[0], argv[1:] | |||
|
312 | if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands: | |||
|
313 | # it's a subcommand, and *not* a flag or class parameter | |||
|
314 | return self.initialize_subcommand(subc, subargv) | |||
|
315 | ||||
|
316 | if '-h' in argv or '--help' in argv or '--help-all' in argv: | |||
211 | self.print_description() |
|
317 | self.print_description() | |
212 | self.print_help() |
|
318 | self.print_help('--help-all' in argv) | |
213 |
s |
|
319 | self.exit(0) | |
214 |
|
320 | |||
215 | if '--version' in argv: |
|
321 | if '--version' in argv: | |
216 | self.print_version() |
|
322 | self.print_version() | |
217 |
s |
|
323 | self.exit(0) | |
218 |
|
324 | |||
219 | loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases, |
|
325 | loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases, | |
220 | flags=self.flags) |
|
326 | flags=self.flags) | |
221 | config = loader.load_config() |
|
327 | try: | |
|
328 | config = loader.load_config() | |||
|
329 | except ArgumentError as e: | |||
|
330 | self.log.fatal(str(e)) | |||
|
331 | self.print_description() | |||
|
332 | self.print_help() | |||
|
333 | self.exit(1) | |||
222 | self.update_config(config) |
|
334 | self.update_config(config) | |
|
335 | # store unparsed args in extra_args | |||
|
336 | self.extra_args = loader.extra_args | |||
223 |
|
337 | |||
224 | def load_config_file(self, filename, path=None): |
|
338 | def load_config_file(self, filename, path=None): | |
225 | """Load a .py based config file by filename and path.""" |
|
339 | """Load a .py based config file by filename and path.""" | |
226 | # TODO: this raises IOError if filename does not exist. |
|
|||
227 | loader = PyFileConfigLoader(filename, path=path) |
|
340 | loader = PyFileConfigLoader(filename, path=path) | |
228 | config = loader.load_config() |
|
341 | config = loader.load_config() | |
229 | self.update_config(config) |
|
342 | self.update_config(config) | |
|
343 | ||||
|
344 | def generate_config_file(self): | |||
|
345 | """generate default config file from Configurables""" | |||
|
346 | lines = ["# Configuration file for %s."%self.name] | |||
|
347 | lines.append('') | |||
|
348 | lines.append('c = get_config()') | |||
|
349 | lines.append('') | |||
|
350 | for cls in self.classes: | |||
|
351 | lines.append(cls.class_config_section()) | |||
|
352 | return '\n'.join(lines) | |||
|
353 | ||||
|
354 | def exit(self, exit_status=0): | |||
|
355 | self.log.debug("Exiting application: %s" % self.name) | |||
|
356 | sys.exit(exit_status) | |||
|
357 | ||||
|
358 | #----------------------------------------------------------------------------- | |||
|
359 | # utility functions, for convenience | |||
|
360 | #----------------------------------------------------------------------------- | |||
|
361 | ||||
|
362 | def boolean_flag(name, configurable, set_help='', unset_help=''): | |||
|
363 | """Helper for building basic --trait, --no-trait flags. | |||
|
364 | ||||
|
365 | Parameters | |||
|
366 | ---------- | |||
|
367 | ||||
|
368 | name : str | |||
|
369 | The name of the flag. | |||
|
370 | configurable : str | |||
|
371 | The 'Class.trait' string of the trait to be set/unset with the flag | |||
|
372 | set_help : unicode | |||
|
373 | help string for --name flag | |||
|
374 | unset_help : unicode | |||
|
375 | help string for --no-name flag | |||
|
376 | ||||
|
377 | Returns | |||
|
378 | ------- | |||
|
379 | ||||
|
380 | cfg : dict | |||
|
381 | A dict with two keys: 'name', and 'no-name', for setting and unsetting | |||
|
382 | the trait, respectively. | |||
|
383 | """ | |||
|
384 | # default helpstrings | |||
|
385 | set_help = set_help or "set %s=True"%configurable | |||
|
386 | unset_help = unset_help or "set %s=False"%configurable | |||
|
387 | ||||
|
388 | cls,trait = configurable.split('.') | |||
|
389 | ||||
|
390 | setter = {cls : {trait : True}} | |||
|
391 | unsetter = {cls : {trait : False}} | |||
|
392 | return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)} | |||
230 |
|
393 |
@@ -7,10 +7,11 b' Authors:' | |||||
7 |
|
7 | |||
8 | * Brian Granger |
|
8 | * Brian Granger | |
9 | * Fernando Perez |
|
9 | * Fernando Perez | |
|
10 | * Min RK | |||
10 | """ |
|
11 | """ | |
11 |
|
12 | |||
12 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
13 |
# Copyright (C) 2008-201 |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
14 | # |
|
15 | # | |
15 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
16 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
@@ -20,12 +21,12 b' Authors:' | |||||
20 | # Imports |
|
21 | # Imports | |
21 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
22 |
|
23 | |||
23 | from copy import deepcopy |
|
|||
24 | import datetime |
|
24 | import datetime | |
|
25 | from copy import deepcopy | |||
25 |
|
26 | |||
26 | from loader import Config |
|
27 | from loader import Config | |
27 | from IPython.utils.traitlets import HasTraits, Instance |
|
28 | from IPython.utils.traitlets import HasTraits, Instance | |
28 | from IPython.utils.text import indent |
|
29 | from IPython.utils.text import indent, wrap_paragraphs | |
29 |
|
30 | |||
30 |
|
31 | |||
31 | #----------------------------------------------------------------------------- |
|
32 | #----------------------------------------------------------------------------- | |
@@ -44,14 +45,13 b' class MultipleInstanceError(ConfigurableError):' | |||||
44 | # Configurable implementation |
|
45 | # Configurable implementation | |
45 | #----------------------------------------------------------------------------- |
|
46 | #----------------------------------------------------------------------------- | |
46 |
|
47 | |||
47 |
|
||||
48 | class Configurable(HasTraits): |
|
48 | class Configurable(HasTraits): | |
49 |
|
49 | |||
50 | config = Instance(Config,(),{}) |
|
50 | config = Instance(Config,(),{}) | |
51 | created = None |
|
51 | created = None | |
52 |
|
52 | |||
53 | def __init__(self, **kwargs): |
|
53 | def __init__(self, **kwargs): | |
54 | """Create a conigurable given a config config. |
|
54 | """Create a configurable given a config config. | |
55 |
|
55 | |||
56 | Parameters |
|
56 | Parameters | |
57 | ---------- |
|
57 | ---------- | |
@@ -146,18 +146,72 b' class Configurable(HasTraits):' | |||||
146 | final_help = [] |
|
146 | final_help = [] | |
147 | final_help.append(u'%s options' % cls.__name__) |
|
147 | final_help.append(u'%s options' % cls.__name__) | |
148 | final_help.append(len(final_help[0])*u'-') |
|
148 | final_help.append(len(final_help[0])*u'-') | |
149 |
for k, |
|
149 | for k,v in cls.class_traits(config=True).iteritems(): | |
150 |
help = v |
|
150 | help = cls.class_get_trait_help(v) | |
151 | header = "%s.%s : %s" % (cls.__name__, k, v.__class__.__name__) |
|
151 | final_help.append(help) | |
152 | final_help.append(header) |
|
|||
153 | if help is not None: |
|
|||
154 | final_help.append(indent(help)) |
|
|||
155 | return '\n'.join(final_help) |
|
152 | return '\n'.join(final_help) | |
|
153 | ||||
|
154 | @classmethod | |||
|
155 | def class_get_trait_help(cls, trait): | |||
|
156 | """Get the help string for a single trait.""" | |||
|
157 | lines = [] | |||
|
158 | header = "%s.%s : %s" % (cls.__name__, trait.name, trait.__class__.__name__) | |||
|
159 | lines.append(header) | |||
|
160 | try: | |||
|
161 | dvr = repr(trait.get_default_value()) | |||
|
162 | except Exception: | |||
|
163 | dvr = None # ignore defaults we can't construct | |||
|
164 | if dvr is not None: | |||
|
165 | if len(dvr) > 64: | |||
|
166 | dvr = dvr[:61]+'...' | |||
|
167 | lines.append(indent('Default: %s'%dvr, 4)) | |||
|
168 | if 'Enum' in trait.__class__.__name__: | |||
|
169 | # include Enum choices | |||
|
170 | lines.append(indent('Choices: %r'%(trait.values,))) | |||
|
171 | ||||
|
172 | help = trait.get_metadata('help') | |||
|
173 | if help is not None: | |||
|
174 | help = '\n'.join(wrap_paragraphs(help, 76)) | |||
|
175 | lines.append(indent(help, 4)) | |||
|
176 | return '\n'.join(lines) | |||
156 |
|
177 | |||
157 | @classmethod |
|
178 | @classmethod | |
158 | def class_print_help(cls): |
|
179 | def class_print_help(cls): | |
|
180 | """Get the help string for a single trait and print it.""" | |||
159 | print cls.class_get_help() |
|
181 | print cls.class_get_help() | |
160 |
|
182 | |||
|
183 | @classmethod | |||
|
184 | def class_config_section(cls): | |||
|
185 | """Get the config class config section""" | |||
|
186 | def c(s): | |||
|
187 | """return a commented, wrapped block.""" | |||
|
188 | s = '\n\n'.join(wrap_paragraphs(s, 78)) | |||
|
189 | ||||
|
190 | return '# ' + s.replace('\n', '\n# ') | |||
|
191 | ||||
|
192 | # section header | |||
|
193 | breaker = '#' + '-'*78 | |||
|
194 | s = "# %s configuration"%cls.__name__ | |||
|
195 | lines = [breaker, s, breaker, ''] | |||
|
196 | # get the description trait | |||
|
197 | desc = cls.class_traits().get('description') | |||
|
198 | if desc: | |||
|
199 | desc = desc.default_value | |||
|
200 | else: | |||
|
201 | # no description trait, use __doc__ | |||
|
202 | desc = getattr(cls, '__doc__', '') | |||
|
203 | if desc: | |||
|
204 | lines.append(c(desc)) | |||
|
205 | lines.append('') | |||
|
206 | ||||
|
207 | for name,trait in cls.class_traits(config=True).iteritems(): | |||
|
208 | help = trait.get_metadata('help') or '' | |||
|
209 | lines.append(c(help)) | |||
|
210 | lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value())) | |||
|
211 | lines.append('') | |||
|
212 | return '\n'.join(lines) | |||
|
213 | ||||
|
214 | ||||
161 |
|
215 | |||
162 | class SingletonConfigurable(Configurable): |
|
216 | class SingletonConfigurable(Configurable): | |
163 | """A configurable that only allows one instance. |
|
217 | """A configurable that only allows one instance. | |
@@ -168,7 +222,32 b' class SingletonConfigurable(Configurable):' | |||||
168 | """ |
|
222 | """ | |
169 |
|
223 | |||
170 | _instance = None |
|
224 | _instance = None | |
171 |
|
225 | |||
|
226 | @classmethod | |||
|
227 | def _walk_mro(cls): | |||
|
228 | """Walk the cls.mro() for parent classes that are also singletons | |||
|
229 | ||||
|
230 | For use in instance() | |||
|
231 | """ | |||
|
232 | ||||
|
233 | for subclass in cls.mro(): | |||
|
234 | if issubclass(cls, subclass) and \ | |||
|
235 | issubclass(subclass, SingletonConfigurable) and \ | |||
|
236 | subclass != SingletonConfigurable: | |||
|
237 | yield subclass | |||
|
238 | ||||
|
239 | @classmethod | |||
|
240 | def clear_instance(cls): | |||
|
241 | """unset _instance for this class and singleton parents. | |||
|
242 | """ | |||
|
243 | if not cls.initialized(): | |||
|
244 | return | |||
|
245 | for subclass in cls._walk_mro(): | |||
|
246 | if isinstance(subclass._instance, cls): | |||
|
247 | # only clear instances that are instances | |||
|
248 | # of the calling class | |||
|
249 | subclass._instance = None | |||
|
250 | ||||
172 | @classmethod |
|
251 | @classmethod | |
173 | def instance(cls, *args, **kwargs): |
|
252 | def instance(cls, *args, **kwargs): | |
174 | """Returns a global instance of this class. |
|
253 | """Returns a global instance of this class. | |
@@ -202,14 +281,10 b' class SingletonConfigurable(Configurable):' | |||||
202 | if cls._instance is None: |
|
281 | if cls._instance is None: | |
203 | inst = cls(*args, **kwargs) |
|
282 | inst = cls(*args, **kwargs) | |
204 | # Now make sure that the instance will also be returned by |
|
283 | # Now make sure that the instance will also be returned by | |
205 |
# |
|
284 | # parent classes' _instance attribute. | |
206 | for subclass in cls.mro(): |
|
285 | for subclass in cls._walk_mro(): | |
207 |
|
|
286 | subclass._instance = inst | |
208 | issubclass(subclass, SingletonConfigurable) and \ |
|
287 | ||
209 | subclass != SingletonConfigurable: |
|
|||
210 | subclass._instance = inst |
|
|||
211 | else: |
|
|||
212 | break |
|
|||
213 | if isinstance(cls._instance, cls): |
|
288 | if isinstance(cls._instance, cls): | |
214 | return cls._instance |
|
289 | return cls._instance | |
215 | else: |
|
290 | else: | |
@@ -223,3 +298,18 b' class SingletonConfigurable(Configurable):' | |||||
223 | """Has an instance been created?""" |
|
298 | """Has an instance been created?""" | |
224 | return hasattr(cls, "_instance") and cls._instance is not None |
|
299 | return hasattr(cls, "_instance") and cls._instance is not None | |
225 |
|
300 | |||
|
301 | ||||
|
302 | class LoggingConfigurable(Configurable): | |||
|
303 | """A parent class for Configurables that log. | |||
|
304 | ||||
|
305 | Subclasses have a log trait, and the default behavior | |||
|
306 | is to get the logger from the currently running Application | |||
|
307 | via Application.instance().log. | |||
|
308 | """ | |||
|
309 | ||||
|
310 | log = Instance('logging.Logger') | |||
|
311 | def _log_default(self): | |||
|
312 | from IPython.config.application import Application | |||
|
313 | return Application.instance().log | |||
|
314 | ||||
|
315 |
@@ -4,10 +4,11 b' Authors' | |||||
4 | ------- |
|
4 | ------- | |
5 | * Brian Granger |
|
5 | * Brian Granger | |
6 | * Fernando Perez |
|
6 | * Fernando Perez | |
|
7 | * Min RK | |||
7 | """ |
|
8 | """ | |
8 |
|
9 | |||
9 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
10 |
# Copyright (C) 2008-20 |
|
11 | # Copyright (C) 2008-2011 The IPython Development Team | |
11 | # |
|
12 | # | |
12 | # Distributed under the terms of the BSD License. The full license is in |
|
13 | # Distributed under the terms of the BSD License. The full license is in | |
13 | # the file COPYING, distributed as part of this software. |
|
14 | # the file COPYING, distributed as part of this software. | |
@@ -22,7 +23,7 b' import re' | |||||
22 | import sys |
|
23 | import sys | |
23 |
|
24 | |||
24 | from IPython.external import argparse |
|
25 | from IPython.external import argparse | |
25 | from IPython.utils.path import filefind |
|
26 | from IPython.utils.path import filefind, get_ipython_dir | |
26 |
|
27 | |||
27 | #----------------------------------------------------------------------------- |
|
28 | #----------------------------------------------------------------------------- | |
28 | # Exceptions |
|
29 | # Exceptions | |
@@ -36,6 +37,9 b' class ConfigError(Exception):' | |||||
36 | class ConfigLoaderError(ConfigError): |
|
37 | class ConfigLoaderError(ConfigError): | |
37 | pass |
|
38 | pass | |
38 |
|
39 | |||
|
40 | class ArgumentError(ConfigLoaderError): | |||
|
41 | pass | |||
|
42 | ||||
39 | #----------------------------------------------------------------------------- |
|
43 | #----------------------------------------------------------------------------- | |
40 | # Argparse fix |
|
44 | # Argparse fix | |
41 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
@@ -265,23 +269,40 b' class PyFileConfigLoader(FileConfigLoader):' | |||||
265 | def _read_file_as_dict(self): |
|
269 | def _read_file_as_dict(self): | |
266 | """Load the config file into self.config, with recursive loading.""" |
|
270 | """Load the config file into self.config, with recursive loading.""" | |
267 | # This closure is made available in the namespace that is used |
|
271 | # This closure is made available in the namespace that is used | |
268 |
# to exec the config file. |
|
272 | # to exec the config file. It allows users to call | |
269 | # load_subconfig('myconfig.py') to load config files recursively. |
|
273 | # load_subconfig('myconfig.py') to load config files recursively. | |
270 | # It needs to be a closure because it has references to self.path |
|
274 | # It needs to be a closure because it has references to self.path | |
271 | # and self.config. The sub-config is loaded with the same path |
|
275 | # and self.config. The sub-config is loaded with the same path | |
272 | # as the parent, but it uses an empty config which is then merged |
|
276 | # as the parent, but it uses an empty config which is then merged | |
273 | # with the parents. |
|
277 | # with the parents. | |
274 | def load_subconfig(fname): |
|
278 | ||
275 | loader = PyFileConfigLoader(fname, self.path) |
|
279 | # If a profile is specified, the config file will be loaded | |
|
280 | # from that profile | |||
|
281 | ||||
|
282 | def load_subconfig(fname, profile=None): | |||
|
283 | # import here to prevent circular imports | |||
|
284 | from IPython.core.profiledir import ProfileDir, ProfileDirError | |||
|
285 | if profile is not None: | |||
|
286 | try: | |||
|
287 | profile_dir = ProfileDir.find_profile_dir_by_name( | |||
|
288 | get_ipython_dir(), | |||
|
289 | profile, | |||
|
290 | ) | |||
|
291 | except ProfileDirError: | |||
|
292 | return | |||
|
293 | path = profile_dir.location | |||
|
294 | else: | |||
|
295 | path = self.path | |||
|
296 | loader = PyFileConfigLoader(fname, path) | |||
276 | try: |
|
297 | try: | |
277 | sub_config = loader.load_config() |
|
298 | sub_config = loader.load_config() | |
278 | except IOError: |
|
299 | except IOError: | |
279 | # Pass silently if the sub config is not there. This happens |
|
300 | # Pass silently if the sub config is not there. This happens | |
280 |
# when a user |
|
301 | # when a user s using a profile, but not the default config. | |
281 | pass |
|
302 | pass | |
282 | else: |
|
303 | else: | |
283 | self.config._merge(sub_config) |
|
304 | self.config._merge(sub_config) | |
284 |
|
305 | |||
285 | # Again, this needs to be a closure and should be used in config |
|
306 | # Again, this needs to be a closure and should be used in config | |
286 | # files to get the config being loaded. |
|
307 | # files to get the config being loaded. | |
287 | def get_config(): |
|
308 | def get_config(): | |
@@ -304,7 +325,7 b' class CommandLineConfigLoader(ConfigLoader):' | |||||
304 | here. |
|
325 | here. | |
305 | """ |
|
326 | """ | |
306 |
|
327 | |||
307 |
kv_pattern = re.compile(r'[A-Za-z]\w*(\.\w+)*\=. |
|
328 | kv_pattern = re.compile(r'[A-Za-z]\w*(\.\w+)*\=.*') | |
308 | flag_pattern = re.compile(r'\-\-\w+(\-\w)*') |
|
329 | flag_pattern = re.compile(r'\-\-\w+(\-\w)*') | |
309 |
|
330 | |||
310 | class KeyValueConfigLoader(CommandLineConfigLoader): |
|
331 | class KeyValueConfigLoader(CommandLineConfigLoader): | |
@@ -346,15 +367,27 b' class KeyValueConfigLoader(CommandLineConfigLoader):' | |||||
346 | >>> cl.load_config(["foo='bar'","A.name='brian'","B.number=0"]) |
|
367 | >>> cl.load_config(["foo='bar'","A.name='brian'","B.number=0"]) | |
347 | {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'} |
|
368 | {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'} | |
348 | """ |
|
369 | """ | |
|
370 | self.clear() | |||
349 | if argv is None: |
|
371 | if argv is None: | |
350 | argv = sys.argv[1:] |
|
372 | argv = sys.argv[1:] | |
351 | self.argv = argv |
|
373 | self.argv = argv | |
352 | self.aliases = aliases or {} |
|
374 | self.aliases = aliases or {} | |
353 | self.flags = flags or {} |
|
375 | self.flags = flags or {} | |
|
376 | ||||
|
377 | ||||
|
378 | def clear(self): | |||
|
379 | super(KeyValueConfigLoader, self).clear() | |||
|
380 | self.extra_args = [] | |||
|
381 | ||||
354 |
|
382 | |||
355 | def load_config(self, argv=None, aliases=None, flags=None): |
|
383 | def load_config(self, argv=None, aliases=None, flags=None): | |
356 | """Parse the configuration and generate the Config object. |
|
384 | """Parse the configuration and generate the Config object. | |
357 |
|
385 | |||
|
386 | After loading, any arguments that are not key-value or | |||
|
387 | flags will be stored in self.extra_args - a list of | |||
|
388 | unparsed command-line arguments. This is used for | |||
|
389 | arguments such as input files or subcommands. | |||
|
390 | ||||
358 | Parameters |
|
391 | Parameters | |
359 | ---------- |
|
392 | ---------- | |
360 | argv : list, optional |
|
393 | argv : list, optional | |
@@ -379,7 +412,7 b' class KeyValueConfigLoader(CommandLineConfigLoader):' | |||||
379 | aliases = self.aliases |
|
412 | aliases = self.aliases | |
380 | if flags is None: |
|
413 | if flags is None: | |
381 | flags = self.flags |
|
414 | flags = self.flags | |
382 |
|
415 | |||
383 | for item in argv: |
|
416 | for item in argv: | |
384 | if kv_pattern.match(item): |
|
417 | if kv_pattern.match(item): | |
385 | lhs,rhs = item.split('=',1) |
|
418 | lhs,rhs = item.split('=',1) | |
@@ -403,14 +436,21 b' class KeyValueConfigLoader(CommandLineConfigLoader):' | |||||
403 | m = item[2:] |
|
436 | m = item[2:] | |
404 | cfg,_ = flags.get(m, (None,None)) |
|
437 | cfg,_ = flags.get(m, (None,None)) | |
405 | if cfg is None: |
|
438 | if cfg is None: | |
406 |
raise |
|
439 | raise ArgumentError("Unrecognized flag: %r"%item) | |
407 | elif isinstance(cfg, (dict, Config)): |
|
440 | elif isinstance(cfg, (dict, Config)): | |
408 |
# |
|
441 | # don't clobber whole config sections, update | |
409 |
|
|
442 | # each section from config: | |
|
443 | for sec,c in cfg.iteritems(): | |||
|
444 | self.config[sec].update(c) | |||
410 | else: |
|
445 | else: | |
411 | raise ValueError("Invalid flag: %r"%flag) |
|
446 | raise ValueError("Invalid flag: %r"%flag) | |
|
447 | elif item.startswith('-'): | |||
|
448 | # this shouldn't ever be valid | |||
|
449 | raise ArgumentError("Invalid argument: %r"%item) | |||
412 | else: |
|
450 | else: | |
413 | raise ValueError("Invalid argument: %r"%item) |
|
451 | # keep all args that aren't valid in a list, | |
|
452 | # in case our parent knows what to do with them. | |||
|
453 | self.extra_args.append(item) | |||
414 | return self.config |
|
454 | return self.config | |
415 |
|
455 | |||
416 | class ArgParseConfigLoader(CommandLineConfigLoader): |
|
456 | class ArgParseConfigLoader(CommandLineConfigLoader): |
@@ -1,24 +1,25 b'' | |||||
1 | c = get_config() |
|
1 | c = get_config() | |
|
2 | app = c.InteractiveShellApp | |||
2 |
|
3 | |||
3 | # This can be used at any point in a config file to load a sub config |
|
4 | # This can be used at any point in a config file to load a sub config | |
4 | # and merge it into the current one. |
|
5 | # and merge it into the current one. | |
5 | load_subconfig('ipython_config.py') |
|
6 | load_subconfig('ipython_config.py', profile='default') | |
6 |
|
7 | |||
7 | lines = """ |
|
8 | lines = """ | |
8 |
from IPython. |
|
9 | from IPython.parallel import * | |
9 | """ |
|
10 | """ | |
10 |
|
11 | |||
11 | # You have to make sure that attributes that are containers already |
|
12 | # You have to make sure that attributes that are containers already | |
12 | # exist before using them. Simple assigning a new list will override |
|
13 | # exist before using them. Simple assigning a new list will override | |
13 | # all previous values. |
|
14 | # all previous values. | |
14 |
if hasattr( |
|
15 | if hasattr(app, 'exec_lines'): | |
15 |
|
|
16 | app.exec_lines.append(lines) | |
16 | else: |
|
17 | else: | |
17 |
|
|
18 | app.exec_lines = [lines] | |
18 |
|
19 | |||
19 | # Load the parallelmagic extension to enable %result, %px, %autopx magics. |
|
20 | # Load the parallelmagic extension to enable %result, %px, %autopx magics. | |
20 |
if hasattr( |
|
21 | if hasattr(app, 'extensions'): | |
21 |
|
|
22 | app.extensions.append('parallelmagic') | |
22 | else: |
|
23 | else: | |
23 |
|
|
24 | app.extensions = ['parallelmagic'] | |
24 |
|
25 |
@@ -1,8 +1,9 b'' | |||||
1 | c = get_config() |
|
1 | c = get_config() | |
|
2 | app = c.InteractiveShellApp | |||
2 |
|
3 | |||
3 | # This can be used at any point in a config file to load a sub config |
|
4 | # This can be used at any point in a config file to load a sub config | |
4 | # and merge it into the current one. |
|
5 | # and merge it into the current one. | |
5 | load_subconfig('ipython_config.py') |
|
6 | load_subconfig('ipython_config.py', profile='default') | |
6 |
|
7 | |||
7 | lines = """ |
|
8 | lines = """ | |
8 | import cmath |
|
9 | import cmath | |
@@ -12,8 +13,9 b' from math import *' | |||||
12 | # You have to make sure that attributes that are containers already |
|
13 | # You have to make sure that attributes that are containers already | |
13 | # exist before using them. Simple assigning a new list will override |
|
14 | # exist before using them. Simple assigning a new list will override | |
14 | # all previous values. |
|
15 | # all previous values. | |
15 | if hasattr(c.Global, 'exec_lines'): |
|
16 | ||
16 | c.Global.exec_lines.append(lines) |
|
17 | if hasattr(app, 'exec_lines'): | |
|
18 | app.exec_lines.append(lines) | |||
17 | else: |
|
19 | else: | |
18 |
|
|
20 | app.exec_lines = [lines] | |
19 |
|
21 |
@@ -1,13 +1,14 b'' | |||||
1 | c = get_config() |
|
1 | c = get_config() | |
|
2 | app = c.InteractiveShellApp | |||
2 |
|
3 | |||
3 | # This can be used at any point in a config file to load a sub config |
|
4 | # This can be used at any point in a config file to load a sub config | |
4 | # and merge it into the current one. |
|
5 | # and merge it into the current one. | |
5 | load_subconfig('ipython_config.py') |
|
6 | load_subconfig('ipython_config.py', profile='default') | |
6 |
|
7 | |||
7 | lines = """ |
|
8 | lines = """ | |
8 | import matplotlib |
|
9 | import matplotlib | |
9 | %gui -a wx |
|
10 | %gui qt | |
10 |
matplotlib.use(' |
|
11 | matplotlib.use('qtagg') | |
11 | matplotlib.interactive(True) |
|
12 | matplotlib.interactive(True) | |
12 | from matplotlib import pyplot as plt |
|
13 | from matplotlib import pyplot as plt | |
13 | from matplotlib.pyplot import * |
|
14 | from matplotlib.pyplot import * | |
@@ -16,7 +17,7 b' from matplotlib.pyplot import *' | |||||
16 | # You have to make sure that attributes that are containers already |
|
17 | # You have to make sure that attributes that are containers already | |
17 | # exist before using them. Simple assigning a new list will override |
|
18 | # exist before using them. Simple assigning a new list will override | |
18 | # all previous values. |
|
19 | # all previous values. | |
19 |
if hasattr( |
|
20 | if hasattr(app, 'exec_lines'): | |
20 |
|
|
21 | app.exec_lines.append(lines) | |
21 | else: |
|
22 | else: | |
22 |
|
|
23 | app.exec_lines = [lines] No newline at end of file |
@@ -1,8 +1,9 b'' | |||||
1 | c = get_config() |
|
1 | c = get_config() | |
|
2 | app = c.InteractiveShellApp | |||
2 |
|
3 | |||
3 | # This can be used at any point in a config file to load a sub config |
|
4 | # This can be used at any point in a config file to load a sub config | |
4 | # and merge it into the current one. |
|
5 | # and merge it into the current one. | |
5 | load_subconfig('ipython_config.py') |
|
6 | load_subconfig('ipython_config.py', profile='default') | |
6 |
|
7 | |||
7 | c.InteractiveShell.prompt_in1 = '\C_LightGreen\u@\h\C_LightBlue[\C_LightCyan\Y1\C_LightBlue]\C_Green|\#> ' |
|
8 | c.InteractiveShell.prompt_in1 = '\C_LightGreen\u@\h\C_LightBlue[\C_LightCyan\Y1\C_LightBlue]\C_Green|\#> ' | |
8 | c.InteractiveShell.prompt_in2 = '\C_Green|\C_LightGreen\D\C_Green> ' |
|
9 | c.InteractiveShell.prompt_in2 = '\C_Green|\C_LightGreen\D\C_Green> ' | |
@@ -23,7 +24,7 b' lines = """' | |||||
23 | # You have to make sure that attributes that are containers already |
|
24 | # You have to make sure that attributes that are containers already | |
24 | # exist before using them. Simple assigning a new list will override |
|
25 | # exist before using them. Simple assigning a new list will override | |
25 | # all previous values. |
|
26 | # all previous values. | |
26 |
if hasattr( |
|
27 | if hasattr(app, 'exec_lines'): | |
27 |
|
|
28 | app.exec_lines.append(lines) | |
28 | else: |
|
29 | else: | |
29 |
|
|
30 | app.exec_lines = [lines] No newline at end of file |
@@ -1,8 +1,9 b'' | |||||
1 | c = get_config() |
|
1 | c = get_config() | |
|
2 | app = c.InteractiveShellApp | |||
2 |
|
3 | |||
3 | # This can be used at any point in a config file to load a sub config |
|
4 | # This can be used at any point in a config file to load a sub config | |
4 | # and merge it into the current one. |
|
5 | # and merge it into the current one. | |
5 | load_subconfig('ipython_config.py') |
|
6 | load_subconfig('ipython_config.py', profile='default') | |
6 |
|
7 | |||
7 | lines = """ |
|
8 | lines = """ | |
8 | from __future__ import division |
|
9 | from __future__ import division | |
@@ -16,14 +17,14 b" f, g, h = map(Function, 'fgh')" | |||||
16 | # exist before using them. Simple assigning a new list will override |
|
17 | # exist before using them. Simple assigning a new list will override | |
17 | # all previous values. |
|
18 | # all previous values. | |
18 |
|
19 | |||
19 |
if hasattr( |
|
20 | if hasattr(app, 'exec_lines'): | |
20 |
|
|
21 | app.exec_lines.append(lines) | |
21 | else: |
|
22 | else: | |
22 |
|
|
23 | app.exec_lines = [lines] | |
23 |
|
24 | |||
24 | # Load the sympy_printing extension to enable nice printing of sympy expr's. |
|
25 | # Load the sympy_printing extension to enable nice printing of sympy expr's. | |
25 |
if hasattr( |
|
26 | if hasattr(app, 'extensions'): | |
26 |
|
|
27 | app.extensions.append('sympyprinting') | |
27 | else: |
|
28 | else: | |
28 |
|
|
29 | app.extensions = ['sympyprinting'] | |
29 |
|
30 |
@@ -42,12 +42,13 b' class Foo(Configurable):' | |||||
42 |
|
42 | |||
43 | class Bar(Configurable): |
|
43 | class Bar(Configurable): | |
44 |
|
44 | |||
|
45 | b = Int(0, config=True, help="The integer b.") | |||
45 | enabled = Bool(True, config=True, help="Enable bar.") |
|
46 | enabled = Bool(True, config=True, help="Enable bar.") | |
46 |
|
47 | |||
47 |
|
48 | |||
48 | class MyApp(Application): |
|
49 | class MyApp(Application): | |
49 |
|
50 | |||
50 |
|
|
51 | name = Unicode(u'myapp') | |
51 | running = Bool(False, config=True, |
|
52 | running = Bool(False, config=True, | |
52 | help="Is the app running?") |
|
53 | help="Is the app running?") | |
53 | classes = List([Bar, Foo]) |
|
54 | classes = List([Bar, Foo]) | |
@@ -71,30 +72,30 b' class TestApplication(TestCase):' | |||||
71 |
|
72 | |||
72 | def test_basic(self): |
|
73 | def test_basic(self): | |
73 | app = MyApp() |
|
74 | app = MyApp() | |
74 |
self.assertEquals(app. |
|
75 | self.assertEquals(app.name, u'myapp') | |
75 | self.assertEquals(app.running, False) |
|
76 | self.assertEquals(app.running, False) | |
76 | self.assertEquals(app.classes, [MyApp,Bar,Foo]) |
|
77 | self.assertEquals(app.classes, [MyApp,Bar,Foo]) | |
77 | self.assertEquals(app.config_file, u'') |
|
78 | self.assertEquals(app.config_file, u'') | |
78 |
|
79 | |||
79 | def test_config(self): |
|
80 | def test_config(self): | |
80 | app = MyApp() |
|
81 | app = MyApp() | |
81 | app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=0"]) |
|
82 | app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=50"]) | |
82 | config = app.config |
|
83 | config = app.config | |
83 | self.assertEquals(config.Foo.i, 10) |
|
84 | self.assertEquals(config.Foo.i, 10) | |
84 | self.assertEquals(config.Foo.j, 10) |
|
85 | self.assertEquals(config.Foo.j, 10) | |
85 | self.assertEquals(config.Bar.enabled, False) |
|
86 | self.assertEquals(config.Bar.enabled, False) | |
86 | self.assertEquals(config.MyApp.log_level,0) |
|
87 | self.assertEquals(config.MyApp.log_level,50) | |
87 |
|
88 | |||
88 | def test_config_propagation(self): |
|
89 | def test_config_propagation(self): | |
89 | app = MyApp() |
|
90 | app = MyApp() | |
90 | app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=0"]) |
|
91 | app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=50"]) | |
91 | app.init_foo() |
|
92 | app.init_foo() | |
92 | app.init_bar() |
|
93 | app.init_bar() | |
93 | self.assertEquals(app.foo.i, 10) |
|
94 | self.assertEquals(app.foo.i, 10) | |
94 | self.assertEquals(app.foo.j, 10) |
|
95 | self.assertEquals(app.foo.j, 10) | |
95 | self.assertEquals(app.bar.enabled, False) |
|
96 | self.assertEquals(app.bar.enabled, False) | |
96 |
|
97 | |||
97 |
def test_ |
|
98 | def test_flags(self): | |
98 | app = MyApp() |
|
99 | app = MyApp() | |
99 | app.parse_command_line(["--disable"]) |
|
100 | app.parse_command_line(["--disable"]) | |
100 | app.init_bar() |
|
101 | app.init_bar() | |
@@ -103,3 +104,32 b' class TestApplication(TestCase):' | |||||
103 | app.init_bar() |
|
104 | app.init_bar() | |
104 | self.assertEquals(app.bar.enabled, True) |
|
105 | self.assertEquals(app.bar.enabled, True) | |
105 |
|
106 | |||
|
107 | def test_aliases(self): | |||
|
108 | app = MyApp() | |||
|
109 | app.parse_command_line(["i=5", "j=10"]) | |||
|
110 | app.init_foo() | |||
|
111 | self.assertEquals(app.foo.i, 5) | |||
|
112 | app.init_foo() | |||
|
113 | self.assertEquals(app.foo.j, 10) | |||
|
114 | ||||
|
115 | def test_flag_clobber(self): | |||
|
116 | """test that setting flags doesn't clobber existing settings""" | |||
|
117 | app = MyApp() | |||
|
118 | app.parse_command_line(["Bar.b=5", "--disable"]) | |||
|
119 | app.init_bar() | |||
|
120 | self.assertEquals(app.bar.enabled, False) | |||
|
121 | self.assertEquals(app.bar.b, 5) | |||
|
122 | app.parse_command_line(["--enable", "Bar.b=10"]) | |||
|
123 | app.init_bar() | |||
|
124 | self.assertEquals(app.bar.enabled, True) | |||
|
125 | self.assertEquals(app.bar.b, 10) | |||
|
126 | ||||
|
127 | def test_extra_args(self): | |||
|
128 | app = MyApp() | |||
|
129 | app.parse_command_line(['extra', "Bar.b=5", "--disable", 'args']) | |||
|
130 | app.init_bar() | |||
|
131 | self.assertEquals(app.bar.enabled, False) | |||
|
132 | self.assertEquals(app.bar.b, 5) | |||
|
133 | self.assertEquals(app.extra_args, ['extra', 'args']) | |||
|
134 | ||||
|
135 |
@@ -48,8 +48,10 b' class MyConfigurable(Configurable):' | |||||
48 | mc_help=u"""MyConfigurable options |
|
48 | mc_help=u"""MyConfigurable options | |
49 | ---------------------- |
|
49 | ---------------------- | |
50 | MyConfigurable.a : Int |
|
50 | MyConfigurable.a : Int | |
|
51 | Default: 1 | |||
51 | The integer a. |
|
52 | The integer a. | |
52 | MyConfigurable.b : Float |
|
53 | MyConfigurable.b : Float | |
|
54 | Default: 1.0 | |||
53 | The integer b.""" |
|
55 | The integer b.""" | |
54 |
|
56 | |||
55 | class Foo(Configurable): |
|
57 | class Foo(Configurable): |
@@ -123,6 +123,13 b' class TestKeyValueCL(TestCase):' | |||||
123 | self.assertEquals(config.Foo.Bar.value, 10) |
|
123 | self.assertEquals(config.Foo.Bar.value, 10) | |
124 | self.assertEquals(config.Foo.Bam.value, range(10)) |
|
124 | self.assertEquals(config.Foo.Bam.value, range(10)) | |
125 | self.assertEquals(config.D.C.value, 'hi there') |
|
125 | self.assertEquals(config.D.C.value, 'hi there') | |
|
126 | ||||
|
127 | def test_extra_args(self): | |||
|
128 | cl = KeyValueConfigLoader() | |||
|
129 | config = cl.load_config(['a=5', 'b', 'c=10', 'd']) | |||
|
130 | self.assertEquals(cl.extra_args, ['b', 'd']) | |||
|
131 | self.assertEquals(config.a, 5) | |||
|
132 | self.assertEquals(config.c, 10) | |||
126 |
|
133 | |||
127 |
|
134 | |||
128 | class TestConfig(TestCase): |
|
135 | class TestConfig(TestCase): |
This diff has been collapsed as it changes many lines, (616 lines changed) Show them Hide them | |||||
@@ -12,13 +12,12 b' Authors:' | |||||
12 |
|
12 | |||
13 | * Brian Granger |
|
13 | * Brian Granger | |
14 | * Fernando Perez |
|
14 | * Fernando Perez | |
|
15 | * Min RK | |||
15 |
|
16 | |||
16 | Notes |
|
|||
17 | ----- |
|
|||
18 | """ |
|
17 | """ | |
19 |
|
18 | |||
20 | #----------------------------------------------------------------------------- |
|
19 | #----------------------------------------------------------------------------- | |
21 |
# Copyright (C) 2008-20 |
|
20 | # Copyright (C) 2008-2011 The IPython Development Team | |
22 | # |
|
21 | # | |
23 | # Distributed under the terms of the BSD License. The full license is in |
|
22 | # Distributed under the terms of the BSD License. The full license is in | |
24 | # the file COPYING, distributed as part of this software. |
|
23 | # the file COPYING, distributed as part of this software. | |
@@ -30,439 +29,262 b' Notes' | |||||
30 |
|
29 | |||
31 | import logging |
|
30 | import logging | |
32 | import os |
|
31 | import os | |
|
32 | import shutil | |||
33 | import sys |
|
33 | import sys | |
34 |
|
34 | |||
|
35 | from IPython.config.application import Application | |||
|
36 | from IPython.config.configurable import Configurable | |||
|
37 | from IPython.config.loader import Config | |||
35 | from IPython.core import release, crashhandler |
|
38 | from IPython.core import release, crashhandler | |
|
39 | from IPython.core.profiledir import ProfileDir, ProfileDirError | |||
36 | from IPython.utils.path import get_ipython_dir, get_ipython_package_dir |
|
40 | from IPython.utils.path import get_ipython_dir, get_ipython_package_dir | |
37 | from IPython.config.loader import ( |
|
41 | from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict | |
38 | PyFileConfigLoader, |
|
|||
39 | ArgParseConfigLoader, |
|
|||
40 | Config, |
|
|||
41 | ) |
|
|||
42 |
|
42 | |||
43 | #----------------------------------------------------------------------------- |
|
43 | #----------------------------------------------------------------------------- | |
44 | # Classes and functions |
|
44 | # Classes and functions | |
45 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
46 |
|
46 | |||
47 | class ApplicationError(Exception): |
|
|||
48 | pass |
|
|||
49 |
|
||||
50 |
|
||||
51 | class BaseAppConfigLoader(ArgParseConfigLoader): |
|
|||
52 | """Default command line options for IPython based applications.""" |
|
|||
53 |
|
||||
54 | def _add_ipython_dir(self, parser): |
|
|||
55 | """Add the --ipython-dir option to the parser.""" |
|
|||
56 | paa = parser.add_argument |
|
|||
57 | paa('--ipython-dir', |
|
|||
58 | dest='Global.ipython_dir',type=unicode, |
|
|||
59 | help= |
|
|||
60 | """Set to override default location of the IPython directory |
|
|||
61 | IPYTHON_DIR, stored as Global.ipython_dir. This can also be |
|
|||
62 | specified through the environment variable IPYTHON_DIR.""", |
|
|||
63 | metavar='Global.ipython_dir') |
|
|||
64 |
|
||||
65 | def _add_log_level(self, parser): |
|
|||
66 | """Add the --log-level option to the parser.""" |
|
|||
67 | paa = parser.add_argument |
|
|||
68 | paa('--log-level', |
|
|||
69 | dest="Global.log_level",type=int, |
|
|||
70 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
|||
71 | metavar='Global.log_level') |
|
|||
72 |
|
||||
73 | def _add_version(self, parser): |
|
|||
74 | """Add the --version option to the parser.""" |
|
|||
75 | parser.add_argument('--version', action="version", |
|
|||
76 | version=self.version) |
|
|||
77 |
|
||||
78 | def _add_arguments(self): |
|
|||
79 | self._add_ipython_dir(self.parser) |
|
|||
80 | self._add_log_level(self.parser) |
|
|||
81 | try: # Old versions of argparse don't have a version action |
|
|||
82 | self._add_version(self.parser) |
|
|||
83 | except Exception: |
|
|||
84 | pass |
|
|||
85 |
|
||||
86 |
|
||||
87 | class Application(object): |
|
|||
88 | """Load a config, construct configurables and set them running. |
|
|||
89 |
|
||||
90 | The configuration of an application can be done via three different Config |
|
|||
91 | objects, which are loaded and ultimately merged into a single one used |
|
|||
92 | from that point on by the app. These are: |
|
|||
93 |
|
||||
94 | 1. default_config: internal defaults, implemented in code. |
|
|||
95 | 2. file_config: read from the filesystem. |
|
|||
96 | 3. command_line_config: read from the system's command line flags. |
|
|||
97 |
|
||||
98 | During initialization, 3 is actually read before 2, since at the |
|
|||
99 | command-line one may override the location of the file to be read. But the |
|
|||
100 | above is the order in which the merge is made. |
|
|||
101 | """ |
|
|||
102 |
|
||||
103 | name = u'ipython' |
|
|||
104 | description = 'IPython: an enhanced interactive Python shell.' |
|
|||
105 | #: Usage message printed by argparse. If None, auto-generate |
|
|||
106 | usage = None |
|
|||
107 | #: The command line config loader. Subclass of ArgParseConfigLoader. |
|
|||
108 | command_line_loader = BaseAppConfigLoader |
|
|||
109 | #: The name of the config file to load, determined at runtime |
|
|||
110 | config_file_name = None |
|
|||
111 | #: The name of the default config file. Track separately from the actual |
|
|||
112 | #: name because some logic happens only if we aren't using the default. |
|
|||
113 | default_config_file_name = u'ipython_config.py' |
|
|||
114 | default_log_level = logging.WARN |
|
|||
115 | #: Set by --profile option |
|
|||
116 | profile_name = None |
|
|||
117 | #: User's ipython directory, typically ~/.ipython or ~/.config/ipython/ |
|
|||
118 | ipython_dir = None |
|
|||
119 | #: Internal defaults, implemented in code. |
|
|||
120 | default_config = None |
|
|||
121 | #: Read from the filesystem. |
|
|||
122 | file_config = None |
|
|||
123 | #: Read from the system's command line flags. |
|
|||
124 | command_line_config = None |
|
|||
125 | #: The final config that will be passed to the main object. |
|
|||
126 | master_config = None |
|
|||
127 | #: A reference to the argv to be used (typically ends up being sys.argv[1:]) |
|
|||
128 | argv = None |
|
|||
129 | #: extra arguments computed by the command-line loader |
|
|||
130 | extra_args = None |
|
|||
131 | #: The class to use as the crash handler. |
|
|||
132 | crash_handler_class = crashhandler.CrashHandler |
|
|||
133 |
|
||||
134 | # Private attributes |
|
|||
135 | _exiting = False |
|
|||
136 | _initialized = False |
|
|||
137 |
|
||||
138 | def __init__(self, argv=None): |
|
|||
139 | self.argv = sys.argv[1:] if argv is None else argv |
|
|||
140 | self.init_logger() |
|
|||
141 |
|
||||
142 | def init_logger(self): |
|
|||
143 | self.log = logging.getLogger(self.__class__.__name__) |
|
|||
144 | # This is used as the default until the command line arguments are read. |
|
|||
145 | self.log.setLevel(self.default_log_level) |
|
|||
146 | self._log_handler = logging.StreamHandler() |
|
|||
147 | self._log_formatter = logging.Formatter("[%(name)s] %(message)s") |
|
|||
148 | self._log_handler.setFormatter(self._log_formatter) |
|
|||
149 | self.log.addHandler(self._log_handler) |
|
|||
150 |
|
||||
151 | def _set_log_level(self, level): |
|
|||
152 | self.log.setLevel(level) |
|
|||
153 |
|
||||
154 | def _get_log_level(self): |
|
|||
155 | return self.log.level |
|
|||
156 |
|
||||
157 | log_level = property(_get_log_level, _set_log_level) |
|
|||
158 |
|
||||
159 | def initialize(self): |
|
|||
160 | """Initialize the application. |
|
|||
161 |
|
||||
162 | Loads all configuration information and sets all application state, but |
|
|||
163 | does not start any relevant processing (typically some kind of event |
|
|||
164 | loop). |
|
|||
165 |
|
||||
166 | Once this method has been called, the application is flagged as |
|
|||
167 | initialized and the method becomes a no-op.""" |
|
|||
168 |
|
||||
169 | if self._initialized: |
|
|||
170 | return |
|
|||
171 |
|
||||
172 | # The first part is protected with an 'attempt' wrapper, that will log |
|
|||
173 | # failures with the basic system traceback machinery. Once our crash |
|
|||
174 | # handler is in place, we can let any subsequent exception propagate, |
|
|||
175 | # as our handler will log it with much better detail than the default. |
|
|||
176 | self.attempt(self.create_crash_handler) |
|
|||
177 |
|
||||
178 | # Configuration phase |
|
|||
179 | # Default config (internally hardwired in application code) |
|
|||
180 | self.create_default_config() |
|
|||
181 | self.log_default_config() |
|
|||
182 | self.set_default_config_log_level() |
|
|||
183 |
|
||||
184 | # Command-line config |
|
|||
185 | self.pre_load_command_line_config() |
|
|||
186 | self.load_command_line_config() |
|
|||
187 | self.set_command_line_config_log_level() |
|
|||
188 | self.post_load_command_line_config() |
|
|||
189 | self.log_command_line_config() |
|
|||
190 |
|
47 | |||
191 | # Find resources needed for filesystem access, using information from |
|
48 | #----------------------------------------------------------------------------- | |
192 | # the above two |
|
49 | # Base Application Class | |
193 | self.find_ipython_dir() |
|
50 | #----------------------------------------------------------------------------- | |
194 | self.find_resources() |
|
|||
195 | self.find_config_file_name() |
|
|||
196 | self.find_config_file_paths() |
|
|||
197 |
|
||||
198 | # File-based config |
|
|||
199 | self.pre_load_file_config() |
|
|||
200 | self.load_file_config() |
|
|||
201 | self.set_file_config_log_level() |
|
|||
202 | self.post_load_file_config() |
|
|||
203 | self.log_file_config() |
|
|||
204 |
|
51 | |||
205 | # Merge all config objects into a single one the app can then use |
|
52 | # aliases and flags | |
206 | self.merge_configs() |
|
|||
207 | self.log_master_config() |
|
|||
208 |
|
53 | |||
209 | # Construction phase |
|
54 | base_aliases = dict( | |
210 | self.pre_construct() |
|
55 | profile='BaseIPythonApplication.profile', | |
211 | self.construct() |
|
56 | ipython_dir='BaseIPythonApplication.ipython_dir', | |
212 | self.post_construct() |
|
57 | ) | |
213 |
|
58 | |||
214 | # Done, flag as such and |
|
59 | base_flags = dict( | |
215 | self._initialized = True |
|
60 | debug = ({'Application' : {'log_level' : logging.DEBUG}}, | |
|
61 | "set log level to logging.DEBUG (maximize logging output)"), | |||
|
62 | quiet = ({'Application' : {'log_level' : logging.CRITICAL}}, | |||
|
63 | "set log level to logging.CRITICAL (minimize logging output)"), | |||
|
64 | init = ({'BaseIPythonApplication' : { | |||
|
65 | 'copy_config_files' : True, | |||
|
66 | 'auto_create' : True} | |||
|
67 | }, "Initialize profile with default config files") | |||
|
68 | ) | |||
216 |
|
69 | |||
217 | def start(self): |
|
|||
218 | """Start the application.""" |
|
|||
219 | self.initialize() |
|
|||
220 | self.start_app() |
|
|||
221 |
|
70 | |||
|
71 | class BaseIPythonApplication(Application): | |||
|
72 | ||||
|
73 | name = Unicode(u'ipython') | |||
|
74 | description = Unicode(u'IPython: an enhanced interactive Python shell.') | |||
|
75 | version = Unicode(release.version) | |||
|
76 | ||||
|
77 | aliases = Dict(base_aliases) | |||
|
78 | flags = Dict(base_flags) | |||
|
79 | ||||
|
80 | # Track whether the config_file has changed, | |||
|
81 | # because some logic happens only if we aren't using the default. | |||
|
82 | config_file_specified = Bool(False) | |||
|
83 | ||||
|
84 | config_file_name = Unicode(u'ipython_config.py') | |||
|
85 | def _config_file_name_default(self): | |||
|
86 | return self.name.replace('-','_') + u'_config.py' | |||
|
87 | def _config_file_name_changed(self, name, old, new): | |||
|
88 | if new != old: | |||
|
89 | self.config_file_specified = True | |||
|
90 | ||||
|
91 | # The directory that contains IPython's builtin profiles. | |||
|
92 | builtin_profile_dir = Unicode( | |||
|
93 | os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default') | |||
|
94 | ) | |||
|
95 | ||||
|
96 | config_file_paths = List(Unicode) | |||
|
97 | def _config_file_paths_default(self): | |||
|
98 | return [os.getcwdu()] | |||
|
99 | ||||
|
100 | profile = Unicode(u'default', config=True, | |||
|
101 | help="""The IPython profile to use.""" | |||
|
102 | ) | |||
|
103 | def _profile_changed(self, name, old, new): | |||
|
104 | self.builtin_profile_dir = os.path.join( | |||
|
105 | get_ipython_package_dir(), u'config', u'profile', new | |||
|
106 | ) | |||
|
107 | ||||
|
108 | ipython_dir = Unicode(get_ipython_dir(), config=True, | |||
|
109 | help=""" | |||
|
110 | The name of the IPython directory. This directory is used for logging | |||
|
111 | configuration (through profiles), history storage, etc. The default | |||
|
112 | is usually $HOME/.ipython. This options can also be specified through | |||
|
113 | the environment variable IPYTHON_DIR. | |||
|
114 | """ | |||
|
115 | ) | |||
|
116 | ||||
|
117 | overwrite = Bool(False, config=True, | |||
|
118 | help="""Whether to overwrite existing config files when copying""") | |||
|
119 | auto_create = Bool(False, config=True, | |||
|
120 | help="""Whether to create profile dir if it doesn't exist""") | |||
|
121 | ||||
|
122 | config_files = List(Unicode) | |||
|
123 | def _config_files_default(self): | |||
|
124 | return [u'ipython_config.py'] | |||
|
125 | ||||
|
126 | copy_config_files = Bool(False, config=True, | |||
|
127 | help="""Whether to install the default config files into the profile dir. | |||
|
128 | If a new profile is being created, and IPython contains config files for that | |||
|
129 | profile, then they will be staged into the new directory. Otherwise, | |||
|
130 | default config files will be automatically generated. | |||
|
131 | """) | |||
|
132 | ||||
|
133 | # The class to use as the crash handler. | |||
|
134 | crash_handler_class = Type(crashhandler.CrashHandler) | |||
|
135 | ||||
|
136 | def __init__(self, **kwargs): | |||
|
137 | super(BaseIPythonApplication, self).__init__(**kwargs) | |||
|
138 | # ensure even default IPYTHON_DIR exists | |||
|
139 | if not os.path.exists(self.ipython_dir): | |||
|
140 | self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir) | |||
|
141 | ||||
222 | #------------------------------------------------------------------------- |
|
142 | #------------------------------------------------------------------------- | |
223 | # Various stages of Application creation |
|
143 | # Various stages of Application creation | |
224 | #------------------------------------------------------------------------- |
|
144 | #------------------------------------------------------------------------- | |
225 |
|
145 | |||
226 |
def |
|
146 | def init_crash_handler(self): | |
227 | """Create a crash handler, typically setting sys.excepthook to it.""" |
|
147 | """Create a crash handler, typically setting sys.excepthook to it.""" | |
228 | self.crash_handler = self.crash_handler_class(self) |
|
148 | self.crash_handler = self.crash_handler_class(self) | |
229 | sys.excepthook = self.crash_handler |
|
149 | sys.excepthook = self.crash_handler | |
230 |
|
150 | |||
231 | def create_default_config(self): |
|
151 | def _ipython_dir_changed(self, name, old, new): | |
232 | """Create defaults that can't be set elsewhere. |
|
152 | if old in sys.path: | |
233 |
|
153 | sys.path.remove(old) | ||
234 | For the most part, we try to set default in the class attributes |
|
154 | sys.path.append(os.path.abspath(new)) | |
235 | of Configurables. But, defaults the top-level Application (which is |
|
155 | if not os.path.isdir(new): | |
236 | not a HasTraits or Configurables) are not set in this way. Instead |
|
156 | os.makedirs(new, mode=0777) | |
237 | we set them here. The Global section is for variables like this that |
|
157 | readme = os.path.join(new, 'README') | |
238 | don't belong to a particular configurable. |
|
158 | if not os.path.exists(readme): | |
239 | """ |
|
159 | path = os.path.join(get_ipython_package_dir(), u'config', u'profile') | |
240 | c = Config() |
|
160 | shutil.copy(os.path.join(path, 'README'), readme) | |
241 | c.Global.ipython_dir = get_ipython_dir() |
|
161 | self.log.debug("IPYTHON_DIR set to: %s" % new) | |
242 | c.Global.log_level = self.log_level |
|
162 | ||
243 | self.default_config = c |
|
163 | def load_config_file(self, suppress_errors=True): | |
244 |
|
||||
245 | def log_default_config(self): |
|
|||
246 | self.log.debug('Default config loaded:') |
|
|||
247 | self.log.debug(repr(self.default_config)) |
|
|||
248 |
|
||||
249 | def set_default_config_log_level(self): |
|
|||
250 | try: |
|
|||
251 | self.log_level = self.default_config.Global.log_level |
|
|||
252 | except AttributeError: |
|
|||
253 | # Fallback to the default_log_level class attribute |
|
|||
254 | pass |
|
|||
255 |
|
||||
256 | def create_command_line_config(self): |
|
|||
257 | """Create and return a command line config loader.""" |
|
|||
258 | return self.command_line_loader( |
|
|||
259 | self.argv, |
|
|||
260 | description=self.description, |
|
|||
261 | version=release.version, |
|
|||
262 | usage=self.usage |
|
|||
263 | ) |
|
|||
264 |
|
||||
265 | def pre_load_command_line_config(self): |
|
|||
266 | """Do actions just before loading the command line config.""" |
|
|||
267 | pass |
|
|||
268 |
|
||||
269 | def load_command_line_config(self): |
|
|||
270 | """Load the command line config.""" |
|
|||
271 | loader = self.create_command_line_config() |
|
|||
272 | self.command_line_config = loader.load_config() |
|
|||
273 | self.extra_args = loader.get_extra_args() |
|
|||
274 |
|
||||
275 | def set_command_line_config_log_level(self): |
|
|||
276 | try: |
|
|||
277 | self.log_level = self.command_line_config.Global.log_level |
|
|||
278 | except AttributeError: |
|
|||
279 | pass |
|
|||
280 |
|
||||
281 | def post_load_command_line_config(self): |
|
|||
282 | """Do actions just after loading the command line config.""" |
|
|||
283 | pass |
|
|||
284 |
|
||||
285 | def log_command_line_config(self): |
|
|||
286 | self.log.debug("Command line config loaded:") |
|
|||
287 | self.log.debug(repr(self.command_line_config)) |
|
|||
288 |
|
||||
289 | def find_ipython_dir(self): |
|
|||
290 | """Set the IPython directory. |
|
|||
291 |
|
||||
292 | This sets ``self.ipython_dir``, but the actual value that is passed to |
|
|||
293 | the application is kept in either ``self.default_config`` or |
|
|||
294 | ``self.command_line_config``. This also adds ``self.ipython_dir`` to |
|
|||
295 | ``sys.path`` so config files there can be referenced by other config |
|
|||
296 | files. |
|
|||
297 | """ |
|
|||
298 |
|
||||
299 | try: |
|
|||
300 | self.ipython_dir = self.command_line_config.Global.ipython_dir |
|
|||
301 | except AttributeError: |
|
|||
302 | self.ipython_dir = self.default_config.Global.ipython_dir |
|
|||
303 | sys.path.append(os.path.abspath(self.ipython_dir)) |
|
|||
304 | if not os.path.isdir(self.ipython_dir): |
|
|||
305 | os.makedirs(self.ipython_dir, mode=0777) |
|
|||
306 | self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir) |
|
|||
307 |
|
||||
308 | def find_resources(self): |
|
|||
309 | """Find other resources that need to be in place. |
|
|||
310 |
|
||||
311 | Things like cluster directories need to be in place to find the |
|
|||
312 | config file. These happen right after the IPython directory has |
|
|||
313 | been set. |
|
|||
314 | """ |
|
|||
315 | pass |
|
|||
316 |
|
||||
317 | def find_config_file_name(self): |
|
|||
318 | """Find the config file name for this application. |
|
|||
319 |
|
||||
320 | This must set ``self.config_file_name`` to the filename of the |
|
|||
321 | config file to use (just the filename). The search paths for the |
|
|||
322 | config file are set in :meth:`find_config_file_paths` and then passed |
|
|||
323 | to the config file loader where they are resolved to an absolute path. |
|
|||
324 |
|
||||
325 | If a profile has been set at the command line, this will resolve it. |
|
|||
326 | """ |
|
|||
327 | try: |
|
|||
328 | self.config_file_name = self.command_line_config.Global.config_file |
|
|||
329 | except AttributeError: |
|
|||
330 | pass |
|
|||
331 | else: |
|
|||
332 | return |
|
|||
333 |
|
||||
334 | try: |
|
|||
335 | self.profile_name = self.command_line_config.Global.profile |
|
|||
336 | except AttributeError: |
|
|||
337 | # Just use the default as there is no profile |
|
|||
338 | self.config_file_name = self.default_config_file_name |
|
|||
339 | else: |
|
|||
340 | # Use the default config file name and profile name if set |
|
|||
341 | # to determine the used config file name. |
|
|||
342 | name_parts = self.default_config_file_name.split('.') |
|
|||
343 | name_parts.insert(1, u'_' + self.profile_name + u'.') |
|
|||
344 | self.config_file_name = ''.join(name_parts) |
|
|||
345 |
|
||||
346 | def find_config_file_paths(self): |
|
|||
347 | """Set the search paths for resolving the config file. |
|
|||
348 |
|
||||
349 | This must set ``self.config_file_paths`` to a sequence of search |
|
|||
350 | paths to pass to the config file loader. |
|
|||
351 | """ |
|
|||
352 | # Include our own profiles directory last, so that users can still find |
|
|||
353 | # our shipped copies of builtin profiles even if they don't have them |
|
|||
354 | # in their local ipython directory. |
|
|||
355 | prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile') |
|
|||
356 | self.config_file_paths = (os.getcwdu(), self.ipython_dir, prof_dir) |
|
|||
357 |
|
||||
358 | def pre_load_file_config(self): |
|
|||
359 | """Do actions before the config file is loaded.""" |
|
|||
360 | pass |
|
|||
361 |
|
||||
362 | def load_file_config(self, suppress_errors=True): |
|
|||
363 | """Load the config file. |
|
164 | """Load the config file. | |
364 |
|
165 | |||
365 | This tries to load the config file from disk. If successful, the |
|
|||
366 | ``CONFIG_FILE`` config variable is set to the resolved config file |
|
|||
367 | location. If not successful, an empty config is used. |
|
|||
368 |
|
||||
369 | By default, errors in loading config are handled, and a warning |
|
166 | By default, errors in loading config are handled, and a warning | |
370 | printed on screen. For testing, the suppress_errors option is set |
|
167 | printed on screen. For testing, the suppress_errors option is set | |
371 | to False, so errors will make tests fail. |
|
168 | to False, so errors will make tests fail. | |
372 | """ |
|
169 | """ | |
|
170 | base_config = 'ipython_config.py' | |||
|
171 | self.log.debug("Attempting to load config file: %s" % | |||
|
172 | base_config) | |||
|
173 | try: | |||
|
174 | Application.load_config_file( | |||
|
175 | self, | |||
|
176 | base_config, | |||
|
177 | path=self.config_file_paths | |||
|
178 | ) | |||
|
179 | except IOError: | |||
|
180 | # ignore errors loading parent | |||
|
181 | pass | |||
|
182 | if self.config_file_name == base_config: | |||
|
183 | # don't load secondary config | |||
|
184 | return | |||
373 | self.log.debug("Attempting to load config file: %s" % |
|
185 | self.log.debug("Attempting to load config file: %s" % | |
374 | self.config_file_name) |
|
186 | self.config_file_name) | |
375 | loader = PyFileConfigLoader(self.config_file_name, |
|
|||
376 | path=self.config_file_paths) |
|
|||
377 | try: |
|
187 | try: | |
378 |
|
|
188 | Application.load_config_file( | |
379 | self.file_config.Global.config_file = loader.full_filename |
|
189 | self, | |
|
190 | self.config_file_name, | |||
|
191 | path=self.config_file_paths | |||
|
192 | ) | |||
380 | except IOError: |
|
193 | except IOError: | |
381 | # Only warn if the default config file was NOT being used. |
|
194 | # Only warn if the default config file was NOT being used. | |
382 |
if |
|
195 | if self.config_file_specified: | |
383 | self.log.warn("Config file not found, skipping: %s" % |
|
196 | self.log.warn("Config file not found, skipping: %s" % | |
384 |
self.config_file_name |
|
197 | self.config_file_name) | |
385 | self.file_config = Config() |
|
|||
386 | except: |
|
198 | except: | |
387 |
|
|
199 | # For testing purposes. | |
|
200 | if not suppress_errors: | |||
388 | raise |
|
201 | raise | |
389 | self.log.warn("Error loading config file: %s" % |
|
202 | self.log.warn("Error loading config file: %s" % | |
390 | self.config_file_name, exc_info=True) |
|
203 | self.config_file_name, exc_info=True) | |
391 | self.file_config = Config() |
|
|||
392 |
|
204 | |||
393 |
def |
|
205 | def init_profile_dir(self): | |
394 | # We need to keeep self.log_level updated. But we only use the value |
|
206 | """initialize the profile dir""" | |
395 | # of the file_config if a value was not specified at the command |
|
207 | try: | |
396 | # line, because the command line overrides everything. |
|
208 | # location explicitly specified: | |
397 | if not hasattr(self.command_line_config.Global, 'log_level'): |
|
209 | location = self.config.ProfileDir.location | |
|
210 | except AttributeError: | |||
|
211 | # location not specified, find by profile name | |||
398 | try: |
|
212 | try: | |
399 | self.log_level = self.file_config.Global.log_level |
|
213 | p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config) | |
400 |
except |
|
214 | except ProfileDirError: | |
401 | pass # Use existing value |
|
215 | # not found, maybe create it (always create default profile) | |
402 |
|
216 | if self.auto_create or self.profile=='default': | ||
403 | def post_load_file_config(self): |
|
217 | try: | |
404 | """Do actions after the config file is loaded.""" |
|
218 | p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config) | |
405 | pass |
|
219 | except ProfileDirError: | |
406 |
|
220 | self.log.fatal("Could not create profile: %r"%self.profile) | ||
407 | def log_file_config(self): |
|
221 | self.exit(1) | |
408 | if hasattr(self.file_config.Global, 'config_file'): |
|
222 | else: | |
409 | self.log.debug("Config file loaded: %s" % |
|
223 | self.log.info("Created profile dir: %r"%p.location) | |
410 | self.file_config.Global.config_file) |
|
224 | else: | |
411 | self.log.debug(repr(self.file_config)) |
|
225 | self.log.fatal("Profile %r not found."%self.profile) | |
412 |
|
226 | self.exit(1) | ||
413 | def merge_configs(self): |
|
227 | else: | |
414 | """Merge the default, command line and file config objects.""" |
|
228 | self.log.info("Using existing profile dir: %r"%p.location) | |
415 | config = Config() |
|
|||
416 | config._merge(self.default_config) |
|
|||
417 | config._merge(self.file_config) |
|
|||
418 | config._merge(self.command_line_config) |
|
|||
419 |
|
||||
420 | # XXX fperez - propose to Brian we rename master_config to simply |
|
|||
421 | # config, I think this is going to be heavily used in examples and |
|
|||
422 | # application code and the name is shorter/easier to find/remember. |
|
|||
423 | # For now, just alias it... |
|
|||
424 | self.master_config = config |
|
|||
425 | self.config = config |
|
|||
426 |
|
||||
427 | def log_master_config(self): |
|
|||
428 | self.log.debug("Master config created:") |
|
|||
429 | self.log.debug(repr(self.master_config)) |
|
|||
430 |
|
||||
431 | def pre_construct(self): |
|
|||
432 | """Do actions after the config has been built, but before construct.""" |
|
|||
433 | pass |
|
|||
434 |
|
||||
435 | def construct(self): |
|
|||
436 | """Construct the main objects that make up this app.""" |
|
|||
437 | self.log.debug("Constructing main objects for application") |
|
|||
438 |
|
||||
439 | def post_construct(self): |
|
|||
440 | """Do actions after construct, but before starting the app.""" |
|
|||
441 | pass |
|
|||
442 |
|
||||
443 | def start_app(self): |
|
|||
444 | """Actually start the app.""" |
|
|||
445 | self.log.debug("Starting application") |
|
|||
446 |
|
||||
447 | #------------------------------------------------------------------------- |
|
|||
448 | # Utility methods |
|
|||
449 | #------------------------------------------------------------------------- |
|
|||
450 |
|
||||
451 | def exit(self, exit_status=0): |
|
|||
452 | if self._exiting: |
|
|||
453 | pass |
|
|||
454 | else: |
|
229 | else: | |
455 | self.log.debug("Exiting application: %s" % self.name) |
|
230 | # location is fully specified | |
456 | self._exiting = True |
|
231 | try: | |
457 | sys.exit(exit_status) |
|
232 | p = ProfileDir.find_profile_dir(location, self.config) | |
458 |
|
233 | except ProfileDirError: | ||
459 | def attempt(self, func): |
|
234 | # not found, maybe create it | |
460 | try: |
|
235 | if self.auto_create: | |
461 | func() |
|
236 | try: | |
462 | except SystemExit: |
|
237 | p = ProfileDir.create_profile_dir(location, self.config) | |
463 | raise |
|
238 | except ProfileDirError: | |
464 | except: |
|
239 | self.log.fatal("Could not create profile directory: %r"%location) | |
465 | self.log.critical("Aborting application: %s" % self.name, |
|
240 | self.exit(1) | |
466 |
|
|
241 | else: | |
467 | self.exit(0) |
|
242 | self.log.info("Creating new profile dir: %r"%location) | |
|
243 | else: | |||
|
244 | self.log.fatal("Profile directory %r not found."%location) | |||
|
245 | self.exit(1) | |||
|
246 | else: | |||
|
247 | self.log.info("Using existing profile dir: %r"%location) | |||
|
248 | ||||
|
249 | self.profile_dir = p | |||
|
250 | self.config_file_paths.append(p.location) | |||
|
251 | ||||
|
252 | def init_config_files(self): | |||
|
253 | """[optionally] copy default config files into profile dir.""" | |||
|
254 | # copy config files | |||
|
255 | if self.copy_config_files: | |||
|
256 | path = self.builtin_profile_dir | |||
|
257 | src = self.profile | |||
|
258 | ||||
|
259 | cfg = self.config_file_name | |||
|
260 | if path and os.path.exists(os.path.join(path, cfg)): | |||
|
261 | self.log.warn("Staging %r from %s into %r [overwrite=%s]"%( | |||
|
262 | cfg, src, self.profile_dir.location, self.overwrite) | |||
|
263 | ) | |||
|
264 | self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite) | |||
|
265 | else: | |||
|
266 | self.stage_default_config_file() | |||
|
267 | ||||
|
268 | def stage_default_config_file(self): | |||
|
269 | """auto generate default config file, and stage it into the profile.""" | |||
|
270 | s = self.generate_config_file() | |||
|
271 | fname = os.path.join(self.profile_dir.location, self.config_file_name) | |||
|
272 | if self.overwrite or not os.path.exists(fname): | |||
|
273 | self.log.warn("Generating default config file: %r"%(fname)) | |||
|
274 | with open(fname, 'w') as f: | |||
|
275 | f.write(s) | |||
|
276 | ||||
|
277 | ||||
|
278 | def initialize(self, argv=None): | |||
|
279 | self.init_crash_handler() | |||
|
280 | self.parse_command_line(argv) | |||
|
281 | if self.subapp is not None: | |||
|
282 | # stop here if subapp is taking over | |||
|
283 | return | |||
|
284 | cl_config = self.config | |||
|
285 | self.init_profile_dir() | |||
|
286 | self.init_config_files() | |||
|
287 | self.load_config_file() | |||
|
288 | # enforce cl-opts override configfile opts: | |||
|
289 | self.update_config(cl_config) | |||
468 |
|
290 |
@@ -101,18 +101,15 b' class HistoryManager(Configurable):' | |||||
101 |
|
101 | |||
102 | if self.hist_file == u'': |
|
102 | if self.hist_file == u'': | |
103 | # No one has set the hist_file, yet. |
|
103 | # No one has set the hist_file, yet. | |
104 | if shell.profile: |
|
104 | histfname = 'history' | |
105 | histfname = 'history-%s' % shell.profile |
|
105 | self.hist_file = os.path.join(shell.profile_dir.location, histfname + '.sqlite') | |
106 | else: |
|
|||
107 | histfname = 'history' |
|
|||
108 | self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite') |
|
|||
109 |
|
106 | |||
110 | try: |
|
107 | try: | |
111 | self.init_db() |
|
108 | self.init_db() | |
112 | except sqlite3.DatabaseError: |
|
109 | except sqlite3.DatabaseError: | |
113 | if os.path.isfile(self.hist_file): |
|
110 | if os.path.isfile(self.hist_file): | |
114 | # Try to move the file out of the way. |
|
111 | # Try to move the file out of the way. | |
115 |
newpath = os.path.join(self.shell. |
|
112 | newpath = os.path.join(self.shell.profile_dir.location, "hist-corrupt.sqlite") | |
116 | os.rename(self.hist_file, newpath) |
|
113 | os.rename(self.hist_file, newpath) | |
117 | print("ERROR! History file wasn't a valid SQLite database.", |
|
114 | print("ERROR! History file wasn't a valid SQLite database.", | |
118 | "It was moved to %s" % newpath, "and a new file created.") |
|
115 | "It was moved to %s" % newpath, "and a new file created.") |
@@ -57,6 +57,7 b' from IPython.core.magic import Magic' | |||||
57 | from IPython.core.payload import PayloadManager |
|
57 | from IPython.core.payload import PayloadManager | |
58 | from IPython.core.plugin import PluginManager |
|
58 | from IPython.core.plugin import PluginManager | |
59 | from IPython.core.prefilter import PrefilterManager, ESC_MAGIC |
|
59 | from IPython.core.prefilter import PrefilterManager, ESC_MAGIC | |
|
60 | from IPython.core.profiledir import ProfileDir | |||
60 | from IPython.external.Itpl import ItplNS |
|
61 | from IPython.external.Itpl import ItplNS | |
61 | from IPython.utils import PyColorize |
|
62 | from IPython.utils import PyColorize | |
62 | from IPython.utils import io |
|
63 | from IPython.utils import io | |
@@ -238,7 +239,9 b' class InteractiveShell(SingletonConfigurable, Magic):' | |||||
238 | """ |
|
239 | """ | |
239 | ) |
|
240 | ) | |
240 | colors = CaselessStrEnum(('NoColor','LightBG','Linux'), |
|
241 | colors = CaselessStrEnum(('NoColor','LightBG','Linux'), | |
241 |
default_value=get_default_colors(), config=True |
|
242 | default_value=get_default_colors(), config=True, | |
|
243 | help="Set the color scheme (NoColor, Linux, or LightBG)." | |||
|
244 | ) | |||
242 | debug = CBool(False, config=True) |
|
245 | debug = CBool(False, config=True) | |
243 | deep_reload = CBool(False, config=True, help= |
|
246 | deep_reload = CBool(False, config=True, help= | |
244 | """ |
|
247 | """ | |
@@ -291,7 +294,6 b' class InteractiveShell(SingletonConfigurable, Magic):' | |||||
291 | """ |
|
294 | """ | |
292 | ) |
|
295 | ) | |
293 |
|
296 | |||
294 | profile = Unicode('', config=True) |
|
|||
295 | prompt_in1 = Str('In [\\#]: ', config=True) |
|
297 | prompt_in1 = Str('In [\\#]: ', config=True) | |
296 | prompt_in2 = Str(' .\\D.: ', config=True) |
|
298 | prompt_in2 = Str(' .\\D.: ', config=True) | |
297 | prompt_out = Str('Out[\\#]: ', config=True) |
|
299 | prompt_out = Str('Out[\\#]: ', config=True) | |
@@ -342,10 +344,18 b' class InteractiveShell(SingletonConfigurable, Magic):' | |||||
342 | payload_manager = Instance('IPython.core.payload.PayloadManager') |
|
344 | payload_manager = Instance('IPython.core.payload.PayloadManager') | |
343 | history_manager = Instance('IPython.core.history.HistoryManager') |
|
345 | history_manager = Instance('IPython.core.history.HistoryManager') | |
344 |
|
346 | |||
|
347 | profile_dir = Instance('IPython.core.application.ProfileDir') | |||
|
348 | @property | |||
|
349 | def profile(self): | |||
|
350 | if self.profile_dir is not None: | |||
|
351 | name = os.path.basename(self.profile_dir.location) | |||
|
352 | return name.replace('profile_','') | |||
|
353 | ||||
|
354 | ||||
345 | # Private interface |
|
355 | # Private interface | |
346 | _post_execute = Instance(dict) |
|
356 | _post_execute = Instance(dict) | |
347 |
|
357 | |||
348 | def __init__(self, config=None, ipython_dir=None, |
|
358 | def __init__(self, config=None, ipython_dir=None, profile_dir=None, | |
349 | user_ns=None, user_global_ns=None, |
|
359 | user_ns=None, user_global_ns=None, | |
350 | custom_exceptions=((), None)): |
|
360 | custom_exceptions=((), None)): | |
351 |
|
361 | |||
@@ -355,6 +365,7 b' class InteractiveShell(SingletonConfigurable, Magic):' | |||||
355 |
|
365 | |||
356 | # These are relatively independent and stateless |
|
366 | # These are relatively independent and stateless | |
357 | self.init_ipython_dir(ipython_dir) |
|
367 | self.init_ipython_dir(ipython_dir) | |
|
368 | self.init_profile_dir(profile_dir) | |||
358 | self.init_instance_attrs() |
|
369 | self.init_instance_attrs() | |
359 | self.init_environment() |
|
370 | self.init_environment() | |
360 |
|
371 | |||
@@ -372,7 +383,7 b' class InteractiveShell(SingletonConfigurable, Magic):' | |||||
372 | # While we're trying to have each part of the code directly access what |
|
383 | # While we're trying to have each part of the code directly access what | |
373 | # it needs without keeping redundant references to objects, we have too |
|
384 | # it needs without keeping redundant references to objects, we have too | |
374 | # much legacy code that expects ip.db to exist. |
|
385 | # much legacy code that expects ip.db to exist. | |
375 |
self.db = PickleShareDB(os.path.join(self. |
|
386 | self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db')) | |
376 |
|
387 | |||
377 | self.init_history() |
|
388 | self.init_history() | |
378 | self.init_encoding() |
|
389 | self.init_encoding() | |
@@ -457,16 +468,16 b' class InteractiveShell(SingletonConfigurable, Magic):' | |||||
457 | def init_ipython_dir(self, ipython_dir): |
|
468 | def init_ipython_dir(self, ipython_dir): | |
458 | if ipython_dir is not None: |
|
469 | if ipython_dir is not None: | |
459 | self.ipython_dir = ipython_dir |
|
470 | self.ipython_dir = ipython_dir | |
460 | self.config.Global.ipython_dir = self.ipython_dir |
|
|||
461 | return |
|
471 | return | |
462 |
|
472 | |||
463 | if hasattr(self.config.Global, 'ipython_dir'): |
|
473 | self.ipython_dir = get_ipython_dir() | |
464 | self.ipython_dir = self.config.Global.ipython_dir |
|
|||
465 | else: |
|
|||
466 | self.ipython_dir = get_ipython_dir() |
|
|||
467 |
|
474 | |||
468 | # All children can just read this |
|
475 | def init_profile_dir(self, profile_dir): | |
469 | self.config.Global.ipython_dir = self.ipython_dir |
|
476 | if profile_dir is not None: | |
|
477 | self.profile_dir = profile_dir | |||
|
478 | return | |||
|
479 | self.profile_dir =\ | |||
|
480 | ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default') | |||
470 |
|
481 | |||
471 | def init_instance_attrs(self): |
|
482 | def init_instance_attrs(self): | |
472 | self.more = False |
|
483 | self.more = False |
@@ -46,6 +46,7 b' from IPython.core import debugger, oinspect' | |||||
46 | from IPython.core.error import TryNext |
|
46 | from IPython.core.error import TryNext | |
47 | from IPython.core.error import UsageError |
|
47 | from IPython.core.error import UsageError | |
48 | from IPython.core.fakemodule import FakeModule |
|
48 | from IPython.core.fakemodule import FakeModule | |
|
49 | from IPython.core.profiledir import ProfileDir | |||
49 | from IPython.core.macro import Macro |
|
50 | from IPython.core.macro import Macro | |
50 | from IPython.core import page |
|
51 | from IPython.core import page | |
51 | from IPython.core.prefilter import ESC_MAGIC |
|
52 | from IPython.core.prefilter import ESC_MAGIC | |
@@ -533,10 +534,7 b' Currently the magic system has the following functions:\\n"""' | |||||
533 |
|
534 | |||
534 | def magic_profile(self, parameter_s=''): |
|
535 | def magic_profile(self, parameter_s=''): | |
535 | """Print your currently active IPython profile.""" |
|
536 | """Print your currently active IPython profile.""" | |
536 |
|
|
537 | print self.shell.profile | |
537 | printpl('Current IPython profile: $self.shell.profile.') |
|
|||
538 | else: |
|
|||
539 | print 'No profile active.' |
|
|||
540 |
|
538 | |||
541 | def magic_pinfo(self, parameter_s='', namespaces=None): |
|
539 | def magic_pinfo(self, parameter_s='', namespaces=None): | |
542 | """Provide detailed information about an object. |
|
540 | """Provide detailed information about an object. | |
@@ -3373,22 +3371,16 b' Defaulting color scheme to \'NoColor\'"""' | |||||
3373 | else: |
|
3371 | else: | |
3374 | overwrite = False |
|
3372 | overwrite = False | |
3375 | from IPython.config import profile |
|
3373 | from IPython.config import profile | |
3376 |
profile_dir = os.path. |
|
3374 | profile_dir = os.path.dirname(profile.__file__) | |
3377 | ipython_dir = self.ipython_dir |
|
3375 | ipython_dir = self.ipython_dir | |
3378 | files = os.listdir(profile_dir) |
|
3376 | print "Installing profiles to: %s [overwrite=%s]"%(ipython_dir,overwrite) | |
3379 |
|
3377 | for src in os.listdir(profile_dir): | ||
3380 | to_install = [] |
|
3378 | if src.startswith('profile_'): | |
3381 | for f in files: |
|
3379 | name = src.replace('profile_', '') | |
3382 | if f.startswith('ipython_config'): |
|
3380 | print " %s"%name | |
3383 | src = os.path.join(profile_dir, f) |
|
3381 | pd = ProfileDir.create_profile_dir_by_name(ipython_dir, name) | |
3384 | dst = os.path.join(ipython_dir, f) |
|
3382 | pd.copy_config_file('ipython_config.py', path=src, | |
3385 | if (not os.path.isfile(dst)) or overwrite: |
|
3383 | overwrite=overwrite) | |
3386 | to_install.append((f, src, dst)) |
|
|||
3387 | if len(to_install)>0: |
|
|||
3388 | print "Installing profiles to: ", ipython_dir |
|
|||
3389 | for (f, src, dst) in to_install: |
|
|||
3390 | shutil.copy(src, dst) |
|
|||
3391 | print " %s" % f |
|
|||
3392 |
|
3384 | |||
3393 | @skip_doctest |
|
3385 | @skip_doctest | |
3394 | def magic_install_default_config(self, s): |
|
3386 | def magic_install_default_config(self, s): | |
@@ -3404,15 +3396,9 b' Defaulting color scheme to \'NoColor\'"""' | |||||
3404 | overwrite = True |
|
3396 | overwrite = True | |
3405 | else: |
|
3397 | else: | |
3406 | overwrite = False |
|
3398 | overwrite = False | |
3407 | from IPython.config import default |
|
3399 | pd = self.shell.profile_dir | |
3408 | config_dir = os.path.split(default.__file__)[0] |
|
3400 | print "Installing default config file in: %s" % pd.location | |
3409 | ipython_dir = self.ipython_dir |
|
3401 | pd.copy_config_file('ipython_config.py', overwrite=overwrite) | |
3410 | default_config_file_name = 'ipython_config.py' |
|
|||
3411 | src = os.path.join(config_dir, default_config_file_name) |
|
|||
3412 | dst = os.path.join(ipython_dir, default_config_file_name) |
|
|||
3413 | if (not os.path.isfile(dst)) or overwrite: |
|
|||
3414 | shutil.copy(src, dst) |
|
|||
3415 | print "Installing default config file: %s" % dst |
|
|||
3416 |
|
3402 | |||
3417 | # Pylab support: simple wrappers that activate pylab, load gui input |
|
3403 | # Pylab support: simple wrappers that activate pylab, load gui input | |
3418 | # handling and modify slightly %run |
|
3404 | # handling and modify slightly %run |
@@ -4,7 +4,7 b'' | |||||
4 | import os |
|
4 | import os | |
5 | import tempfile |
|
5 | import tempfile | |
6 |
|
6 | |||
7 | from IPython.core.application import Application |
|
7 | from IPython.core.application import BaseIPythonApplication | |
8 | from IPython.testing import decorators as testdec |
|
8 | from IPython.testing import decorators as testdec | |
9 |
|
9 | |||
10 | @testdec.onlyif_unicode_paths |
|
10 | @testdec.onlyif_unicode_paths | |
@@ -16,22 +16,11 b' def test_unicode_cwd():' | |||||
16 | os.chdir(wd) |
|
16 | os.chdir(wd) | |
17 | #raise Exception(repr(os.getcwd())) |
|
17 | #raise Exception(repr(os.getcwd())) | |
18 | try: |
|
18 | try: | |
19 | app = Application() |
|
19 | app = BaseIPythonApplication() | |
20 | # The lines below are copied from Application.initialize() |
|
20 | # The lines below are copied from Application.initialize() | |
21 | app.create_default_config() |
|
21 | app.init_profile_dir() | |
22 |
app. |
|
22 | app.init_config_files() | |
23 | app.set_default_config_log_level() |
|
23 | app.load_config_file(suppress_errors=False) | |
24 |
|
||||
25 | # Find resources needed for filesystem access, using information from |
|
|||
26 | # the above two |
|
|||
27 | app.find_ipython_dir() |
|
|||
28 | app.find_resources() |
|
|||
29 | app.find_config_file_name() |
|
|||
30 | app.find_config_file_paths() |
|
|||
31 |
|
||||
32 | # File-based config |
|
|||
33 | app.pre_load_file_config() |
|
|||
34 | app.load_file_config(suppress_errors=False) |
|
|||
35 | finally: |
|
24 | finally: | |
36 | os.chdir(old_wd) |
|
25 | os.chdir(old_wd) | |
37 |
|
26 | |||
@@ -48,22 +37,11 b' def test_unicode_ipdir():' | |||||
48 | old_ipdir2 = os.environ.pop("IPYTHON_DIR", None) |
|
37 | old_ipdir2 = os.environ.pop("IPYTHON_DIR", None) | |
49 | os.environ["IPYTHONDIR"] = ipdir.encode("utf-8") |
|
38 | os.environ["IPYTHONDIR"] = ipdir.encode("utf-8") | |
50 | try: |
|
39 | try: | |
51 | app = Application() |
|
40 | app = BaseIPythonApplication() | |
52 | # The lines below are copied from Application.initialize() |
|
41 | # The lines below are copied from Application.initialize() | |
53 | app.create_default_config() |
|
42 | app.init_profile_dir() | |
54 |
app. |
|
43 | app.init_config_files() | |
55 | app.set_default_config_log_level() |
|
44 | app.load_config_file(suppress_errors=False) | |
56 |
|
||||
57 | # Find resources needed for filesystem access, using information from |
|
|||
58 | # the above two |
|
|||
59 | app.find_ipython_dir() |
|
|||
60 | app.find_resources() |
|
|||
61 | app.find_config_file_name() |
|
|||
62 | app.find_config_file_paths() |
|
|||
63 |
|
||||
64 | # File-based config |
|
|||
65 | app.pre_load_file_config() |
|
|||
66 | app.load_file_config(suppress_errors=False) |
|
|||
67 | finally: |
|
45 | finally: | |
68 | if old_ipdir1: |
|
46 | if old_ipdir1: | |
69 | os.environ["IPYTHONDIR"] = old_ipdir1 |
|
47 | os.environ["IPYTHONDIR"] = old_ipdir1 |
@@ -92,7 +92,7 b' class ParalleMagic(Plugin):' | |||||
92 | Then you can do the following:: |
|
92 | Then you can do the following:: | |
93 |
|
93 | |||
94 | In [24]: %px a = 5 |
|
94 | In [24]: %px a = 5 | |
95 | Parallel execution on engines: all |
|
95 | Parallel execution on engine(s): all | |
96 | Out[24]: |
|
96 | Out[24]: | |
97 | <Results List> |
|
97 | <Results List> | |
98 | [0] In [7]: a = 5 |
|
98 | [0] In [7]: a = 5 | |
@@ -102,7 +102,7 b' class ParalleMagic(Plugin):' | |||||
102 | if self.active_view is None: |
|
102 | if self.active_view is None: | |
103 | print NO_ACTIVE_VIEW |
|
103 | print NO_ACTIVE_VIEW | |
104 | return |
|
104 | return | |
105 | print "Parallel execution on engines: %s" % self.active_view.targets |
|
105 | print "Parallel execution on engine(s): %s" % self.active_view.targets | |
106 | result = self.active_view.execute(parameter_s, block=False) |
|
106 | result = self.active_view.execute(parameter_s, block=False) | |
107 | if self.active_view.block: |
|
107 | if self.active_view.block: | |
108 | result.get() |
|
108 | result.get() | |
@@ -125,9 +125,9 b' class ParalleMagic(Plugin):' | |||||
125 | %autopx to enabled |
|
125 | %autopx to enabled | |
126 |
|
126 | |||
127 | In [26]: a = 10 |
|
127 | In [26]: a = 10 | |
128 | Parallel execution on engines: [0,1,2,3] |
|
128 | Parallel execution on engine(s): [0,1,2,3] | |
129 | In [27]: print a |
|
129 | In [27]: print a | |
130 | Parallel execution on engines: [0,1,2,3] |
|
130 | Parallel execution on engine(s): [0,1,2,3] | |
131 | [stdout:0] 10 |
|
131 | [stdout:0] 10 | |
132 | [stdout:1] 10 |
|
132 | [stdout:1] 10 | |
133 | [stdout:2] 10 |
|
133 | [stdout:2] 10 | |
@@ -174,15 +174,21 b' class ParalleMagic(Plugin):' | |||||
174 | If self.active_view.block is True, wait for the result |
|
174 | If self.active_view.block is True, wait for the result | |
175 | and display the result. Otherwise, this is a noop. |
|
175 | and display the result. Otherwise, this is a noop. | |
176 | """ |
|
176 | """ | |
|
177 | if isinstance(result.stdout, basestring): | |||
|
178 | # single result | |||
|
179 | stdouts = [result.stdout.rstrip()] | |||
|
180 | else: | |||
|
181 | stdouts = [s.rstrip() for s in result.stdout] | |||
|
182 | ||||
177 | targets = self.active_view.targets |
|
183 | targets = self.active_view.targets | |
178 | if isinstance(targets, int): |
|
184 | if isinstance(targets, int): | |
179 | targets = [targets] |
|
185 | targets = [targets] | |
180 | if targets == 'all': |
|
186 | elif targets == 'all': | |
181 | targets = self.active_view.client.ids |
|
187 | targets = self.active_view.client.ids | |
182 | stdout = [s.rstrip() for s in result.stdout] |
|
188 | ||
183 | if any(stdout): |
|
189 | if any(stdouts): | |
184 |
for i, |
|
190 | for eid,stdout in zip(targets, stdouts): | |
185 |
print '[stdout:%i]'%eid, stdout |
|
191 | print '[stdout:%i]'%eid, stdout | |
186 |
|
192 | |||
187 |
|
193 | |||
188 | def pxrun_cell(self, raw_cell, store_history=True): |
|
194 | def pxrun_cell(self, raw_cell, store_history=True): |
@@ -30,8 +30,8 b' class BaseFrontendMixin(object):' | |||||
30 |
|
30 | |||
31 | # Disconnect the old kernel manager's channels. |
|
31 | # Disconnect the old kernel manager's channels. | |
32 | old_manager.sub_channel.message_received.disconnect(self._dispatch) |
|
32 | old_manager.sub_channel.message_received.disconnect(self._dispatch) | |
33 |
old_manager. |
|
33 | old_manager.shell_channel.message_received.disconnect(self._dispatch) | |
34 |
old_manager. |
|
34 | old_manager.stdin_channel.message_received.disconnect(self._dispatch) | |
35 | old_manager.hb_channel.kernel_died.disconnect( |
|
35 | old_manager.hb_channel.kernel_died.disconnect( | |
36 | self._handle_kernel_died) |
|
36 | self._handle_kernel_died) | |
37 |
|
37 | |||
@@ -50,8 +50,8 b' class BaseFrontendMixin(object):' | |||||
50 |
|
50 | |||
51 | # Connect the new kernel manager's channels. |
|
51 | # Connect the new kernel manager's channels. | |
52 | kernel_manager.sub_channel.message_received.connect(self._dispatch) |
|
52 | kernel_manager.sub_channel.message_received.connect(self._dispatch) | |
53 |
kernel_manager. |
|
53 | kernel_manager.shell_channel.message_received.connect(self._dispatch) | |
54 |
kernel_manager. |
|
54 | kernel_manager.stdin_channel.message_received.connect(self._dispatch) | |
55 | kernel_manager.hb_channel.kernel_died.connect(self._handle_kernel_died) |
|
55 | kernel_manager.hb_channel.kernel_died.connect(self._handle_kernel_died) | |
56 |
|
56 | |||
57 | # Handle the case where the kernel manager started channels before |
|
57 | # Handle the case where the kernel manager started channels before |
@@ -19,7 +19,7 b' from IPython.external.qt import QtCore, QtGui' | |||||
19 | from IPython.config.configurable import Configurable |
|
19 | from IPython.config.configurable import Configurable | |
20 | from IPython.frontend.qt.rich_text import HtmlExporter |
|
20 | from IPython.frontend.qt.rich_text import HtmlExporter | |
21 | from IPython.frontend.qt.util import MetaQObjectHasTraits, get_font |
|
21 | from IPython.frontend.qt.util import MetaQObjectHasTraits, get_font | |
22 | from IPython.utils.traitlets import Bool, Enum, Int |
|
22 | from IPython.utils.traitlets import Bool, Enum, Int, Unicode | |
23 | from ansi_code_processor import QtAnsiCodeProcessor |
|
23 | from ansi_code_processor import QtAnsiCodeProcessor | |
24 | from completion_widget import CompletionWidget |
|
24 | from completion_widget import CompletionWidget | |
25 | from kill_ring import QtKillRing |
|
25 | from kill_ring import QtKillRing | |
@@ -55,33 +55,62 b' class ConsoleWidget(Configurable, QtGui.QWidget):' | |||||
55 |
|
55 | |||
56 | #------ Configuration ------------------------------------------------------ |
|
56 | #------ Configuration ------------------------------------------------------ | |
57 |
|
57 | |||
58 | # Whether to process ANSI escape codes. |
|
58 | ansi_codes = Bool(True, config=True, | |
59 | ansi_codes = Bool(True, config=True) |
|
59 | help="Whether to process ANSI escape codes." | |
60 |
|
60 | ) | ||
61 | # The maximum number of lines of text before truncation. Specifying a |
|
61 | buffer_size = Int(500, config=True, | |
62 | # non-positive number disables text truncation (not recommended). |
|
62 | help=""" | |
63 | buffer_size = Int(500, config=True) |
|
63 | The maximum number of lines of text before truncation. Specifying a | |
64 |
|
64 | non-positive number disables text truncation (not recommended). | ||
65 | # Whether to use a list widget or plain text output for tab completion. |
|
65 | """ | |
66 | gui_completion = Bool(False, config=True) |
|
66 | ) | |
67 |
|
67 | gui_completion = Bool(False, config=True, | ||
68 | # The type of underlying text widget to use. Valid values are 'plain', which |
|
68 | help="Use a list widget instead of plain text output for tab completion." | |
69 | # specifies a QPlainTextEdit, and 'rich', which specifies a QTextEdit. |
|
69 | ) | |
70 | # NOTE: this value can only be specified during initialization. |
|
70 | # NOTE: this value can only be specified during initialization. | |
71 |
kind = Enum(['plain', 'rich'], default_value='plain', config=True |
|
71 | kind = Enum(['plain', 'rich'], default_value='plain', config=True, | |
72 |
|
72 | help=""" | ||
73 |
|
|
73 | The type of underlying text widget to use. Valid values are 'plain', which | |
74 | # 'inside' : The widget pages like a traditional terminal. |
|
74 | specifies a QPlainTextEdit, and 'rich', which specifies a QTextEdit. | |
75 | # 'hsplit' : When paging is requested, the widget is split |
|
75 | """ | |
76 | # horizontally. The top pane contains the console, and the |
|
76 | ) | |
77 | # bottom pane contains the paged text. |
|
|||
78 | # 'vsplit' : Similar to 'hsplit', except that a vertical splitter used. |
|
|||
79 | # 'custom' : No action is taken by the widget beyond emitting a |
|
|||
80 | # 'custom_page_requested(str)' signal. |
|
|||
81 | # 'none' : The text is written directly to the console. |
|
|||
82 | # NOTE: this value can only be specified during initialization. |
|
77 | # NOTE: this value can only be specified during initialization. | |
83 | paging = Enum(['inside', 'hsplit', 'vsplit', 'custom', 'none'], |
|
78 | paging = Enum(['inside', 'hsplit', 'vsplit', 'custom', 'none'], | |
84 |
default_value='inside', config=True |
|
79 | default_value='inside', config=True, | |
|
80 | help=""" | |||
|
81 | The type of paging to use. Valid values are: | |||
|
82 | ||||
|
83 | 'inside' : The widget pages like a traditional terminal. | |||
|
84 | 'hsplit' : When paging is requested, the widget is split | |||
|
85 | horizontally. The top pane contains the console, and the | |||
|
86 | bottom pane contains the paged text. | |||
|
87 | 'vsplit' : Similar to 'hsplit', except that a vertical splitter used. | |||
|
88 | 'custom' : No action is taken by the widget beyond emitting a | |||
|
89 | 'custom_page_requested(str)' signal. | |||
|
90 | 'none' : The text is written directly to the console. | |||
|
91 | """) | |||
|
92 | ||||
|
93 | font_family = Unicode(config=True, | |||
|
94 | help="""The font family to use for the console. | |||
|
95 | On OSX this defaults to Monaco, on Windows the default is | |||
|
96 | Consolas with fallback of Courier, and on other platforms | |||
|
97 | the default is Monospace. | |||
|
98 | """) | |||
|
99 | def _font_family_default(self): | |||
|
100 | if sys.platform == 'win32': | |||
|
101 | # Consolas ships with Vista/Win7, fallback to Courier if needed | |||
|
102 | return 'Consolas' | |||
|
103 | elif sys.platform == 'darwin': | |||
|
104 | # OSX always has Monaco, no need for a fallback | |||
|
105 | return 'Monaco' | |||
|
106 | else: | |||
|
107 | # Monospace should always exist, no need for a fallback | |||
|
108 | return 'Monospace' | |||
|
109 | ||||
|
110 | font_size = Int(config=True, | |||
|
111 | help="""The font size. If unconfigured, Qt will be entrusted | |||
|
112 | with the size of the font. | |||
|
113 | """) | |||
85 |
|
114 | |||
86 | # Whether to override ShortcutEvents for the keybindings defined by this |
|
115 | # Whether to override ShortcutEvents for the keybindings defined by this | |
87 | # widget (Ctrl+n, Ctrl+a, etc). Enable this if you want this widget to take |
|
116 | # widget (Ctrl+n, Ctrl+a, etc). Enable this if you want this widget to take | |
@@ -591,16 +620,18 b' class ConsoleWidget(Configurable, QtGui.QWidget):' | |||||
591 | """ |
|
620 | """ | |
592 | if sys.platform == 'win32': |
|
621 | if sys.platform == 'win32': | |
593 | # Consolas ships with Vista/Win7, fallback to Courier if needed |
|
622 | # Consolas ships with Vista/Win7, fallback to Courier if needed | |
594 |
|
|
623 | fallback = 'Courier' | |
595 | elif sys.platform == 'darwin': |
|
624 | elif sys.platform == 'darwin': | |
596 |
# OSX always has Monaco |
|
625 | # OSX always has Monaco | |
597 |
|
|
626 | fallback = 'Monaco' | |
|
627 | else: | |||
|
628 | # Monospace should always exist | |||
|
629 | fallback = 'Monospace' | |||
|
630 | font = get_font(self.font_family, fallback) | |||
|
631 | if self.font_size: | |||
|
632 | font.setPointSize(self.font_size) | |||
598 | else: |
|
633 | else: | |
599 | # FIXME: remove Consolas as a default on Linux once our font |
|
634 | font.setPointSize(QtGui.qApp.font().pointSize()) | |
600 | # selections are configurable by the user. |
|
|||
601 | family, fallback = 'Consolas', 'Monospace' |
|
|||
602 | font = get_font(family, fallback) |
|
|||
603 | font.setPointSize(QtGui.qApp.font().pointSize()) |
|
|||
604 | font.setStyleHint(QtGui.QFont.TypeWriter) |
|
635 | font.setStyleHint(QtGui.QFont.TypeWriter) | |
605 | self._set_font(font) |
|
636 | self._set_font(font) | |
606 |
|
637 |
@@ -13,7 +13,7 b' from IPython.external.qt import QtCore, QtGui' | |||||
13 | from IPython.core.inputsplitter import InputSplitter, transform_classic_prompt |
|
13 | from IPython.core.inputsplitter import InputSplitter, transform_classic_prompt | |
14 | from IPython.core.oinspect import call_tip |
|
14 | from IPython.core.oinspect import call_tip | |
15 | from IPython.frontend.qt.base_frontend_mixin import BaseFrontendMixin |
|
15 | from IPython.frontend.qt.base_frontend_mixin import BaseFrontendMixin | |
16 | from IPython.utils.traitlets import Bool |
|
16 | from IPython.utils.traitlets import Bool, Instance | |
17 | from bracket_matcher import BracketMatcher |
|
17 | from bracket_matcher import BracketMatcher | |
18 | from call_tip_widget import CallTipWidget |
|
18 | from call_tip_widget import CallTipWidget | |
19 | from completion_lexer import CompletionLexer |
|
19 | from completion_lexer import CompletionLexer | |
@@ -106,6 +106,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):' | |||||
106 | _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) |
|
106 | _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) | |
107 | _input_splitter_class = InputSplitter |
|
107 | _input_splitter_class = InputSplitter | |
108 | _local_kernel = False |
|
108 | _local_kernel = False | |
|
109 | _highlighter = Instance(FrontendHighlighter) | |||
109 |
|
110 | |||
110 | #--------------------------------------------------------------------------- |
|
111 | #--------------------------------------------------------------------------- | |
111 | # 'object' interface |
|
112 | # 'object' interface | |
@@ -183,7 +184,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):' | |||||
183 |
|
184 | |||
184 | See parent class :meth:`execute` docstring for full details. |
|
185 | See parent class :meth:`execute` docstring for full details. | |
185 | """ |
|
186 | """ | |
186 |
msg_id = self.kernel_manager. |
|
187 | msg_id = self.kernel_manager.shell_channel.execute(source, hidden) | |
187 | self._request_info['execute'] = self._ExecutionRequest(msg_id, 'user') |
|
188 | self._request_info['execute'] = self._ExecutionRequest(msg_id, 'user') | |
188 | self._hidden = hidden |
|
189 | self._hidden = hidden | |
189 | if not hidden: |
|
190 | if not hidden: | |
@@ -329,7 +330,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):' | |||||
329 | self.kernel_manager.sub_channel.flush() |
|
330 | self.kernel_manager.sub_channel.flush() | |
330 |
|
331 | |||
331 | def callback(line): |
|
332 | def callback(line): | |
332 |
self.kernel_manager. |
|
333 | self.kernel_manager.stdin_channel.input(line) | |
333 | self._readline(msg['content']['prompt'], callback=callback) |
|
334 | self._readline(msg['content']['prompt'], callback=callback) | |
334 |
|
335 | |||
335 | def _handle_kernel_died(self, since_last_heartbeat): |
|
336 | def _handle_kernel_died(self, since_last_heartbeat): | |
@@ -526,7 +527,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):' | |||||
526 |
|
527 | |||
527 | # Send the metadata request to the kernel |
|
528 | # Send the metadata request to the kernel | |
528 | name = '.'.join(context) |
|
529 | name = '.'.join(context) | |
529 |
msg_id = self.kernel_manager. |
|
530 | msg_id = self.kernel_manager.shell_channel.object_info(name) | |
530 | pos = self._get_cursor().position() |
|
531 | pos = self._get_cursor().position() | |
531 | self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) |
|
532 | self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) | |
532 | return True |
|
533 | return True | |
@@ -537,7 +538,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):' | |||||
537 | context = self._get_context() |
|
538 | context = self._get_context() | |
538 | if context: |
|
539 | if context: | |
539 | # Send the completion request to the kernel |
|
540 | # Send the completion request to the kernel | |
540 |
msg_id = self.kernel_manager. |
|
541 | msg_id = self.kernel_manager.shell_channel.complete( | |
541 | '.'.join(context), # text |
|
542 | '.'.join(context), # text | |
542 | self._get_input_buffer_cursor_line(), # line |
|
543 | self._get_input_buffer_cursor_line(), # line | |
543 | self._get_input_buffer_cursor_column(), # cursor_pos |
|
544 | self._get_input_buffer_cursor_column(), # cursor_pos |
@@ -23,9 +23,7 b' from IPython.core.inputsplitter import IPythonInputSplitter, \\' | |||||
23 | from IPython.core.usage import default_gui_banner |
|
23 | from IPython.core.usage import default_gui_banner | |
24 | from IPython.utils.traitlets import Bool, Str, Unicode |
|
24 | from IPython.utils.traitlets import Bool, Str, Unicode | |
25 | from frontend_widget import FrontendWidget |
|
25 | from frontend_widget import FrontendWidget | |
26 | from styles import (default_light_style_sheet, default_light_syntax_style, |
|
26 | import styles | |
27 | default_dark_style_sheet, default_dark_syntax_style, |
|
|||
28 | default_bw_style_sheet, default_bw_syntax_style) |
|
|||
29 |
|
27 | |||
30 | #----------------------------------------------------------------------------- |
|
28 | #----------------------------------------------------------------------------- | |
31 | # Constants |
|
29 | # Constants | |
@@ -42,6 +40,11 b" default_output_sep2 = ''" | |||||
42 | # Base path for most payload sources. |
|
40 | # Base path for most payload sources. | |
43 | zmq_shell_source = 'IPython.zmq.zmqshell.ZMQInteractiveShell' |
|
41 | zmq_shell_source = 'IPython.zmq.zmqshell.ZMQInteractiveShell' | |
44 |
|
42 | |||
|
43 | if sys.platform.startswith('win'): | |||
|
44 | default_editor = 'notepad' | |||
|
45 | else: | |||
|
46 | default_editor = '' | |||
|
47 | ||||
45 | #----------------------------------------------------------------------------- |
|
48 | #----------------------------------------------------------------------------- | |
46 | # IPythonWidget class |
|
49 | # IPythonWidget class | |
47 | #----------------------------------------------------------------------------- |
|
50 | #----------------------------------------------------------------------------- | |
@@ -56,26 +59,35 b' class IPythonWidget(FrontendWidget):' | |||||
56 | custom_edit = Bool(False) |
|
59 | custom_edit = Bool(False) | |
57 | custom_edit_requested = QtCore.Signal(object, object) |
|
60 | custom_edit_requested = QtCore.Signal(object, object) | |
58 |
|
61 | |||
59 | # A command for invoking a system text editor. If the string contains a |
|
62 | editor = Unicode(default_editor, config=True, | |
60 | # {filename} format specifier, it will be used. Otherwise, the filename will |
|
63 | help=""" | |
61 | # be appended to the end the command. |
|
64 | A command for invoking a system text editor. If the string contains a | |
62 | editor = Unicode('default', config=True) |
|
65 | {filename} format specifier, it will be used. Otherwise, the filename will | |
63 |
|
66 | be appended to the end the command. | ||
64 | # The editor command to use when a specific line number is requested. The |
|
67 | """) | |
65 | # string should contain two format specifiers: {line} and {filename}. If |
|
68 | ||
66 | # this parameter is not specified, the line number option to the %edit magic |
|
69 | editor_line = Unicode(config=True, | |
67 | # will be ignored. |
|
70 | help=""" | |
68 | editor_line = Unicode(config=True) |
|
71 | The editor command to use when a specific line number is requested. The | |
69 |
|
72 | string should contain two format specifiers: {line} and {filename}. If | ||
70 | # A CSS stylesheet. The stylesheet can contain classes for: |
|
73 | this parameter is not specified, the line number option to the %edit magic | |
71 | # 1. Qt: QPlainTextEdit, QFrame, QWidget, etc |
|
74 | will be ignored. | |
72 | # 2. Pygments: .c, .k, .o, etc (see PygmentsHighlighter) |
|
75 | """) | |
73 | # 3. IPython: .error, .in-prompt, .out-prompt, etc |
|
76 | ||
74 |
style_sheet = Unicode(config=True |
|
77 | style_sheet = Unicode(config=True, | |
|
78 | help=""" | |||
|
79 | A CSS stylesheet. The stylesheet can contain classes for: | |||
|
80 | 1. Qt: QPlainTextEdit, QFrame, QWidget, etc | |||
|
81 | 2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter) | |||
|
82 | 3. IPython: .error, .in-prompt, .out-prompt, etc | |||
|
83 | """) | |||
75 |
|
84 | |||
76 | # If not empty, use this Pygments style for syntax highlighting. Otherwise, |
|
85 | ||
77 | # the style sheet is queried for Pygments style information. |
|
86 | syntax_style = Str(config=True, | |
78 | syntax_style = Str(config=True) |
|
87 | help=""" | |
|
88 | If not empty, use this Pygments style for syntax highlighting. Otherwise, | |||
|
89 | the style sheet is queried for Pygments style information. | |||
|
90 | """) | |||
79 |
|
91 | |||
80 | # Prompts. |
|
92 | # Prompts. | |
81 | in_prompt = Str(default_in_prompt, config=True) |
|
93 | in_prompt = Str(default_in_prompt, config=True) | |
@@ -215,7 +227,7 b' class IPythonWidget(FrontendWidget):' | |||||
215 | """ Reimplemented to make a history request. |
|
227 | """ Reimplemented to make a history request. | |
216 | """ |
|
228 | """ | |
217 | super(IPythonWidget, self)._started_channels() |
|
229 | super(IPythonWidget, self)._started_channels() | |
218 |
self.kernel_manager. |
|
230 | self.kernel_manager.shell_channel.history(hist_access_type='tail', n=1000) | |
219 |
|
231 | |||
220 | #--------------------------------------------------------------------------- |
|
232 | #--------------------------------------------------------------------------- | |
221 | # 'ConsoleWidget' public interface |
|
233 | # 'ConsoleWidget' public interface | |
@@ -257,7 +269,7 b' class IPythonWidget(FrontendWidget):' | |||||
257 | text = '' |
|
269 | text = '' | |
258 |
|
270 | |||
259 | # Send the completion request to the kernel |
|
271 | # Send the completion request to the kernel | |
260 |
msg_id = self.kernel_manager. |
|
272 | msg_id = self.kernel_manager.shell_channel.complete( | |
261 | text, # text |
|
273 | text, # text | |
262 | self._get_input_buffer_cursor_line(), # line |
|
274 | self._get_input_buffer_cursor_line(), # line | |
263 | self._get_input_buffer_cursor_column(), # cursor_pos |
|
275 | self._get_input_buffer_cursor_column(), # cursor_pos | |
@@ -308,7 +320,7 b' class IPythonWidget(FrontendWidget):' | |||||
308 | """ |
|
320 | """ | |
309 | # If a number was not specified, make a prompt number request. |
|
321 | # If a number was not specified, make a prompt number request. | |
310 | if number is None: |
|
322 | if number is None: | |
311 |
msg_id = self.kernel_manager. |
|
323 | msg_id = self.kernel_manager.shell_channel.execute('', silent=True) | |
312 | info = self._ExecutionRequest(msg_id, 'prompt') |
|
324 | info = self._ExecutionRequest(msg_id, 'prompt') | |
313 | self._request_info['execute'] = info |
|
325 | self._request_info['execute'] = info | |
314 | return |
|
326 | return | |
@@ -371,14 +383,14 b' class IPythonWidget(FrontendWidget):' | |||||
371 | """ |
|
383 | """ | |
372 | colors = colors.lower() |
|
384 | colors = colors.lower() | |
373 | if colors=='lightbg': |
|
385 | if colors=='lightbg': | |
374 | self.style_sheet = default_light_style_sheet |
|
386 | self.style_sheet = styles.default_light_style_sheet | |
375 | self.syntax_style = default_light_syntax_style |
|
387 | self.syntax_style = styles.default_light_syntax_style | |
376 | elif colors=='linux': |
|
388 | elif colors=='linux': | |
377 | self.style_sheet = default_dark_style_sheet |
|
389 | self.style_sheet = styles.default_dark_style_sheet | |
378 | self.syntax_style = default_dark_syntax_style |
|
390 | self.syntax_style = styles.default_dark_syntax_style | |
379 | elif colors=='nocolor': |
|
391 | elif colors=='nocolor': | |
380 | self.style_sheet = default_bw_style_sheet |
|
392 | self.style_sheet = styles.default_bw_style_sheet | |
381 | self.syntax_style = default_bw_syntax_style |
|
393 | self.syntax_style = styles.default_bw_syntax_style | |
382 | else: |
|
394 | else: | |
383 | raise KeyError("No such color scheme: %s"%colors) |
|
395 | raise KeyError("No such color scheme: %s"%colors) | |
384 |
|
396 | |||
@@ -399,8 +411,10 b' class IPythonWidget(FrontendWidget):' | |||||
399 | """ |
|
411 | """ | |
400 | if self.custom_edit: |
|
412 | if self.custom_edit: | |
401 | self.custom_edit_requested.emit(filename, line) |
|
413 | self.custom_edit_requested.emit(filename, line) | |
402 |
elif self.editor |
|
414 | elif not self.editor: | |
403 |
self._append_plain_text('No default editor available.\n' |
|
415 | self._append_plain_text('No default editor available.\n' | |
|
416 | 'Specify a GUI text editor in the `IPythonWidget.editor` configurable\n' | |||
|
417 | 'to enable the %edit magic') | |||
404 | else: |
|
418 | else: | |
405 | try: |
|
419 | try: | |
406 | filename = '"%s"' % filename |
|
420 | filename = '"%s"' % filename | |
@@ -482,9 +496,13 b' class IPythonWidget(FrontendWidget):' | |||||
482 | bg_color = self._control.palette().window().color() |
|
496 | bg_color = self._control.palette().window().color() | |
483 | self._ansi_processor.set_background_color(bg_color) |
|
497 | self._ansi_processor.set_background_color(bg_color) | |
484 |
|
498 | |||
|
499 | ||||
485 | def _syntax_style_changed(self): |
|
500 | def _syntax_style_changed(self): | |
486 | """ Set the style for the syntax highlighter. |
|
501 | """ Set the style for the syntax highlighter. | |
487 | """ |
|
502 | """ | |
|
503 | if self._highlighter is None: | |||
|
504 | # ignore premature calls | |||
|
505 | return | |||
488 | if self.syntax_style: |
|
506 | if self.syntax_style: | |
489 | self._highlighter.set_style(self.syntax_style) |
|
507 | self._highlighter.set_style(self.syntax_style) | |
490 | else: |
|
508 | else: |
@@ -1,21 +1,50 b'' | |||||
1 | """ A minimal application using the Qt console-style IPython frontend. |
|
1 | """ A minimal application using the Qt console-style IPython frontend. | |
|
2 | ||||
|
3 | This is not a complete console app, as subprocess will not be able to receive | |||
|
4 | input, there is no real readline support, among other limitations. | |||
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Evan Patterson | |||
|
9 | * Min RK | |||
|
10 | * Erik Tollerud | |||
|
11 | * Fernando Perez | |||
|
12 | ||||
2 | """ |
|
13 | """ | |
3 |
|
14 | |||
4 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
5 | # Imports |
|
16 | # Imports | |
6 | #----------------------------------------------------------------------------- |
|
17 | #----------------------------------------------------------------------------- | |
7 |
|
18 | |||
8 |
# |
|
19 | # stdlib imports | |
|
20 | import os | |||
|
21 | import signal | |||
|
22 | import sys | |||
|
23 | ||||
|
24 | # System library imports | |||
9 | from IPython.external.qt import QtGui |
|
25 | from IPython.external.qt import QtGui | |
10 | from pygments.styles import get_all_styles |
|
26 | from pygments.styles import get_all_styles | |
11 |
|
27 | |||
12 | # Local imports |
|
28 | # Local imports | |
13 | from IPython.external.argparse import ArgumentParser |
|
29 | from IPython.config.application import boolean_flag | |
|
30 | from IPython.core.application import BaseIPythonApplication | |||
|
31 | from IPython.core.profiledir import ProfileDir | |||
14 | from IPython.frontend.qt.console.frontend_widget import FrontendWidget |
|
32 | from IPython.frontend.qt.console.frontend_widget import FrontendWidget | |
15 | from IPython.frontend.qt.console.ipython_widget import IPythonWidget |
|
33 | from IPython.frontend.qt.console.ipython_widget import IPythonWidget | |
16 | from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget |
|
34 | from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget | |
17 | from IPython.frontend.qt.console import styles |
|
35 | from IPython.frontend.qt.console import styles | |
18 | from IPython.frontend.qt.kernelmanager import QtKernelManager |
|
36 | from IPython.frontend.qt.kernelmanager import QtKernelManager | |
|
37 | from IPython.utils.traitlets import ( | |||
|
38 | Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any | |||
|
39 | ) | |||
|
40 | from IPython.zmq.ipkernel import ( | |||
|
41 | flags as ipkernel_flags, | |||
|
42 | aliases as ipkernel_aliases, | |||
|
43 | IPKernelApp | |||
|
44 | ) | |||
|
45 | from IPython.zmq.session import Session | |||
|
46 | from IPython.zmq.zmqshell import ZMQInteractiveShell | |||
|
47 | ||||
19 |
|
48 | |||
20 | #----------------------------------------------------------------------------- |
|
49 | #----------------------------------------------------------------------------- | |
21 | # Network Constants |
|
50 | # Network Constants | |
@@ -33,7 +62,8 b' class MainWindow(QtGui.QMainWindow):' | |||||
33 | # 'object' interface |
|
62 | # 'object' interface | |
34 | #--------------------------------------------------------------------------- |
|
63 | #--------------------------------------------------------------------------- | |
35 |
|
64 | |||
36 |
def __init__(self, app, frontend, existing=False, may_close=True |
|
65 | def __init__(self, app, frontend, existing=False, may_close=True, | |
|
66 | confirm_exit=True): | |||
37 | """ Create a MainWindow for the specified FrontendWidget. |
|
67 | """ Create a MainWindow for the specified FrontendWidget. | |
38 |
|
68 | |||
39 | The app is passed as an argument to allow for different |
|
69 | The app is passed as an argument to allow for different | |
@@ -52,6 +82,7 b' class MainWindow(QtGui.QMainWindow):' | |||||
52 | else: |
|
82 | else: | |
53 | self._may_close = True |
|
83 | self._may_close = True | |
54 | self._frontend.exit_requested.connect(self.close) |
|
84 | self._frontend.exit_requested.connect(self.close) | |
|
85 | self._confirm_exit = confirm_exit | |||
55 | self.setCentralWidget(frontend) |
|
86 | self.setCentralWidget(frontend) | |
56 |
|
87 | |||
57 | #--------------------------------------------------------------------------- |
|
88 | #--------------------------------------------------------------------------- | |
@@ -71,6 +102,11 b' class MainWindow(QtGui.QMainWindow):' | |||||
71 |
|
102 | |||
72 | kernel_manager = self._frontend.kernel_manager |
|
103 | kernel_manager = self._frontend.kernel_manager | |
73 |
|
104 | |||
|
105 | if keepkernel is None and not self._confirm_exit: | |||
|
106 | # don't prompt, just terminate the kernel if we own it | |||
|
107 | # or leave it alone if we don't | |||
|
108 | keepkernel = not self._existing | |||
|
109 | ||||
74 | if keepkernel is None: #show prompt |
|
110 | if keepkernel is None: #show prompt | |
75 | if kernel_manager and kernel_manager.channels_running: |
|
111 | if kernel_manager and kernel_manager.channels_running: | |
76 | title = self.window().windowTitle() |
|
112 | title = self.window().windowTitle() | |
@@ -127,123 +163,216 b' class MainWindow(QtGui.QMainWindow):' | |||||
127 | event.accept() |
|
163 | event.accept() | |
128 |
|
164 | |||
129 | #----------------------------------------------------------------------------- |
|
165 | #----------------------------------------------------------------------------- | |
130 | # Main entry point |
|
166 | # Aliases and Flags | |
131 | #----------------------------------------------------------------------------- |
|
167 | #----------------------------------------------------------------------------- | |
132 |
|
168 | |||
133 | def main(): |
|
169 | flags = dict(ipkernel_flags) | |
134 | """ Entry point for application. |
|
170 | ||
|
171 | flags.update({ | |||
|
172 | 'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}}, | |||
|
173 | "Connect to an existing kernel."), | |||
|
174 | 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}}, | |||
|
175 | "Use a pure Python kernel instead of an IPython kernel."), | |||
|
176 | 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}}, | |||
|
177 | "Disable rich text support."), | |||
|
178 | }) | |||
|
179 | flags.update(boolean_flag( | |||
|
180 | 'gui-completion', 'ConsoleWidget.gui_completion', | |||
|
181 | "use a GUI widget for tab completion", | |||
|
182 | "use plaintext output for completion" | |||
|
183 | )) | |||
|
184 | flags.update(boolean_flag( | |||
|
185 | 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit', | |||
|
186 | """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', | |||
|
187 | to force a direct exit without any confirmation. | |||
|
188 | """, | |||
|
189 | """Don't prompt the user when exiting. This will terminate the kernel | |||
|
190 | if it is owned by the frontend, and leave it alive if it is external. | |||
135 | """ |
|
191 | """ | |
136 | # Parse command line arguments. |
|
192 | )) | |
137 | parser = ArgumentParser() |
|
193 | # the flags that are specific to the frontend | |
138 | kgroup = parser.add_argument_group('kernel options') |
|
194 | # these must be scrubbed before being passed to the kernel, | |
139 | kgroup.add_argument('-e', '--existing', action='store_true', |
|
195 | # or it will raise an error on unrecognized flags | |
140 | help='connect to an existing kernel') |
|
196 | qt_flags = ['existing', 'pure', 'plain', 'gui-completion', 'no-gui-completion', | |
141 | kgroup.add_argument('--ip', type=str, default=LOCALHOST, |
|
197 | 'confirm-exit', 'no-confirm-exit'] | |
142 | help=\ |
|
198 | ||
143 | "set the kernel\'s IP address [default localhost].\ |
|
199 | aliases = dict(ipkernel_aliases) | |
144 | If the IP address is something other than localhost, then \ |
|
200 | ||
145 | Consoles on other machines will be able to connect\ |
|
201 | aliases.update(dict( | |
146 | to the Kernel, so be careful!") |
|
202 | hb = 'IPythonQtConsoleApp.hb_port', | |
147 | kgroup.add_argument('--xreq', type=int, metavar='PORT', default=0, |
|
203 | shell = 'IPythonQtConsoleApp.shell_port', | |
148 | help='set the XREQ channel port [default random]') |
|
204 | iopub = 'IPythonQtConsoleApp.iopub_port', | |
149 | kgroup.add_argument('--sub', type=int, metavar='PORT', default=0, |
|
205 | stdin = 'IPythonQtConsoleApp.stdin_port', | |
150 | help='set the SUB channel port [default random]') |
|
206 | ip = 'IPythonQtConsoleApp.ip', | |
151 | kgroup.add_argument('--rep', type=int, metavar='PORT', default=0, |
|
207 | ||
152 | help='set the REP channel port [default random]') |
|
208 | plain = 'IPythonQtConsoleApp.plain', | |
153 | kgroup.add_argument('--hb', type=int, metavar='PORT', default=0, |
|
209 | pure = 'IPythonQtConsoleApp.pure', | |
154 | help='set the heartbeat port [default random]') |
|
210 | gui_completion = 'ConsoleWidget.gui_completion', | |
155 |
|
211 | style = 'IPythonWidget.syntax_style', | ||
156 | egroup = kgroup.add_mutually_exclusive_group() |
|
212 | stylesheet = 'IPythonQtConsoleApp.stylesheet', | |
157 | egroup.add_argument('--pure', action='store_true', help = \ |
|
213 | colors = 'ZMQInteractiveShell.colors', | |
158 | 'use a pure Python kernel instead of an IPython kernel') |
|
214 | ||
159 | egroup.add_argument('--pylab', type=str, metavar='GUI', nargs='?', |
|
215 | editor = 'IPythonWidget.editor', | |
160 | const='auto', help = \ |
|
216 | )) | |
161 | "Pre-load matplotlib and numpy for interactive use. If GUI is not \ |
|
217 | ||
162 | given, the GUI backend is matplotlib's, otherwise use one of: \ |
|
218 | #----------------------------------------------------------------------------- | |
163 | ['tk', 'gtk', 'qt', 'wx', 'inline'].") |
|
219 | # IPythonQtConsole | |
164 |
|
220 | #----------------------------------------------------------------------------- | ||
165 | wgroup = parser.add_argument_group('widget options') |
|
221 | class IPythonQtConsoleApp(BaseIPythonApplication): | |
166 | wgroup.add_argument('--paging', type=str, default='inside', |
|
222 | name = 'ipython-qtconsole' | |
167 | choices = ['inside', 'hsplit', 'vsplit', 'none'], |
|
223 | default_config_file_name='ipython_config.py' | |
168 | help='set the paging style [default inside]') |
|
224 | ||
169 | wgroup.add_argument('--plain', action='store_true', |
|
225 | description = """ | |
170 | help='disable rich text support') |
|
226 | The IPython QtConsole. | |
171 | wgroup.add_argument('--gui-completion', action='store_true', |
|
227 | ||
172 | help='use a GUI widget for tab completion') |
|
228 | This launches a Console-style application using Qt. It is not a full | |
173 | wgroup.add_argument('--style', type=str, |
|
229 | console, in that launched terminal subprocesses will not. | |
174 | choices = list(get_all_styles()), |
|
230 | ||
175 | help='specify a pygments style for by name') |
|
231 | The QtConsole supports various extra features beyond the | |
176 | wgroup.add_argument('--stylesheet', type=str, |
|
232 | ||
177 | help='path to a custom CSS stylesheet') |
|
233 | """ | |
178 | wgroup.add_argument('--colors', type=str, help = \ |
|
234 | ||
179 | "Set the color scheme (LightBG,Linux,NoColor). This is guessed \ |
|
235 | classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session] | |
180 | based on the pygments style if not set.") |
|
236 | flags = Dict(flags) | |
181 |
|
237 | aliases = Dict(aliases) | ||
182 | args = parser.parse_args() |
|
238 | ||
183 |
|
239 | kernel_argv = List(Unicode) | ||
184 | # parse the colors arg down to current known labels |
|
240 | ||
185 | if args.colors: |
|
241 | # connection info: | |
186 | colors=args.colors.lower() |
|
242 | ip = Unicode(LOCALHOST, config=True, | |
187 | if colors in ('lightbg', 'light'): |
|
243 | help="""Set the kernel\'s IP address [default localhost]. | |
188 | colors='lightbg' |
|
244 | If the IP address is something other than localhost, then | |
189 | elif colors in ('dark', 'linux'): |
|
245 | Consoles on other machines will be able to connect | |
190 | colors='linux' |
|
246 | to the Kernel, so be careful!""" | |
191 | else: |
|
247 | ) | |
192 | colors='nocolor' |
|
248 | hb_port = Int(0, config=True, | |
193 | elif args.style: |
|
249 | help="set the heartbeat port [default: random]") | |
194 | if args.style=='bw': |
|
250 | shell_port = Int(0, config=True, | |
195 | colors='nocolor' |
|
251 | help="set the shell (XREP) port [default: random]") | |
196 | elif styles.dark_style(args.style): |
|
252 | iopub_port = Int(0, config=True, | |
197 | colors='linux' |
|
253 | help="set the iopub (PUB) port [default: random]") | |
|
254 | stdin_port = Int(0, config=True, | |||
|
255 | help="set the stdin (XREQ) port [default: random]") | |||
|
256 | ||||
|
257 | existing = CBool(False, config=True, | |||
|
258 | help="Whether to connect to an already running Kernel.") | |||
|
259 | ||||
|
260 | stylesheet = Unicode('', config=True, | |||
|
261 | help="path to a custom CSS stylesheet") | |||
|
262 | ||||
|
263 | pure = CBool(False, config=True, | |||
|
264 | help="Use a pure Python kernel instead of an IPython kernel.") | |||
|
265 | plain = CBool(False, config=True, | |||
|
266 | help="Use a plaintext widget instead of rich text (plain can't print/save).") | |||
|
267 | ||||
|
268 | def _pure_changed(self, name, old, new): | |||
|
269 | kind = 'plain' if self.plain else 'rich' | |||
|
270 | self.config.ConsoleWidget.kind = kind | |||
|
271 | if self.pure: | |||
|
272 | self.widget_factory = FrontendWidget | |||
|
273 | elif self.plain: | |||
|
274 | self.widget_factory = IPythonWidget | |||
198 | else: |
|
275 | else: | |
199 | colors='lightbg' |
|
276 | self.widget_factory = RichIPythonWidget | |
200 | else: |
|
277 | ||
201 | colors=None |
|
278 | _plain_changed = _pure_changed | |
202 |
|
279 | |||
203 | # Don't let Qt or ZMQ swallow KeyboardInterupts. |
|
280 | confirm_exit = CBool(True, config=True, | |
204 | import signal |
|
281 | help=""" | |
205 | signal.signal(signal.SIGINT, signal.SIG_DFL) |
|
282 | Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', | |
206 |
|
283 | to force a direct exit without any confirmation.""", | ||
207 | # Create a KernelManager and start a kernel. |
|
284 | ) | |
208 | kernel_manager = QtKernelManager(xreq_address=(args.ip, args.xreq), |
|
285 | ||
209 | sub_address=(args.ip, args.sub), |
|
286 | # the factory for creating a widget | |
210 | rep_address=(args.ip, args.rep), |
|
287 | widget_factory = Any(RichIPythonWidget) | |
211 | hb_address=(args.ip, args.hb)) |
|
288 | ||
212 | if not args.existing: |
|
289 | def parse_command_line(self, argv=None): | |
213 | # if not args.ip in LOCAL_IPS+ALL_ALIAS: |
|
290 | super(IPythonQtConsoleApp, self).parse_command_line(argv) | |
214 | # raise ValueError("Must bind a local ip, such as: %s"%LOCAL_IPS) |
|
291 | if argv is None: | |
215 |
|
292 | argv = sys.argv[1:] | ||
216 | kwargs = dict(ip=args.ip) |
|
293 | ||
217 | if args.pure: |
|
294 | self.kernel_argv = list(argv) # copy | |
218 | kwargs['ipython']=False |
|
295 | ||
|
296 | # scrub frontend-specific flags | |||
|
297 | for a in argv: | |||
|
298 | if a.startswith('--') and a[2:] in qt_flags: | |||
|
299 | self.kernel_argv.remove(a) | |||
|
300 | ||||
|
301 | def init_kernel_manager(self): | |||
|
302 | # Don't let Qt or ZMQ swallow KeyboardInterupts. | |||
|
303 | signal.signal(signal.SIGINT, signal.SIG_DFL) | |||
|
304 | ||||
|
305 | # Create a KernelManager and start a kernel. | |||
|
306 | self.kernel_manager = QtKernelManager( | |||
|
307 | shell_address=(self.ip, self.shell_port), | |||
|
308 | sub_address=(self.ip, self.iopub_port), | |||
|
309 | stdin_address=(self.ip, self.stdin_port), | |||
|
310 | hb_address=(self.ip, self.hb_port), | |||
|
311 | config=self.config | |||
|
312 | ) | |||
|
313 | # start the kernel | |||
|
314 | if not self.existing: | |||
|
315 | kwargs = dict(ip=self.ip, ipython=not self.pure) | |||
|
316 | kwargs['extra_arguments'] = self.kernel_argv | |||
|
317 | self.kernel_manager.start_kernel(**kwargs) | |||
|
318 | self.kernel_manager.start_channels() | |||
|
319 | ||||
|
320 | ||||
|
321 | def init_qt_elements(self): | |||
|
322 | # Create the widget. | |||
|
323 | self.app = QtGui.QApplication([]) | |||
|
324 | local_kernel = (not self.existing) or self.ip in LOCAL_IPS | |||
|
325 | self.widget = self.widget_factory(config=self.config, | |||
|
326 | local_kernel=local_kernel) | |||
|
327 | self.widget.kernel_manager = self.kernel_manager | |||
|
328 | self.window = MainWindow(self.app, self.widget, self.existing, | |||
|
329 | may_close=local_kernel, | |||
|
330 | confirm_exit=self.confirm_exit) | |||
|
331 | self.window.setWindowTitle('Python' if self.pure else 'IPython') | |||
|
332 | ||||
|
333 | def init_colors(self): | |||
|
334 | """Configure the coloring of the widget""" | |||
|
335 | # Note: This will be dramatically simplified when colors | |||
|
336 | # are removed from the backend. | |||
|
337 | ||||
|
338 | if self.pure: | |||
|
339 | # only IPythonWidget supports styling | |||
|
340 | return | |||
|
341 | ||||
|
342 | # parse the colors arg down to current known labels | |||
|
343 | try: | |||
|
344 | colors = self.config.ZMQInteractiveShell.colors | |||
|
345 | except AttributeError: | |||
|
346 | colors = None | |||
|
347 | try: | |||
|
348 | style = self.config.IPythonWidget.colors | |||
|
349 | except AttributeError: | |||
|
350 | style = None | |||
|
351 | ||||
|
352 | # find the value for colors: | |||
|
353 | if colors: | |||
|
354 | colors=colors.lower() | |||
|
355 | if colors in ('lightbg', 'light'): | |||
|
356 | colors='lightbg' | |||
|
357 | elif colors in ('dark', 'linux'): | |||
|
358 | colors='linux' | |||
|
359 | else: | |||
|
360 | colors='nocolor' | |||
|
361 | elif style: | |||
|
362 | if style=='bw': | |||
|
363 | colors='nocolor' | |||
|
364 | elif styles.dark_style(style): | |||
|
365 | colors='linux' | |||
|
366 | else: | |||
|
367 | colors='lightbg' | |||
219 | else: |
|
368 | else: | |
220 |
|
|
369 | colors=None | |
221 | if args.pylab: |
|
370 | ||
222 | kwargs['pylab']=args.pylab |
|
371 | # Configure the style. | |
223 |
|
372 | widget = self.widget | ||
224 | kernel_manager.start_kernel(**kwargs) |
|
373 | if style: | |
225 | kernel_manager.start_channels() |
|
374 | widget.style_sheet = styles.sheet_from_template(style, colors) | |
226 |
|
375 | widget.syntax_style = style | ||
227 | # Create the widget. |
|
|||
228 | app = QtGui.QApplication([]) |
|
|||
229 | local_kernel = (not args.existing) or args.ip in LOCAL_IPS |
|
|||
230 | if args.pure: |
|
|||
231 | kind = 'plain' if args.plain else 'rich' |
|
|||
232 | widget = FrontendWidget(kind=kind, paging=args.paging, |
|
|||
233 | local_kernel=local_kernel) |
|
|||
234 | elif args.plain: |
|
|||
235 | widget = IPythonWidget(paging=args.paging, local_kernel=local_kernel) |
|
|||
236 | else: |
|
|||
237 | widget = RichIPythonWidget(paging=args.paging, |
|
|||
238 | local_kernel=local_kernel) |
|
|||
239 | widget.gui_completion = args.gui_completion |
|
|||
240 | widget.kernel_manager = kernel_manager |
|
|||
241 |
|
||||
242 | # Configure the style. |
|
|||
243 | if not args.pure: # only IPythonWidget supports styles |
|
|||
244 | if args.style: |
|
|||
245 | widget.syntax_style = args.style |
|
|||
246 | widget.style_sheet = styles.sheet_from_template(args.style, colors) |
|
|||
247 | widget._syntax_style_changed() |
|
376 | widget._syntax_style_changed() | |
248 | widget._style_sheet_changed() |
|
377 | widget._style_sheet_changed() | |
249 | elif colors: |
|
378 | elif colors: | |
@@ -254,23 +383,38 b' def main():' | |||||
254 | # defaults to change |
|
383 | # defaults to change | |
255 | widget.set_default_style() |
|
384 | widget.set_default_style() | |
256 |
|
385 | |||
257 |
if |
|
386 | if self.stylesheet: | |
258 | # we got an expicit stylesheet |
|
387 | # we got an expicit stylesheet | |
259 |
if os.path.isfile( |
|
388 | if os.path.isfile(self.stylesheet): | |
260 |
with open( |
|
389 | with open(self.stylesheet) as f: | |
261 | sheet = f.read() |
|
390 | sheet = f.read() | |
262 | widget.style_sheet = sheet |
|
391 | widget.style_sheet = sheet | |
263 | widget._style_sheet_changed() |
|
392 | widget._style_sheet_changed() | |
264 | else: |
|
393 | else: | |
265 |
raise IOError("Stylesheet %r not found."% |
|
394 | raise IOError("Stylesheet %r not found."%self.stylesheet) | |
|
395 | ||||
|
396 | def initialize(self, argv=None): | |||
|
397 | super(IPythonQtConsoleApp, self).initialize(argv) | |||
|
398 | self.init_kernel_manager() | |||
|
399 | self.init_qt_elements() | |||
|
400 | self.init_colors() | |||
|
401 | ||||
|
402 | def start(self): | |||
266 |
|
403 | |||
267 |
# |
|
404 | # draw the window | |
268 | window = MainWindow(app, widget, args.existing, may_close=local_kernel) |
|
405 | self.window.show() | |
269 | window.setWindowTitle('Python' if args.pure else 'IPython') |
|
|||
270 | window.show() |
|
|||
271 |
|
406 | |||
272 | # Start the application main loop. |
|
407 | # Start the application main loop. | |
273 | app.exec_() |
|
408 | self.app.exec_() | |
|
409 | ||||
|
410 | #----------------------------------------------------------------------------- | |||
|
411 | # Main entry point | |||
|
412 | #----------------------------------------------------------------------------- | |||
|
413 | ||||
|
414 | def main(): | |||
|
415 | app = IPythonQtConsoleApp() | |||
|
416 | app.initialize() | |||
|
417 | app.start() | |||
274 |
|
418 | |||
275 |
|
419 | |||
276 | if __name__ == '__main__': |
|
420 | if __name__ == '__main__': |
@@ -7,7 +7,7 b' from IPython.external.qt import QtCore' | |||||
7 | # IPython imports. |
|
7 | # IPython imports. | |
8 | from IPython.utils.traitlets import Type |
|
8 | from IPython.utils.traitlets import Type | |
9 | from IPython.zmq.kernelmanager import KernelManager, SubSocketChannel, \ |
|
9 | from IPython.zmq.kernelmanager import KernelManager, SubSocketChannel, \ | |
10 |
|
|
10 | ShellSocketChannel, StdInSocketChannel, HBSocketChannel | |
11 | from util import MetaQObjectHasTraits, SuperQObject |
|
11 | from util import MetaQObjectHasTraits, SuperQObject | |
12 |
|
12 | |||
13 |
|
13 | |||
@@ -20,7 +20,7 b' class SocketChannelQObject(SuperQObject):' | |||||
20 | stopped = QtCore.Signal() |
|
20 | stopped = QtCore.Signal() | |
21 |
|
21 | |||
22 | #--------------------------------------------------------------------------- |
|
22 | #--------------------------------------------------------------------------- | |
23 |
# 'Z |
|
23 | # 'ZMQSocketChannel' interface | |
24 | #--------------------------------------------------------------------------- |
|
24 | #--------------------------------------------------------------------------- | |
25 |
|
25 | |||
26 | def start(self): |
|
26 | def start(self): | |
@@ -36,7 +36,7 b' class SocketChannelQObject(SuperQObject):' | |||||
36 | self.stopped.emit() |
|
36 | self.stopped.emit() | |
37 |
|
37 | |||
38 |
|
38 | |||
39 |
class Qt |
|
39 | class QtShellSocketChannel(SocketChannelQObject, ShellSocketChannel): | |
40 |
|
40 | |||
41 | # Emitted when any message is received. |
|
41 | # Emitted when any message is received. | |
42 | message_received = QtCore.Signal(object) |
|
42 | message_received = QtCore.Signal(object) | |
@@ -56,7 +56,7 b' class QtXReqSocketChannel(SocketChannelQObject, XReqSocketChannel):' | |||||
56 | _handlers_called = False |
|
56 | _handlers_called = False | |
57 |
|
57 | |||
58 | #--------------------------------------------------------------------------- |
|
58 | #--------------------------------------------------------------------------- | |
59 |
# ' |
|
59 | # 'ShellSocketChannel' interface | |
60 | #--------------------------------------------------------------------------- |
|
60 | #--------------------------------------------------------------------------- | |
61 |
|
61 | |||
62 | def call_handlers(self, msg): |
|
62 | def call_handlers(self, msg): | |
@@ -76,7 +76,7 b' class QtXReqSocketChannel(SocketChannelQObject, XReqSocketChannel):' | |||||
76 | self._handlers_called = True |
|
76 | self._handlers_called = True | |
77 |
|
77 | |||
78 | #--------------------------------------------------------------------------- |
|
78 | #--------------------------------------------------------------------------- | |
79 |
# 'Qt |
|
79 | # 'QtShellSocketChannel' interface | |
80 | #--------------------------------------------------------------------------- |
|
80 | #--------------------------------------------------------------------------- | |
81 |
|
81 | |||
82 | def reset_first_reply(self): |
|
82 | def reset_first_reply(self): | |
@@ -136,7 +136,7 b' class QtSubSocketChannel(SocketChannelQObject, SubSocketChannel):' | |||||
136 | QtCore.QCoreApplication.instance().processEvents() |
|
136 | QtCore.QCoreApplication.instance().processEvents() | |
137 |
|
137 | |||
138 |
|
138 | |||
139 |
class Qt |
|
139 | class QtStdInSocketChannel(SocketChannelQObject, StdInSocketChannel): | |
140 |
|
140 | |||
141 | # Emitted when any message is received. |
|
141 | # Emitted when any message is received. | |
142 | message_received = QtCore.Signal(object) |
|
142 | message_received = QtCore.Signal(object) | |
@@ -145,7 +145,7 b' class QtRepSocketChannel(SocketChannelQObject, RepSocketChannel):' | |||||
145 | input_requested = QtCore.Signal(object) |
|
145 | input_requested = QtCore.Signal(object) | |
146 |
|
146 | |||
147 | #--------------------------------------------------------------------------- |
|
147 | #--------------------------------------------------------------------------- | |
148 |
# ' |
|
148 | # 'StdInSocketChannel' interface | |
149 | #--------------------------------------------------------------------------- |
|
149 | #--------------------------------------------------------------------------- | |
150 |
|
150 | |||
151 | def call_handlers(self, msg): |
|
151 | def call_handlers(self, msg): | |
@@ -190,8 +190,8 b' class QtKernelManager(KernelManager, SuperQObject):' | |||||
190 |
|
190 | |||
191 | # Use Qt-specific channel classes that emit signals. |
|
191 | # Use Qt-specific channel classes that emit signals. | |
192 | sub_channel_class = Type(QtSubSocketChannel) |
|
192 | sub_channel_class = Type(QtSubSocketChannel) | |
193 |
|
|
193 | shell_channel_class = Type(QtShellSocketChannel) | |
194 |
|
|
194 | stdin_channel_class = Type(QtStdInSocketChannel) | |
195 | hb_channel_class = Type(QtHBSocketChannel) |
|
195 | hb_channel_class = Type(QtHBSocketChannel) | |
196 |
|
196 | |||
197 | #--------------------------------------------------------------------------- |
|
197 | #--------------------------------------------------------------------------- | |
@@ -203,8 +203,8 b' class QtKernelManager(KernelManager, SuperQObject):' | |||||
203 | def start_kernel(self, *args, **kw): |
|
203 | def start_kernel(self, *args, **kw): | |
204 | """ Reimplemented for proper heartbeat management. |
|
204 | """ Reimplemented for proper heartbeat management. | |
205 | """ |
|
205 | """ | |
206 |
if self._ |
|
206 | if self._shell_channel is not None: | |
207 |
self._ |
|
207 | self._shell_channel.reset_first_reply() | |
208 | super(QtKernelManager, self).start_kernel(*args, **kw) |
|
208 | super(QtKernelManager, self).start_kernel(*args, **kw) | |
209 |
|
209 | |||
210 | #------ Channel management ------------------------------------------------- |
|
210 | #------ Channel management ------------------------------------------------- | |
@@ -222,13 +222,13 b' class QtKernelManager(KernelManager, SuperQObject):' | |||||
222 | self.stopped_channels.emit() |
|
222 | self.stopped_channels.emit() | |
223 |
|
223 | |||
224 | @property |
|
224 | @property | |
225 |
def |
|
225 | def shell_channel(self): | |
226 | """ Reimplemented for proper heartbeat management. |
|
226 | """ Reimplemented for proper heartbeat management. | |
227 | """ |
|
227 | """ | |
228 |
if self._ |
|
228 | if self._shell_channel is None: | |
229 |
self._ |
|
229 | self._shell_channel = super(QtKernelManager, self).shell_channel | |
230 |
self._ |
|
230 | self._shell_channel.first_reply.connect(self._first_reply) | |
231 |
return self._ |
|
231 | return self._shell_channel | |
232 |
|
232 | |||
233 | #--------------------------------------------------------------------------- |
|
233 | #--------------------------------------------------------------------------- | |
234 | # Protected interface |
|
234 | # Protected interface |
@@ -58,11 +58,21 b' raw_input_original = raw_input' | |||||
58 |
|
58 | |||
59 | class TerminalInteractiveShell(InteractiveShell): |
|
59 | class TerminalInteractiveShell(InteractiveShell): | |
60 |
|
60 | |||
61 |
autoedit_syntax = CBool(False, config=True |
|
61 | autoedit_syntax = CBool(False, config=True, | |
|
62 | help="auto editing of files with syntax errors.") | |||
62 | banner = Unicode('') |
|
63 | banner = Unicode('') | |
63 |
banner1 = Unicode(default_banner, config=True |
|
64 | banner1 = Unicode(default_banner, config=True, | |
64 | banner2 = Unicode('', config=True) |
|
65 | help="""The part of the banner to be printed before the profile""" | |
65 | confirm_exit = CBool(True, config=True) |
|
66 | ) | |
|
67 | banner2 = Unicode('', config=True, | |||
|
68 | help="""The part of the banner to be printed after the profile""" | |||
|
69 | ) | |||
|
70 | confirm_exit = CBool(True, config=True, | |||
|
71 | help=""" | |||
|
72 | Set to confirm when you try to exit IPython with an EOF (Control-D | |||
|
73 | in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', | |||
|
74 | you can force a direct exit without any confirmation.""", | |||
|
75 | ) | |||
66 | # This display_banner only controls whether or not self.show_banner() |
|
76 | # This display_banner only controls whether or not self.show_banner() | |
67 | # is called when mainloop/interact are called. The default is False |
|
77 | # is called when mainloop/interact are called. The default is False | |
68 | # because for the terminal based application, the banner behavior |
|
78 | # because for the terminal based application, the banner behavior | |
@@ -71,19 +81,35 b' class TerminalInteractiveShell(InteractiveShell):' | |||||
71 | display_banner = CBool(False) # This isn't configurable! |
|
81 | display_banner = CBool(False) # This isn't configurable! | |
72 | embedded = CBool(False) |
|
82 | embedded = CBool(False) | |
73 | embedded_active = CBool(False) |
|
83 | embedded_active = CBool(False) | |
74 |
editor = Unicode(get_default_editor(), config=True |
|
84 | editor = Unicode(get_default_editor(), config=True, | |
75 | pager = Unicode('less', config=True) |
|
85 | help="Set the editor used by IPython (default to $EDITOR/vi/notepad)." | |
76 |
|
86 | ) | ||
77 | screen_length = Int(0, config=True) |
|
87 | pager = Unicode('less', config=True, | |
78 | term_title = CBool(False, config=True) |
|
88 | help="The shell program to be used for paging.") | |
79 |
|
89 | |||
80 | def __init__(self, config=None, ipython_dir=None, user_ns=None, |
|
90 | screen_length = Int(0, config=True, | |
|
91 | help= | |||
|
92 | """Number of lines of your screen, used to control printing of very | |||
|
93 | long strings. Strings longer than this number of lines will be sent | |||
|
94 | through a pager instead of directly printed. The default value for | |||
|
95 | this is 0, which means IPython will auto-detect your screen size every | |||
|
96 | time it needs to print certain potentially long strings (this doesn't | |||
|
97 | change the behavior of the 'print' keyword, it's only triggered | |||
|
98 | internally). If for some reason this isn't working well (it needs | |||
|
99 | curses support), specify it yourself. Otherwise don't change the | |||
|
100 | default.""", | |||
|
101 | ) | |||
|
102 | term_title = CBool(False, config=True, | |||
|
103 | help="Enable auto setting the terminal title." | |||
|
104 | ) | |||
|
105 | ||||
|
106 | def __init__(self, config=None, ipython_dir=None, profile_dir=None, user_ns=None, | |||
81 | user_global_ns=None, custom_exceptions=((),None), |
|
107 | user_global_ns=None, custom_exceptions=((),None), | |
82 | usage=None, banner1=None, banner2=None, |
|
108 | usage=None, banner1=None, banner2=None, | |
83 | display_banner=None): |
|
109 | display_banner=None): | |
84 |
|
110 | |||
85 | super(TerminalInteractiveShell, self).__init__( |
|
111 | super(TerminalInteractiveShell, self).__init__( | |
86 |
config=config, |
|
112 | config=config, profile_dir=profile_dir, user_ns=user_ns, | |
87 | user_global_ns=user_global_ns, custom_exceptions=custom_exceptions |
|
113 | user_global_ns=user_global_ns, custom_exceptions=custom_exceptions | |
88 | ) |
|
114 | ) | |
89 | # use os.system instead of utils.process.system by default, except on Windows |
|
115 | # use os.system instead of utils.process.system by default, except on Windows | |
@@ -167,7 +193,7 b' class TerminalInteractiveShell(InteractiveShell):' | |||||
167 |
|
193 | |||
168 | def compute_banner(self): |
|
194 | def compute_banner(self): | |
169 | self.banner = self.banner1 |
|
195 | self.banner = self.banner1 | |
170 | if self.profile: |
|
196 | if self.profile and self.profile != 'default': | |
171 | self.banner += '\nIPython profile: %s\n' % self.profile |
|
197 | self.banner += '\nIPython profile: %s\n' % self.profile | |
172 | if self.banner2: |
|
198 | if self.banner2: | |
173 | self.banner += '\n' + self.banner2 |
|
199 | self.banner += '\n' + self.banner2 |
This diff has been collapsed as it changes many lines, (695 lines changed) Show them Hide them | |||||
@@ -9,6 +9,7 b' Authors' | |||||
9 |
|
9 | |||
10 | * Brian Granger |
|
10 | * Brian Granger | |
11 | * Fernando Perez |
|
11 | * Fernando Perez | |
|
12 | * Min Ragan-Kelley | |||
12 | """ |
|
13 | """ | |
13 |
|
14 | |||
14 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
@@ -28,17 +29,26 b' import logging' | |||||
28 | import os |
|
29 | import os | |
29 | import sys |
|
30 | import sys | |
30 |
|
31 | |||
|
32 | from IPython.config.loader import ( | |||
|
33 | Config, PyFileConfigLoader | |||
|
34 | ) | |||
|
35 | from IPython.config.application import boolean_flag | |||
31 | from IPython.core import release |
|
36 | from IPython.core import release | |
|
37 | from IPython.core import usage | |||
32 | from IPython.core.crashhandler import CrashHandler |
|
38 | from IPython.core.crashhandler import CrashHandler | |
33 | from IPython.core.application import Application, BaseAppConfigLoader |
|
39 | from IPython.core.formatters import PlainTextFormatter | |
34 | from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell |
|
40 | from IPython.core.application import ( | |
35 | from IPython.config.loader import ( |
|
41 | ProfileDir, BaseIPythonApplication, base_flags, base_aliases | |
36 | Config, |
|
42 | ) | |
37 | PyFileConfigLoader |
|
43 | from IPython.core.shellapp import ( | |
|
44 | InteractiveShellApp, shell_flags, shell_aliases | |||
38 | ) |
|
45 | ) | |
|
46 | from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell | |||
39 | from IPython.lib import inputhook |
|
47 | from IPython.lib import inputhook | |
40 |
from IPython.utils.path import |
|
48 | from IPython.utils.path import get_ipython_dir, check_for_old_config | |
41 |
from IPython. |
|
49 | from IPython.utils.traitlets import ( | |
|
50 | Bool, Dict, CaselessStrEnum | |||
|
51 | ) | |||
42 |
|
52 | |||
43 | #----------------------------------------------------------------------------- |
|
53 | #----------------------------------------------------------------------------- | |
44 | # Globals, utilities and helpers |
|
54 | # Globals, utilities and helpers | |
@@ -48,276 +58,6 b' from IPython.core import usage' | |||||
48 | default_config_file_name = u'ipython_config.py' |
|
58 | default_config_file_name = u'ipython_config.py' | |
49 |
|
59 | |||
50 |
|
60 | |||
51 | class IPAppConfigLoader(BaseAppConfigLoader): |
|
|||
52 |
|
||||
53 | def _add_arguments(self): |
|
|||
54 | super(IPAppConfigLoader, self)._add_arguments() |
|
|||
55 | paa = self.parser.add_argument |
|
|||
56 | paa('-p', |
|
|||
57 | '--profile', dest='Global.profile', type=unicode, |
|
|||
58 | help= |
|
|||
59 | """The string name of the ipython profile to be used. Assume that your |
|
|||
60 | config file is ipython_config-<name>.py (looks in current dir first, |
|
|||
61 | then in IPYTHON_DIR). This is a quick way to keep and load multiple |
|
|||
62 | config files for different tasks, especially if include your basic one |
|
|||
63 | in your more specialized ones. You can keep a basic |
|
|||
64 | IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which |
|
|||
65 | include this one and load extra things for particular tasks.""", |
|
|||
66 | metavar='Global.profile') |
|
|||
67 | paa('--config-file', |
|
|||
68 | dest='Global.config_file', type=unicode, |
|
|||
69 | help= |
|
|||
70 | """Set the config file name to override default. Normally IPython |
|
|||
71 | loads ipython_config.py (from current directory) or |
|
|||
72 | IPYTHON_DIR/ipython_config.py. If the loading of your config file |
|
|||
73 | fails, IPython starts with a bare bones configuration (no modules |
|
|||
74 | loaded at all).""", |
|
|||
75 | metavar='Global.config_file') |
|
|||
76 | paa('--autocall', |
|
|||
77 | dest='InteractiveShell.autocall', type=int, |
|
|||
78 | help= |
|
|||
79 | """Make IPython automatically call any callable object even if you |
|
|||
80 | didn't type explicit parentheses. For example, 'str 43' becomes |
|
|||
81 | 'str(43)' automatically. The value can be '0' to disable the feature, |
|
|||
82 | '1' for 'smart' autocall, where it is not applied if there are no more |
|
|||
83 | arguments on the line, and '2' for 'full' autocall, where all callable |
|
|||
84 | objects are automatically called (even if no arguments are present). |
|
|||
85 | The default is '1'.""", |
|
|||
86 | metavar='InteractiveShell.autocall') |
|
|||
87 | paa('--autoindent', |
|
|||
88 | action='store_true', dest='InteractiveShell.autoindent', |
|
|||
89 | help='Turn on autoindenting.') |
|
|||
90 | paa('--no-autoindent', |
|
|||
91 | action='store_false', dest='InteractiveShell.autoindent', |
|
|||
92 | help='Turn off autoindenting.') |
|
|||
93 | paa('--automagic', |
|
|||
94 | action='store_true', dest='InteractiveShell.automagic', |
|
|||
95 | help= |
|
|||
96 | """Turn on the auto calling of magic commands. Type %%magic at the |
|
|||
97 | IPython prompt for more information.""") |
|
|||
98 | paa('--no-automagic', |
|
|||
99 | action='store_false', dest='InteractiveShell.automagic', |
|
|||
100 | help='Turn off the auto calling of magic commands.') |
|
|||
101 | paa('--autoedit-syntax', |
|
|||
102 | action='store_true', dest='TerminalInteractiveShell.autoedit_syntax', |
|
|||
103 | help='Turn on auto editing of files with syntax errors.') |
|
|||
104 | paa('--no-autoedit-syntax', |
|
|||
105 | action='store_false', dest='TerminalInteractiveShell.autoedit_syntax', |
|
|||
106 | help='Turn off auto editing of files with syntax errors.') |
|
|||
107 | paa('--banner', |
|
|||
108 | action='store_true', dest='Global.display_banner', |
|
|||
109 | help='Display a banner upon starting IPython.') |
|
|||
110 | paa('--no-banner', |
|
|||
111 | action='store_false', dest='Global.display_banner', |
|
|||
112 | help="Don't display a banner upon starting IPython.") |
|
|||
113 | paa('--cache-size', |
|
|||
114 | type=int, dest='InteractiveShell.cache_size', |
|
|||
115 | help= |
|
|||
116 | """Set the size of the output cache. The default is 1000, you can |
|
|||
117 | change it permanently in your config file. Setting it to 0 completely |
|
|||
118 | disables the caching system, and the minimum value accepted is 20 (if |
|
|||
119 | you provide a value less than 20, it is reset to 0 and a warning is |
|
|||
120 | issued). This limit is defined because otherwise you'll spend more |
|
|||
121 | time re-flushing a too small cache than working""", |
|
|||
122 | metavar='InteractiveShell.cache_size') |
|
|||
123 | paa('--classic', |
|
|||
124 | action='store_true', dest='Global.classic', |
|
|||
125 | help="Gives IPython a similar feel to the classic Python prompt.") |
|
|||
126 | paa('--colors', |
|
|||
127 | type=str, dest='InteractiveShell.colors', |
|
|||
128 | help="Set the color scheme (NoColor, Linux, and LightBG).", |
|
|||
129 | metavar='InteractiveShell.colors') |
|
|||
130 | paa('--color-info', |
|
|||
131 | action='store_true', dest='InteractiveShell.color_info', |
|
|||
132 | help= |
|
|||
133 | """IPython can display information about objects via a set of func- |
|
|||
134 | tions, and optionally can use colors for this, syntax highlighting |
|
|||
135 | source code and various other elements. However, because this |
|
|||
136 | information is passed through a pager (like 'less') and many pagers get |
|
|||
137 | confused with color codes, this option is off by default. You can test |
|
|||
138 | it and turn it on permanently in your ipython_config.py file if it |
|
|||
139 | works for you. Test it and turn it on permanently if it works with |
|
|||
140 | your system. The magic function %%color_info allows you to toggle this |
|
|||
141 | inter- actively for testing.""") |
|
|||
142 | paa('--no-color-info', |
|
|||
143 | action='store_false', dest='InteractiveShell.color_info', |
|
|||
144 | help="Disable using colors for info related things.") |
|
|||
145 | paa('--confirm-exit', |
|
|||
146 | action='store_true', dest='TerminalInteractiveShell.confirm_exit', |
|
|||
147 | help= |
|
|||
148 | """Set to confirm when you try to exit IPython with an EOF (Control-D |
|
|||
149 | in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or |
|
|||
150 | '%%Exit', you can force a direct exit without any confirmation.""") |
|
|||
151 | paa('--no-confirm-exit', |
|
|||
152 | action='store_false', dest='TerminalInteractiveShell.confirm_exit', |
|
|||
153 | help="Don't prompt the user when exiting.") |
|
|||
154 | paa('--deep-reload', |
|
|||
155 | action='store_true', dest='InteractiveShell.deep_reload', |
|
|||
156 | help= |
|
|||
157 | """Enable deep (recursive) reloading by default. IPython can use the |
|
|||
158 | deep_reload module which reloads changes in modules recursively (it |
|
|||
159 | replaces the reload() function, so you don't need to change anything to |
|
|||
160 | use it). deep_reload() forces a full reload of modules whose code may |
|
|||
161 | have changed, which the default reload() function does not. When |
|
|||
162 | deep_reload is off, IPython will use the normal reload(), but |
|
|||
163 | deep_reload will still be available as dreload(). This fea- ture is off |
|
|||
164 | by default [which means that you have both normal reload() and |
|
|||
165 | dreload()].""") |
|
|||
166 | paa('--no-deep-reload', |
|
|||
167 | action='store_false', dest='InteractiveShell.deep_reload', |
|
|||
168 | help="Disable deep (recursive) reloading by default.") |
|
|||
169 | paa('--editor', |
|
|||
170 | type=str, dest='TerminalInteractiveShell.editor', |
|
|||
171 | help="Set the editor used by IPython (default to $EDITOR/vi/notepad).", |
|
|||
172 | metavar='TerminalInteractiveShell.editor') |
|
|||
173 | paa('--log','-l', |
|
|||
174 | action='store_true', dest='InteractiveShell.logstart', |
|
|||
175 | help="Start logging to the default log file (./ipython_log.py).") |
|
|||
176 | paa('--logfile','-lf', |
|
|||
177 | type=unicode, dest='InteractiveShell.logfile', |
|
|||
178 | help="Start logging to logfile with this name.", |
|
|||
179 | metavar='InteractiveShell.logfile') |
|
|||
180 | paa('--log-append','-la', |
|
|||
181 | type=unicode, dest='InteractiveShell.logappend', |
|
|||
182 | help="Start logging to the given file in append mode.", |
|
|||
183 | metavar='InteractiveShell.logfile') |
|
|||
184 | paa('--pdb', |
|
|||
185 | action='store_true', dest='InteractiveShell.pdb', |
|
|||
186 | help="Enable auto calling the pdb debugger after every exception.") |
|
|||
187 | paa('--no-pdb', |
|
|||
188 | action='store_false', dest='InteractiveShell.pdb', |
|
|||
189 | help="Disable auto calling the pdb debugger after every exception.") |
|
|||
190 | paa('--pprint', |
|
|||
191 | action='store_true', dest='PlainTextFormatter.pprint', |
|
|||
192 | help="Enable auto pretty printing of results.") |
|
|||
193 | paa('--no-pprint', |
|
|||
194 | action='store_false', dest='PlainTextFormatter.pprint', |
|
|||
195 | help="Disable auto auto pretty printing of results.") |
|
|||
196 | paa('--prompt-in1','-pi1', |
|
|||
197 | type=str, dest='InteractiveShell.prompt_in1', |
|
|||
198 | help= |
|
|||
199 | """Set the main input prompt ('In [\#]: '). Note that if you are using |
|
|||
200 | numbered prompts, the number is represented with a '\#' in the string. |
|
|||
201 | Don't forget to quote strings with spaces embedded in them. Most |
|
|||
202 | bash-like escapes can be used to customize IPython's prompts, as well |
|
|||
203 | as a few additional ones which are IPython-spe- cific. All valid |
|
|||
204 | prompt escapes are described in detail in the Customization section of |
|
|||
205 | the IPython manual.""", |
|
|||
206 | metavar='InteractiveShell.prompt_in1') |
|
|||
207 | paa('--prompt-in2','-pi2', |
|
|||
208 | type=str, dest='InteractiveShell.prompt_in2', |
|
|||
209 | help= |
|
|||
210 | """Set the secondary input prompt (' .\D.: '). Similar to the previous |
|
|||
211 | option, but used for the continuation prompts. The special sequence |
|
|||
212 | '\D' is similar to '\#', but with all digits replaced by dots (so you |
|
|||
213 | can have your continuation prompt aligned with your input prompt). |
|
|||
214 | Default: ' .\D.: ' (note three spaces at the start for alignment with |
|
|||
215 | 'In [\#]')""", |
|
|||
216 | metavar='InteractiveShell.prompt_in2') |
|
|||
217 | paa('--prompt-out','-po', |
|
|||
218 | type=str, dest='InteractiveShell.prompt_out', |
|
|||
219 | help="Set the output prompt ('Out[\#]:')", |
|
|||
220 | metavar='InteractiveShell.prompt_out') |
|
|||
221 | paa('--quick', |
|
|||
222 | action='store_true', dest='Global.quick', |
|
|||
223 | help="Enable quick startup with no config files.") |
|
|||
224 | paa('--readline', |
|
|||
225 | action='store_true', dest='InteractiveShell.readline_use', |
|
|||
226 | help="Enable readline for command line usage.") |
|
|||
227 | paa('--no-readline', |
|
|||
228 | action='store_false', dest='InteractiveShell.readline_use', |
|
|||
229 | help="Disable readline for command line usage.") |
|
|||
230 | paa('--screen-length','-sl', |
|
|||
231 | type=int, dest='TerminalInteractiveShell.screen_length', |
|
|||
232 | help= |
|
|||
233 | """Number of lines of your screen, used to control printing of very |
|
|||
234 | long strings. Strings longer than this number of lines will be sent |
|
|||
235 | through a pager instead of directly printed. The default value for |
|
|||
236 | this is 0, which means IPython will auto-detect your screen size every |
|
|||
237 | time it needs to print certain potentially long strings (this doesn't |
|
|||
238 | change the behavior of the 'print' keyword, it's only triggered |
|
|||
239 | internally). If for some reason this isn't working well (it needs |
|
|||
240 | curses support), specify it yourself. Otherwise don't change the |
|
|||
241 | default.""", |
|
|||
242 | metavar='TerminalInteractiveShell.screen_length') |
|
|||
243 | paa('--separate-in','-si', |
|
|||
244 | type=str, dest='InteractiveShell.separate_in', |
|
|||
245 | help="Separator before input prompts. Default '\\n'.", |
|
|||
246 | metavar='InteractiveShell.separate_in') |
|
|||
247 | paa('--separate-out','-so', |
|
|||
248 | type=str, dest='InteractiveShell.separate_out', |
|
|||
249 | help="Separator before output prompts. Default 0 (nothing).", |
|
|||
250 | metavar='InteractiveShell.separate_out') |
|
|||
251 | paa('--separate-out2','-so2', |
|
|||
252 | type=str, dest='InteractiveShell.separate_out2', |
|
|||
253 | help="Separator after output prompts. Default 0 (nonight).", |
|
|||
254 | metavar='InteractiveShell.separate_out2') |
|
|||
255 | paa('--no-sep', |
|
|||
256 | action='store_true', dest='Global.nosep', |
|
|||
257 | help="Eliminate all spacing between prompts.") |
|
|||
258 | paa('--term-title', |
|
|||
259 | action='store_true', dest='TerminalInteractiveShell.term_title', |
|
|||
260 | help="Enable auto setting the terminal title.") |
|
|||
261 | paa('--no-term-title', |
|
|||
262 | action='store_false', dest='TerminalInteractiveShell.term_title', |
|
|||
263 | help="Disable auto setting the terminal title.") |
|
|||
264 | paa('--xmode', |
|
|||
265 | type=str, dest='InteractiveShell.xmode', |
|
|||
266 | help= |
|
|||
267 | """Exception reporting mode ('Plain','Context','Verbose'). Plain: |
|
|||
268 | similar to python's normal traceback printing. Context: prints 5 lines |
|
|||
269 | of context source code around each line in the traceback. Verbose: |
|
|||
270 | similar to Context, but additionally prints the variables currently |
|
|||
271 | visible where the exception happened (shortening their strings if too |
|
|||
272 | long). This can potentially be very slow, if you happen to have a huge |
|
|||
273 | data structure whose string representation is complex to compute. |
|
|||
274 | Your computer may appear to freeze for a while with cpu usage at 100%%. |
|
|||
275 | If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting |
|
|||
276 | it more than once). |
|
|||
277 | """, |
|
|||
278 | metavar='InteractiveShell.xmode') |
|
|||
279 | paa('--ext', |
|
|||
280 | type=str, dest='Global.extra_extension', |
|
|||
281 | help="The dotted module name of an IPython extension to load.", |
|
|||
282 | metavar='Global.extra_extension') |
|
|||
283 | paa('-c', |
|
|||
284 | type=str, dest='Global.code_to_run', |
|
|||
285 | help="Execute the given command string.", |
|
|||
286 | metavar='Global.code_to_run') |
|
|||
287 | paa('-i', |
|
|||
288 | action='store_true', dest='Global.force_interact', |
|
|||
289 | help= |
|
|||
290 | "If running code from the command line, become interactive afterwards.") |
|
|||
291 |
|
||||
292 | # Options to start with GUI control enabled from the beginning |
|
|||
293 | paa('--gui', |
|
|||
294 | type=str, dest='Global.gui', |
|
|||
295 | help="Enable GUI event loop integration ('qt', 'wx', 'gtk').", |
|
|||
296 | metavar='gui-mode') |
|
|||
297 | paa('--pylab','-pylab', |
|
|||
298 | type=str, dest='Global.pylab', |
|
|||
299 | nargs='?', const='auto', metavar='gui-mode', |
|
|||
300 | help="Pre-load matplotlib and numpy for interactive use. "+ |
|
|||
301 | "If no value is given, the gui backend is matplotlib's, else use "+ |
|
|||
302 | "one of: ['tk', 'qt', 'wx', 'gtk', 'osx'].") |
|
|||
303 |
|
||||
304 | # Legacy GUI options. Leave them in for backwards compatibility, but the |
|
|||
305 | # 'thread' names are really a misnomer now. |
|
|||
306 | paa('--wthread', '-wthread', |
|
|||
307 | action='store_true', dest='Global.wthread', |
|
|||
308 | help= |
|
|||
309 | """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""") |
|
|||
310 | paa('--q4thread', '--qthread', '-q4thread', '-qthread', |
|
|||
311 | action='store_true', dest='Global.q4thread', |
|
|||
312 | help= |
|
|||
313 | """Enable Qt4 event loop integration. Qt3 is no longer supported. |
|
|||
314 | (DEPRECATED, use --gui qt)""") |
|
|||
315 | paa('--gthread', '-gthread', |
|
|||
316 | action='store_true', dest='Global.gthread', |
|
|||
317 | help= |
|
|||
318 | """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""") |
|
|||
319 |
|
||||
320 |
|
||||
321 | #----------------------------------------------------------------------------- |
|
61 | #----------------------------------------------------------------------------- | |
322 | # Crash handler for this application |
|
62 | # Crash handler for this application | |
323 | #----------------------------------------------------------------------------- |
|
63 | #----------------------------------------------------------------------------- | |
@@ -377,271 +117,219 b' class IPAppCrashHandler(CrashHandler):' | |||||
377 |
|
117 | |||
378 | return ''.join(report) |
|
118 | return ''.join(report) | |
379 |
|
119 | |||
|
120 | #----------------------------------------------------------------------------- | |||
|
121 | # Aliases and Flags | |||
|
122 | #----------------------------------------------------------------------------- | |||
|
123 | flags = dict(base_flags) | |||
|
124 | flags.update(shell_flags) | |||
|
125 | addflag = lambda *args: flags.update(boolean_flag(*args)) | |||
|
126 | addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax', | |||
|
127 | 'Turn on auto editing of files with syntax errors.', | |||
|
128 | 'Turn off auto editing of files with syntax errors.' | |||
|
129 | ) | |||
|
130 | addflag('banner', 'TerminalIPythonApp.display_banner', | |||
|
131 | "Display a banner upon starting IPython.", | |||
|
132 | "Don't display a banner upon starting IPython." | |||
|
133 | ) | |||
|
134 | addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit', | |||
|
135 | """Set to confirm when you try to exit IPython with an EOF (Control-D | |||
|
136 | in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', | |||
|
137 | you can force a direct exit without any confirmation.""", | |||
|
138 | "Don't prompt the user when exiting." | |||
|
139 | ) | |||
|
140 | addflag('term-title', 'TerminalInteractiveShell.term_title', | |||
|
141 | "Enable auto setting the terminal title.", | |||
|
142 | "Disable auto setting the terminal title." | |||
|
143 | ) | |||
|
144 | classic_config = Config() | |||
|
145 | classic_config.InteractiveShell.cache_size = 0 | |||
|
146 | classic_config.PlainTextFormatter.pprint = False | |||
|
147 | classic_config.InteractiveShell.prompt_in1 = '>>> ' | |||
|
148 | classic_config.InteractiveShell.prompt_in2 = '... ' | |||
|
149 | classic_config.InteractiveShell.prompt_out = '' | |||
|
150 | classic_config.InteractiveShell.separate_in = '' | |||
|
151 | classic_config.InteractiveShell.separate_out = '' | |||
|
152 | classic_config.InteractiveShell.separate_out2 = '' | |||
|
153 | classic_config.InteractiveShell.colors = 'NoColor' | |||
|
154 | classic_config.InteractiveShell.xmode = 'Plain' | |||
|
155 | ||||
|
156 | flags['classic']=( | |||
|
157 | classic_config, | |||
|
158 | "Gives IPython a similar feel to the classic Python prompt." | |||
|
159 | ) | |||
|
160 | # # log doesn't make so much sense this way anymore | |||
|
161 | # paa('--log','-l', | |||
|
162 | # action='store_true', dest='InteractiveShell.logstart', | |||
|
163 | # help="Start logging to the default log file (./ipython_log.py).") | |||
|
164 | # | |||
|
165 | # # quick is harder to implement | |||
|
166 | flags['quick']=( | |||
|
167 | {'TerminalIPythonApp' : {'quick' : True}}, | |||
|
168 | "Enable quick startup with no config files." | |||
|
169 | ) | |||
|
170 | ||||
|
171 | flags['i'] = ( | |||
|
172 | {'TerminalIPythonApp' : {'force_interact' : True}}, | |||
|
173 | "If running code from the command line, become interactive afterwards." | |||
|
174 | ) | |||
|
175 | flags['pylab'] = ( | |||
|
176 | {'TerminalIPythonApp' : {'pylab' : 'auto'}}, | |||
|
177 | """Pre-load matplotlib and numpy for interactive use with | |||
|
178 | the default matplotlib backend.""" | |||
|
179 | ) | |||
|
180 | ||||
|
181 | aliases = dict(base_aliases) | |||
|
182 | aliases.update(shell_aliases) | |||
|
183 | ||||
|
184 | # it's possible we don't want short aliases for *all* of these: | |||
|
185 | aliases.update(dict( | |||
|
186 | gui='TerminalIPythonApp.gui', | |||
|
187 | pylab='TerminalIPythonApp.pylab', | |||
|
188 | )) | |||
380 |
|
189 | |||
381 | #----------------------------------------------------------------------------- |
|
190 | #----------------------------------------------------------------------------- | |
382 | # Main classes and functions |
|
191 | # Main classes and functions | |
383 | #----------------------------------------------------------------------------- |
|
192 | #----------------------------------------------------------------------------- | |
384 |
|
193 | |||
385 | class IPythonApp(Application): |
|
194 | class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): | |
386 | name = u'ipython' |
|
195 | name = u'ipython' | |
387 | #: argparse formats better the 'usage' than the 'description' field |
|
196 | description = usage.cl_usage | |
388 | description = None |
|
|||
389 | usage = usage.cl_usage |
|
|||
390 | command_line_loader = IPAppConfigLoader |
|
|||
391 | default_config_file_name = default_config_file_name |
|
197 | default_config_file_name = default_config_file_name | |
392 | crash_handler_class = IPAppCrashHandler |
|
198 | crash_handler_class = IPAppCrashHandler | |
|
199 | ||||
|
200 | flags = Dict(flags) | |||
|
201 | aliases = Dict(aliases) | |||
|
202 | classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter] | |||
|
203 | subcommands = Dict(dict( | |||
|
204 | qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp', | |||
|
205 | """Launch the IPython Qt Console.""" | |||
|
206 | ), | |||
|
207 | profile = ("IPython.core.profileapp.ProfileApp", | |||
|
208 | "Create and manage IPython profiles.") | |||
|
209 | )) | |||
|
210 | ||||
|
211 | # *do* autocreate requested profile, but don't create the config file. | |||
|
212 | auto_create=Bool(True) | |||
|
213 | # configurables | |||
|
214 | ignore_old_config=Bool(False, config=True, | |||
|
215 | help="Suppress warning messages about legacy config files" | |||
|
216 | ) | |||
|
217 | quick = Bool(False, config=True, | |||
|
218 | help="""Start IPython quickly by skipping the loading of config files.""" | |||
|
219 | ) | |||
|
220 | def _quick_changed(self, name, old, new): | |||
|
221 | if new: | |||
|
222 | self.load_config_file = lambda *a, **kw: None | |||
|
223 | self.ignore_old_config=True | |||
|
224 | ||||
|
225 | gui = CaselessStrEnum(('qt','wx','gtk'), config=True, | |||
|
226 | help="Enable GUI event loop integration ('qt', 'wx', 'gtk')." | |||
|
227 | ) | |||
|
228 | pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'], | |||
|
229 | config=True, | |||
|
230 | help="""Pre-load matplotlib and numpy for interactive use, | |||
|
231 | selecting a particular matplotlib backend and loop integration. | |||
|
232 | """ | |||
|
233 | ) | |||
|
234 | display_banner = Bool(True, config=True, | |||
|
235 | help="Whether to display a banner upon starting IPython." | |||
|
236 | ) | |||
|
237 | ||||
|
238 | # if there is code of files to run from the cmd line, don't interact | |||
|
239 | # unless the --i flag (App.force_interact) is true. | |||
|
240 | force_interact = Bool(False, config=True, | |||
|
241 | help="""If a command or file is given via the command-line, | |||
|
242 | e.g. 'ipython foo.py""" | |||
|
243 | ) | |||
|
244 | def _force_interact_changed(self, name, old, new): | |||
|
245 | if new: | |||
|
246 | self.interact = True | |||
|
247 | ||||
|
248 | def _file_to_run_changed(self, name, old, new): | |||
|
249 | if new and not self.force_interact: | |||
|
250 | self.interact = False | |||
|
251 | _code_to_run_changed = _file_to_run_changed | |||
|
252 | ||||
|
253 | # internal, not-configurable | |||
|
254 | interact=Bool(True) | |||
|
255 | ||||
|
256 | ||||
|
257 | def initialize(self, argv=None): | |||
|
258 | """Do actions after construct, but before starting the app.""" | |||
|
259 | super(TerminalIPythonApp, self).initialize(argv) | |||
|
260 | if self.subapp is not None: | |||
|
261 | # don't bother initializing further, starting subapp | |||
|
262 | return | |||
|
263 | if not self.ignore_old_config: | |||
|
264 | check_for_old_config(self.ipython_dir) | |||
|
265 | # print self.extra_args | |||
|
266 | if self.extra_args: | |||
|
267 | self.file_to_run = self.extra_args[0] | |||
|
268 | # create the shell | |||
|
269 | self.init_shell() | |||
|
270 | # and draw the banner | |||
|
271 | self.init_banner() | |||
|
272 | # Now a variety of things that happen after the banner is printed. | |||
|
273 | self.init_gui_pylab() | |||
|
274 | self.init_extensions() | |||
|
275 | self.init_code() | |||
393 |
|
276 | |||
394 | def create_default_config(self): |
|
277 | def init_shell(self): | |
395 | super(IPythonApp, self).create_default_config() |
|
278 | """initialize the InteractiveShell instance""" | |
396 | # Eliminate multiple lookups |
|
|||
397 | Global = self.default_config.Global |
|
|||
398 |
|
||||
399 | # Set all default values |
|
|||
400 | Global.display_banner = True |
|
|||
401 |
|
||||
402 | # If the -c flag is given or a file is given to run at the cmd line |
|
|||
403 | # like "ipython foo.py", normally we exit without starting the main |
|
|||
404 | # loop. The force_interact config variable allows a user to override |
|
|||
405 | # this and interact. It is also set by the -i cmd line flag, just |
|
|||
406 | # like Python. |
|
|||
407 | Global.force_interact = False |
|
|||
408 |
|
||||
409 | # By default always interact by starting the IPython mainloop. |
|
|||
410 | Global.interact = True |
|
|||
411 |
|
||||
412 | # No GUI integration by default |
|
|||
413 | Global.gui = False |
|
|||
414 | # Pylab off by default |
|
|||
415 | Global.pylab = False |
|
|||
416 |
|
||||
417 | # Deprecated versions of gui support that used threading, we support |
|
|||
418 | # them just for bacwards compatibility as an alternate spelling for |
|
|||
419 | # '--gui X' |
|
|||
420 | Global.qthread = False |
|
|||
421 | Global.q4thread = False |
|
|||
422 | Global.wthread = False |
|
|||
423 | Global.gthread = False |
|
|||
424 |
|
||||
425 | def load_file_config(self): |
|
|||
426 | if hasattr(self.command_line_config.Global, 'quick'): |
|
|||
427 | if self.command_line_config.Global.quick: |
|
|||
428 | self.file_config = Config() |
|
|||
429 | return |
|
|||
430 | super(IPythonApp, self).load_file_config() |
|
|||
431 |
|
||||
432 | def post_load_file_config(self): |
|
|||
433 | if hasattr(self.command_line_config.Global, 'extra_extension'): |
|
|||
434 | if not hasattr(self.file_config.Global, 'extensions'): |
|
|||
435 | self.file_config.Global.extensions = [] |
|
|||
436 | self.file_config.Global.extensions.append( |
|
|||
437 | self.command_line_config.Global.extra_extension) |
|
|||
438 | del self.command_line_config.Global.extra_extension |
|
|||
439 |
|
||||
440 | def pre_construct(self): |
|
|||
441 | config = self.master_config |
|
|||
442 |
|
||||
443 | if hasattr(config.Global, 'classic'): |
|
|||
444 | if config.Global.classic: |
|
|||
445 | config.InteractiveShell.cache_size = 0 |
|
|||
446 | config.PlainTextFormatter.pprint = False |
|
|||
447 | config.InteractiveShell.prompt_in1 = '>>> ' |
|
|||
448 | config.InteractiveShell.prompt_in2 = '... ' |
|
|||
449 | config.InteractiveShell.prompt_out = '' |
|
|||
450 | config.InteractiveShell.separate_in = \ |
|
|||
451 | config.InteractiveShell.separate_out = \ |
|
|||
452 | config.InteractiveShell.separate_out2 = '' |
|
|||
453 | config.InteractiveShell.colors = 'NoColor' |
|
|||
454 | config.InteractiveShell.xmode = 'Plain' |
|
|||
455 |
|
||||
456 | if hasattr(config.Global, 'nosep'): |
|
|||
457 | if config.Global.nosep: |
|
|||
458 | config.InteractiveShell.separate_in = \ |
|
|||
459 | config.InteractiveShell.separate_out = \ |
|
|||
460 | config.InteractiveShell.separate_out2 = '' |
|
|||
461 |
|
||||
462 | # if there is code of files to run from the cmd line, don't interact |
|
|||
463 | # unless the -i flag (Global.force_interact) is true. |
|
|||
464 | code_to_run = config.Global.get('code_to_run','') |
|
|||
465 | file_to_run = False |
|
|||
466 | if self.extra_args and self.extra_args[0]: |
|
|||
467 | file_to_run = True |
|
|||
468 | if file_to_run or code_to_run: |
|
|||
469 | if not config.Global.force_interact: |
|
|||
470 | config.Global.interact = False |
|
|||
471 |
|
||||
472 | def construct(self): |
|
|||
473 | # I am a little hesitant to put these into InteractiveShell itself. |
|
279 | # I am a little hesitant to put these into InteractiveShell itself. | |
474 | # But that might be the place for them |
|
280 | # But that might be the place for them | |
475 | sys.path.insert(0, '') |
|
281 | sys.path.insert(0, '') | |
476 |
|
282 | |||
477 | # Create an InteractiveShell instance. |
|
283 | # Create an InteractiveShell instance. | |
478 | self.shell = TerminalInteractiveShell.instance(config=self.master_config) |
|
|||
479 |
|
||||
480 | def post_construct(self): |
|
|||
481 | """Do actions after construct, but before starting the app.""" |
|
|||
482 | config = self.master_config |
|
|||
483 |
|
||||
484 | # shell.display_banner should always be False for the terminal |
|
284 | # shell.display_banner should always be False for the terminal | |
485 | # based app, because we call shell.show_banner() by hand below |
|
285 | # based app, because we call shell.show_banner() by hand below | |
486 | # so the banner shows *before* all extension loading stuff. |
|
286 | # so the banner shows *before* all extension loading stuff. | |
487 | self.shell.display_banner = False |
|
287 | self.shell = TerminalInteractiveShell.instance(config=self.config, | |
488 | if config.Global.display_banner and \ |
|
288 | display_banner=False, profile_dir=self.profile_dir, | |
489 | config.Global.interact: |
|
289 | ipython_dir=self.ipython_dir) | |
490 | self.shell.show_banner() |
|
|||
491 |
|
290 | |||
|
291 | def init_banner(self): | |||
|
292 | """optionally display the banner""" | |||
|
293 | if self.display_banner and self.interact: | |||
|
294 | self.shell.show_banner() | |||
492 | # Make sure there is a space below the banner. |
|
295 | # Make sure there is a space below the banner. | |
493 | if self.log_level <= logging.INFO: print |
|
296 | if self.log_level <= logging.INFO: print | |
494 |
|
297 | |||
495 | # Now a variety of things that happen after the banner is printed. |
|
|||
496 | self._enable_gui_pylab() |
|
|||
497 | self._load_extensions() |
|
|||
498 | self._run_exec_lines() |
|
|||
499 | self._run_exec_files() |
|
|||
500 | self._run_cmd_line_code() |
|
|||
501 |
|
298 | |||
502 |
def |
|
299 | def init_gui_pylab(self): | |
503 | """Enable GUI event loop integration, taking pylab into account.""" |
|
300 | """Enable GUI event loop integration, taking pylab into account.""" | |
504 | Global = self.master_config.Global |
|
301 | gui = self.gui | |
505 |
|
||||
506 | # Select which gui to use |
|
|||
507 | if Global.gui: |
|
|||
508 | gui = Global.gui |
|
|||
509 | # The following are deprecated, but there's likely to be a lot of use |
|
|||
510 | # of this form out there, so we might as well support it for now. But |
|
|||
511 | # the --gui option above takes precedence. |
|
|||
512 | elif Global.wthread: |
|
|||
513 | gui = inputhook.GUI_WX |
|
|||
514 | elif Global.qthread: |
|
|||
515 | gui = inputhook.GUI_QT |
|
|||
516 | elif Global.gthread: |
|
|||
517 | gui = inputhook.GUI_GTK |
|
|||
518 | else: |
|
|||
519 | gui = None |
|
|||
520 |
|
302 | |||
521 |
# Using |
|
303 | # Using `pylab` will also require gui activation, though which toolkit | |
522 | # to use may be chosen automatically based on mpl configuration. |
|
304 | # to use may be chosen automatically based on mpl configuration. | |
523 |
if |
|
305 | if self.pylab: | |
524 | activate = self.shell.enable_pylab |
|
306 | activate = self.shell.enable_pylab | |
525 |
if |
|
307 | if self.pylab == 'auto': | |
526 | gui = None |
|
308 | gui = None | |
527 | else: |
|
309 | else: | |
528 |
gui = |
|
310 | gui = self.pylab | |
529 | else: |
|
311 | else: | |
530 | # Enable only GUI integration, no pylab |
|
312 | # Enable only GUI integration, no pylab | |
531 | activate = inputhook.enable_gui |
|
313 | activate = inputhook.enable_gui | |
532 |
|
314 | |||
533 |
if gui or |
|
315 | if gui or self.pylab: | |
534 | try: |
|
316 | try: | |
535 | self.log.info("Enabling GUI event loop integration, " |
|
317 | self.log.info("Enabling GUI event loop integration, " | |
536 |
"toolkit=%s, pylab=%s" % (gui, |
|
318 | "toolkit=%s, pylab=%s" % (gui, self.pylab) ) | |
537 | activate(gui) |
|
319 | activate(gui) | |
538 | except: |
|
320 | except: | |
539 | self.log.warn("Error in enabling GUI event loop integration:") |
|
321 | self.log.warn("Error in enabling GUI event loop integration:") | |
540 | self.shell.showtraceback() |
|
322 | self.shell.showtraceback() | |
541 |
|
323 | |||
542 |
def |
|
324 | def start(self): | |
543 | """Load all IPython extensions in Global.extensions. |
|
325 | if self.subapp is not None: | |
544 |
|
326 | return self.subapp.start() | ||
545 | This uses the :meth:`ExtensionManager.load_extensions` to load all |
|
327 | # perform any prexec steps: | |
546 | the extensions listed in ``self.master_config.Global.extensions``. |
|
328 | if self.interact: | |
547 | """ |
|
|||
548 | try: |
|
|||
549 | if hasattr(self.master_config.Global, 'extensions'): |
|
|||
550 | self.log.debug("Loading IPython extensions...") |
|
|||
551 | extensions = self.master_config.Global.extensions |
|
|||
552 | for ext in extensions: |
|
|||
553 | try: |
|
|||
554 | self.log.info("Loading IPython extension: %s" % ext) |
|
|||
555 | self.shell.extension_manager.load_extension(ext) |
|
|||
556 | except: |
|
|||
557 | self.log.warn("Error in loading extension: %s" % ext) |
|
|||
558 | self.shell.showtraceback() |
|
|||
559 | except: |
|
|||
560 | self.log.warn("Unknown error in loading extensions:") |
|
|||
561 | self.shell.showtraceback() |
|
|||
562 |
|
||||
563 | def _run_exec_lines(self): |
|
|||
564 | """Run lines of code in Global.exec_lines in the user's namespace.""" |
|
|||
565 | try: |
|
|||
566 | if hasattr(self.master_config.Global, 'exec_lines'): |
|
|||
567 | self.log.debug("Running code from Global.exec_lines...") |
|
|||
568 | exec_lines = self.master_config.Global.exec_lines |
|
|||
569 | for line in exec_lines: |
|
|||
570 | try: |
|
|||
571 | self.log.info("Running code in user namespace: %s" % |
|
|||
572 | line) |
|
|||
573 | self.shell.run_cell(line, store_history=False) |
|
|||
574 | except: |
|
|||
575 | self.log.warn("Error in executing line in user " |
|
|||
576 | "namespace: %s" % line) |
|
|||
577 | self.shell.showtraceback() |
|
|||
578 | except: |
|
|||
579 | self.log.warn("Unknown error in handling Global.exec_lines:") |
|
|||
580 | self.shell.showtraceback() |
|
|||
581 |
|
||||
582 | def _exec_file(self, fname): |
|
|||
583 | full_filename = filefind(fname, [u'.', self.ipython_dir]) |
|
|||
584 | if os.path.isfile(full_filename): |
|
|||
585 | if full_filename.endswith(u'.py'): |
|
|||
586 | self.log.info("Running file in user namespace: %s" % |
|
|||
587 | full_filename) |
|
|||
588 | # Ensure that __file__ is always defined to match Python behavior |
|
|||
589 | self.shell.user_ns['__file__'] = fname |
|
|||
590 | try: |
|
|||
591 | self.shell.safe_execfile(full_filename, self.shell.user_ns) |
|
|||
592 | finally: |
|
|||
593 | del self.shell.user_ns['__file__'] |
|
|||
594 | elif full_filename.endswith('.ipy'): |
|
|||
595 | self.log.info("Running file in user namespace: %s" % |
|
|||
596 | full_filename) |
|
|||
597 | self.shell.safe_execfile_ipy(full_filename) |
|
|||
598 | else: |
|
|||
599 | self.log.warn("File does not have a .py or .ipy extension: <%s>" |
|
|||
600 | % full_filename) |
|
|||
601 | def _run_exec_files(self): |
|
|||
602 | try: |
|
|||
603 | if hasattr(self.master_config.Global, 'exec_files'): |
|
|||
604 | self.log.debug("Running files in Global.exec_files...") |
|
|||
605 | exec_files = self.master_config.Global.exec_files |
|
|||
606 | for fname in exec_files: |
|
|||
607 | self._exec_file(fname) |
|
|||
608 | except: |
|
|||
609 | self.log.warn("Unknown error in handling Global.exec_files:") |
|
|||
610 | self.shell.showtraceback() |
|
|||
611 |
|
||||
612 | def _run_cmd_line_code(self): |
|
|||
613 | if hasattr(self.master_config.Global, 'code_to_run'): |
|
|||
614 | line = self.master_config.Global.code_to_run |
|
|||
615 | try: |
|
|||
616 | self.log.info("Running code given at command line (-c): %s" % |
|
|||
617 | line) |
|
|||
618 | self.shell.run_cell(line, store_history=False) |
|
|||
619 | except: |
|
|||
620 | self.log.warn("Error in executing line in user namespace: %s" % |
|
|||
621 | line) |
|
|||
622 | self.shell.showtraceback() |
|
|||
623 | return |
|
|||
624 | # Like Python itself, ignore the second if the first of these is present |
|
|||
625 | try: |
|
|||
626 | fname = self.extra_args[0] |
|
|||
627 | except: |
|
|||
628 | pass |
|
|||
629 | else: |
|
|||
630 | try: |
|
|||
631 | self._exec_file(fname) |
|
|||
632 | except: |
|
|||
633 | self.log.warn("Error in executing file in user namespace: %s" % |
|
|||
634 | fname) |
|
|||
635 | self.shell.showtraceback() |
|
|||
636 |
|
||||
637 | def start_app(self): |
|
|||
638 | if not getattr(self.master_config.Global, 'ignore_old_config', False): |
|
|||
639 | check_for_old_config(self.ipython_dir) |
|
|||
640 | if self.master_config.Global.interact: |
|
|||
641 | self.log.debug("Starting IPython's mainloop...") |
|
329 | self.log.debug("Starting IPython's mainloop...") | |
642 | self.shell.mainloop() |
|
330 | self.shell.mainloop() | |
643 | else: |
|
331 | else: | |
644 |
self.log.debug("IPython not interactive |
|
332 | self.log.debug("IPython not interactive...") | |
645 |
|
333 | |||
646 |
|
334 | |||
647 | def load_default_config(ipython_dir=None): |
|
335 | def load_default_config(ipython_dir=None): | |
@@ -651,16 +339,19 b' def load_default_config(ipython_dir=None):' | |||||
651 | """ |
|
339 | """ | |
652 | if ipython_dir is None: |
|
340 | if ipython_dir is None: | |
653 | ipython_dir = get_ipython_dir() |
|
341 | ipython_dir = get_ipython_dir() | |
654 | cl = PyFileConfigLoader(default_config_file_name, ipython_dir) |
|
342 | profile_dir = os.path.join(ipython_dir, 'profile_default') | |
|
343 | cl = PyFileConfigLoader(default_config_file_name, profile_dir) | |||
655 | config = cl.load_config() |
|
344 | config = cl.load_config() | |
656 | return config |
|
345 | return config | |
657 |
|
346 | |||
658 |
|
347 | |||
659 | def launch_new_instance(): |
|
348 | def launch_new_instance(): | |
660 | """Create and run a full blown IPython instance""" |
|
349 | """Create and run a full blown IPython instance""" | |
661 | app = IPythonApp() |
|
350 | app = TerminalIPythonApp.instance() | |
|
351 | app.initialize() | |||
662 | app.start() |
|
352 | app.start() | |
663 |
|
353 | |||
664 |
|
354 | |||
665 | if __name__ == '__main__': |
|
355 | if __name__ == '__main__': | |
666 | launch_new_instance() |
|
356 | launch_new_instance() | |
|
357 |
@@ -304,9 +304,7 b' class IPythonRunner(InteractiveRunner):' | |||||
304 | def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True): |
|
304 | def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True): | |
305 | """New runner, optionally passing the ipython command to use.""" |
|
305 | """New runner, optionally passing the ipython command to use.""" | |
306 |
|
306 | |||
307 |
args0 = [' |
|
307 | args0 = ['colors=NoColor', | |
308 | '-pi1','In [\\#]: ', |
|
|||
309 | '-pi2',' .\\D.: ', |
|
|||
310 | '--no-term-title', |
|
308 | '--no-term-title', | |
311 | '--no-autoindent'] |
|
309 | '--no-autoindent'] | |
312 | if args is None: args = args0 |
|
310 | if args is None: args = args0 |
@@ -87,8 +87,8 b' def figsize(sizex, sizey):' | |||||
87 | matplotlib.rcParams['figure.figsize'] = [sizex, sizey] |
|
87 | matplotlib.rcParams['figure.figsize'] = [sizex, sizey] | |
88 |
|
88 | |||
89 |
|
89 | |||
90 | def figure_to_svg(fig): |
|
90 | def print_figure(fig, fmt='png'): | |
91 | """Convert a figure to svg for inline display.""" |
|
91 | """Convert a figure to svg or png for inline display.""" | |
92 | # When there's an empty figure, we shouldn't return anything, otherwise we |
|
92 | # When there's an empty figure, we shouldn't return anything, otherwise we | |
93 | # get big blank areas in the qt console. |
|
93 | # get big blank areas in the qt console. | |
94 | if not fig.axes: |
|
94 | if not fig.axes: | |
@@ -100,12 +100,13 b' def figure_to_svg(fig):' | |||||
100 | fig.set_edgecolor('white') |
|
100 | fig.set_edgecolor('white') | |
101 | try: |
|
101 | try: | |
102 | string_io = StringIO() |
|
102 | string_io = StringIO() | |
103 | fig.canvas.print_figure(string_io, format='svg') |
|
103 | # use 72 dpi to match QTConsole's dpi | |
104 | svg = string_io.getvalue() |
|
104 | fig.canvas.print_figure(string_io, format=fmt, dpi=72) | |
|
105 | data = string_io.getvalue() | |||
105 | finally: |
|
106 | finally: | |
106 | fig.set_facecolor(fc) |
|
107 | fig.set_facecolor(fc) | |
107 | fig.set_edgecolor(ec) |
|
108 | fig.set_edgecolor(ec) | |
108 |
return |
|
109 | return data | |
109 |
|
110 | |||
110 |
|
111 | |||
111 | # We need a little factory function here to create the closure where |
|
112 | # We need a little factory function here to create the closure where | |
@@ -150,6 +151,29 b' def mpl_runner(safe_execfile):' | |||||
150 | return mpl_execfile |
|
151 | return mpl_execfile | |
151 |
|
152 | |||
152 |
|
153 | |||
|
154 | def select_figure_format(shell, fmt): | |||
|
155 | """Select figure format for inline backend, either 'png' or 'svg'. | |||
|
156 | ||||
|
157 | Using this method ensures only one figure format is active at a time. | |||
|
158 | """ | |||
|
159 | from matplotlib.figure import Figure | |||
|
160 | from IPython.zmq.pylab import backend_inline | |||
|
161 | ||||
|
162 | svg_formatter = shell.display_formatter.formatters['image/svg+xml'] | |||
|
163 | png_formatter = shell.display_formatter.formatters['image/png'] | |||
|
164 | ||||
|
165 | if fmt=='png': | |||
|
166 | svg_formatter.type_printers.pop(Figure, None) | |||
|
167 | png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png')) | |||
|
168 | elif fmt=='svg': | |||
|
169 | png_formatter.type_printers.pop(Figure, None) | |||
|
170 | svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg')) | |||
|
171 | else: | |||
|
172 | raise ValueError("supported formats are: 'png', 'svg', not %r"%fmt) | |||
|
173 | ||||
|
174 | # set the format to be used in the backend() | |||
|
175 | backend_inline._figure_format = fmt | |||
|
176 | ||||
153 | #----------------------------------------------------------------------------- |
|
177 | #----------------------------------------------------------------------------- | |
154 | # Code for initializing matplotlib and importing pylab |
|
178 | # Code for initializing matplotlib and importing pylab | |
155 | #----------------------------------------------------------------------------- |
|
179 | #----------------------------------------------------------------------------- | |
@@ -208,7 +232,6 b' def activate_matplotlib(backend):' | |||||
208 | # For this, we wrap it into a decorator which adds a 'called' flag. |
|
232 | # For this, we wrap it into a decorator which adds a 'called' flag. | |
209 | pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive) |
|
233 | pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive) | |
210 |
|
234 | |||
211 |
|
||||
212 | def import_pylab(user_ns, backend, import_all=True, shell=None): |
|
235 | def import_pylab(user_ns, backend, import_all=True, shell=None): | |
213 | """Import the standard pylab symbols into user_ns.""" |
|
236 | """Import the standard pylab symbols into user_ns.""" | |
214 |
|
237 | |||
@@ -228,43 +251,32 b' def import_pylab(user_ns, backend, import_all=True, shell=None):' | |||||
228 | # If using our svg payload backend, register the post-execution |
|
251 | # If using our svg payload backend, register the post-execution | |
229 | # function that will pick up the results for display. This can only be |
|
252 | # function that will pick up the results for display. This can only be | |
230 | # done with access to the real shell object. |
|
253 | # done with access to the real shell object. | |
|
254 | # | |||
|
255 | from IPython.zmq.pylab.backend_inline import InlineBackendConfig | |||
|
256 | ||||
|
257 | cfg = InlineBackendConfig.instance(config=shell.config) | |||
|
258 | cfg.shell = shell | |||
|
259 | ||||
231 | if backend == backends['inline']: |
|
260 | if backend == backends['inline']: | |
232 |
from IPython.zmq.pylab.backend_inline import flush_ |
|
261 | from IPython.zmq.pylab.backend_inline import flush_figures | |
233 | from matplotlib import pyplot |
|
262 | from matplotlib import pyplot | |
234 |
shell.register_post_execute(flush_ |
|
263 | shell.register_post_execute(flush_figures) | |
235 | # The typical default figure size is too large for inline use, |
|
264 | # load inline_rc | |
236 | # so we shrink the figure size to 6x4, and tweak fonts to |
|
265 | pyplot.rcParams.update(cfg.rc) | |
237 | # make that fit. This is configurable via Global.pylab_inline_rc, |
|
|||
238 | # or rather it will be once the zmq kernel is hooked up to |
|
|||
239 | # the config system. |
|
|||
240 |
|
||||
241 | default_rc = { |
|
|||
242 | 'figure.figsize': (6.0,4.0), |
|
|||
243 | # 12pt labels get cutoff on 6x4 logplots, so use 10pt. |
|
|||
244 | 'font.size': 10, |
|
|||
245 | # 10pt still needs a little more room on the xlabel: |
|
|||
246 | 'figure.subplot.bottom' : .125 |
|
|||
247 | } |
|
|||
248 | rc = getattr(shell.config.Global, 'pylab_inline_rc', default_rc) |
|
|||
249 | pyplot.rcParams.update(rc) |
|
|||
250 | shell.config.Global.pylab_inline_rc = rc |
|
|||
251 |
|
266 | |||
252 | # Add 'figsize' to pyplot and to the user's namespace |
|
267 | # Add 'figsize' to pyplot and to the user's namespace | |
253 | user_ns['figsize'] = pyplot.figsize = figsize |
|
268 | user_ns['figsize'] = pyplot.figsize = figsize | |
254 | shell.user_ns_hidden['figsize'] = figsize |
|
269 | shell.user_ns_hidden['figsize'] = figsize | |
255 |
|
270 | |||
|
271 | # Setup the default figure format | |||
|
272 | fmt = cfg.figure_format | |||
|
273 | select_figure_format(shell, fmt) | |||
|
274 | ||||
256 | # The old pastefig function has been replaced by display |
|
275 | # The old pastefig function has been replaced by display | |
257 | # Always add this svg formatter so display works. |
|
276 | from IPython.core.display import display | |
258 | from IPython.core.display import display, display_svg |
|
|||
259 | svg_formatter = shell.display_formatter.formatters['image/svg+xml'] |
|
|||
260 | svg_formatter.for_type_by_name( |
|
|||
261 | 'matplotlib.figure','Figure',figure_to_svg |
|
|||
262 | ) |
|
|||
263 | # Add display and display_png to the user's namespace |
|
277 | # Add display and display_png to the user's namespace | |
264 | user_ns['display'] = display |
|
278 | user_ns['display'] = display | |
265 | shell.user_ns_hidden['display'] = display |
|
279 | shell.user_ns_hidden['display'] = display | |
266 | user_ns['display_svg'] = display_svg |
|
|||
267 | shell.user_ns_hidden['display_svg'] = display_svg |
|
|||
268 | user_ns['getfigs'] = getfigs |
|
280 | user_ns['getfigs'] = getfigs | |
269 | shell.user_ns_hidden['getfigs'] = getfigs |
|
281 | shell.user_ns_hidden['getfigs'] = getfigs | |
270 |
|
282 |
@@ -41,7 +41,7 b' from .. import pylabtools as pt' | |||||
41 | def test_figure_to_svg(): |
|
41 | def test_figure_to_svg(): | |
42 | # simple empty-figure test |
|
42 | # simple empty-figure test | |
43 | fig = plt.figure() |
|
43 | fig = plt.figure() | |
44 |
yield nt.assert_equal(pt.figure |
|
44 | yield nt.assert_equal(pt.print_figure(fig, 'svg'), None) | |
45 |
|
45 | |||
46 | plt.close('all') |
|
46 | plt.close('all') | |
47 |
|
47 | |||
@@ -50,5 +50,5 b' def test_figure_to_svg():' | |||||
50 | ax = fig.add_subplot(1,1,1) |
|
50 | ax = fig.add_subplot(1,1,1) | |
51 | ax.plot([1,2,3]) |
|
51 | ax.plot([1,2,3]) | |
52 | plt.draw() |
|
52 | plt.draw() | |
53 |
svg = pt.figure |
|
53 | svg = pt.print_figure(fig, 'svg')[:100].lower() | |
54 | yield nt.assert_true('doctype svg' in svg) |
|
54 | yield nt.assert_true('doctype svg' in svg) |
@@ -1,4 +1,9 b'' | |||||
1 |
"""The IPython ZMQ-based parallel computing interface. |
|
1 | """The IPython ZMQ-based parallel computing interface. | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * MinRK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2011 The IPython Development Team |
|
8 | # Copyright (C) 2011 The IPython Development Team | |
4 | # |
|
9 | # |
This diff has been collapsed as it changes many lines, (753 lines changed) Show them Hide them | |||||
@@ -2,10 +2,16 b'' | |||||
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The ipcluster application. |
|
4 | The ipcluster application. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Brian Granger | |||
|
9 | * MinRK | |||
|
10 | ||||
5 | """ |
|
11 | """ | |
6 |
|
12 | |||
7 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
8 |
# Copyright (C) 2008-20 |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
9 | # |
|
15 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
@@ -25,12 +31,18 b' from subprocess import check_call, CalledProcessError, PIPE' | |||||
25 | import zmq |
|
31 | import zmq | |
26 | from zmq.eventloop import ioloop |
|
32 | from zmq.eventloop import ioloop | |
27 |
|
33 | |||
28 | from IPython.external.argparse import ArgumentParser, SUPPRESS |
|
34 | from IPython.config.application import Application, boolean_flag | |
|
35 | from IPython.config.loader import Config | |||
|
36 | from IPython.core.application import BaseIPythonApplication | |||
|
37 | from IPython.core.profiledir import ProfileDir | |||
|
38 | from IPython.utils.daemonize import daemonize | |||
29 | from IPython.utils.importstring import import_item |
|
39 | from IPython.utils.importstring import import_item | |
|
40 | from IPython.utils.traitlets import Int, Unicode, Bool, CFloat, Dict, List | |||
30 |
|
41 | |||
31 |
from IPython.parallel.apps. |
|
42 | from IPython.parallel.apps.baseapp import ( | |
32 | ApplicationWithClusterDir, ClusterDirConfigLoader, |
|
43 | BaseParallelApplication, | |
33 |
|
|
44 | PIDFileError, | |
|
45 | base_flags, base_aliases | |||
34 | ) |
|
46 | ) | |
35 |
|
47 | |||
36 |
|
48 | |||
@@ -42,16 +54,15 b' from IPython.parallel.apps.clusterdir import (' | |||||
42 | default_config_file_name = u'ipcluster_config.py' |
|
54 | default_config_file_name = u'ipcluster_config.py' | |
43 |
|
55 | |||
44 |
|
56 | |||
45 | _description = """\ |
|
57 | _description = """Start an IPython cluster for parallel computing. | |
46 | Start an IPython cluster for parallel computing.\n\n |
|
|||
47 |
|
58 | |||
48 | An IPython cluster consists of 1 controller and 1 or more engines. |
|
59 | An IPython cluster consists of 1 controller and 1 or more engines. | |
49 | This command automates the startup of these processes using a wide |
|
60 | This command automates the startup of these processes using a wide | |
50 | range of startup methods (SSH, local processes, PBS, mpiexec, |
|
61 | range of startup methods (SSH, local processes, PBS, mpiexec, | |
51 | Windows HPC Server 2008). To start a cluster with 4 engines on your |
|
62 | Windows HPC Server 2008). To start a cluster with 4 engines on your | |
52 |
local host simply do 'ipcluster start |
|
63 | local host simply do 'ipcluster start n=4'. For more complex usage | |
53 |
you will typically do 'ipcluster create |
|
64 | you will typically do 'ipcluster create profile=mycluster', then edit | |
54 |
configuration files, followed by 'ipcluster start |
|
65 | configuration files, followed by 'ipcluster start profile=mycluster n=4'. | |
55 | """ |
|
66 | """ | |
56 |
|
67 | |||
57 |
|
68 | |||
@@ -72,426 +83,286 b' NO_CLUSTER = 12' | |||||
72 |
|
83 | |||
73 |
|
84 | |||
74 | #----------------------------------------------------------------------------- |
|
85 | #----------------------------------------------------------------------------- | |
75 | # Command line options |
|
86 | # Main application | |
76 | #----------------------------------------------------------------------------- |
|
87 | #----------------------------------------------------------------------------- | |
|
88 | start_help = """Start an IPython cluster for parallel computing | |||
|
89 | ||||
|
90 | Start an ipython cluster by its profile name or cluster | |||
|
91 | directory. Cluster directories contain configuration, log and | |||
|
92 | security related files and are named using the convention | |||
|
93 | 'profile_<name>' and should be creating using the 'start' | |||
|
94 | subcommand of 'ipcluster'. If your cluster directory is in | |||
|
95 | the cwd or the ipython directory, you can simply refer to it | |||
|
96 | using its profile name, 'ipcluster start n=4 profile=<profile>`, | |||
|
97 | otherwise use the 'profile_dir' option. | |||
|
98 | """ | |||
|
99 | stop_help = """Stop a running IPython cluster | |||
|
100 | ||||
|
101 | Stop a running ipython cluster by its profile name or cluster | |||
|
102 | directory. Cluster directories are named using the convention | |||
|
103 | 'profile_<name>'. If your cluster directory is in | |||
|
104 | the cwd or the ipython directory, you can simply refer to it | |||
|
105 | using its profile name, 'ipcluster stop profile=<profile>`, otherwise | |||
|
106 | use the 'profile_dir' option. | |||
|
107 | """ | |||
|
108 | engines_help = """Start engines connected to an existing IPython cluster | |||
|
109 | ||||
|
110 | Start one or more engines to connect to an existing Cluster | |||
|
111 | by profile name or cluster directory. | |||
|
112 | Cluster directories contain configuration, log and | |||
|
113 | security related files and are named using the convention | |||
|
114 | 'profile_<name>' and should be creating using the 'start' | |||
|
115 | subcommand of 'ipcluster'. If your cluster directory is in | |||
|
116 | the cwd or the ipython directory, you can simply refer to it | |||
|
117 | using its profile name, 'ipcluster engines n=4 profile=<profile>`, | |||
|
118 | otherwise use the 'profile_dir' option. | |||
|
119 | """ | |||
|
120 | stop_aliases = dict( | |||
|
121 | signal='IPClusterStop.signal', | |||
|
122 | profile='BaseIPythonApplication.profile', | |||
|
123 | profile_dir='ProfileDir.location', | |||
|
124 | ) | |||
77 |
|
125 | |||
78 |
|
126 | class IPClusterStop(BaseParallelApplication): | ||
79 | class IPClusterAppConfigLoader(ClusterDirConfigLoader): |
|
127 | name = u'ipcluster' | |
80 |
|
128 | description = stop_help | ||
81 | def _add_arguments(self): |
|
129 | config_file_name = Unicode(default_config_file_name) | |
82 | # Don't call ClusterDirConfigLoader._add_arguments as we don't want |
|
130 | ||
83 | # its defaults on self.parser. Instead, we will put those on |
|
131 | signal = Int(signal.SIGINT, config=True, | |
84 | # default options on our subparsers. |
|
132 | help="signal to use for stopping processes.") | |
85 |
|
||||
86 | # This has all the common options that all subcommands use |
|
|||
87 | parent_parser1 = ArgumentParser( |
|
|||
88 | add_help=False, |
|
|||
89 | argument_default=SUPPRESS |
|
|||
90 | ) |
|
|||
91 | self._add_ipython_dir(parent_parser1) |
|
|||
92 | self._add_log_level(parent_parser1) |
|
|||
93 |
|
||||
94 | # This has all the common options that other subcommands use |
|
|||
95 | parent_parser2 = ArgumentParser( |
|
|||
96 | add_help=False, |
|
|||
97 | argument_default=SUPPRESS |
|
|||
98 | ) |
|
|||
99 | self._add_cluster_profile(parent_parser2) |
|
|||
100 | self._add_cluster_dir(parent_parser2) |
|
|||
101 | self._add_work_dir(parent_parser2) |
|
|||
102 | paa = parent_parser2.add_argument |
|
|||
103 | paa('--log-to-file', |
|
|||
104 | action='store_true', dest='Global.log_to_file', |
|
|||
105 | help='Log to a file in the log directory (default is stdout)') |
|
|||
106 |
|
||||
107 | # Create the object used to create the subparsers. |
|
|||
108 | subparsers = self.parser.add_subparsers( |
|
|||
109 | dest='Global.subcommand', |
|
|||
110 | title='ipcluster subcommands', |
|
|||
111 | description= |
|
|||
112 | """ipcluster has a variety of subcommands. The general way of |
|
|||
113 | running ipcluster is 'ipcluster <cmd> [options]'. To get help |
|
|||
114 | on a particular subcommand do 'ipcluster <cmd> -h'.""" |
|
|||
115 | # help="For more help, type 'ipcluster <cmd> -h'", |
|
|||
116 | ) |
|
|||
117 |
|
||||
118 | # The "list" subcommand parser |
|
|||
119 | parser_list = subparsers.add_parser( |
|
|||
120 | 'list', |
|
|||
121 | parents=[parent_parser1], |
|
|||
122 | argument_default=SUPPRESS, |
|
|||
123 | help="List all clusters in cwd and ipython_dir.", |
|
|||
124 | description= |
|
|||
125 | """List all available clusters, by cluster directory, that can |
|
|||
126 | be found in the current working directly or in the ipython |
|
|||
127 | directory. Cluster directories are named using the convention |
|
|||
128 | 'cluster_<profile>'.""" |
|
|||
129 | ) |
|
|||
130 |
|
||||
131 | # The "create" subcommand parser |
|
|||
132 | parser_create = subparsers.add_parser( |
|
|||
133 | 'create', |
|
|||
134 | parents=[parent_parser1, parent_parser2], |
|
|||
135 | argument_default=SUPPRESS, |
|
|||
136 | help="Create a new cluster directory.", |
|
|||
137 | description= |
|
|||
138 | """Create an ipython cluster directory by its profile name or |
|
|||
139 | cluster directory path. Cluster directories contain |
|
|||
140 | configuration, log and security related files and are named |
|
|||
141 | using the convention 'cluster_<profile>'. By default they are |
|
|||
142 | located in your ipython directory. Once created, you will |
|
|||
143 | probably need to edit the configuration files in the cluster |
|
|||
144 | directory to configure your cluster. Most users will create a |
|
|||
145 | cluster directory by profile name, |
|
|||
146 | 'ipcluster create -p mycluster', which will put the directory |
|
|||
147 | in '<ipython_dir>/cluster_mycluster'. |
|
|||
148 | """ |
|
|||
149 | ) |
|
|||
150 | paa = parser_create.add_argument |
|
|||
151 | paa('--reset-config', |
|
|||
152 | dest='Global.reset_config', action='store_true', |
|
|||
153 | help= |
|
|||
154 | """Recopy the default config files to the cluster directory. |
|
|||
155 | You will loose any modifications you have made to these files.""") |
|
|||
156 |
|
||||
157 | # The "start" subcommand parser |
|
|||
158 | parser_start = subparsers.add_parser( |
|
|||
159 | 'start', |
|
|||
160 | parents=[parent_parser1, parent_parser2], |
|
|||
161 | argument_default=SUPPRESS, |
|
|||
162 | help="Start a cluster.", |
|
|||
163 | description= |
|
|||
164 | """Start an ipython cluster by its profile name or cluster |
|
|||
165 | directory. Cluster directories contain configuration, log and |
|
|||
166 | security related files and are named using the convention |
|
|||
167 | 'cluster_<profile>' and should be creating using the 'start' |
|
|||
168 | subcommand of 'ipcluster'. If your cluster directory is in |
|
|||
169 | the cwd or the ipython directory, you can simply refer to it |
|
|||
170 | using its profile name, 'ipcluster start -n 4 -p <profile>`, |
|
|||
171 | otherwise use the '--cluster-dir' option. |
|
|||
172 | """ |
|
|||
173 | ) |
|
|||
174 |
|
133 | |||
175 | paa = parser_start.add_argument |
|
134 | aliases = Dict(stop_aliases) | |
176 | paa('-n', '--number', |
|
135 | ||
177 | type=int, dest='Global.n', |
|
136 | def start(self): | |
178 | help='The number of engines to start.', |
|
137 | """Start the app for the stop subcommand.""" | |
179 | metavar='Global.n') |
|
138 | try: | |
180 | paa('--clean-logs', |
|
139 | pid = self.get_pid_from_file() | |
181 | dest='Global.clean_logs', action='store_true', |
|
140 | except PIDFileError: | |
182 | help='Delete old log flies before starting.') |
|
141 | self.log.critical( | |
183 | paa('--no-clean-logs', |
|
142 | 'Could not read pid file, cluster is probably not running.' | |
184 | dest='Global.clean_logs', action='store_false', |
|
143 | ) | |
185 | help="Don't delete old log flies before starting.") |
|
144 | # Here I exit with a unusual exit status that other processes | |
186 | paa('--daemon', |
|
145 | # can watch for to learn how I existed. | |
187 | dest='Global.daemonize', action='store_true', |
|
146 | self.remove_pid_file() | |
188 | help='Daemonize the ipcluster program. This implies --log-to-file') |
|
147 | self.exit(ALREADY_STOPPED) | |
189 | paa('--no-daemon', |
|
|||
190 | dest='Global.daemonize', action='store_false', |
|
|||
191 | help="Dont't daemonize the ipcluster program.") |
|
|||
192 | paa('--delay', |
|
|||
193 | type=float, dest='Global.delay', |
|
|||
194 | help="Specify the delay (in seconds) between starting the controller and starting the engine(s).") |
|
|||
195 |
|
||||
196 | # The "stop" subcommand parser |
|
|||
197 | parser_stop = subparsers.add_parser( |
|
|||
198 | 'stop', |
|
|||
199 | parents=[parent_parser1, parent_parser2], |
|
|||
200 | argument_default=SUPPRESS, |
|
|||
201 | help="Stop a running cluster.", |
|
|||
202 | description= |
|
|||
203 | """Stop a running ipython cluster by its profile name or cluster |
|
|||
204 | directory. Cluster directories are named using the convention |
|
|||
205 | 'cluster_<profile>'. If your cluster directory is in |
|
|||
206 | the cwd or the ipython directory, you can simply refer to it |
|
|||
207 | using its profile name, 'ipcluster stop -p <profile>`, otherwise |
|
|||
208 | use the '--cluster-dir' option. |
|
|||
209 | """ |
|
|||
210 | ) |
|
|||
211 | paa = parser_stop.add_argument |
|
|||
212 | paa('--signal', |
|
|||
213 | dest='Global.signal', type=int, |
|
|||
214 | help="The signal number to use in stopping the cluster (default=2).", |
|
|||
215 | metavar="Global.signal") |
|
|||
216 |
|
148 | |||
217 | # the "engines" subcommand parser |
|
149 | if not self.check_pid(pid): | |
218 | parser_engines = subparsers.add_parser( |
|
150 | self.log.critical( | |
219 | 'engines', |
|
151 | 'Cluster [pid=%r] is not running.' % pid | |
220 | parents=[parent_parser1, parent_parser2], |
|
152 | ) | |
221 | argument_default=SUPPRESS, |
|
153 | self.remove_pid_file() | |
222 | help="Attach some engines to an existing controller or cluster.", |
|
154 | # Here I exit with a unusual exit status that other processes | |
223 | description= |
|
155 | # can watch for to learn how I existed. | |
224 | """Start one or more engines to connect to an existing Cluster |
|
156 | self.exit(ALREADY_STOPPED) | |
225 | by profile name or cluster directory. |
|
157 | ||
226 | Cluster directories contain configuration, log and |
|
158 | elif os.name=='posix': | |
227 | security related files and are named using the convention |
|
159 | sig = self.signal | |
228 | 'cluster_<profile>' and should be creating using the 'start' |
|
160 | self.log.info( | |
229 | subcommand of 'ipcluster'. If your cluster directory is in |
|
161 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) | |
230 | the cwd or the ipython directory, you can simply refer to it |
|
162 | ) | |
231 | using its profile name, 'ipcluster engines -n 4 -p <profile>`, |
|
163 | try: | |
232 | otherwise use the '--cluster-dir' option. |
|
164 | os.kill(pid, sig) | |
233 | """ |
|
165 | except OSError: | |
234 | ) |
|
166 | self.log.error("Stopping cluster failed, assuming already dead.", | |
235 | paa = parser_engines.add_argument |
|
167 | exc_info=True) | |
236 | paa('-n', '--number', |
|
168 | self.remove_pid_file() | |
237 | type=int, dest='Global.n', |
|
169 | elif os.name=='nt': | |
238 | help='The number of engines to start.', |
|
170 | try: | |
239 | metavar='Global.n') |
|
171 | # kill the whole tree | |
240 | paa('--daemon', |
|
172 | p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE) | |
241 | dest='Global.daemonize', action='store_true', |
|
173 | except (CalledProcessError, OSError): | |
242 | help='Daemonize the ipcluster program. This implies --log-to-file') |
|
174 | self.log.error("Stopping cluster failed, assuming already dead.", | |
243 | paa('--no-daemon', |
|
175 | exc_info=True) | |
244 | dest='Global.daemonize', action='store_false', |
|
176 | self.remove_pid_file() | |
245 | help="Dont't daemonize the ipcluster program.") |
|
177 | ||
246 |
|
178 | engine_aliases = {} | ||
247 | #----------------------------------------------------------------------------- |
|
179 | engine_aliases.update(base_aliases) | |
248 | # Main application |
|
180 | engine_aliases.update(dict( | |
249 | #----------------------------------------------------------------------------- |
|
181 | n='IPClusterEngines.n', | |
250 |
|
182 | elauncher = 'IPClusterEngines.engine_launcher_class', | ||
251 |
|
183 | )) | ||
252 |
class IPCluster |
|
184 | class IPClusterEngines(BaseParallelApplication): | |
253 |
|
185 | |||
254 | name = u'ipcluster' |
|
186 | name = u'ipcluster' | |
255 |
description = |
|
187 | description = engines_help | |
256 | usage = None |
|
188 | usage = None | |
257 | command_line_loader = IPClusterAppConfigLoader |
|
189 | config_file_name = Unicode(default_config_file_name) | |
258 | default_config_file_name = default_config_file_name |
|
|||
259 | default_log_level = logging.INFO |
|
190 | default_log_level = logging.INFO | |
260 | auto_create_cluster_dir = False |
|
191 | classes = List() | |
261 |
|
192 | def _classes_default(self): | ||
262 | def create_default_config(self): |
|
193 | from IPython.parallel.apps import launcher | |
263 | super(IPClusterApp, self).create_default_config() |
|
194 | launchers = launcher.all_launchers | |
264 | self.default_config.Global.controller_launcher = \ |
|
195 | eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__] | |
265 | 'IPython.parallel.apps.launcher.LocalControllerLauncher' |
|
196 | return [ProfileDir]+eslaunchers | |
266 | self.default_config.Global.engine_launcher = \ |
|
197 | ||
267 | 'IPython.parallel.apps.launcher.LocalEngineSetLauncher' |
|
198 | n = Int(2, config=True, | |
268 | self.default_config.Global.n = 2 |
|
199 | help="The number of engines to start.") | |
269 | self.default_config.Global.delay = 2 |
|
|||
270 | self.default_config.Global.reset_config = False |
|
|||
271 | self.default_config.Global.clean_logs = True |
|
|||
272 | self.default_config.Global.signal = signal.SIGINT |
|
|||
273 | self.default_config.Global.daemonize = False |
|
|||
274 |
|
||||
275 | def find_resources(self): |
|
|||
276 | subcommand = self.command_line_config.Global.subcommand |
|
|||
277 | if subcommand=='list': |
|
|||
278 | self.list_cluster_dirs() |
|
|||
279 | # Exit immediately because there is nothing left to do. |
|
|||
280 | self.exit() |
|
|||
281 | elif subcommand=='create': |
|
|||
282 | self.auto_create_cluster_dir = True |
|
|||
283 | super(IPClusterApp, self).find_resources() |
|
|||
284 | elif subcommand=='start' or subcommand=='stop': |
|
|||
285 | self.auto_create_cluster_dir = True |
|
|||
286 | try: |
|
|||
287 | super(IPClusterApp, self).find_resources() |
|
|||
288 | except ClusterDirError: |
|
|||
289 | raise ClusterDirError( |
|
|||
290 | "Could not find a cluster directory. A cluster dir must " |
|
|||
291 | "be created before running 'ipcluster start'. Do " |
|
|||
292 | "'ipcluster create -h' or 'ipcluster list -h' for more " |
|
|||
293 | "information about creating and listing cluster dirs." |
|
|||
294 | ) |
|
|||
295 | elif subcommand=='engines': |
|
|||
296 | self.auto_create_cluster_dir = False |
|
|||
297 | try: |
|
|||
298 | super(IPClusterApp, self).find_resources() |
|
|||
299 | except ClusterDirError: |
|
|||
300 | raise ClusterDirError( |
|
|||
301 | "Could not find a cluster directory. A cluster dir must " |
|
|||
302 | "be created before running 'ipcluster start'. Do " |
|
|||
303 | "'ipcluster create -h' or 'ipcluster list -h' for more " |
|
|||
304 | "information about creating and listing cluster dirs." |
|
|||
305 | ) |
|
|||
306 |
|
200 | |||
307 | def list_cluster_dirs(self): |
|
201 | engine_launcher_class = Unicode('LocalEngineSetLauncher', | |
308 | # Find the search paths |
|
202 | config=True, | |
309 | cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','') |
|
203 | help="The class for launching a set of Engines." | |
310 | if cluster_dir_paths: |
|
204 | ) | |
311 | cluster_dir_paths = cluster_dir_paths.split(':') |
|
205 | daemonize = Bool(False, config=True, | |
312 | else: |
|
206 | help='Daemonize the ipcluster program. This implies --log-to-file') | |
313 | cluster_dir_paths = [] |
|
|||
314 | try: |
|
|||
315 | ipython_dir = self.command_line_config.Global.ipython_dir |
|
|||
316 | except AttributeError: |
|
|||
317 | ipython_dir = self.default_config.Global.ipython_dir |
|
|||
318 | paths = [os.getcwd(), ipython_dir] + \ |
|
|||
319 | cluster_dir_paths |
|
|||
320 | paths = list(set(paths)) |
|
|||
321 |
|
||||
322 | self.log.info('Searching for cluster dirs in paths: %r' % paths) |
|
|||
323 | for path in paths: |
|
|||
324 | files = os.listdir(path) |
|
|||
325 | for f in files: |
|
|||
326 | full_path = os.path.join(path, f) |
|
|||
327 | if os.path.isdir(full_path) and f.startswith('cluster_'): |
|
|||
328 | profile = full_path.split('_')[-1] |
|
|||
329 | start_cmd = 'ipcluster start -p %s -n 4' % profile |
|
|||
330 | print start_cmd + " ==> " + full_path |
|
|||
331 |
|
||||
332 | def pre_construct(self): |
|
|||
333 | # IPClusterApp.pre_construct() is where we cd to the working directory. |
|
|||
334 | super(IPClusterApp, self).pre_construct() |
|
|||
335 | config = self.master_config |
|
|||
336 | try: |
|
|||
337 | daemon = config.Global.daemonize |
|
|||
338 | if daemon: |
|
|||
339 | config.Global.log_to_file = True |
|
|||
340 | except AttributeError: |
|
|||
341 | pass |
|
|||
342 |
|
207 | |||
343 | def construct(self): |
|
208 | def _daemonize_changed(self, name, old, new): | |
344 | config = self.master_config |
|
209 | if new: | |
345 | subcmd = config.Global.subcommand |
|
210 | self.log_to_file = True | |
346 | reset = config.Global.reset_config |
|
|||
347 | if subcmd == 'list': |
|
|||
348 | return |
|
|||
349 | if subcmd == 'create': |
|
|||
350 | self.log.info('Copying default config files to cluster directory ' |
|
|||
351 | '[overwrite=%r]' % (reset,)) |
|
|||
352 | self.cluster_dir_obj.copy_all_config_files(overwrite=reset) |
|
|||
353 | if subcmd =='start': |
|
|||
354 | self.cluster_dir_obj.copy_all_config_files(overwrite=False) |
|
|||
355 | self.start_logging() |
|
|||
356 | self.loop = ioloop.IOLoop.instance() |
|
|||
357 | # reactor.callWhenRunning(self.start_launchers) |
|
|||
358 | dc = ioloop.DelayedCallback(self.start_launchers, 0, self.loop) |
|
|||
359 | dc.start() |
|
|||
360 | if subcmd == 'engines': |
|
|||
361 | self.start_logging() |
|
|||
362 | self.loop = ioloop.IOLoop.instance() |
|
|||
363 | # reactor.callWhenRunning(self.start_launchers) |
|
|||
364 | engine_only = lambda : self.start_launchers(controller=False) |
|
|||
365 | dc = ioloop.DelayedCallback(engine_only, 0, self.loop) |
|
|||
366 | dc.start() |
|
|||
367 |
|
211 | |||
368 | def start_launchers(self, controller=True): |
|
212 | aliases = Dict(engine_aliases) | |
369 | config = self.master_config |
|
213 | # flags = Dict(flags) | |
370 |
|
214 | _stopping = False | ||
371 | # Create the launchers. In both bases, we set the work_dir of |
|
|||
372 | # the launcher to the cluster_dir. This is where the launcher's |
|
|||
373 | # subprocesses will be launched. It is not where the controller |
|
|||
374 | # and engine will be launched. |
|
|||
375 | if controller: |
|
|||
376 | cl_class = import_item(config.Global.controller_launcher) |
|
|||
377 | self.controller_launcher = cl_class( |
|
|||
378 | work_dir=self.cluster_dir, config=config, |
|
|||
379 | logname=self.log.name |
|
|||
380 | ) |
|
|||
381 | # Setup the observing of stopping. If the controller dies, shut |
|
|||
382 | # everything down as that will be completely fatal for the engines. |
|
|||
383 | self.controller_launcher.on_stop(self.stop_launchers) |
|
|||
384 | # But, we don't monitor the stopping of engines. An engine dying |
|
|||
385 | # is just fine and in principle a user could start a new engine. |
|
|||
386 | # Also, if we did monitor engine stopping, it is difficult to |
|
|||
387 | # know what to do when only some engines die. Currently, the |
|
|||
388 | # observing of engine stopping is inconsistent. Some launchers |
|
|||
389 | # might trigger on a single engine stopping, other wait until |
|
|||
390 | # all stop. TODO: think more about how to handle this. |
|
|||
391 | else: |
|
|||
392 | self.controller_launcher = None |
|
|||
393 |
|
||||
394 | el_class = import_item(config.Global.engine_launcher) |
|
|||
395 | self.engine_launcher = el_class( |
|
|||
396 | work_dir=self.cluster_dir, config=config, logname=self.log.name |
|
|||
397 | ) |
|
|||
398 |
|
215 | |||
|
216 | def initialize(self, argv=None): | |||
|
217 | super(IPClusterEngines, self).initialize(argv) | |||
|
218 | self.init_signal() | |||
|
219 | self.init_launchers() | |||
|
220 | ||||
|
221 | def init_launchers(self): | |||
|
222 | self.engine_launcher = self.build_launcher(self.engine_launcher_class) | |||
|
223 | self.engine_launcher.on_stop(lambda r: self.loop.stop()) | |||
|
224 | ||||
|
225 | def init_signal(self): | |||
399 | # Setup signals |
|
226 | # Setup signals | |
400 | signal.signal(signal.SIGINT, self.sigint_handler) |
|
227 | signal.signal(signal.SIGINT, self.sigint_handler) | |
401 |
|
228 | |||
402 | # Start the controller and engines |
|
229 | def build_launcher(self, clsname): | |
403 | self._stopping = False # Make sure stop_launchers is not called 2x. |
|
230 | """import and instantiate a Launcher based on importstring""" | |
404 | if controller: |
|
231 | if '.' not in clsname: | |
405 | self.start_controller() |
|
232 | # not a module, presume it's the raw name in apps.launcher | |
406 | dc = ioloop.DelayedCallback(self.start_engines, 1000*config.Global.delay*controller, self.loop) |
|
233 | clsname = 'IPython.parallel.apps.launcher.'+clsname | |
407 | dc.start() |
|
234 | # print repr(clsname) | |
408 | self.startup_message() |
|
235 | klass = import_item(clsname) | |
409 |
|
236 | |||
410 | def startup_message(self, r=None): |
|
237 | launcher = klass( | |
411 | self.log.info("IPython cluster: started") |
|
238 | work_dir=self.profile_dir.location, config=self.config, log=self.log | |
412 | return r |
|
|||
413 |
|
||||
414 | def start_controller(self, r=None): |
|
|||
415 | # self.log.info("In start_controller") |
|
|||
416 | config = self.master_config |
|
|||
417 | d = self.controller_launcher.start( |
|
|||
418 | cluster_dir=config.Global.cluster_dir |
|
|||
419 | ) |
|
239 | ) | |
420 |
return |
|
240 | return launcher | |
421 |
|
241 | |||
422 |
def start_engines(self |
|
242 | def start_engines(self): | |
423 |
|
|
243 | self.log.info("Starting %i engines"%self.n) | |
424 | config = self.master_config |
|
244 | self.engine_launcher.start( | |
425 |
|
245 | self.n, | ||
426 | d = self.engine_launcher.start( |
|
246 | self.profile_dir.location | |
427 | config.Global.n, |
|
|||
428 | cluster_dir=config.Global.cluster_dir |
|
|||
429 | ) |
|
247 | ) | |
430 | return d |
|
|||
431 |
|
||||
432 | def stop_controller(self, r=None): |
|
|||
433 | # self.log.info("In stop_controller") |
|
|||
434 | if self.controller_launcher and self.controller_launcher.running: |
|
|||
435 | return self.controller_launcher.stop() |
|
|||
436 |
|
248 | |||
437 |
def stop_engines(self |
|
249 | def stop_engines(self): | |
438 |
|
|
250 | self.log.info("Stopping Engines...") | |
439 | if self.engine_launcher.running: |
|
251 | if self.engine_launcher.running: | |
440 | d = self.engine_launcher.stop() |
|
252 | d = self.engine_launcher.stop() | |
441 | # d.addErrback(self.log_err) |
|
|||
442 | return d |
|
253 | return d | |
443 | else: |
|
254 | else: | |
444 | return None |
|
255 | return None | |
445 |
|
256 | |||
446 | def log_err(self, f): |
|
|||
447 | self.log.error(f.getTraceback()) |
|
|||
448 | return None |
|
|||
449 |
|
||||
450 | def stop_launchers(self, r=None): |
|
257 | def stop_launchers(self, r=None): | |
451 | if not self._stopping: |
|
258 | if not self._stopping: | |
452 | self._stopping = True |
|
259 | self._stopping = True | |
453 | # if isinstance(r, failure.Failure): |
|
|||
454 | # self.log.error('Unexpected error in ipcluster:') |
|
|||
455 | # self.log.info(r.getTraceback()) |
|
|||
456 | self.log.error("IPython cluster: stopping") |
|
260 | self.log.error("IPython cluster: stopping") | |
457 | # These return deferreds. We are not doing anything with them |
|
261 | self.stop_engines() | |
458 | # but we are holding refs to them as a reminder that they |
|
|||
459 | # do return deferreds. |
|
|||
460 | d1 = self.stop_engines() |
|
|||
461 | d2 = self.stop_controller() |
|
|||
462 | # Wait a few seconds to let things shut down. |
|
262 | # Wait a few seconds to let things shut down. | |
463 | dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop) |
|
263 | dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop) | |
464 | dc.start() |
|
264 | dc.start() | |
465 | # reactor.callLater(4.0, reactor.stop) |
|
|||
466 |
|
265 | |||
467 | def sigint_handler(self, signum, frame): |
|
266 | def sigint_handler(self, signum, frame): | |
|
267 | self.log.debug("SIGINT received, stopping launchers...") | |||
468 | self.stop_launchers() |
|
268 | self.stop_launchers() | |
469 |
|
269 | |||
470 | def start_logging(self): |
|
270 | def start_logging(self): | |
471 | # Remove old log files of the controller and engine |
|
271 | # Remove old log files of the controller and engine | |
472 |
if self. |
|
272 | if self.clean_logs: | |
473 |
log_dir = self. |
|
273 | log_dir = self.profile_dir.log_dir | |
474 | for f in os.listdir(log_dir): |
|
274 | for f in os.listdir(log_dir): | |
475 | if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f): |
|
275 | if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f): | |
476 | os.remove(os.path.join(log_dir, f)) |
|
276 | os.remove(os.path.join(log_dir, f)) | |
477 | # This will remove old log files for ipcluster itself |
|
277 | # This will remove old log files for ipcluster itself | |
478 |
super(IP |
|
278 | # super(IPBaseParallelApplication, self).start_logging() | |
479 |
|
279 | |||
480 |
def start |
|
280 | def start(self): | |
481 |
"""Start the app |
|
281 | """Start the app for the engines subcommand.""" | |
482 | subcmd = self.master_config.Global.subcommand |
|
282 | self.log.info("IPython cluster: started") | |
483 | if subcmd=='create' or subcmd=='list': |
|
283 | # First see if the cluster is already running | |
484 |
|
|
284 | ||
485 | elif subcmd=='start': |
|
285 | # Now log and daemonize | |
486 | self.start_app_start() |
|
286 | self.log.info( | |
487 | elif subcmd=='stop': |
|
287 | 'Starting engines with [daemon=%r]' % self.daemonize | |
488 | self.start_app_stop() |
|
288 | ) | |
489 | elif subcmd=='engines': |
|
289 | # TODO: Get daemonize working on Windows or as a Windows Server. | |
490 |
|
|
290 | if self.daemonize: | |
491 |
|
291 | if os.name=='posix': | ||
492 | def start_app_start(self): |
|
292 | daemonize() | |
|
293 | ||||
|
294 | dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop) | |||
|
295 | dc.start() | |||
|
296 | # Now write the new pid file AFTER our new forked pid is active. | |||
|
297 | # self.write_pid_file() | |||
|
298 | try: | |||
|
299 | self.loop.start() | |||
|
300 | except KeyboardInterrupt: | |||
|
301 | pass | |||
|
302 | except zmq.ZMQError as e: | |||
|
303 | if e.errno == errno.EINTR: | |||
|
304 | pass | |||
|
305 | else: | |||
|
306 | raise | |||
|
307 | ||||
|
308 | start_aliases = {} | |||
|
309 | start_aliases.update(engine_aliases) | |||
|
310 | start_aliases.update(dict( | |||
|
311 | delay='IPClusterStart.delay', | |||
|
312 | clean_logs='IPClusterStart.clean_logs', | |||
|
313 | )) | |||
|
314 | ||||
|
315 | class IPClusterStart(IPClusterEngines): | |||
|
316 | ||||
|
317 | name = u'ipcluster' | |||
|
318 | description = start_help | |||
|
319 | default_log_level = logging.INFO | |||
|
320 | auto_create = Bool(True, config=True, | |||
|
321 | help="whether to create the profile_dir if it doesn't exist") | |||
|
322 | classes = List() | |||
|
323 | def _classes_default(self,): | |||
|
324 | from IPython.parallel.apps import launcher | |||
|
325 | return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers | |||
|
326 | ||||
|
327 | clean_logs = Bool(True, config=True, | |||
|
328 | help="whether to cleanup old logs before starting") | |||
|
329 | ||||
|
330 | delay = CFloat(1., config=True, | |||
|
331 | help="delay (in s) between starting the controller and the engines") | |||
|
332 | ||||
|
333 | controller_launcher_class = Unicode('LocalControllerLauncher', | |||
|
334 | config=True, | |||
|
335 | help="The class for launching a Controller." | |||
|
336 | ) | |||
|
337 | reset = Bool(False, config=True, | |||
|
338 | help="Whether to reset config files as part of '--create'." | |||
|
339 | ) | |||
|
340 | ||||
|
341 | # flags = Dict(flags) | |||
|
342 | aliases = Dict(start_aliases) | |||
|
343 | ||||
|
344 | def init_launchers(self): | |||
|
345 | self.controller_launcher = self.build_launcher(self.controller_launcher_class) | |||
|
346 | self.engine_launcher = self.build_launcher(self.engine_launcher_class) | |||
|
347 | self.controller_launcher.on_stop(self.stop_launchers) | |||
|
348 | ||||
|
349 | def start_controller(self): | |||
|
350 | self.controller_launcher.start( | |||
|
351 | self.profile_dir.location | |||
|
352 | ) | |||
|
353 | ||||
|
354 | def stop_controller(self): | |||
|
355 | # self.log.info("In stop_controller") | |||
|
356 | if self.controller_launcher and self.controller_launcher.running: | |||
|
357 | return self.controller_launcher.stop() | |||
|
358 | ||||
|
359 | def stop_launchers(self, r=None): | |||
|
360 | if not self._stopping: | |||
|
361 | self.stop_controller() | |||
|
362 | super(IPClusterStart, self).stop_launchers() | |||
|
363 | ||||
|
364 | def start(self): | |||
493 | """Start the app for the start subcommand.""" |
|
365 | """Start the app for the start subcommand.""" | |
494 | config = self.master_config |
|
|||
495 | # First see if the cluster is already running |
|
366 | # First see if the cluster is already running | |
496 | try: |
|
367 | try: | |
497 | pid = self.get_pid_from_file() |
|
368 | pid = self.get_pid_from_file() | |
@@ -512,14 +383,17 b' class IPClusterApp(ApplicationWithClusterDir):' | |||||
512 |
|
383 | |||
513 | # Now log and daemonize |
|
384 | # Now log and daemonize | |
514 | self.log.info( |
|
385 | self.log.info( | |
515 |
'Starting ipcluster with [daemon=%r]' % |
|
386 | 'Starting ipcluster with [daemon=%r]' % self.daemonize | |
516 | ) |
|
387 | ) | |
517 | # TODO: Get daemonize working on Windows or as a Windows Server. |
|
388 | # TODO: Get daemonize working on Windows or as a Windows Server. | |
518 |
if |
|
389 | if self.daemonize: | |
519 | if os.name=='posix': |
|
390 | if os.name=='posix': | |
520 | from twisted.scripts._twistd_unix import daemonize |
|
|||
521 | daemonize() |
|
391 | daemonize() | |
522 |
|
392 | |||
|
393 | dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop) | |||
|
394 | dc.start() | |||
|
395 | dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop) | |||
|
396 | dc.start() | |||
523 | # Now write the new pid file AFTER our new forked pid is active. |
|
397 | # Now write the new pid file AFTER our new forked pid is active. | |
524 | self.write_pid_file() |
|
398 | self.write_pid_file() | |
525 | try: |
|
399 | try: | |
@@ -534,81 +408,36 b' class IPClusterApp(ApplicationWithClusterDir):' | |||||
534 | finally: |
|
408 | finally: | |
535 | self.remove_pid_file() |
|
409 | self.remove_pid_file() | |
536 |
|
410 | |||
537 | def start_app_engines(self): |
|
411 | base='IPython.parallel.apps.ipclusterapp.IPCluster' | |
538 | """Start the app for the start subcommand.""" |
|
|||
539 | config = self.master_config |
|
|||
540 | # First see if the cluster is already running |
|
|||
541 |
|
||||
542 | # Now log and daemonize |
|
|||
543 | self.log.info( |
|
|||
544 | 'Starting engines with [daemon=%r]' % config.Global.daemonize |
|
|||
545 | ) |
|
|||
546 | # TODO: Get daemonize working on Windows or as a Windows Server. |
|
|||
547 | if config.Global.daemonize: |
|
|||
548 | if os.name=='posix': |
|
|||
549 | from twisted.scripts._twistd_unix import daemonize |
|
|||
550 | daemonize() |
|
|||
551 |
|
412 | |||
552 | # Now write the new pid file AFTER our new forked pid is active. |
|
413 | class IPClusterApp(Application): | |
553 | # self.write_pid_file() |
|
414 | name = u'ipcluster' | |
554 | try: |
|
415 | description = _description | |
555 | self.loop.start() |
|
|||
556 | except KeyboardInterrupt: |
|
|||
557 | pass |
|
|||
558 | except zmq.ZMQError as e: |
|
|||
559 | if e.errno == errno.EINTR: |
|
|||
560 | pass |
|
|||
561 | else: |
|
|||
562 | raise |
|
|||
563 | # self.remove_pid_file() |
|
|||
564 |
|
416 | |||
565 | def start_app_stop(self): |
|
417 | subcommands = { | |
566 | """Start the app for the stop subcommand.""" |
|
418 | 'start' : (base+'Start', start_help), | |
567 | config = self.master_config |
|
419 | 'stop' : (base+'Stop', stop_help), | |
568 | try: |
|
420 | 'engines' : (base+'Engines', engines_help), | |
569 | pid = self.get_pid_from_file() |
|
421 | } | |
570 | except PIDFileError: |
|
422 | ||
571 | self.log.critical( |
|
423 | # no aliases or flags for parent App | |
572 | 'Could not read pid file, cluster is probably not running.' |
|
424 | aliases = Dict() | |
573 | ) |
|
425 | flags = Dict() | |
574 | # Here I exit with a unusual exit status that other processes |
|
426 | ||
575 | # can watch for to learn how I existed. |
|
427 | def start(self): | |
576 | self.remove_pid_file() |
|
428 | if self.subapp is None: | |
577 | self.exit(ALREADY_STOPPED) |
|
429 | print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()) | |
578 |
|
430 | |||
579 | if not self.check_pid(pid): |
|
431 | self.print_description() | |
580 |
self. |
|
432 | self.print_subcommands() | |
581 | 'Cluster [pid=%r] is not running.' % pid |
|
433 | self.exit(1) | |
582 |
|
|
434 | else: | |
583 | self.remove_pid_file() |
|
435 | return self.subapp.start() | |
584 | # Here I exit with a unusual exit status that other processes |
|
|||
585 | # can watch for to learn how I existed. |
|
|||
586 | self.exit(ALREADY_STOPPED) |
|
|||
587 |
|
||||
588 | elif os.name=='posix': |
|
|||
589 | sig = config.Global.signal |
|
|||
590 | self.log.info( |
|
|||
591 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) |
|
|||
592 | ) |
|
|||
593 | try: |
|
|||
594 | os.kill(pid, sig) |
|
|||
595 | except OSError: |
|
|||
596 | self.log.error("Stopping cluster failed, assuming already dead.", |
|
|||
597 | exc_info=True) |
|
|||
598 | self.remove_pid_file() |
|
|||
599 | elif os.name=='nt': |
|
|||
600 | try: |
|
|||
601 | # kill the whole tree |
|
|||
602 | p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE) |
|
|||
603 | except (CalledProcessError, OSError): |
|
|||
604 | self.log.error("Stopping cluster failed, assuming already dead.", |
|
|||
605 | exc_info=True) |
|
|||
606 | self.remove_pid_file() |
|
|||
607 |
|
||||
608 |
|
436 | |||
609 | def launch_new_instance(): |
|
437 | def launch_new_instance(): | |
610 | """Create and run the IPython cluster.""" |
|
438 | """Create and run the IPython cluster.""" | |
611 | app = IPClusterApp() |
|
439 | app = IPClusterApp.instance() | |
|
440 | app.initialize() | |||
612 | app.start() |
|
441 | app.start() | |
613 |
|
442 | |||
614 |
|
443 |
This diff has been collapsed as it changes many lines, (509 lines changed) Show them Hide them | |||||
@@ -2,10 +2,16 b'' | |||||
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython controller application. |
|
4 | The IPython controller application. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Brian Granger | |||
|
9 | * MinRK | |||
|
10 | ||||
5 | """ |
|
11 | """ | |
6 |
|
12 | |||
7 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
8 |
# Copyright (C) 2008-20 |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
9 | # |
|
15 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
@@ -17,31 +23,46 b' The IPython controller application.' | |||||
17 |
|
23 | |||
18 | from __future__ import with_statement |
|
24 | from __future__ import with_statement | |
19 |
|
25 | |||
20 | import copy |
|
|||
21 | import os |
|
26 | import os | |
22 | import logging |
|
|||
23 | import socket |
|
27 | import socket | |
24 | import stat |
|
28 | import stat | |
25 | import sys |
|
29 | import sys | |
26 | import uuid |
|
30 | import uuid | |
27 |
|
31 | |||
|
32 | from multiprocessing import Process | |||
|
33 | ||||
28 | import zmq |
|
34 | import zmq | |
|
35 | from zmq.devices import ProcessMonitoredQueue | |||
29 | from zmq.log.handlers import PUBHandler |
|
36 | from zmq.log.handlers import PUBHandler | |
30 | from zmq.utils import jsonapi as json |
|
37 | from zmq.utils import jsonapi as json | |
31 |
|
38 | |||
32 |
from IPython.config. |
|
39 | from IPython.config.application import boolean_flag | |
33 |
|
40 | from IPython.core.profiledir import ProfileDir | ||
34 | from IPython.parallel import factory |
|
|||
35 |
|
41 | |||
36 |
from IPython.parallel.apps. |
|
42 | from IPython.parallel.apps.baseapp import ( | |
37 | ApplicationWithClusterDir, |
|
43 | BaseParallelApplication, | |
38 | ClusterDirConfigLoader |
|
44 | base_flags | |
39 | ) |
|
45 | ) | |
40 | from IPython.parallel.util import disambiguate_ip_address, split_url |
|
46 | from IPython.utils.importstring import import_item | |
41 | # from IPython.kernel.fcutil import FCServiceFactory, FURLError |
|
47 | from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict | |
42 | from IPython.utils.traitlets import Instance, Unicode |
|
48 | ||
|
49 | # from IPython.parallel.controller.controller import ControllerFactory | |||
|
50 | from IPython.zmq.session import Session | |||
|
51 | from IPython.parallel.controller.heartmonitor import HeartMonitor | |||
|
52 | from IPython.parallel.controller.hub import HubFactory | |||
|
53 | from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler | |||
|
54 | from IPython.parallel.controller.sqlitedb import SQLiteDB | |||
43 |
|
55 | |||
44 | from IPython.parallel.controller.controller import ControllerFactory |
|
56 | from IPython.parallel.util import signal_children, split_url | |
|
57 | ||||
|
58 | # conditional import of MongoDB backend class | |||
|
59 | ||||
|
60 | try: | |||
|
61 | from IPython.parallel.controller.mongodb import MongoDB | |||
|
62 | except ImportError: | |||
|
63 | maybe_mongo = [] | |||
|
64 | else: | |||
|
65 | maybe_mongo = [MongoDB] | |||
45 |
|
66 | |||
46 |
|
67 | |||
47 | #----------------------------------------------------------------------------- |
|
68 | #----------------------------------------------------------------------------- | |
@@ -59,238 +80,112 b' The IPython controller provides a gateway between the IPython engines and' | |||||
59 | clients. The controller needs to be started before the engines and can be |
|
80 | clients. The controller needs to be started before the engines and can be | |
60 | configured using command line options or using a cluster directory. Cluster |
|
81 | configured using command line options or using a cluster directory. Cluster | |
61 | directories contain config, log and security files and are usually located in |
|
82 | directories contain config, log and security files and are usually located in | |
62 |
your ipython directory and named as " |
|
83 | your ipython directory and named as "profile_name". See the `profile` | |
63 |
and |
|
84 | and `profile_dir` options for details. | |
64 | """ |
|
85 | """ | |
65 |
|
86 | |||
66 | #----------------------------------------------------------------------------- |
|
|||
67 | # Default interfaces |
|
|||
68 | #----------------------------------------------------------------------------- |
|
|||
69 |
|
||||
70 | # The default client interfaces for FCClientServiceFactory.interfaces |
|
|||
71 | default_client_interfaces = Config() |
|
|||
72 | default_client_interfaces.Default.url_file = 'ipcontroller-client.url' |
|
|||
73 |
|
||||
74 | # Make this a dict we can pass to Config.__init__ for the default |
|
|||
75 | default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items())) |
|
|||
76 |
|
87 | |||
77 |
|
88 | |||
78 |
|
89 | |||
79 | # The default engine interfaces for FCEngineServiceFactory.interfaces |
|
|||
80 | default_engine_interfaces = Config() |
|
|||
81 | default_engine_interfaces.Default.url_file = u'ipcontroller-engine.url' |
|
|||
82 |
|
||||
83 | # Make this a dict we can pass to Config.__init__ for the default |
|
|||
84 | default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) |
|
|||
85 |
|
||||
86 |
|
||||
87 | #----------------------------------------------------------------------------- |
|
|||
88 | # Service factories |
|
|||
89 | #----------------------------------------------------------------------------- |
|
|||
90 |
|
||||
91 | # |
|
|||
92 | # class FCClientServiceFactory(FCServiceFactory): |
|
|||
93 | # """A Foolscap implementation of the client services.""" |
|
|||
94 | # |
|
|||
95 | # cert_file = Unicode(u'ipcontroller-client.pem', config=True) |
|
|||
96 | # interfaces = Instance(klass=Config, kw=default_client_interfaces, |
|
|||
97 | # allow_none=False, config=True) |
|
|||
98 | # |
|
|||
99 | # |
|
|||
100 | # class FCEngineServiceFactory(FCServiceFactory): |
|
|||
101 | # """A Foolscap implementation of the engine services.""" |
|
|||
102 | # |
|
|||
103 | # cert_file = Unicode(u'ipcontroller-engine.pem', config=True) |
|
|||
104 | # interfaces = Instance(klass=dict, kw=default_engine_interfaces, |
|
|||
105 | # allow_none=False, config=True) |
|
|||
106 | # |
|
|||
107 |
|
||||
108 | #----------------------------------------------------------------------------- |
|
|||
109 | # Command line options |
|
|||
110 | #----------------------------------------------------------------------------- |
|
|||
111 |
|
||||
112 |
|
||||
113 | class IPControllerAppConfigLoader(ClusterDirConfigLoader): |
|
|||
114 |
|
||||
115 | def _add_arguments(self): |
|
|||
116 | super(IPControllerAppConfigLoader, self)._add_arguments() |
|
|||
117 | paa = self.parser.add_argument |
|
|||
118 |
|
||||
119 | ## Hub Config: |
|
|||
120 | paa('--mongodb', |
|
|||
121 | dest='HubFactory.db_class', action='store_const', |
|
|||
122 | const='IPython.parallel.controller.mongodb.MongoDB', |
|
|||
123 | help='Use MongoDB for task storage [default: in-memory]') |
|
|||
124 | paa('--sqlite', |
|
|||
125 | dest='HubFactory.db_class', action='store_const', |
|
|||
126 | const='IPython.parallel.controller.sqlitedb.SQLiteDB', |
|
|||
127 | help='Use SQLite3 for DB task storage [default: in-memory]') |
|
|||
128 | paa('--hb', |
|
|||
129 | type=int, dest='HubFactory.hb', nargs=2, |
|
|||
130 | help='The (2) ports the Hub\'s Heartmonitor will use for the heartbeat ' |
|
|||
131 | 'connections [default: random]', |
|
|||
132 | metavar='Hub.hb_ports') |
|
|||
133 | paa('--ping', |
|
|||
134 | type=int, dest='HubFactory.ping', |
|
|||
135 | help='The frequency at which the Hub pings the engines for heartbeats ' |
|
|||
136 | ' (in ms) [default: 100]', |
|
|||
137 | metavar='Hub.ping') |
|
|||
138 |
|
||||
139 | # Client config |
|
|||
140 | paa('--client-ip', |
|
|||
141 | type=str, dest='HubFactory.client_ip', |
|
|||
142 | help='The IP address or hostname the Hub will listen on for ' |
|
|||
143 | 'client connections. Both engine-ip and client-ip can be set simultaneously ' |
|
|||
144 | 'via --ip [default: loopback]', |
|
|||
145 | metavar='Hub.client_ip') |
|
|||
146 | paa('--client-transport', |
|
|||
147 | type=str, dest='HubFactory.client_transport', |
|
|||
148 | help='The ZeroMQ transport the Hub will use for ' |
|
|||
149 | 'client connections. Both engine-transport and client-transport can be set simultaneously ' |
|
|||
150 | 'via --transport [default: tcp]', |
|
|||
151 | metavar='Hub.client_transport') |
|
|||
152 | paa('--query', |
|
|||
153 | type=int, dest='HubFactory.query_port', |
|
|||
154 | help='The port on which the Hub XREP socket will listen for result queries from clients [default: random]', |
|
|||
155 | metavar='Hub.query_port') |
|
|||
156 | paa('--notifier', |
|
|||
157 | type=int, dest='HubFactory.notifier_port', |
|
|||
158 | help='The port on which the Hub PUB socket will listen for notification connections [default: random]', |
|
|||
159 | metavar='Hub.notifier_port') |
|
|||
160 |
|
||||
161 | # Engine config |
|
|||
162 | paa('--engine-ip', |
|
|||
163 | type=str, dest='HubFactory.engine_ip', |
|
|||
164 | help='The IP address or hostname the Hub will listen on for ' |
|
|||
165 | 'engine connections. This applies to the Hub and its schedulers' |
|
|||
166 | 'engine-ip and client-ip can be set simultaneously ' |
|
|||
167 | 'via --ip [default: loopback]', |
|
|||
168 | metavar='Hub.engine_ip') |
|
|||
169 | paa('--engine-transport', |
|
|||
170 | type=str, dest='HubFactory.engine_transport', |
|
|||
171 | help='The ZeroMQ transport the Hub will use for ' |
|
|||
172 | 'client connections. Both engine-transport and client-transport can be set simultaneously ' |
|
|||
173 | 'via --transport [default: tcp]', |
|
|||
174 | metavar='Hub.engine_transport') |
|
|||
175 |
|
||||
176 | # Scheduler config |
|
|||
177 | paa('--mux', |
|
|||
178 | type=int, dest='ControllerFactory.mux', nargs=2, |
|
|||
179 | help='The (2) ports the MUX scheduler will listen on for client,engine ' |
|
|||
180 | 'connections, respectively [default: random]', |
|
|||
181 | metavar='Scheduler.mux_ports') |
|
|||
182 | paa('--task', |
|
|||
183 | type=int, dest='ControllerFactory.task', nargs=2, |
|
|||
184 | help='The (2) ports the Task scheduler will listen on for client,engine ' |
|
|||
185 | 'connections, respectively [default: random]', |
|
|||
186 | metavar='Scheduler.task_ports') |
|
|||
187 | paa('--control', |
|
|||
188 | type=int, dest='ControllerFactory.control', nargs=2, |
|
|||
189 | help='The (2) ports the Control scheduler will listen on for client,engine ' |
|
|||
190 | 'connections, respectively [default: random]', |
|
|||
191 | metavar='Scheduler.control_ports') |
|
|||
192 | paa('--iopub', |
|
|||
193 | type=int, dest='ControllerFactory.iopub', nargs=2, |
|
|||
194 | help='The (2) ports the IOPub scheduler will listen on for client,engine ' |
|
|||
195 | 'connections, respectively [default: random]', |
|
|||
196 | metavar='Scheduler.iopub_ports') |
|
|||
197 |
|
||||
198 | paa('--scheme', |
|
|||
199 | type=str, dest='HubFactory.scheme', |
|
|||
200 | choices = ['pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'], |
|
|||
201 | help='select the task scheduler scheme [default: Python LRU]', |
|
|||
202 | metavar='Scheduler.scheme') |
|
|||
203 | paa('--usethreads', |
|
|||
204 | dest='ControllerFactory.usethreads', action="store_true", |
|
|||
205 | help='Use threads instead of processes for the schedulers', |
|
|||
206 | ) |
|
|||
207 | paa('--hwm', |
|
|||
208 | dest='TaskScheduler.hwm', type=int, |
|
|||
209 | help='specify the High Water Mark (HWM) ' |
|
|||
210 | 'in the Python scheduler. This is the maximum number ' |
|
|||
211 | 'of allowed outstanding tasks on each engine.', |
|
|||
212 | ) |
|
|||
213 |
|
||||
214 | ## Global config |
|
|||
215 | paa('--log-to-file', |
|
|||
216 | action='store_true', dest='Global.log_to_file', |
|
|||
217 | help='Log to a file in the log directory (default is stdout)') |
|
|||
218 | paa('--log-url', |
|
|||
219 | type=str, dest='Global.log_url', |
|
|||
220 | help='Broadcast logs to an iploggerz process [default: disabled]') |
|
|||
221 | paa('-r','--reuse-files', |
|
|||
222 | action='store_true', dest='Global.reuse_files', |
|
|||
223 | help='Try to reuse existing json connection files.') |
|
|||
224 | paa('--no-secure', |
|
|||
225 | action='store_false', dest='Global.secure', |
|
|||
226 | help='Turn off execution keys (default).') |
|
|||
227 | paa('--secure', |
|
|||
228 | action='store_true', dest='Global.secure', |
|
|||
229 | help='Turn on execution keys.') |
|
|||
230 | paa('--execkey', |
|
|||
231 | type=str, dest='Global.exec_key', |
|
|||
232 | help='path to a file containing an execution key.', |
|
|||
233 | metavar='keyfile') |
|
|||
234 | paa('--ssh', |
|
|||
235 | type=str, dest='Global.sshserver', |
|
|||
236 | help='ssh url for clients to use when connecting to the Controller ' |
|
|||
237 | 'processes. It should be of the form: [user@]server[:port]. The ' |
|
|||
238 | 'Controller\'s listening addresses must be accessible from the ssh server', |
|
|||
239 | metavar='Global.sshserver') |
|
|||
240 | paa('--location', |
|
|||
241 | type=str, dest='Global.location', |
|
|||
242 | help="The external IP or domain name of this machine, used for disambiguating " |
|
|||
243 | "engine and client connections.", |
|
|||
244 | metavar='Global.location') |
|
|||
245 | factory.add_session_arguments(self.parser) |
|
|||
246 | factory.add_registration_arguments(self.parser) |
|
|||
247 |
|
||||
248 |
|
||||
249 | #----------------------------------------------------------------------------- |
|
90 | #----------------------------------------------------------------------------- | |
250 | # The main application |
|
91 | # The main application | |
251 | #----------------------------------------------------------------------------- |
|
92 | #----------------------------------------------------------------------------- | |
252 |
|
93 | flags = {} | ||
253 |
|
94 | flags.update(base_flags) | ||
254 | class IPControllerApp(ApplicationWithClusterDir): |
|
95 | flags.update({ | |
|
96 | 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}}, | |||
|
97 | 'Use threads instead of processes for the schedulers'), | |||
|
98 | 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}}, | |||
|
99 | 'use the SQLiteDB backend'), | |||
|
100 | 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}}, | |||
|
101 | 'use the MongoDB backend'), | |||
|
102 | 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}}, | |||
|
103 | 'use the in-memory DictDB backend'), | |||
|
104 | 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}}, | |||
|
105 | 'reuse existing json connection files') | |||
|
106 | }) | |||
|
107 | ||||
|
108 | flags.update(boolean_flag('secure', 'IPControllerApp.secure', | |||
|
109 | "Use HMAC digests for authentication of messages.", | |||
|
110 | "Don't authenticate messages." | |||
|
111 | )) | |||
|
112 | ||||
|
113 | class IPControllerApp(BaseParallelApplication): | |||
255 |
|
114 | |||
256 | name = u'ipcontroller' |
|
115 | name = u'ipcontroller' | |
257 | description = _description |
|
116 | description = _description | |
258 | command_line_loader = IPControllerAppConfigLoader |
|
117 | config_file_name = Unicode(default_config_file_name) | |
259 | default_config_file_name = default_config_file_name |
|
118 | classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo | |
260 | auto_create_cluster_dir = True |
|
|||
261 |
|
119 | |||
262 |
|
120 | # change default to True | ||
263 | def create_default_config(self): |
|
121 | auto_create = Bool(True, config=True, | |
264 | super(IPControllerApp, self).create_default_config() |
|
122 | help="""Whether to create profile dir if it doesn't exist.""") | |
265 | # Don't set defaults for Global.secure or Global.reuse_furls |
|
123 | ||
266 | # as those are set in a component. |
|
124 | reuse_files = Bool(False, config=True, | |
267 | self.default_config.Global.import_statements = [] |
|
125 | help='Whether to reuse existing json connection files.' | |
268 | self.default_config.Global.clean_logs = True |
|
126 | ) | |
269 | self.default_config.Global.secure = True |
|
127 | secure = Bool(True, config=True, | |
270 | self.default_config.Global.reuse_files = False |
|
128 | help='Whether to use HMAC digests for extra message authentication.' | |
271 | self.default_config.Global.exec_key = "exec_key.key" |
|
129 | ) | |
272 | self.default_config.Global.sshserver = None |
|
130 | ssh_server = Unicode(u'', config=True, | |
273 | self.default_config.Global.location = None |
|
131 | help="""ssh url for clients to use when connecting to the Controller | |
274 |
|
132 | processes. It should be of the form: [user@]server[:port]. The | ||
275 | def pre_construct(self): |
|
133 | Controller's listening addresses must be accessible from the ssh server""", | |
276 | super(IPControllerApp, self).pre_construct() |
|
134 | ) | |
277 | c = self.master_config |
|
135 | location = Unicode(u'', config=True, | |
278 | # The defaults for these are set in FCClientServiceFactory and |
|
136 | help="""The external IP or domain name of the Controller, used for disambiguating | |
279 | # FCEngineServiceFactory, so we only set them here if the global |
|
137 | engine and client connections.""", | |
280 | # options have be set to override the class level defaults. |
|
138 | ) | |
|
139 | import_statements = List([], config=True, | |||
|
140 | help="import statements to be run at startup. Necessary in some environments" | |||
|
141 | ) | |||
|
142 | ||||
|
143 | use_threads = Bool(False, config=True, | |||
|
144 | help='Use threads instead of processes for the schedulers', | |||
|
145 | ) | |||
|
146 | ||||
|
147 | # internal | |||
|
148 | children = List() | |||
|
149 | mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') | |||
|
150 | ||||
|
151 | def _use_threads_changed(self, name, old, new): | |||
|
152 | self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') | |||
|
153 | ||||
|
154 | aliases = Dict(dict( | |||
|
155 | log_level = 'IPControllerApp.log_level', | |||
|
156 | log_url = 'IPControllerApp.log_url', | |||
|
157 | reuse_files = 'IPControllerApp.reuse_files', | |||
|
158 | secure = 'IPControllerApp.secure', | |||
|
159 | ssh = 'IPControllerApp.ssh_server', | |||
|
160 | use_threads = 'IPControllerApp.use_threads', | |||
|
161 | import_statements = 'IPControllerApp.import_statements', | |||
|
162 | location = 'IPControllerApp.location', | |||
|
163 | ||||
|
164 | ident = 'Session.session', | |||
|
165 | user = 'Session.username', | |||
|
166 | exec_key = 'Session.keyfile', | |||
|
167 | ||||
|
168 | url = 'HubFactory.url', | |||
|
169 | ip = 'HubFactory.ip', | |||
|
170 | transport = 'HubFactory.transport', | |||
|
171 | port = 'HubFactory.regport', | |||
|
172 | ||||
|
173 | ping = 'HeartMonitor.period', | |||
|
174 | ||||
|
175 | scheme = 'TaskScheduler.scheme_name', | |||
|
176 | hwm = 'TaskScheduler.hwm', | |||
|
177 | ||||
|
178 | ||||
|
179 | profile = "BaseIPythonApplication.profile", | |||
|
180 | profile_dir = 'ProfileDir.location', | |||
281 |
|
181 | |||
282 | # if hasattr(c.Global, 'reuse_furls'): |
|
182 | )) | |
283 | # c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls |
|
183 | flags = Dict(flags) | |
284 | # c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls |
|
|||
285 | # del c.Global.reuse_furls |
|
|||
286 | # if hasattr(c.Global, 'secure'): |
|
|||
287 | # c.FCClientServiceFactory.secure = c.Global.secure |
|
|||
288 | # c.FCEngineServiceFactory.secure = c.Global.secure |
|
|||
289 | # del c.Global.secure |
|
|||
290 |
|
184 | |||
|
185 | ||||
291 | def save_connection_dict(self, fname, cdict): |
|
186 | def save_connection_dict(self, fname, cdict): | |
292 | """save a connection dict to json file.""" |
|
187 | """save a connection dict to json file.""" | |
293 |
c = self. |
|
188 | c = self.config | |
294 | url = cdict['url'] |
|
189 | url = cdict['url'] | |
295 | location = cdict['location'] |
|
190 | location = cdict['location'] | |
296 | if not location: |
|
191 | if not location: | |
@@ -301,43 +196,41 b' class IPControllerApp(ApplicationWithClusterDir):' | |||||
301 | else: |
|
196 | else: | |
302 | location = socket.gethostbyname_ex(socket.gethostname())[2][-1] |
|
197 | location = socket.gethostbyname_ex(socket.gethostname())[2][-1] | |
303 | cdict['location'] = location |
|
198 | cdict['location'] = location | |
304 |
fname = os.path.join( |
|
199 | fname = os.path.join(self.profile_dir.security_dir, fname) | |
305 | with open(fname, 'w') as f: |
|
200 | with open(fname, 'w') as f: | |
306 | f.write(json.dumps(cdict, indent=2)) |
|
201 | f.write(json.dumps(cdict, indent=2)) | |
307 | os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) |
|
202 | os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) | |
308 |
|
203 | |||
309 | def load_config_from_json(self): |
|
204 | def load_config_from_json(self): | |
310 | """load config from existing json connector files.""" |
|
205 | """load config from existing json connector files.""" | |
311 |
c = self. |
|
206 | c = self.config | |
312 | # load from engine config |
|
207 | # load from engine config | |
313 |
with open(os.path.join( |
|
208 | with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f: | |
314 | cfg = json.loads(f.read()) |
|
209 | cfg = json.loads(f.read()) | |
315 |
key = c.Session |
|
210 | key = c.Session.key = cfg['exec_key'] | |
316 | xport,addr = cfg['url'].split('://') |
|
211 | xport,addr = cfg['url'].split('://') | |
317 | c.HubFactory.engine_transport = xport |
|
212 | c.HubFactory.engine_transport = xport | |
318 | ip,ports = addr.split(':') |
|
213 | ip,ports = addr.split(':') | |
319 | c.HubFactory.engine_ip = ip |
|
214 | c.HubFactory.engine_ip = ip | |
320 | c.HubFactory.regport = int(ports) |
|
215 | c.HubFactory.regport = int(ports) | |
321 |
|
|
216 | self.location = cfg['location'] | |
322 |
|
217 | |||
323 | # load client config |
|
218 | # load client config | |
324 |
with open(os.path.join( |
|
219 | with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f: | |
325 | cfg = json.loads(f.read()) |
|
220 | cfg = json.loads(f.read()) | |
326 | assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys" |
|
221 | assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys" | |
327 | xport,addr = cfg['url'].split('://') |
|
222 | xport,addr = cfg['url'].split('://') | |
328 | c.HubFactory.client_transport = xport |
|
223 | c.HubFactory.client_transport = xport | |
329 | ip,ports = addr.split(':') |
|
224 | ip,ports = addr.split(':') | |
330 | c.HubFactory.client_ip = ip |
|
225 | c.HubFactory.client_ip = ip | |
331 |
|
|
226 | self.ssh_server = cfg['ssh'] | |
332 | assert int(ports) == c.HubFactory.regport, "regport mismatch" |
|
227 | assert int(ports) == c.HubFactory.regport, "regport mismatch" | |
333 |
|
228 | |||
334 |
def |
|
229 | def init_hub(self): | |
335 | # This is the working dir by now. |
|
230 | c = self.config | |
336 | sys.path.insert(0, '') |
|
|||
337 | c = self.master_config |
|
|||
338 |
|
231 | |||
339 | self.import_statements() |
|
232 | self.do_import_statements() | |
340 |
reusing = |
|
233 | reusing = self.reuse_files | |
341 | if reusing: |
|
234 | if reusing: | |
342 | try: |
|
235 | try: | |
343 | self.load_config_from_json() |
|
236 | self.load_config_from_json() | |
@@ -346,21 +239,20 b' class IPControllerApp(ApplicationWithClusterDir):' | |||||
346 | # check again, because reusing may have failed: |
|
239 | # check again, because reusing may have failed: | |
347 | if reusing: |
|
240 | if reusing: | |
348 | pass |
|
241 | pass | |
349 |
elif |
|
242 | elif self.secure: | |
350 | keyfile = os.path.join(c.Global.security_dir, c.Global.exec_key) |
|
|||
351 | key = str(uuid.uuid4()) |
|
243 | key = str(uuid.uuid4()) | |
352 | with open(keyfile, 'w') as f: |
|
244 | # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key) | |
353 |
|
|
245 | # with open(keyfile, 'w') as f: | |
354 | os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR) |
|
246 | # f.write(key) | |
355 | c.SessionFactory.exec_key = key |
|
247 | # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR) | |
|
248 | c.Session.key = key | |||
356 | else: |
|
249 | else: | |
357 |
c.Session |
|
250 | key = c.Session.key = '' | |
358 | key = None |
|
|||
359 |
|
251 | |||
360 | try: |
|
252 | try: | |
361 |
self.factory = |
|
253 | self.factory = HubFactory(config=c, log=self.log) | |
362 | self.start_logging() |
|
254 | # self.start_logging() | |
363 |
self.factory. |
|
255 | self.factory.init_hub() | |
364 | except: |
|
256 | except: | |
365 | self.log.error("Couldn't construct the Controller", exc_info=True) |
|
257 | self.log.error("Couldn't construct the Controller", exc_info=True) | |
366 | self.exit(1) |
|
258 | self.exit(1) | |
@@ -369,21 +261,82 b' class IPControllerApp(ApplicationWithClusterDir):' | |||||
369 | # save to new json config files |
|
261 | # save to new json config files | |
370 | f = self.factory |
|
262 | f = self.factory | |
371 | cdict = {'exec_key' : key, |
|
263 | cdict = {'exec_key' : key, | |
372 |
'ssh' : |
|
264 | 'ssh' : self.ssh_server, | |
373 | 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport), |
|
265 | 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport), | |
374 |
'location' : |
|
266 | 'location' : self.location | |
375 | } |
|
267 | } | |
376 | self.save_connection_dict('ipcontroller-client.json', cdict) |
|
268 | self.save_connection_dict('ipcontroller-client.json', cdict) | |
377 | edict = cdict |
|
269 | edict = cdict | |
378 | edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport)) |
|
270 | edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport)) | |
379 | self.save_connection_dict('ipcontroller-engine.json', edict) |
|
271 | self.save_connection_dict('ipcontroller-engine.json', edict) | |
|
272 | ||||
|
273 | # | |||
|
274 | def init_schedulers(self): | |||
|
275 | children = self.children | |||
|
276 | mq = import_item(str(self.mq_class)) | |||
380 |
|
277 | |||
|
278 | hub = self.factory | |||
|
279 | # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url | |||
|
280 | # IOPub relay (in a Process) | |||
|
281 | q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub') | |||
|
282 | q.bind_in(hub.client_info['iopub']) | |||
|
283 | q.bind_out(hub.engine_info['iopub']) | |||
|
284 | q.setsockopt_out(zmq.SUBSCRIBE, '') | |||
|
285 | q.connect_mon(hub.monitor_url) | |||
|
286 | q.daemon=True | |||
|
287 | children.append(q) | |||
|
288 | ||||
|
289 | # Multiplexer Queue (in a Process) | |||
|
290 | q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out') | |||
|
291 | q.bind_in(hub.client_info['mux']) | |||
|
292 | q.setsockopt_in(zmq.IDENTITY, 'mux') | |||
|
293 | q.bind_out(hub.engine_info['mux']) | |||
|
294 | q.connect_mon(hub.monitor_url) | |||
|
295 | q.daemon=True | |||
|
296 | children.append(q) | |||
|
297 | ||||
|
298 | # Control Queue (in a Process) | |||
|
299 | q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol') | |||
|
300 | q.bind_in(hub.client_info['control']) | |||
|
301 | q.setsockopt_in(zmq.IDENTITY, 'control') | |||
|
302 | q.bind_out(hub.engine_info['control']) | |||
|
303 | q.connect_mon(hub.monitor_url) | |||
|
304 | q.daemon=True | |||
|
305 | children.append(q) | |||
|
306 | try: | |||
|
307 | scheme = self.config.TaskScheduler.scheme_name | |||
|
308 | except AttributeError: | |||
|
309 | scheme = TaskScheduler.scheme_name.get_default_value() | |||
|
310 | # Task Queue (in a Process) | |||
|
311 | if scheme == 'pure': | |||
|
312 | self.log.warn("task::using pure XREQ Task scheduler") | |||
|
313 | q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask') | |||
|
314 | # q.setsockopt_out(zmq.HWM, hub.hwm) | |||
|
315 | q.bind_in(hub.client_info['task'][1]) | |||
|
316 | q.setsockopt_in(zmq.IDENTITY, 'task') | |||
|
317 | q.bind_out(hub.engine_info['task']) | |||
|
318 | q.connect_mon(hub.monitor_url) | |||
|
319 | q.daemon=True | |||
|
320 | children.append(q) | |||
|
321 | elif scheme == 'none': | |||
|
322 | self.log.warn("task::using no Task scheduler") | |||
|
323 | ||||
|
324 | else: | |||
|
325 | self.log.info("task::using Python %s Task scheduler"%scheme) | |||
|
326 | sargs = (hub.client_info['task'][1], hub.engine_info['task'], | |||
|
327 | hub.monitor_url, hub.client_info['notification']) | |||
|
328 | kwargs = dict(logname='scheduler', loglevel=self.log_level, | |||
|
329 | log_url = self.log_url, config=dict(self.config)) | |||
|
330 | q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) | |||
|
331 | q.daemon=True | |||
|
332 | children.append(q) | |||
|
333 | ||||
381 |
|
334 | |||
382 | def save_urls(self): |
|
335 | def save_urls(self): | |
383 | """save the registration urls to files.""" |
|
336 | """save the registration urls to files.""" | |
384 |
c = self. |
|
337 | c = self.config | |
385 |
|
338 | |||
386 |
sec_dir = |
|
339 | sec_dir = self.profile_dir.security_dir | |
387 | cf = self.factory |
|
340 | cf = self.factory | |
388 |
|
341 | |||
389 | with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f: |
|
342 | with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f: | |
@@ -393,8 +346,8 b' class IPControllerApp(ApplicationWithClusterDir):' | |||||
393 | f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport)) |
|
346 | f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport)) | |
394 |
|
347 | |||
395 |
|
348 | |||
396 | def import_statements(self): |
|
349 | def do_import_statements(self): | |
397 |
statements = self. |
|
350 | statements = self.import_statements | |
398 | for s in statements: |
|
351 | for s in statements: | |
399 | try: |
|
352 | try: | |
400 | self.log.msg("Executing statement: '%s'" % s) |
|
353 | self.log.msg("Executing statement: '%s'" % s) | |
@@ -402,30 +355,52 b' class IPControllerApp(ApplicationWithClusterDir):' | |||||
402 | except: |
|
355 | except: | |
403 | self.log.msg("Error running statement: %s" % s) |
|
356 | self.log.msg("Error running statement: %s" % s) | |
404 |
|
357 | |||
405 |
def |
|
358 | def forward_logging(self): | |
406 | super(IPControllerApp, self).start_logging() |
|
359 | if self.log_url: | |
407 | if self.master_config.Global.log_url: |
|
360 | self.log.info("Forwarding logging to %s"%self.log_url) | |
408 |
context = |
|
361 | context = zmq.Context.instance() | |
409 | lsock = context.socket(zmq.PUB) |
|
362 | lsock = context.socket(zmq.PUB) | |
410 |
lsock.connect(self. |
|
363 | lsock.connect(self.log_url) | |
411 | handler = PUBHandler(lsock) |
|
364 | handler = PUBHandler(lsock) | |
|
365 | self.log.removeHandler(self._log_handler) | |||
412 | handler.root_topic = 'controller' |
|
366 | handler.root_topic = 'controller' | |
413 | handler.setLevel(self.log_level) |
|
367 | handler.setLevel(self.log_level) | |
414 | self.log.addHandler(handler) |
|
368 | self.log.addHandler(handler) | |
415 | # |
|
369 | self._log_handler = handler | |
416 | def start_app(self): |
|
370 | # # | |
|
371 | ||||
|
372 | def initialize(self, argv=None): | |||
|
373 | super(IPControllerApp, self).initialize(argv) | |||
|
374 | self.forward_logging() | |||
|
375 | self.init_hub() | |||
|
376 | self.init_schedulers() | |||
|
377 | ||||
|
378 | def start(self): | |||
417 | # Start the subprocesses: |
|
379 | # Start the subprocesses: | |
418 | self.factory.start() |
|
380 | self.factory.start() | |
|
381 | child_procs = [] | |||
|
382 | for child in self.children: | |||
|
383 | child.start() | |||
|
384 | if isinstance(child, ProcessMonitoredQueue): | |||
|
385 | child_procs.append(child.launcher) | |||
|
386 | elif isinstance(child, Process): | |||
|
387 | child_procs.append(child) | |||
|
388 | if child_procs: | |||
|
389 | signal_children(child_procs) | |||
|
390 | ||||
419 | self.write_pid_file(overwrite=True) |
|
391 | self.write_pid_file(overwrite=True) | |
|
392 | ||||
420 | try: |
|
393 | try: | |
421 | self.factory.loop.start() |
|
394 | self.factory.loop.start() | |
422 | except KeyboardInterrupt: |
|
395 | except KeyboardInterrupt: | |
423 | self.log.critical("Interrupted, Exiting...\n") |
|
396 | self.log.critical("Interrupted, Exiting...\n") | |
|
397 | ||||
424 |
|
398 | |||
425 |
|
399 | |||
426 | def launch_new_instance(): |
|
400 | def launch_new_instance(): | |
427 | """Create and run the IPython controller""" |
|
401 | """Create and run the IPython controller""" | |
428 | app = IPControllerApp() |
|
402 | app = IPControllerApp.instance() | |
|
403 | app.initialize() | |||
429 | app.start() |
|
404 | app.start() | |
430 |
|
405 | |||
431 |
|
406 |
@@ -2,10 +2,16 b'' | |||||
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython engine application |
|
4 | The IPython engine application | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Brian Granger | |||
|
9 | * MinRK | |||
|
10 | ||||
5 | """ |
|
11 | """ | |
6 |
|
12 | |||
7 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
8 |
# Copyright (C) 2008-20 |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
9 | # |
|
15 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
@@ -22,18 +28,18 b' import sys' | |||||
22 | import zmq |
|
28 | import zmq | |
23 | from zmq.eventloop import ioloop |
|
29 | from zmq.eventloop import ioloop | |
24 |
|
30 | |||
25 |
from IPython. |
|
31 | from IPython.core.profiledir import ProfileDir | |
26 | ApplicationWithClusterDir, |
|
32 | from IPython.parallel.apps.baseapp import BaseParallelApplication | |
27 | ClusterDirConfigLoader |
|
|||
28 | ) |
|
|||
29 | from IPython.zmq.log import EnginePUBHandler |
|
33 | from IPython.zmq.log import EnginePUBHandler | |
30 |
|
34 | |||
31 |
from IPython. |
|
35 | from IPython.config.configurable import Configurable | |
|
36 | from IPython.zmq.session import Session | |||
32 | from IPython.parallel.engine.engine import EngineFactory |
|
37 | from IPython.parallel.engine.engine import EngineFactory | |
33 | from IPython.parallel.engine.streamkernel import Kernel |
|
38 | from IPython.parallel.engine.streamkernel import Kernel | |
34 | from IPython.parallel.util import disambiguate_url |
|
39 | from IPython.parallel.util import disambiguate_url | |
35 |
|
40 | |||
36 | from IPython.utils.importstring import import_item |
|
41 | from IPython.utils.importstring import import_item | |
|
42 | from IPython.utils.traitlets import Bool, Unicode, Dict, List | |||
37 |
|
43 | |||
38 |
|
44 | |||
39 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
@@ -43,6 +49,20 b' from IPython.utils.importstring import import_item' | |||||
43 | #: The default config file name for this application |
|
49 | #: The default config file name for this application | |
44 | default_config_file_name = u'ipengine_config.py' |
|
50 | default_config_file_name = u'ipengine_config.py' | |
45 |
|
51 | |||
|
52 | _description = """Start an IPython engine for parallel computing. | |||
|
53 | ||||
|
54 | IPython engines run in parallel and perform computations on behalf of a client | |||
|
55 | and controller. A controller needs to be started before the engines. The | |||
|
56 | engine can be configured using command line options or using a cluster | |||
|
57 | directory. Cluster directories contain config, log and security files and are | |||
|
58 | usually located in your ipython directory and named as "profile_name". | |||
|
59 | See the `profile` and `profile_dir` options for details. | |||
|
60 | """ | |||
|
61 | ||||
|
62 | ||||
|
63 | #----------------------------------------------------------------------------- | |||
|
64 | # MPI configuration | |||
|
65 | #----------------------------------------------------------------------------- | |||
46 |
|
66 | |||
47 | mpi4py_init = """from mpi4py import MPI as mpi |
|
67 | mpi4py_init = """from mpi4py import MPI as mpi | |
48 | mpi.size = mpi.COMM_WORLD.Get_size() |
|
68 | mpi.size = mpi.COMM_WORLD.Get_size() | |
@@ -58,123 +78,78 b' mpi.rank = 0' | |||||
58 | mpi.size = 0 |
|
78 | mpi.size = 0 | |
59 | """ |
|
79 | """ | |
60 |
|
80 | |||
|
81 | class MPI(Configurable): | |||
|
82 | """Configurable for MPI initialization""" | |||
|
83 | use = Unicode('', config=True, | |||
|
84 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).' | |||
|
85 | ) | |||
61 |
|
86 | |||
62 | _description = """Start an IPython engine for parallel computing.\n\n |
|
87 | def _on_use_changed(self, old, new): | |
|
88 | # load default init script if it's not set | |||
|
89 | if not self.init_script: | |||
|
90 | self.init_script = self.default_inits.get(new, '') | |||
|
91 | ||||
|
92 | init_script = Unicode('', config=True, | |||
|
93 | help="Initialization code for MPI") | |||
|
94 | ||||
|
95 | default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init}, | |||
|
96 | config=True) | |||
63 |
|
97 | |||
64 | IPython engines run in parallel and perform computations on behalf of a client |
|
|||
65 | and controller. A controller needs to be started before the engines. The |
|
|||
66 | engine can be configured using command line options or using a cluster |
|
|||
67 | directory. Cluster directories contain config, log and security files and are |
|
|||
68 | usually located in your ipython directory and named as "cluster_<profile>". |
|
|||
69 | See the --profile and --cluster-dir options for details. |
|
|||
70 | """ |
|
|||
71 |
|
98 | |||
72 | #----------------------------------------------------------------------------- |
|
99 | #----------------------------------------------------------------------------- | |
73 | # Command line options |
|
100 | # Main application | |
74 | #----------------------------------------------------------------------------- |
|
101 | #----------------------------------------------------------------------------- | |
75 |
|
102 | |||
76 |
|
103 | |||
77 | class IPEngineAppConfigLoader(ClusterDirConfigLoader): |
|
104 | class IPEngineApp(BaseParallelApplication): | |
78 |
|
105 | |||
79 | def _add_arguments(self): |
|
106 | name = Unicode(u'ipengine') | |
80 | super(IPEngineAppConfigLoader, self)._add_arguments() |
|
107 | description = Unicode(_description) | |
81 | paa = self.parser.add_argument |
|
108 | config_file_name = Unicode(default_config_file_name) | |
82 | # Controller config |
|
109 | classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI]) | |
83 | paa('--file', '-f', |
|
110 | ||
84 | type=unicode, dest='Global.url_file', |
|
111 | startup_script = Unicode(u'', config=True, | |
85 | help='The full location of the file containing the connection information fo ' |
|
112 | help='specify a script to be run at startup') | |
86 | 'controller. If this is not given, the file must be in the ' |
|
113 | startup_command = Unicode('', config=True, | |
87 | 'security directory of the cluster directory. This location is ' |
|
|||
88 | 'resolved using the --profile and --app-dir options.', |
|
|||
89 | metavar='Global.url_file') |
|
|||
90 | # MPI |
|
|||
91 | paa('--mpi', |
|
|||
92 | type=str, dest='MPI.use', |
|
|||
93 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', |
|
|||
94 | metavar='MPI.use') |
|
|||
95 | # Global config |
|
|||
96 | paa('--log-to-file', |
|
|||
97 | action='store_true', dest='Global.log_to_file', |
|
|||
98 | help='Log to a file in the log directory (default is stdout)') |
|
|||
99 | paa('--log-url', |
|
|||
100 | dest='Global.log_url', |
|
|||
101 | help="url of ZMQ logger, as started with iploggerz") |
|
|||
102 | # paa('--execkey', |
|
|||
103 | # type=str, dest='Global.exec_key', |
|
|||
104 | # help='path to a file containing an execution key.', |
|
|||
105 | # metavar='keyfile') |
|
|||
106 | # paa('--no-secure', |
|
|||
107 | # action='store_false', dest='Global.secure', |
|
|||
108 | # help='Turn off execution keys.') |
|
|||
109 | # paa('--secure', |
|
|||
110 | # action='store_true', dest='Global.secure', |
|
|||
111 | # help='Turn on execution keys (default).') |
|
|||
112 | # init command |
|
|||
113 | paa('-c', |
|
|||
114 | type=str, dest='Global.extra_exec_lines', |
|
|||
115 | help='specify a command to be run at startup') |
|
114 | help='specify a command to be run at startup') | |
116 | paa('-s', |
|
|||
117 | type=unicode, dest='Global.extra_exec_file', |
|
|||
118 | help='specify a script to be run at startup') |
|
|||
119 |
|
||||
120 | factory.add_session_arguments(self.parser) |
|
|||
121 | factory.add_registration_arguments(self.parser) |
|
|||
122 |
|
115 | |||
|
116 | url_file = Unicode(u'', config=True, | |||
|
117 | help="""The full location of the file containing the connection information for | |||
|
118 | the controller. If this is not given, the file must be in the | |||
|
119 | security directory of the cluster directory. This location is | |||
|
120 | resolved using the `profile` or `profile_dir` options.""", | |||
|
121 | ) | |||
123 |
|
122 | |||
124 | #----------------------------------------------------------------------------- |
|
123 | url_file_name = Unicode(u'ipcontroller-engine.json') | |
125 | # Main application |
|
124 | log_url = Unicode('', config=True, | |
126 | #----------------------------------------------------------------------------- |
|
125 | help="""The URL for the iploggerapp instance, for forwarding | |
|
126 | logging to a central location.""") | |||
127 |
|
127 | |||
|
128 | aliases = Dict(dict( | |||
|
129 | file = 'IPEngineApp.url_file', | |||
|
130 | c = 'IPEngineApp.startup_command', | |||
|
131 | s = 'IPEngineApp.startup_script', | |||
128 |
|
132 | |||
129 | class IPEngineApp(ApplicationWithClusterDir): |
|
133 | ident = 'Session.session', | |
130 |
|
134 | user = 'Session.username', | ||
131 | name = u'ipengine' |
|
135 | exec_key = 'Session.keyfile', | |
132 | description = _description |
|
136 | ||
133 | command_line_loader = IPEngineAppConfigLoader |
|
137 | url = 'EngineFactory.url', | |
134 | default_config_file_name = default_config_file_name |
|
138 | ip = 'EngineFactory.ip', | |
135 | auto_create_cluster_dir = True |
|
139 | transport = 'EngineFactory.transport', | |
136 |
|
140 | port = 'EngineFactory.regport', | ||
137 | def create_default_config(self): |
|
141 | location = 'EngineFactory.location', | |
138 | super(IPEngineApp, self).create_default_config() |
|
142 | ||
139 |
|
143 | timeout = 'EngineFactory.timeout', | ||
140 | # The engine should not clean logs as we don't want to remove the |
|
144 | ||
141 | # active log files of other running engines. |
|
145 | profile = "IPEngineApp.profile", | |
142 | self.default_config.Global.clean_logs = False |
|
146 | profile_dir = 'ProfileDir.location', | |
143 | self.default_config.Global.secure = True |
|
147 | ||
144 |
|
148 | mpi = 'MPI.use', | ||
145 | # Global config attributes |
|
149 | ||
146 | self.default_config.Global.exec_lines = [] |
|
150 | log_level = 'IPEngineApp.log_level', | |
147 | self.default_config.Global.extra_exec_lines = '' |
|
151 | log_url = 'IPEngineApp.log_url' | |
148 | self.default_config.Global.extra_exec_file = u'' |
|
152 | )) | |
149 |
|
||||
150 | # Configuration related to the controller |
|
|||
151 | # This must match the filename (path not included) that the controller |
|
|||
152 | # used for the FURL file. |
|
|||
153 | self.default_config.Global.url_file = u'' |
|
|||
154 | self.default_config.Global.url_file_name = u'ipcontroller-engine.json' |
|
|||
155 | # If given, this is the actual location of the controller's FURL file. |
|
|||
156 | # If not, this is computed using the profile, app_dir and furl_file_name |
|
|||
157 | # self.default_config.Global.key_file_name = u'exec_key.key' |
|
|||
158 | # self.default_config.Global.key_file = u'' |
|
|||
159 |
|
||||
160 | # MPI related config attributes |
|
|||
161 | self.default_config.MPI.use = '' |
|
|||
162 | self.default_config.MPI.mpi4py = mpi4py_init |
|
|||
163 | self.default_config.MPI.pytrilinos = pytrilinos_init |
|
|||
164 |
|
||||
165 | def post_load_command_line_config(self): |
|
|||
166 | pass |
|
|||
167 |
|
||||
168 | def pre_construct(self): |
|
|||
169 | super(IPEngineApp, self).pre_construct() |
|
|||
170 | # self.find_cont_url_file() |
|
|||
171 | self.find_url_file() |
|
|||
172 | if self.master_config.Global.extra_exec_lines: |
|
|||
173 | self.master_config.Global.exec_lines.append(self.master_config.Global.extra_exec_lines) |
|
|||
174 | if self.master_config.Global.extra_exec_file: |
|
|||
175 | enc = sys.getfilesystemencoding() or 'utf8' |
|
|||
176 | cmd="execfile(%r)"%self.master_config.Global.extra_exec_file.encode(enc) |
|
|||
177 | self.master_config.Global.exec_lines.append(cmd) |
|
|||
178 |
|
153 | |||
179 | # def find_key_file(self): |
|
154 | # def find_key_file(self): | |
180 | # """Set the key file. |
|
155 | # """Set the key file. | |
@@ -186,7 +161,7 b' class IPEngineApp(ApplicationWithClusterDir):' | |||||
186 | # # Find the actual controller key file |
|
161 | # # Find the actual controller key file | |
187 | # if not config.Global.key_file: |
|
162 | # if not config.Global.key_file: | |
188 | # try_this = os.path.join( |
|
163 | # try_this = os.path.join( | |
189 |
# config.Global. |
|
164 | # config.Global.profile_dir, | |
190 | # config.Global.security_dir, |
|
165 | # config.Global.security_dir, | |
191 | # config.Global.key_file_name |
|
166 | # config.Global.key_file_name | |
192 | # ) |
|
167 | # ) | |
@@ -198,82 +173,74 b' class IPEngineApp(ApplicationWithClusterDir):' | |||||
198 | Here we don't try to actually see if it exists for is valid as that |
|
173 | Here we don't try to actually see if it exists for is valid as that | |
199 | is hadled by the connection logic. |
|
174 | is hadled by the connection logic. | |
200 | """ |
|
175 | """ | |
201 |
config = self. |
|
176 | config = self.config | |
202 | # Find the actual controller key file |
|
177 | # Find the actual controller key file | |
203 |
if not |
|
178 | if not self.url_file: | |
204 |
|
|
179 | self.url_file = os.path.join( | |
205 |
|
|
180 | self.profile_dir.security_dir, | |
206 | config.Global.security_dir, |
|
181 | self.url_file_name | |
207 | config.Global.url_file_name |
|
|||
208 | ) |
|
182 | ) | |
209 | config.Global.url_file = try_this |
|
183 | def init_engine(self): | |
210 |
|
||||
211 | def construct(self): |
|
|||
212 | # This is the working dir by now. |
|
184 | # This is the working dir by now. | |
213 | sys.path.insert(0, '') |
|
185 | sys.path.insert(0, '') | |
214 |
config = self. |
|
186 | config = self.config | |
|
187 | # print config | |||
|
188 | self.find_url_file() | |||
|
189 | ||||
215 | # if os.path.exists(config.Global.key_file) and config.Global.secure: |
|
190 | # if os.path.exists(config.Global.key_file) and config.Global.secure: | |
216 | # config.SessionFactory.exec_key = config.Global.key_file |
|
191 | # config.SessionFactory.exec_key = config.Global.key_file | |
217 |
if os.path.exists( |
|
192 | if os.path.exists(self.url_file): | |
218 |
with open( |
|
193 | with open(self.url_file) as f: | |
219 | d = json.loads(f.read()) |
|
194 | d = json.loads(f.read()) | |
220 | for k,v in d.iteritems(): |
|
195 | for k,v in d.iteritems(): | |
221 | if isinstance(v, unicode): |
|
196 | if isinstance(v, unicode): | |
222 | d[k] = v.encode() |
|
197 | d[k] = v.encode() | |
223 | if d['exec_key']: |
|
198 | if d['exec_key']: | |
224 |
config.Session |
|
199 | config.Session.key = d['exec_key'] | |
225 | d['url'] = disambiguate_url(d['url'], d['location']) |
|
200 | d['url'] = disambiguate_url(d['url'], d['location']) | |
226 |
config. |
|
201 | config.EngineFactory.url = d['url'] | |
227 | config.EngineFactory.location = d['location'] |
|
202 | config.EngineFactory.location = d['location'] | |
228 |
|
203 | |||
|
204 | try: | |||
|
205 | exec_lines = config.Kernel.exec_lines | |||
|
206 | except AttributeError: | |||
|
207 | config.Kernel.exec_lines = [] | |||
|
208 | exec_lines = config.Kernel.exec_lines | |||
229 |
|
209 | |||
230 |
|
210 | if self.startup_script: | ||
231 | config.Kernel.exec_lines = config.Global.exec_lines |
|
211 | enc = sys.getfilesystemencoding() or 'utf8' | |
232 |
|
212 | cmd="execfile(%r)"%self.startup_script.encode(enc) | ||
233 | self.start_mpi() |
|
213 | exec_lines.append(cmd) | |
|
214 | if self.startup_command: | |||
|
215 | exec_lines.append(self.startup_command) | |||
234 |
|
216 | |||
235 |
# Create the underlying shell class and Engine |
|
217 | # Create the underlying shell class and Engine | |
236 | # shell_class = import_item(self.master_config.Global.shell_class) |
|
218 | # shell_class = import_item(self.master_config.Global.shell_class) | |
|
219 | # print self.config | |||
237 | try: |
|
220 | try: | |
238 |
self.engine = EngineFactory(config=config, log |
|
221 | self.engine = EngineFactory(config=config, log=self.log) | |
239 | except: |
|
222 | except: | |
240 | self.log.error("Couldn't start the Engine", exc_info=True) |
|
223 | self.log.error("Couldn't start the Engine", exc_info=True) | |
241 | self.exit(1) |
|
224 | self.exit(1) | |
242 |
|
225 | |||
243 |
|
|
226 | def forward_logging(self): | |
244 |
|
227 | if self.log_url: | ||
245 | # Create the service hierarchy |
|
228 | self.log.info("Forwarding logging to %s"%self.log_url) | |
246 | # self.main_service = service.MultiService() |
|
|||
247 | # self.engine_service.setServiceParent(self.main_service) |
|
|||
248 | # self.tub_service = Tub() |
|
|||
249 | # self.tub_service.setServiceParent(self.main_service) |
|
|||
250 | # # This needs to be called before the connection is initiated |
|
|||
251 | # self.main_service.startService() |
|
|||
252 |
|
||||
253 | # This initiates the connection to the controller and calls |
|
|||
254 | # register_engine to tell the controller we are ready to do work |
|
|||
255 | # self.engine_connector = EngineConnector(self.tub_service) |
|
|||
256 |
|
||||
257 | # self.log.info("Using furl file: %s" % self.master_config.Global.furl_file) |
|
|||
258 |
|
||||
259 | # reactor.callWhenRunning(self.call_connect) |
|
|||
260 |
|
||||
261 |
|
||||
262 | def start_logging(self): |
|
|||
263 | super(IPEngineApp, self).start_logging() |
|
|||
264 | if self.master_config.Global.log_url: |
|
|||
265 | context = self.engine.context |
|
229 | context = self.engine.context | |
266 | lsock = context.socket(zmq.PUB) |
|
230 | lsock = context.socket(zmq.PUB) | |
267 |
lsock.connect(self. |
|
231 | lsock.connect(self.log_url) | |
|
232 | self.log.removeHandler(self._log_handler) | |||
268 | handler = EnginePUBHandler(self.engine, lsock) |
|
233 | handler = EnginePUBHandler(self.engine, lsock) | |
269 | handler.setLevel(self.log_level) |
|
234 | handler.setLevel(self.log_level) | |
270 | self.log.addHandler(handler) |
|
235 | self.log.addHandler(handler) | |
271 |
|
236 | self._log_handler = handler | ||
272 | def start_mpi(self): |
|
237 | # | |
|
238 | def init_mpi(self): | |||
273 | global mpi |
|
239 | global mpi | |
274 | mpikey = self.master_config.MPI.use |
|
240 | self.mpi = MPI(config=self.config) | |
275 | mpi_import_statement = self.master_config.MPI.get(mpikey, None) |
|
241 | ||
276 |
|
|
242 | mpi_import_statement = self.mpi.init_script | |
|
243 | if mpi_import_statement: | |||
277 | try: |
|
244 | try: | |
278 | self.log.info("Initializing MPI:") |
|
245 | self.log.info("Initializing MPI:") | |
279 | self.log.info(mpi_import_statement) |
|
246 | self.log.info(mpi_import_statement) | |
@@ -283,8 +250,13 b' class IPEngineApp(ApplicationWithClusterDir):' | |||||
283 | else: |
|
250 | else: | |
284 | mpi = None |
|
251 | mpi = None | |
285 |
|
252 | |||
286 |
|
253 | def initialize(self, argv=None): | ||
287 | def start_app(self): |
|
254 | super(IPEngineApp, self).initialize(argv) | |
|
255 | self.init_mpi() | |||
|
256 | self.init_engine() | |||
|
257 | self.forward_logging() | |||
|
258 | ||||
|
259 | def start(self): | |||
288 | self.engine.start() |
|
260 | self.engine.start() | |
289 | try: |
|
261 | try: | |
290 | self.engine.loop.start() |
|
262 | self.engine.loop.start() | |
@@ -293,8 +265,9 b' class IPEngineApp(ApplicationWithClusterDir):' | |||||
293 |
|
265 | |||
294 |
|
266 | |||
295 | def launch_new_instance(): |
|
267 | def launch_new_instance(): | |
296 |
"""Create and run the IPython |
|
268 | """Create and run the IPython engine""" | |
297 | app = IPEngineApp() |
|
269 | app = IPEngineApp.instance() | |
|
270 | app.initialize() | |||
298 | app.start() |
|
271 | app.start() | |
299 |
|
272 | |||
300 |
|
273 |
@@ -2,6 +2,11 b'' | |||||
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | A simple IPython logger application |
|
4 | A simple IPython logger application | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * MinRK | |||
|
9 | ||||
5 | """ |
|
10 | """ | |
6 |
|
11 | |||
7 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
@@ -20,9 +25,12 b' import sys' | |||||
20 |
|
25 | |||
21 | import zmq |
|
26 | import zmq | |
22 |
|
27 | |||
23 |
from IPython. |
|
28 | from IPython.core.profiledir import ProfileDir | |
24 | ApplicationWithClusterDir, |
|
29 | from IPython.utils.traitlets import Bool, Dict, Unicode | |
25 | ClusterDirConfigLoader |
|
30 | ||
|
31 | from IPython.parallel.apps.baseapp import ( | |||
|
32 | BaseParallelApplication, | |||
|
33 | base_aliases | |||
26 | ) |
|
34 | ) | |
27 | from IPython.parallel.apps.logwatcher import LogWatcher |
|
35 | from IPython.parallel.apps.logwatcher import LogWatcher | |
28 |
|
36 | |||
@@ -33,89 +41,49 b' from IPython.parallel.apps.logwatcher import LogWatcher' | |||||
33 | #: The default config file name for this application |
|
41 | #: The default config file name for this application | |
34 | default_config_file_name = u'iplogger_config.py' |
|
42 | default_config_file_name = u'iplogger_config.py' | |
35 |
|
43 | |||
36 |
_description = """Start an IPython logger for parallel computing. |
|
44 | _description = """Start an IPython logger for parallel computing. | |
37 |
|
45 | |||
38 | IPython controllers and engines (and your own processes) can broadcast log messages |
|
46 | IPython controllers and engines (and your own processes) can broadcast log messages | |
39 | by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The |
|
47 | by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The | |
40 | logger can be configured using command line options or using a cluster |
|
48 | logger can be configured using command line options or using a cluster | |
41 | directory. Cluster directories contain config, log and security files and are |
|
49 | directory. Cluster directories contain config, log and security files and are | |
42 |
usually located in your ipython directory and named as " |
|
50 | usually located in your ipython directory and named as "profile_name". | |
43 |
See the |
|
51 | See the `profile` and `profile_dir` options for details. | |
44 | """ |
|
52 | """ | |
45 |
|
53 | |||
46 | #----------------------------------------------------------------------------- |
|
|||
47 | # Command line options |
|
|||
48 | #----------------------------------------------------------------------------- |
|
|||
49 |
|
||||
50 |
|
||||
51 | class IPLoggerAppConfigLoader(ClusterDirConfigLoader): |
|
|||
52 |
|
||||
53 | def _add_arguments(self): |
|
|||
54 | super(IPLoggerAppConfigLoader, self)._add_arguments() |
|
|||
55 | paa = self.parser.add_argument |
|
|||
56 | # Controller config |
|
|||
57 | paa('--url', |
|
|||
58 | type=str, dest='LogWatcher.url', |
|
|||
59 | help='The url the LogWatcher will listen on', |
|
|||
60 | ) |
|
|||
61 | # MPI |
|
|||
62 | paa('--topics', |
|
|||
63 | type=str, dest='LogWatcher.topics', nargs='+', |
|
|||
64 | help='What topics to subscribe to', |
|
|||
65 | metavar='topics') |
|
|||
66 | # Global config |
|
|||
67 | paa('--log-to-file', |
|
|||
68 | action='store_true', dest='Global.log_to_file', |
|
|||
69 | help='Log to a file in the log directory (default is stdout)') |
|
|||
70 |
|
||||
71 |
|
54 | |||
72 | #----------------------------------------------------------------------------- |
|
55 | #----------------------------------------------------------------------------- | |
73 | # Main application |
|
56 | # Main application | |
74 | #----------------------------------------------------------------------------- |
|
57 | #----------------------------------------------------------------------------- | |
|
58 | aliases = {} | |||
|
59 | aliases.update(base_aliases) | |||
|
60 | aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics')) | |||
75 |
|
61 | |||
|
62 | class IPLoggerApp(BaseParallelApplication): | |||
76 |
|
63 | |||
77 | class IPLoggerApp(ApplicationWithClusterDir): |
|
64 | name = u'iplogger' | |
78 |
|
||||
79 | name = u'iploggerz' |
|
|||
80 | description = _description |
|
65 | description = _description | |
81 | command_line_loader = IPLoggerAppConfigLoader |
|
66 | config_file_name = Unicode(default_config_file_name) | |
82 | default_config_file_name = default_config_file_name |
|
67 | ||
83 | auto_create_cluster_dir = True |
|
68 | classes = [LogWatcher, ProfileDir] | |
84 |
|
69 | aliases = Dict(aliases) | ||
85 | def create_default_config(self): |
|
70 | ||
86 | super(IPLoggerApp, self).create_default_config() |
|
71 | def initialize(self, argv=None): | |
87 |
|
72 | super(IPLoggerApp, self).initialize(argv) | ||
88 | # The engine should not clean logs as we don't want to remove the |
|
73 | self.init_watcher() | |
89 | # active log files of other running engines. |
|
74 | ||
90 | self.default_config.Global.clean_logs = False |
|
75 | def init_watcher(self): | |
91 |
|
||||
92 | # If given, this is the actual location of the logger's URL file. |
|
|||
93 | # If not, this is computed using the profile, app_dir and furl_file_name |
|
|||
94 | self.default_config.Global.url_file_name = u'iplogger.url' |
|
|||
95 | self.default_config.Global.url_file = u'' |
|
|||
96 |
|
||||
97 | def post_load_command_line_config(self): |
|
|||
98 | pass |
|
|||
99 |
|
||||
100 | def pre_construct(self): |
|
|||
101 | super(IPLoggerApp, self).pre_construct() |
|
|||
102 |
|
||||
103 | def construct(self): |
|
|||
104 | # This is the working dir by now. |
|
|||
105 | sys.path.insert(0, '') |
|
|||
106 |
|
||||
107 | self.start_logging() |
|
|||
108 |
|
||||
109 | try: |
|
76 | try: | |
110 |
self.watcher = LogWatcher(config=self. |
|
77 | self.watcher = LogWatcher(config=self.config, log=self.log) | |
111 | except: |
|
78 | except: | |
112 | self.log.error("Couldn't start the LogWatcher", exc_info=True) |
|
79 | self.log.error("Couldn't start the LogWatcher", exc_info=True) | |
113 | self.exit(1) |
|
80 | self.exit(1) | |
|
81 | self.log.info("Listening for log messages on %r"%self.watcher.url) | |||
114 |
|
82 | |||
115 |
|
83 | |||
116 |
def start |
|
84 | def start(self): | |
|
85 | self.watcher.start() | |||
117 | try: |
|
86 | try: | |
118 | self.watcher.start() |
|
|||
119 | self.watcher.loop.start() |
|
87 | self.watcher.loop.start() | |
120 | except KeyboardInterrupt: |
|
88 | except KeyboardInterrupt: | |
121 | self.log.critical("Logging Interrupted, shutting down...\n") |
|
89 | self.log.critical("Logging Interrupted, shutting down...\n") | |
@@ -123,7 +91,8 b' class IPLoggerApp(ApplicationWithClusterDir):' | |||||
123 |
|
91 | |||
124 | def launch_new_instance(): |
|
92 | def launch_new_instance(): | |
125 | """Create and run the IPython LogWatcher""" |
|
93 | """Create and run the IPython LogWatcher""" | |
126 | app = IPLoggerApp() |
|
94 | app = IPLoggerApp.instance() | |
|
95 | app.initialize() | |||
127 | app.start() |
|
96 | app.start() | |
128 |
|
97 | |||
129 |
|
98 |
@@ -2,10 +2,15 b'' | |||||
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | Facilities for launching IPython processes asynchronously. |
|
4 | Facilities for launching IPython processes asynchronously. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Brian Granger | |||
|
9 | * MinRK | |||
5 | """ |
|
10 | """ | |
6 |
|
11 | |||
7 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
8 |
# Copyright (C) 2008-20 |
|
13 | # Copyright (C) 2008-2011 The IPython Development Team | |
9 | # |
|
14 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
15 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
16 | # the file COPYING, distributed as part of this software. | |
@@ -49,14 +54,13 b' except ImportError:' | |||||
49 |
|
54 | |||
50 | from zmq.eventloop import ioloop |
|
55 | from zmq.eventloop import ioloop | |
51 |
|
56 | |||
52 |
from IPython. |
|
57 | from IPython.config.application import Application | |
53 |
|
|
58 | from IPython.config.configurable import LoggingConfigurable | |
54 | from IPython.utils.traitlets import Any, Str, Int, List, Unicode, Dict, Instance, CUnicode |
|
59 | from IPython.utils.text import EvalFormatter | |
|
60 | from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance | |||
55 | from IPython.utils.path import get_ipython_module_path |
|
61 | from IPython.utils.path import get_ipython_module_path | |
56 | from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError |
|
62 | from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError | |
57 |
|
63 | |||
58 | from IPython.parallel.factory import LoggingFactory |
|
|||
59 |
|
||||
60 | from .win32support import forward_read_events |
|
64 | from .win32support import forward_read_events | |
61 |
|
65 | |||
62 | from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob |
|
66 | from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob | |
@@ -97,16 +101,16 b' class UnknownStatus(LauncherError):' | |||||
97 | pass |
|
101 | pass | |
98 |
|
102 | |||
99 |
|
103 | |||
100 |
class BaseLauncher(Logging |
|
104 | class BaseLauncher(LoggingConfigurable): | |
101 | """An asbtraction for starting, stopping and signaling a process.""" |
|
105 | """An asbtraction for starting, stopping and signaling a process.""" | |
102 |
|
106 | |||
103 | # In all of the launchers, the work_dir is where child processes will be |
|
107 | # In all of the launchers, the work_dir is where child processes will be | |
104 |
# run. This will usually be the |
|
108 | # run. This will usually be the profile_dir, but may not be. any work_dir | |
105 | # passed into the __init__ method will override the config value. |
|
109 | # passed into the __init__ method will override the config value. | |
106 | # This should not be used to set the work_dir for the actual engine |
|
110 | # This should not be used to set the work_dir for the actual engine | |
107 | # and controller. Instead, use their own config files or the |
|
111 | # and controller. Instead, use their own config files or the | |
108 | # controller_args, engine_args attributes of the launchers to add |
|
112 | # controller_args, engine_args attributes of the launchers to add | |
109 |
# the |
|
113 | # the work_dir option. | |
110 | work_dir = Unicode(u'.') |
|
114 | work_dir = Unicode(u'.') | |
111 | loop = Instance('zmq.eventloop.ioloop.IOLoop') |
|
115 | loop = Instance('zmq.eventloop.ioloop.IOLoop') | |
112 |
|
116 | |||
@@ -153,29 +157,20 b' class BaseLauncher(LoggingFactory):' | |||||
153 | return False |
|
157 | return False | |
154 |
|
158 | |||
155 | def start(self): |
|
159 | def start(self): | |
156 | """Start the process. |
|
160 | """Start the process.""" | |
157 |
|
||||
158 | This must return a deferred that fires with information about the |
|
|||
159 | process starting (like a pid, job id, etc.). |
|
|||
160 | """ |
|
|||
161 | raise NotImplementedError('start must be implemented in a subclass') |
|
161 | raise NotImplementedError('start must be implemented in a subclass') | |
162 |
|
162 | |||
163 | def stop(self): |
|
163 | def stop(self): | |
164 | """Stop the process and notify observers of stopping. |
|
164 | """Stop the process and notify observers of stopping. | |
165 |
|
165 | |||
166 | This must return a deferred that fires with information about the |
|
166 | This method will return None immediately. | |
167 | processing stopping, like errors that occur while the process is |
|
167 | To observe the actual process stopping, see :meth:`on_stop`. | |
168 | attempting to be shut down. This deferred won't fire when the process |
|
|||
169 | actually stops. To observe the actual process stopping, see |
|
|||
170 | :func:`observe_stop`. |
|
|||
171 | """ |
|
168 | """ | |
172 | raise NotImplementedError('stop must be implemented in a subclass') |
|
169 | raise NotImplementedError('stop must be implemented in a subclass') | |
173 |
|
170 | |||
174 | def on_stop(self, f): |
|
171 | def on_stop(self, f): | |
175 | """Get a deferred that will fire when the process stops. |
|
172 | """Register a callback to be called with this Launcher's stop_data | |
176 |
|
173 | when the process actually finishes. | ||
177 | The deferred will fire with data that contains information about |
|
|||
178 | the exit status of the process. |
|
|||
179 | """ |
|
174 | """ | |
180 | if self.state=='after': |
|
175 | if self.state=='after': | |
181 | return f(self.stop_data) |
|
176 | return f(self.stop_data) | |
@@ -198,7 +193,7 b' class BaseLauncher(LoggingFactory):' | |||||
198 | """Call this to trigger process stop actions. |
|
193 | """Call this to trigger process stop actions. | |
199 |
|
194 | |||
200 | This logs the process stopping and sets the state to 'after'. Call |
|
195 | This logs the process stopping and sets the state to 'after'. Call | |
201 |
this to trigger |
|
196 | this to trigger callbacks registered via :meth:`on_stop`.""" | |
202 |
|
197 | |||
203 | self.log.info('Process %r stopped: %r' % (self.args[0], data)) |
|
198 | self.log.info('Process %r stopped: %r' % (self.args[0], data)) | |
204 | self.stop_data = data |
|
199 | self.stop_data = data | |
@@ -211,8 +206,6 b' class BaseLauncher(LoggingFactory):' | |||||
211 | def signal(self, sig): |
|
206 | def signal(self, sig): | |
212 | """Signal the process. |
|
207 | """Signal the process. | |
213 |
|
208 | |||
214 | Return a semi-meaningless deferred after signaling the process. |
|
|||
215 |
|
||||
216 | Parameters |
|
209 | Parameters | |
217 | ---------- |
|
210 | ---------- | |
218 | sig : str or int |
|
211 | sig : str or int | |
@@ -243,7 +236,6 b' class LocalProcessLauncher(BaseLauncher):' | |||||
243 | work_dir=work_dir, config=config, **kwargs |
|
236 | work_dir=work_dir, config=config, **kwargs | |
244 | ) |
|
237 | ) | |
245 | self.process = None |
|
238 | self.process = None | |
246 | self.start_deferred = None |
|
|||
247 | self.poller = None |
|
239 | self.poller = None | |
248 |
|
240 | |||
249 | def find_args(self): |
|
241 | def find_args(self): | |
@@ -328,17 +320,19 b' class LocalProcessLauncher(BaseLauncher):' | |||||
328 | class LocalControllerLauncher(LocalProcessLauncher): |
|
320 | class LocalControllerLauncher(LocalProcessLauncher): | |
329 | """Launch a controller as a regular external process.""" |
|
321 | """Launch a controller as a regular external process.""" | |
330 |
|
322 | |||
331 |
controller_cmd = List(ipcontroller_cmd_argv, config=True |
|
323 | controller_cmd = List(ipcontroller_cmd_argv, config=True, | |
|
324 | help="""Popen command to launch ipcontroller.""") | |||
332 | # Command line arguments to ipcontroller. |
|
325 | # Command line arguments to ipcontroller. | |
333 |
controller_args = List(['--log-to-file',' |
|
326 | controller_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True, | |
|
327 | help="""command-line args to pass to ipcontroller""") | |||
334 |
|
328 | |||
335 | def find_args(self): |
|
329 | def find_args(self): | |
336 | return self.controller_cmd + self.controller_args |
|
330 | return self.controller_cmd + self.controller_args | |
337 |
|
331 | |||
338 |
def start(self, |
|
332 | def start(self, profile_dir): | |
339 |
"""Start the controller by |
|
333 | """Start the controller by profile_dir.""" | |
340 |
self.controller_args.extend([' |
|
334 | self.controller_args.extend(['profile_dir=%s'%profile_dir]) | |
341 |
self. |
|
335 | self.profile_dir = unicode(profile_dir) | |
342 | self.log.info("Starting LocalControllerLauncher: %r" % self.args) |
|
336 | self.log.info("Starting LocalControllerLauncher: %r" % self.args) | |
343 | return super(LocalControllerLauncher, self).start() |
|
337 | return super(LocalControllerLauncher, self).start() | |
344 |
|
338 | |||
@@ -346,19 +340,20 b' class LocalControllerLauncher(LocalProcessLauncher):' | |||||
346 | class LocalEngineLauncher(LocalProcessLauncher): |
|
340 | class LocalEngineLauncher(LocalProcessLauncher): | |
347 | """Launch a single engine as a regular externall process.""" |
|
341 | """Launch a single engine as a regular externall process.""" | |
348 |
|
342 | |||
349 |
engine_cmd = List(ipengine_cmd_argv, config=True |
|
343 | engine_cmd = List(ipengine_cmd_argv, config=True, | |
|
344 | help="""command to launch the Engine.""") | |||
350 | # Command line arguments for ipengine. |
|
345 | # Command line arguments for ipengine. | |
351 | engine_args = List( |
|
346 | engine_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True, | |
352 | ['--log-to-file','--log-level', str(logging.INFO)], config=True |
|
347 | help="command-line arguments to pass to ipengine" | |
353 | ) |
|
348 | ) | |
354 |
|
349 | |||
355 | def find_args(self): |
|
350 | def find_args(self): | |
356 | return self.engine_cmd + self.engine_args |
|
351 | return self.engine_cmd + self.engine_args | |
357 |
|
352 | |||
358 |
def start(self, |
|
353 | def start(self, profile_dir): | |
359 |
"""Start the engine by |
|
354 | """Start the engine by profile_dir.""" | |
360 |
self.engine_args.extend([' |
|
355 | self.engine_args.extend(['profile_dir=%s'%profile_dir]) | |
361 |
self. |
|
356 | self.profile_dir = unicode(profile_dir) | |
362 | return super(LocalEngineLauncher, self).start() |
|
357 | return super(LocalEngineLauncher, self).start() | |
363 |
|
358 | |||
364 |
|
359 | |||
@@ -367,7 +362,8 b' class LocalEngineSetLauncher(BaseLauncher):' | |||||
367 |
|
362 | |||
368 | # Command line arguments for ipengine. |
|
363 | # Command line arguments for ipengine. | |
369 | engine_args = List( |
|
364 | engine_args = List( | |
370 |
['--log-to-file',' |
|
365 | ['--log-to-file','log_level=%i'%logging.INFO], config=True, | |
|
366 | help="command-line arguments to pass to ipengine" | |||
371 | ) |
|
367 | ) | |
372 | # launcher class |
|
368 | # launcher class | |
373 | launcher_class = LocalEngineLauncher |
|
369 | launcher_class = LocalEngineLauncher | |
@@ -381,16 +377,16 b' class LocalEngineSetLauncher(BaseLauncher):' | |||||
381 | ) |
|
377 | ) | |
382 | self.stop_data = {} |
|
378 | self.stop_data = {} | |
383 |
|
379 | |||
384 |
def start(self, n, |
|
380 | def start(self, n, profile_dir): | |
385 |
"""Start n engines by profile or |
|
381 | """Start n engines by profile or profile_dir.""" | |
386 |
self. |
|
382 | self.profile_dir = unicode(profile_dir) | |
387 | dlist = [] |
|
383 | dlist = [] | |
388 | for i in range(n): |
|
384 | for i in range(n): | |
389 |
el = self.launcher_class(work_dir=self.work_dir, config=self.config, log |
|
385 | el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log) | |
390 | # Copy the engine args over to each engine launcher. |
|
386 | # Copy the engine args over to each engine launcher. | |
391 | el.engine_args = copy.deepcopy(self.engine_args) |
|
387 | el.engine_args = copy.deepcopy(self.engine_args) | |
392 | el.on_stop(self._notice_engine_stopped) |
|
388 | el.on_stop(self._notice_engine_stopped) | |
393 |
d = el.start( |
|
389 | d = el.start(profile_dir) | |
394 | if i==0: |
|
390 | if i==0: | |
395 | self.log.info("Starting LocalEngineSetLauncher: %r" % el.args) |
|
391 | self.log.info("Starting LocalEngineSetLauncher: %r" % el.args) | |
396 | self.launchers[i] = el |
|
392 | self.launchers[i] = el | |
@@ -442,16 +438,18 b' class LocalEngineSetLauncher(BaseLauncher):' | |||||
442 | class MPIExecLauncher(LocalProcessLauncher): |
|
438 | class MPIExecLauncher(LocalProcessLauncher): | |
443 | """Launch an external process using mpiexec.""" |
|
439 | """Launch an external process using mpiexec.""" | |
444 |
|
440 | |||
445 | # The mpiexec command to use in starting the process. |
|
441 | mpi_cmd = List(['mpiexec'], config=True, | |
446 | mpi_cmd = List(['mpiexec'], config=True) |
|
442 | help="The mpiexec command to use in starting the process." | |
447 | # The command line arguments to pass to mpiexec. |
|
443 | ) | |
448 |
mpi_args = List([], config=True |
|
444 | mpi_args = List([], config=True, | |
449 | # The program to start using mpiexec. |
|
445 | help="The command line arguments to pass to mpiexec." | |
450 | program = List(['date'], config=True) |
|
446 | ) | |
451 | # The command line argument to the program. |
|
447 | program = List(['date'], config=True, | |
452 | program_args = List([], config=True) |
|
448 | help="The program to start via mpiexec.") | |
453 | # The number of instances of the program to start. |
|
449 | program_args = List([], config=True, | |
454 | n = Int(1, config=True) |
|
450 | help="The command line argument to the program." | |
|
451 | ) | |||
|
452 | n = Int(1) | |||
455 |
|
453 | |||
456 | def find_args(self): |
|
454 | def find_args(self): | |
457 | """Build self.args using all the fields.""" |
|
455 | """Build self.args using all the fields.""" | |
@@ -467,15 +465,18 b' class MPIExecLauncher(LocalProcessLauncher):' | |||||
467 | class MPIExecControllerLauncher(MPIExecLauncher): |
|
465 | class MPIExecControllerLauncher(MPIExecLauncher): | |
468 | """Launch a controller using mpiexec.""" |
|
466 | """Launch a controller using mpiexec.""" | |
469 |
|
467 | |||
470 |
controller_cmd = List(ipcontroller_cmd_argv, config=True |
|
468 | controller_cmd = List(ipcontroller_cmd_argv, config=True, | |
471 | # Command line arguments to ipcontroller. |
|
469 | help="Popen command to launch the Contropper" | |
472 | controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True) |
|
470 | ) | |
473 | n = Int(1, config=False) |
|
471 | controller_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True, | |
|
472 | help="Command line arguments to pass to ipcontroller." | |||
|
473 | ) | |||
|
474 | n = Int(1) | |||
474 |
|
475 | |||
475 |
def start(self, |
|
476 | def start(self, profile_dir): | |
476 |
"""Start the controller by |
|
477 | """Start the controller by profile_dir.""" | |
477 |
self.controller_args.extend([' |
|
478 | self.controller_args.extend(['profile_dir=%s'%profile_dir]) | |
478 |
self. |
|
479 | self.profile_dir = unicode(profile_dir) | |
479 | self.log.info("Starting MPIExecControllerLauncher: %r" % self.args) |
|
480 | self.log.info("Starting MPIExecControllerLauncher: %r" % self.args) | |
480 | return super(MPIExecControllerLauncher, self).start(1) |
|
481 | return super(MPIExecControllerLauncher, self).start(1) | |
481 |
|
482 | |||
@@ -486,17 +487,19 b' class MPIExecControllerLauncher(MPIExecLauncher):' | |||||
486 |
|
487 | |||
487 | class MPIExecEngineSetLauncher(MPIExecLauncher): |
|
488 | class MPIExecEngineSetLauncher(MPIExecLauncher): | |
488 |
|
489 | |||
489 |
program = List(ipengine_cmd_argv, config=True |
|
490 | program = List(ipengine_cmd_argv, config=True, | |
490 | # Command line arguments for ipengine. |
|
491 | help="Popen command for ipengine" | |
|
492 | ) | |||
491 | program_args = List( |
|
493 | program_args = List( | |
492 |
['--log-to-file',' |
|
494 | ['--log-to-file','log_level=%i'%logging.INFO], config=True, | |
|
495 | help="Command line arguments for ipengine." | |||
493 | ) |
|
496 | ) | |
494 |
n = Int(1 |
|
497 | n = Int(1) | |
495 |
|
498 | |||
496 |
def start(self, n, |
|
499 | def start(self, n, profile_dir): | |
497 |
"""Start n engines by profile or |
|
500 | """Start n engines by profile or profile_dir.""" | |
498 |
self.program_args.extend([' |
|
501 | self.program_args.extend(['profile_dir=%s'%profile_dir]) | |
499 |
self. |
|
502 | self.profile_dir = unicode(profile_dir) | |
500 | self.n = n |
|
503 | self.n = n | |
501 | self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args) |
|
504 | self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args) | |
502 | return super(MPIExecEngineSetLauncher, self).start(n) |
|
505 | return super(MPIExecEngineSetLauncher, self).start(n) | |
@@ -505,7 +508,7 b' class MPIExecEngineSetLauncher(MPIExecLauncher):' | |||||
505 | # SSH launchers |
|
508 | # SSH launchers | |
506 | #----------------------------------------------------------------------------- |
|
509 | #----------------------------------------------------------------------------- | |
507 |
|
510 | |||
508 |
# TODO: Get SSH Launcher |
|
511 | # TODO: Get SSH Launcher back to level of sshx in 0.10.2 | |
509 |
|
512 | |||
510 | class SSHLauncher(LocalProcessLauncher): |
|
513 | class SSHLauncher(LocalProcessLauncher): | |
511 | """A minimal launcher for ssh. |
|
514 | """A minimal launcher for ssh. | |
@@ -515,13 +518,20 b' class SSHLauncher(LocalProcessLauncher):' | |||||
515 | as well. |
|
518 | as well. | |
516 | """ |
|
519 | """ | |
517 |
|
520 | |||
518 |
ssh_cmd = List(['ssh'], config=True |
|
521 | ssh_cmd = List(['ssh'], config=True, | |
519 | ssh_args = List(['-tt'], config=True) |
|
522 | help="command for starting ssh") | |
520 |
|
|
523 | ssh_args = List(['-tt'], config=True, | |
521 | program_args = List([], config=True) |
|
524 | help="args to pass to ssh") | |
522 |
|
|
525 | program = List(['date'], config=True, | |
523 | user = CUnicode('', config=True) |
|
526 | help="Program to launch via ssh") | |
524 | location = CUnicode('') |
|
527 | program_args = List([], config=True, | |
|
528 | help="args to pass to remote program") | |||
|
529 | hostname = Unicode('', config=True, | |||
|
530 | help="hostname on which to launch the program") | |||
|
531 | user = Unicode('', config=True, | |||
|
532 | help="username for ssh") | |||
|
533 | location = Unicode('', config=True, | |||
|
534 | help="user@hostname location for ssh in one setting") | |||
525 |
|
535 | |||
526 | def _hostname_changed(self, name, old, new): |
|
536 | def _hostname_changed(self, name, old, new): | |
527 | if self.user: |
|
537 | if self.user: | |
@@ -536,8 +546,8 b' class SSHLauncher(LocalProcessLauncher):' | |||||
536 | return self.ssh_cmd + self.ssh_args + [self.location] + \ |
|
546 | return self.ssh_cmd + self.ssh_args + [self.location] + \ | |
537 | self.program + self.program_args |
|
547 | self.program + self.program_args | |
538 |
|
548 | |||
539 |
def start(self, |
|
549 | def start(self, profile_dir, hostname=None, user=None): | |
540 |
self. |
|
550 | self.profile_dir = unicode(profile_dir) | |
541 | if hostname is not None: |
|
551 | if hostname is not None: | |
542 | self.hostname = hostname |
|
552 | self.hostname = hostname | |
543 | if user is not None: |
|
553 | if user is not None: | |
@@ -555,28 +565,33 b' class SSHLauncher(LocalProcessLauncher):' | |||||
555 |
|
565 | |||
556 | class SSHControllerLauncher(SSHLauncher): |
|
566 | class SSHControllerLauncher(SSHLauncher): | |
557 |
|
567 | |||
558 |
program = List(ipcontroller_cmd_argv, config=True |
|
568 | program = List(ipcontroller_cmd_argv, config=True, | |
559 | # Command line arguments to ipcontroller. |
|
569 | help="remote ipcontroller command.") | |
560 |
program_args = List(['-r', '--log-to-file',' |
|
570 | program_args = List(['--reuse-files', '--log-to-file','log_level=%i'%logging.INFO], config=True, | |
|
571 | help="Command line arguments to ipcontroller.") | |||
561 |
|
572 | |||
562 |
|
573 | |||
563 | class SSHEngineLauncher(SSHLauncher): |
|
574 | class SSHEngineLauncher(SSHLauncher): | |
564 |
program = List(ipengine_cmd_argv, config=True |
|
575 | program = List(ipengine_cmd_argv, config=True, | |
|
576 | help="remote ipengine command.") | |||
565 | # Command line arguments for ipengine. |
|
577 | # Command line arguments for ipengine. | |
566 | program_args = List( |
|
578 | program_args = List( | |
567 |
['--log-to-file',' |
|
579 | ['--log-to-file','log_level=%i'%logging.INFO], config=True, | |
|
580 | help="Command line arguments to ipengine." | |||
568 | ) |
|
581 | ) | |
569 |
|
582 | |||
570 | class SSHEngineSetLauncher(LocalEngineSetLauncher): |
|
583 | class SSHEngineSetLauncher(LocalEngineSetLauncher): | |
571 | launcher_class = SSHEngineLauncher |
|
584 | launcher_class = SSHEngineLauncher | |
572 |
engines = Dict(config=True |
|
585 | engines = Dict(config=True, | |
|
586 | help="""dict of engines to launch. This is a dict by hostname of ints, | |||
|
587 | corresponding to the number of engines to start on that host.""") | |||
573 |
|
588 | |||
574 |
def start(self, n, |
|
589 | def start(self, n, profile_dir): | |
575 |
"""Start engines by profile or |
|
590 | """Start engines by profile or profile_dir. | |
576 | `n` is ignored, and the `engines` config property is used instead. |
|
591 | `n` is ignored, and the `engines` config property is used instead. | |
577 | """ |
|
592 | """ | |
578 |
|
593 | |||
579 |
self. |
|
594 | self.profile_dir = unicode(profile_dir) | |
580 | dlist = [] |
|
595 | dlist = [] | |
581 | for host, n in self.engines.iteritems(): |
|
596 | for host, n in self.engines.iteritems(): | |
582 | if isinstance(n, (tuple, list)): |
|
597 | if isinstance(n, (tuple, list)): | |
@@ -589,13 +604,13 b' class SSHEngineSetLauncher(LocalEngineSetLauncher):' | |||||
589 | else: |
|
604 | else: | |
590 | user=None |
|
605 | user=None | |
591 | for i in range(n): |
|
606 | for i in range(n): | |
592 |
el = self.launcher_class(work_dir=self.work_dir, config=self.config, log |
|
607 | el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log) | |
593 |
|
608 | |||
594 | # Copy the engine args over to each engine launcher. |
|
609 | # Copy the engine args over to each engine launcher. | |
595 | i |
|
610 | i | |
596 | el.program_args = args |
|
611 | el.program_args = args | |
597 | el.on_stop(self._notice_engine_stopped) |
|
612 | el.on_stop(self._notice_engine_stopped) | |
598 |
d = el.start( |
|
613 | d = el.start(profile_dir, user=user, hostname=host) | |
599 | if i==0: |
|
614 | if i==0: | |
600 | self.log.info("Starting SSHEngineSetLauncher: %r" % el.args) |
|
615 | self.log.info("Starting SSHEngineSetLauncher: %r" % el.args) | |
601 | self.launchers[host+str(i)] = el |
|
616 | self.launchers[host+str(i)] = el | |
@@ -624,17 +639,19 b' def find_job_cmd():' | |||||
624 |
|
639 | |||
625 | class WindowsHPCLauncher(BaseLauncher): |
|
640 | class WindowsHPCLauncher(BaseLauncher): | |
626 |
|
641 | |||
627 | # A regular expression used to get the job id from the output of the |
|
642 | job_id_regexp = Unicode(r'\d+', config=True, | |
628 | # submit_command. |
|
643 | help="""A regular expression used to get the job id from the output of the | |
629 | job_id_regexp = Str(r'\d+', config=True) |
|
644 | submit_command. """ | |
630 | # The filename of the instantiated job script. |
|
645 | ) | |
631 |
job_file_name = |
|
646 | job_file_name = Unicode(u'ipython_job.xml', config=True, | |
|
647 | help="The filename of the instantiated job script.") | |||
632 | # The full path to the instantiated job script. This gets made dynamically |
|
648 | # The full path to the instantiated job script. This gets made dynamically | |
633 | # by combining the work_dir with the job_file_name. |
|
649 | # by combining the work_dir with the job_file_name. | |
634 |
job_file = |
|
650 | job_file = Unicode(u'') | |
635 | # The hostname of the scheduler to submit the job to |
|
651 | scheduler = Unicode('', config=True, | |
636 | scheduler = CUnicode('', config=True) |
|
652 | help="The hostname of the scheduler to submit the job to.") | |
637 |
job_cmd = |
|
653 | job_cmd = Unicode(find_job_cmd(), config=True, | |
|
654 | help="The command for submitting jobs.") | |||
638 |
|
655 | |||
639 | def __init__(self, work_dir=u'.', config=None, **kwargs): |
|
656 | def __init__(self, work_dir=u'.', config=None, **kwargs): | |
640 | super(WindowsHPCLauncher, self).__init__( |
|
657 | super(WindowsHPCLauncher, self).__init__( | |
@@ -671,7 +688,7 b' class WindowsHPCLauncher(BaseLauncher):' | |||||
671 | '/scheduler:%s' % self.scheduler |
|
688 | '/scheduler:%s' % self.scheduler | |
672 | ] |
|
689 | ] | |
673 | self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),)) |
|
690 | self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),)) | |
674 | # Twisted will raise DeprecationWarnings if we try to pass unicode to this |
|
691 | ||
675 | output = check_output([self.job_cmd]+args, |
|
692 | output = check_output([self.job_cmd]+args, | |
676 | env=os.environ, |
|
693 | env=os.environ, | |
677 | cwd=self.work_dir, |
|
694 | cwd=self.work_dir, | |
@@ -702,8 +719,10 b' class WindowsHPCLauncher(BaseLauncher):' | |||||
702 |
|
719 | |||
703 | class WindowsHPCControllerLauncher(WindowsHPCLauncher): |
|
720 | class WindowsHPCControllerLauncher(WindowsHPCLauncher): | |
704 |
|
721 | |||
705 |
job_file_name = |
|
722 | job_file_name = Unicode(u'ipcontroller_job.xml', config=True, | |
706 | extra_args = List([], config=False) |
|
723 | help="WinHPC xml job file.") | |
|
724 | extra_args = List([], config=False, | |||
|
725 | help="extra args to pass to ipcontroller") | |||
707 |
|
726 | |||
708 | def write_job_file(self, n): |
|
727 | def write_job_file(self, n): | |
709 | job = IPControllerJob(config=self.config) |
|
728 | job = IPControllerJob(config=self.config) | |
@@ -712,8 +731,8 b' class WindowsHPCControllerLauncher(WindowsHPCLauncher):' | |||||
712 | # The tasks work directory is *not* the actual work directory of |
|
731 | # The tasks work directory is *not* the actual work directory of | |
713 | # the controller. It is used as the base path for the stdout/stderr |
|
732 | # the controller. It is used as the base path for the stdout/stderr | |
714 | # files that the scheduler redirects to. |
|
733 | # files that the scheduler redirects to. | |
715 |
t.work_directory = self. |
|
734 | t.work_directory = self.profile_dir | |
716 |
# Add the |
|
735 | # Add the profile_dir and from self.start(). | |
717 | t.controller_args.extend(self.extra_args) |
|
736 | t.controller_args.extend(self.extra_args) | |
718 | job.add_task(t) |
|
737 | job.add_task(t) | |
719 |
|
738 | |||
@@ -722,19 +741,21 b' class WindowsHPCControllerLauncher(WindowsHPCLauncher):' | |||||
722 |
|
741 | |||
723 | @property |
|
742 | @property | |
724 | def job_file(self): |
|
743 | def job_file(self): | |
725 |
return os.path.join(self. |
|
744 | return os.path.join(self.profile_dir, self.job_file_name) | |
726 |
|
745 | |||
727 |
def start(self, |
|
746 | def start(self, profile_dir): | |
728 |
"""Start the controller by |
|
747 | """Start the controller by profile_dir.""" | |
729 |
self.extra_args = [' |
|
748 | self.extra_args = ['profile_dir=%s'%profile_dir] | |
730 |
self. |
|
749 | self.profile_dir = unicode(profile_dir) | |
731 | return super(WindowsHPCControllerLauncher, self).start(1) |
|
750 | return super(WindowsHPCControllerLauncher, self).start(1) | |
732 |
|
751 | |||
733 |
|
752 | |||
734 | class WindowsHPCEngineSetLauncher(WindowsHPCLauncher): |
|
753 | class WindowsHPCEngineSetLauncher(WindowsHPCLauncher): | |
735 |
|
754 | |||
736 |
job_file_name = |
|
755 | job_file_name = Unicode(u'ipengineset_job.xml', config=True, | |
737 | extra_args = List([], config=False) |
|
756 | help="jobfile for ipengines job") | |
|
757 | extra_args = List([], config=False, | |||
|
758 | help="extra args to pas to ipengine") | |||
738 |
|
759 | |||
739 | def write_job_file(self, n): |
|
760 | def write_job_file(self, n): | |
740 | job = IPEngineSetJob(config=self.config) |
|
761 | job = IPEngineSetJob(config=self.config) | |
@@ -744,8 +765,8 b' class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):' | |||||
744 | # The tasks work directory is *not* the actual work directory of |
|
765 | # The tasks work directory is *not* the actual work directory of | |
745 | # the engine. It is used as the base path for the stdout/stderr |
|
766 | # the engine. It is used as the base path for the stdout/stderr | |
746 | # files that the scheduler redirects to. |
|
767 | # files that the scheduler redirects to. | |
747 |
t.work_directory = self. |
|
768 | t.work_directory = self.profile_dir | |
748 |
# Add the |
|
769 | # Add the profile_dir and from self.start(). | |
749 | t.engine_args.extend(self.extra_args) |
|
770 | t.engine_args.extend(self.extra_args) | |
750 | job.add_task(t) |
|
771 | job.add_task(t) | |
751 |
|
772 | |||
@@ -754,12 +775,12 b' class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):' | |||||
754 |
|
775 | |||
755 | @property |
|
776 | @property | |
756 | def job_file(self): |
|
777 | def job_file(self): | |
757 |
return os.path.join(self. |
|
778 | return os.path.join(self.profile_dir, self.job_file_name) | |
758 |
|
779 | |||
759 |
def start(self, n, |
|
780 | def start(self, n, profile_dir): | |
760 |
"""Start the controller by |
|
781 | """Start the controller by profile_dir.""" | |
761 |
self.extra_args = [' |
|
782 | self.extra_args = ['profile_dir=%s'%profile_dir] | |
762 |
self. |
|
783 | self.profile_dir = unicode(profile_dir) | |
763 | return super(WindowsHPCEngineSetLauncher, self).start(n) |
|
784 | return super(WindowsHPCEngineSetLauncher, self).start(n) | |
764 |
|
785 | |||
765 |
|
786 | |||
@@ -776,41 +797,43 b' class BatchSystemLauncher(BaseLauncher):' | |||||
776 |
|
797 | |||
777 | This class also has the notion of a batch script. The ``batch_template`` |
|
798 | This class also has the notion of a batch script. The ``batch_template`` | |
778 | attribute can be set to a string that is a template for the batch script. |
|
799 | attribute can be set to a string that is a template for the batch script. | |
779 |
This template is instantiated using |
|
800 | This template is instantiated using string formatting. Thus the template can | |
780 |
|
|
801 | use {n} fot the number of instances. Subclasses can add additional variables | |
781 | to the template dict. |
|
802 | to the template dict. | |
782 | """ |
|
803 | """ | |
783 |
|
804 | |||
784 | # Subclasses must fill these in. See PBSEngineSet |
|
805 | # Subclasses must fill these in. See PBSEngineSet | |
785 | # The name of the command line program used to submit jobs. |
|
806 | submit_command = List([''], config=True, | |
786 | submit_command = List([''], config=True) |
|
807 | help="The name of the command line program used to submit jobs.") | |
787 | # The name of the command line program used to delete jobs. |
|
808 | delete_command = List([''], config=True, | |
788 | delete_command = List([''], config=True) |
|
809 | help="The name of the command line program used to delete jobs.") | |
789 | # A regular expression used to get the job id from the output of the |
|
810 | job_id_regexp = Unicode('', config=True, | |
790 | # submit_command. |
|
811 | help="""A regular expression used to get the job id from the output of the | |
791 | job_id_regexp = CUnicode('', config=True) |
|
812 | submit_command.""") | |
792 | # The string that is the batch script template itself. |
|
813 | batch_template = Unicode('', config=True, | |
793 | batch_template = CUnicode('', config=True) |
|
814 | help="The string that is the batch script template itself.") | |
794 | # The file that contains the batch template |
|
815 | batch_template_file = Unicode(u'', config=True, | |
795 | batch_template_file = CUnicode(u'', config=True) |
|
816 | help="The file that contains the batch template.") | |
796 | # The filename of the instantiated batch script. |
|
817 | batch_file_name = Unicode(u'batch_script', config=True, | |
797 | batch_file_name = CUnicode(u'batch_script', config=True) |
|
818 | help="The filename of the instantiated batch script.") | |
798 | # The PBS Queue |
|
819 | queue = Unicode(u'', config=True, | |
799 | queue = CUnicode(u'', config=True) |
|
820 | help="The PBS Queue.") | |
800 |
|
821 | |||
801 | # not configurable, override in subclasses |
|
822 | # not configurable, override in subclasses | |
802 | # PBS Job Array regex |
|
823 | # PBS Job Array regex | |
803 |
job_array_regexp = |
|
824 | job_array_regexp = Unicode('') | |
804 |
job_array_template = |
|
825 | job_array_template = Unicode('') | |
805 | # PBS Queue regex |
|
826 | # PBS Queue regex | |
806 |
queue_regexp = |
|
827 | queue_regexp = Unicode('') | |
807 |
queue_template = |
|
828 | queue_template = Unicode('') | |
808 | # The default batch template, override in subclasses |
|
829 | # The default batch template, override in subclasses | |
809 |
default_template = |
|
830 | default_template = Unicode('') | |
810 | # The full path to the instantiated batch script. |
|
831 | # The full path to the instantiated batch script. | |
811 |
batch_file = |
|
832 | batch_file = Unicode(u'') | |
812 | # the format dict used with batch_template: |
|
833 | # the format dict used with batch_template: | |
813 | context = Dict() |
|
834 | context = Dict() | |
|
835 | # the Formatter instance for rendering the templates: | |||
|
836 | formatter = Instance(EvalFormatter, (), {}) | |||
814 |
|
837 | |||
815 |
|
838 | |||
816 | def find_args(self): |
|
839 | def find_args(self): | |
@@ -837,7 +860,6 b' class BatchSystemLauncher(BaseLauncher):' | |||||
837 | """Instantiate and write the batch script to the work_dir.""" |
|
860 | """Instantiate and write the batch script to the work_dir.""" | |
838 | self.context['n'] = n |
|
861 | self.context['n'] = n | |
839 | self.context['queue'] = self.queue |
|
862 | self.context['queue'] = self.queue | |
840 | print self.context |
|
|||
841 | # first priority is batch_template if set |
|
863 | # first priority is batch_template if set | |
842 | if self.batch_template_file and not self.batch_template: |
|
864 | if self.batch_template_file and not self.batch_template: | |
843 | # second priority is batch_template_file |
|
865 | # second priority is batch_template_file | |
@@ -861,20 +883,19 b' class BatchSystemLauncher(BaseLauncher):' | |||||
861 | firstline, rest = self.batch_template.split('\n',1) |
|
883 | firstline, rest = self.batch_template.split('\n',1) | |
862 | self.batch_template = u'\n'.join([firstline, self.queue_template, rest]) |
|
884 | self.batch_template = u'\n'.join([firstline, self.queue_template, rest]) | |
863 |
|
885 | |||
864 |
script_as_string = |
|
886 | script_as_string = self.formatter.format(self.batch_template, **self.context) | |
865 | self.log.info('Writing instantiated batch script: %s' % self.batch_file) |
|
887 | self.log.info('Writing instantiated batch script: %s' % self.batch_file) | |
866 |
|
888 | |||
867 | with open(self.batch_file, 'w') as f: |
|
889 | with open(self.batch_file, 'w') as f: | |
868 | f.write(script_as_string) |
|
890 | f.write(script_as_string) | |
869 | os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) |
|
891 | os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | |
870 |
|
892 | |||
871 |
def start(self, n, |
|
893 | def start(self, n, profile_dir): | |
872 | """Start n copies of the process using a batch system.""" |
|
894 | """Start n copies of the process using a batch system.""" | |
873 |
# Here we save profile |
|
895 | # Here we save profile_dir in the context so they | |
874 |
# can be used in the batch script template as |
|
896 | # can be used in the batch script template as {profile_dir} | |
875 | # ${cluster_dir} |
|
897 | self.context['profile_dir'] = profile_dir | |
876 |
self. |
|
898 | self.profile_dir = unicode(profile_dir) | |
877 | self.cluster_dir = unicode(cluster_dir) |
|
|||
878 | self.write_batch_script(n) |
|
899 | self.write_batch_script(n) | |
879 | output = check_output(self.args, env=os.environ) |
|
900 | output = check_output(self.args, env=os.environ) | |
880 |
|
901 | |||
@@ -891,84 +912,91 b' class BatchSystemLauncher(BaseLauncher):' | |||||
891 | class PBSLauncher(BatchSystemLauncher): |
|
912 | class PBSLauncher(BatchSystemLauncher): | |
892 | """A BatchSystemLauncher subclass for PBS.""" |
|
913 | """A BatchSystemLauncher subclass for PBS.""" | |
893 |
|
914 | |||
894 |
submit_command = List(['qsub'], config=True |
|
915 | submit_command = List(['qsub'], config=True, | |
895 | delete_command = List(['qdel'], config=True) |
|
916 | help="The PBS submit command ['qsub']") | |
896 | job_id_regexp = CUnicode(r'\d+', config=True) |
|
917 | delete_command = List(['qdel'], config=True, | |
|
918 | help="The PBS delete command ['qsub']") | |||
|
919 | job_id_regexp = Unicode(r'\d+', config=True, | |||
|
920 | help="Regular expresion for identifying the job ID [r'\d+']") | |||
897 |
|
921 | |||
898 |
batch_file = |
|
922 | batch_file = Unicode(u'') | |
899 |
job_array_regexp = |
|
923 | job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+') | |
900 |
job_array_template = |
|
924 | job_array_template = Unicode('#PBS -t 1-{n}') | |
901 |
queue_regexp = |
|
925 | queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+') | |
902 |
queue_template = |
|
926 | queue_template = Unicode('#PBS -q {queue}') | |
903 |
|
927 | |||
904 |
|
928 | |||
905 | class PBSControllerLauncher(PBSLauncher): |
|
929 | class PBSControllerLauncher(PBSLauncher): | |
906 | """Launch a controller using PBS.""" |
|
930 | """Launch a controller using PBS.""" | |
907 |
|
931 | |||
908 |
batch_file_name = |
|
932 | batch_file_name = Unicode(u'pbs_controller', config=True, | |
909 | default_template= CUnicode("""#!/bin/sh |
|
933 | help="batch file name for the controller job.") | |
|
934 | default_template= Unicode("""#!/bin/sh | |||
910 | #PBS -V |
|
935 | #PBS -V | |
911 | #PBS -N ipcontroller |
|
936 | #PBS -N ipcontroller | |
912 | %s --log-to-file --cluster-dir $cluster_dir |
|
937 | %s --log-to-file profile_dir={profile_dir} | |
913 | """%(' '.join(ipcontroller_cmd_argv))) |
|
938 | """%(' '.join(ipcontroller_cmd_argv))) | |
914 |
|
939 | |||
915 |
def start(self, |
|
940 | def start(self, profile_dir): | |
916 |
"""Start the controller by profile or |
|
941 | """Start the controller by profile or profile_dir.""" | |
917 | self.log.info("Starting PBSControllerLauncher: %r" % self.args) |
|
942 | self.log.info("Starting PBSControllerLauncher: %r" % self.args) | |
918 |
return super(PBSControllerLauncher, self).start(1, |
|
943 | return super(PBSControllerLauncher, self).start(1, profile_dir) | |
919 |
|
944 | |||
920 |
|
945 | |||
921 | class PBSEngineSetLauncher(PBSLauncher): |
|
946 | class PBSEngineSetLauncher(PBSLauncher): | |
922 | """Launch Engines using PBS""" |
|
947 | """Launch Engines using PBS""" | |
923 |
batch_file_name = |
|
948 | batch_file_name = Unicode(u'pbs_engines', config=True, | |
924 | default_template= CUnicode(u"""#!/bin/sh |
|
949 | help="batch file name for the engine(s) job.") | |
|
950 | default_template= Unicode(u"""#!/bin/sh | |||
925 | #PBS -V |
|
951 | #PBS -V | |
926 | #PBS -N ipengine |
|
952 | #PBS -N ipengine | |
927 | %s --cluster-dir $cluster_dir |
|
953 | %s profile_dir={profile_dir} | |
928 | """%(' '.join(ipengine_cmd_argv))) |
|
954 | """%(' '.join(ipengine_cmd_argv))) | |
929 |
|
955 | |||
930 |
def start(self, n, |
|
956 | def start(self, n, profile_dir): | |
931 |
"""Start n engines by profile or |
|
957 | """Start n engines by profile or profile_dir.""" | |
932 | self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args)) |
|
958 | self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args)) | |
933 |
return super(PBSEngineSetLauncher, self).start(n, |
|
959 | return super(PBSEngineSetLauncher, self).start(n, profile_dir) | |
934 |
|
960 | |||
935 | #SGE is very similar to PBS |
|
961 | #SGE is very similar to PBS | |
936 |
|
962 | |||
937 | class SGELauncher(PBSLauncher): |
|
963 | class SGELauncher(PBSLauncher): | |
938 | """Sun GridEngine is a PBS clone with slightly different syntax""" |
|
964 | """Sun GridEngine is a PBS clone with slightly different syntax""" | |
939 |
job_array_regexp = |
|
965 | job_array_regexp = Unicode('#\$\W+\-t') | |
940 |
job_array_template = |
|
966 | job_array_template = Unicode('#$ -t 1-{n}') | |
941 |
queue_regexp = |
|
967 | queue_regexp = Unicode('#\$\W+-q\W+\$?\w+') | |
942 |
queue_template = |
|
968 | queue_template = Unicode('#$ -q $queue') | |
943 |
|
969 | |||
944 | class SGEControllerLauncher(SGELauncher): |
|
970 | class SGEControllerLauncher(SGELauncher): | |
945 | """Launch a controller using SGE.""" |
|
971 | """Launch a controller using SGE.""" | |
946 |
|
972 | |||
947 |
batch_file_name = |
|
973 | batch_file_name = Unicode(u'sge_controller', config=True, | |
948 | default_template= CUnicode(u"""#$$ -V |
|
974 | help="batch file name for the ipontroller job.") | |
949 | #$$ -S /bin/sh |
|
975 | default_template= Unicode(u"""#$ -V | |
950 | #$$ -N ipcontroller |
|
976 | #$ -S /bin/sh | |
951 | %s --log-to-file --cluster-dir $cluster_dir |
|
977 | #$ -N ipcontroller | |
|
978 | %s --log-to-file profile_dir={profile_dir} | |||
952 | """%(' '.join(ipcontroller_cmd_argv))) |
|
979 | """%(' '.join(ipcontroller_cmd_argv))) | |
953 |
|
980 | |||
954 |
def start(self, |
|
981 | def start(self, profile_dir): | |
955 |
"""Start the controller by profile or |
|
982 | """Start the controller by profile or profile_dir.""" | |
956 | self.log.info("Starting PBSControllerLauncher: %r" % self.args) |
|
983 | self.log.info("Starting PBSControllerLauncher: %r" % self.args) | |
957 |
return super( |
|
984 | return super(SGEControllerLauncher, self).start(1, profile_dir) | |
958 |
|
985 | |||
959 | class SGEEngineSetLauncher(SGELauncher): |
|
986 | class SGEEngineSetLauncher(SGELauncher): | |
960 | """Launch Engines with SGE""" |
|
987 | """Launch Engines with SGE""" | |
961 |
batch_file_name = |
|
988 | batch_file_name = Unicode(u'sge_engines', config=True, | |
962 | default_template = CUnicode("""#$$ -V |
|
989 | help="batch file name for the engine(s) job.") | |
963 | #$$ -S /bin/sh |
|
990 | default_template = Unicode("""#$ -V | |
964 | #$$ -N ipengine |
|
991 | #$ -S /bin/sh | |
965 | %s --cluster-dir $cluster_dir |
|
992 | #$ -N ipengine | |
|
993 | %s profile_dir={profile_dir} | |||
966 | """%(' '.join(ipengine_cmd_argv))) |
|
994 | """%(' '.join(ipengine_cmd_argv))) | |
967 |
|
995 | |||
968 |
def start(self, n, |
|
996 | def start(self, n, profile_dir): | |
969 |
"""Start n engines by profile or |
|
997 | """Start n engines by profile or profile_dir.""" | |
970 | self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args)) |
|
998 | self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args)) | |
971 |
return super(SGEEngineSetLauncher, self).start(n, |
|
999 | return super(SGEEngineSetLauncher, self).start(n, profile_dir) | |
972 |
|
1000 | |||
973 |
|
1001 | |||
974 | #----------------------------------------------------------------------------- |
|
1002 | #----------------------------------------------------------------------------- | |
@@ -979,18 +1007,57 b' class SGEEngineSetLauncher(SGELauncher):' | |||||
979 | class IPClusterLauncher(LocalProcessLauncher): |
|
1007 | class IPClusterLauncher(LocalProcessLauncher): | |
980 | """Launch the ipcluster program in an external process.""" |
|
1008 | """Launch the ipcluster program in an external process.""" | |
981 |
|
1009 | |||
982 |
ipcluster_cmd = List(ipcluster_cmd_argv, config=True |
|
1010 | ipcluster_cmd = List(ipcluster_cmd_argv, config=True, | |
983 | # Command line arguments to pass to ipcluster. |
|
1011 | help="Popen command for ipcluster") | |
984 | ipcluster_args = List( |
|
1012 | ipcluster_args = List( | |
985 |
['--clean-logs', '--log-to-file', ' |
|
1013 | ['--clean-logs', '--log-to-file', 'log_level=%i'%logging.INFO], config=True, | |
986 | ipcluster_subcommand = Str('start') |
|
1014 | help="Command line arguments to pass to ipcluster.") | |
|
1015 | ipcluster_subcommand = Unicode('start') | |||
987 | ipcluster_n = Int(2) |
|
1016 | ipcluster_n = Int(2) | |
988 |
|
1017 | |||
989 | def find_args(self): |
|
1018 | def find_args(self): | |
990 | return self.ipcluster_cmd + [self.ipcluster_subcommand] + \ |
|
1019 | return self.ipcluster_cmd + ['--'+self.ipcluster_subcommand] + \ | |
991 |
[' |
|
1020 | ['n=%i'%self.ipcluster_n] + self.ipcluster_args | |
992 |
|
1021 | |||
993 | def start(self): |
|
1022 | def start(self): | |
994 | self.log.info("Starting ipcluster: %r" % self.args) |
|
1023 | self.log.info("Starting ipcluster: %r" % self.args) | |
995 | return super(IPClusterLauncher, self).start() |
|
1024 | return super(IPClusterLauncher, self).start() | |
996 |
|
1025 | |||
|
1026 | #----------------------------------------------------------------------------- | |||
|
1027 | # Collections of launchers | |||
|
1028 | #----------------------------------------------------------------------------- | |||
|
1029 | ||||
|
1030 | local_launchers = [ | |||
|
1031 | LocalControllerLauncher, | |||
|
1032 | LocalEngineLauncher, | |||
|
1033 | LocalEngineSetLauncher, | |||
|
1034 | ] | |||
|
1035 | mpi_launchers = [ | |||
|
1036 | MPIExecLauncher, | |||
|
1037 | MPIExecControllerLauncher, | |||
|
1038 | MPIExecEngineSetLauncher, | |||
|
1039 | ] | |||
|
1040 | ssh_launchers = [ | |||
|
1041 | SSHLauncher, | |||
|
1042 | SSHControllerLauncher, | |||
|
1043 | SSHEngineLauncher, | |||
|
1044 | SSHEngineSetLauncher, | |||
|
1045 | ] | |||
|
1046 | winhpc_launchers = [ | |||
|
1047 | WindowsHPCLauncher, | |||
|
1048 | WindowsHPCControllerLauncher, | |||
|
1049 | WindowsHPCEngineSetLauncher, | |||
|
1050 | ] | |||
|
1051 | pbs_launchers = [ | |||
|
1052 | PBSLauncher, | |||
|
1053 | PBSControllerLauncher, | |||
|
1054 | PBSEngineSetLauncher, | |||
|
1055 | ] | |||
|
1056 | sge_launchers = [ | |||
|
1057 | SGELauncher, | |||
|
1058 | SGEControllerLauncher, | |||
|
1059 | SGEEngineSetLauncher, | |||
|
1060 | ] | |||
|
1061 | all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\ | |||
|
1062 | + pbs_launchers + sge_launchers | |||
|
1063 |
@@ -1,5 +1,12 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | """A simple logger object that consolidates messages incoming from ipcluster processes.""" |
|
2 | """ | |
|
3 | A simple logger object that consolidates messages incoming from ipcluster processes. | |||
|
4 | ||||
|
5 | Authors: | |||
|
6 | ||||
|
7 | * MinRK | |||
|
8 | ||||
|
9 | """ | |||
3 |
|
10 | |||
4 | #----------------------------------------------------------------------------- |
|
11 | #----------------------------------------------------------------------------- | |
5 | # Copyright (C) 2011 The IPython Development Team |
|
12 | # Copyright (C) 2011 The IPython Development Team | |
@@ -19,29 +26,35 b' import sys' | |||||
19 | import zmq |
|
26 | import zmq | |
20 | from zmq.eventloop import ioloop, zmqstream |
|
27 | from zmq.eventloop import ioloop, zmqstream | |
21 |
|
28 | |||
22 | from IPython.utils.traitlets import Int, Str, Instance, List |
|
29 | from IPython.config.configurable import LoggingConfigurable | |
23 |
|
30 | from IPython.utils.traitlets import Int, Unicode, Instance, List | ||
24 | from IPython.parallel.factory import LoggingFactory |
|
|||
25 |
|
31 | |||
26 | #----------------------------------------------------------------------------- |
|
32 | #----------------------------------------------------------------------------- | |
27 | # Classes |
|
33 | # Classes | |
28 | #----------------------------------------------------------------------------- |
|
34 | #----------------------------------------------------------------------------- | |
29 |
|
35 | |||
30 |
|
36 | |||
31 |
class LogWatcher(Logging |
|
37 | class LogWatcher(LoggingConfigurable): | |
32 | """A simple class that receives messages on a SUB socket, as published |
|
38 | """A simple class that receives messages on a SUB socket, as published | |
33 | by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself. |
|
39 | by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself. | |
34 |
|
40 | |||
35 | This can subscribe to multiple topics, but defaults to all topics. |
|
41 | This can subscribe to multiple topics, but defaults to all topics. | |
36 | """ |
|
42 | """ | |
|
43 | ||||
37 | # configurables |
|
44 | # configurables | |
38 |
topics = List([''], config=True |
|
45 | topics = List([''], config=True, | |
39 | url = Str('tcp://127.0.0.1:20202', config=True) |
|
46 | help="The ZMQ topics to subscribe to. Default is to subscribe to all messages") | |
|
47 | url = Unicode('tcp://127.0.0.1:20202', config=True, | |||
|
48 | help="ZMQ url on which to listen for log messages") | |||
40 |
|
49 | |||
41 | # internals |
|
50 | # internals | |
42 | context = Instance(zmq.Context, (), {}) |
|
|||
43 | stream = Instance('zmq.eventloop.zmqstream.ZMQStream') |
|
51 | stream = Instance('zmq.eventloop.zmqstream.ZMQStream') | |
44 | loop = Instance('zmq.eventloop.ioloop.IOLoop') |
|
52 | ||
|
53 | context = Instance(zmq.Context) | |||
|
54 | def _context_default(self): | |||
|
55 | return zmq.Context.instance() | |||
|
56 | ||||
|
57 | loop = Instance(zmq.eventloop.ioloop.IOLoop) | |||
45 | def _loop_default(self): |
|
58 | def _loop_default(self): | |
46 | return ioloop.IOLoop.instance() |
|
59 | return ioloop.IOLoop.instance() | |
47 |
|
60 | |||
@@ -62,9 +75,13 b' class LogWatcher(LoggingFactory):' | |||||
62 | def subscribe(self): |
|
75 | def subscribe(self): | |
63 | """Update our SUB socket's subscriptions.""" |
|
76 | """Update our SUB socket's subscriptions.""" | |
64 | self.stream.setsockopt(zmq.UNSUBSCRIBE, '') |
|
77 | self.stream.setsockopt(zmq.UNSUBSCRIBE, '') | |
65 |
f |
|
78 | if '' in self.topics: | |
66 |
self.log.debug("Subscribing to: |
|
79 | self.log.debug("Subscribing to: everything") | |
67 |
self.stream.setsockopt(zmq.SUBSCRIBE, |
|
80 | self.stream.setsockopt(zmq.SUBSCRIBE, '') | |
|
81 | else: | |||
|
82 | for topic in self.topics: | |||
|
83 | self.log.debug("Subscribing to: %r"%(topic)) | |||
|
84 | self.stream.setsockopt(zmq.SUBSCRIBE, topic) | |||
68 |
|
85 | |||
69 | def _extract_level(self, topic_str): |
|
86 | def _extract_level(self, topic_str): | |
70 | """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')""" |
|
87 | """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')""" | |
@@ -94,5 +111,5 b' class LogWatcher(LoggingFactory):' | |||||
94 | level,topic = self._extract_level(topic) |
|
111 | level,topic = self._extract_level(topic) | |
95 | if msg[-1] == '\n': |
|
112 | if msg[-1] == '\n': | |
96 | msg = msg[:-1] |
|
113 | msg = msg[:-1] | |
97 |
log |
|
114 | self.log.log(level, "[%s] %s" % (topic, msg)) | |
98 |
|
115 |
@@ -1,7 +1,13 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | """Utility for forwarding file read events over a zmq socket. |
|
2 | """Utility for forwarding file read events over a zmq socket. | |
3 |
|
3 | |||
4 |
This is necessary because select on Windows only supports |
|
4 | This is necessary because select on Windows only supports sockets, not FDs. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * MinRK | |||
|
9 | ||||
|
10 | """ | |||
5 |
|
11 | |||
6 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
7 | # Copyright (C) 2011 The IPython Development Team |
|
13 | # Copyright (C) 2011 The IPython Development Team | |
@@ -31,7 +37,7 b' class ForwarderThread(Thread):' | |||||
31 | self.fd = fd |
|
37 | self.fd = fd | |
32 |
|
38 | |||
33 | def run(self): |
|
39 | def run(self): | |
34 |
""" |
|
40 | """Loop through lines in self.fd, and send them over self.sock.""" | |
35 | line = self.fd.readline() |
|
41 | line = self.fd.readline() | |
36 | # allow for files opened in unicode mode |
|
42 | # allow for files opened in unicode mode | |
37 | if isinstance(line, unicode): |
|
43 | if isinstance(line, unicode): | |
@@ -46,7 +52,7 b' class ForwarderThread(Thread):' | |||||
46 | self.sock.close() |
|
52 | self.sock.close() | |
47 |
|
53 | |||
48 | def forward_read_events(fd, context=None): |
|
54 | def forward_read_events(fd, context=None): | |
49 |
""" |
|
55 | """Forward read events from an FD over a socket. | |
50 |
|
56 | |||
51 | This method wraps a file in a socket pair, so it can |
|
57 | This method wraps a file in a socket pair, so it can | |
52 | be polled for read events by select (specifically zmq.eventloop.ioloop) |
|
58 | be polled for read events by select (specifically zmq.eventloop.ioloop) | |
@@ -64,4 +70,4 b' def forward_read_events(fd, context=None):' | |||||
64 | return pull |
|
70 | return pull | |
65 |
|
71 | |||
66 |
|
72 | |||
67 | __all__ = ['forward_read_events'] No newline at end of file |
|
73 | __all__ = ['forward_read_events'] |
@@ -3,10 +3,16 b'' | |||||
3 | """ |
|
3 | """ | |
4 | Job and task components for writing .xml files that the Windows HPC Server |
|
4 | Job and task components for writing .xml files that the Windows HPC Server | |
5 | 2008 can use to start jobs. |
|
5 | 2008 can use to start jobs. | |
|
6 | ||||
|
7 | Authors: | |||
|
8 | ||||
|
9 | * Brian Granger | |||
|
10 | * MinRK | |||
|
11 | ||||
6 | """ |
|
12 | """ | |
7 |
|
13 | |||
8 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
9 |
# Copyright (C) 2008-20 |
|
15 | # Copyright (C) 2008-2011 The IPython Development Team | |
10 | # |
|
16 | # | |
11 | # Distributed under the terms of the BSD License. The full license is in |
|
17 | # Distributed under the terms of the BSD License. The full license is in | |
12 | # the file COPYING, distributed as part of this software. |
|
18 | # the file COPYING, distributed as part of this software. | |
@@ -24,8 +30,8 b' from xml.etree import ElementTree as ET' | |||||
24 |
|
30 | |||
25 | from IPython.config.configurable import Configurable |
|
31 | from IPython.config.configurable import Configurable | |
26 | from IPython.utils.traitlets import ( |
|
32 | from IPython.utils.traitlets import ( | |
27 |
|
|
33 | Unicode, Int, List, Instance, | |
28 |
Enum, Bool |
|
34 | Enum, Bool | |
29 | ) |
|
35 | ) | |
30 |
|
36 | |||
31 | #----------------------------------------------------------------------------- |
|
37 | #----------------------------------------------------------------------------- | |
@@ -74,27 +80,27 b' def find_username():' | |||||
74 |
|
80 | |||
75 | class WinHPCJob(Configurable): |
|
81 | class WinHPCJob(Configurable): | |
76 |
|
82 | |||
77 |
job_id = |
|
83 | job_id = Unicode('') | |
78 |
job_name = |
|
84 | job_name = Unicode('MyJob', config=True) | |
79 | min_cores = Int(1, config=True) |
|
85 | min_cores = Int(1, config=True) | |
80 | max_cores = Int(1, config=True) |
|
86 | max_cores = Int(1, config=True) | |
81 | min_sockets = Int(1, config=True) |
|
87 | min_sockets = Int(1, config=True) | |
82 | max_sockets = Int(1, config=True) |
|
88 | max_sockets = Int(1, config=True) | |
83 | min_nodes = Int(1, config=True) |
|
89 | min_nodes = Int(1, config=True) | |
84 | max_nodes = Int(1, config=True) |
|
90 | max_nodes = Int(1, config=True) | |
85 |
unit_type = |
|
91 | unit_type = Unicode("Core", config=True) | |
86 | auto_calculate_min = Bool(True, config=True) |
|
92 | auto_calculate_min = Bool(True, config=True) | |
87 | auto_calculate_max = Bool(True, config=True) |
|
93 | auto_calculate_max = Bool(True, config=True) | |
88 | run_until_canceled = Bool(False, config=True) |
|
94 | run_until_canceled = Bool(False, config=True) | |
89 | is_exclusive = Bool(False, config=True) |
|
95 | is_exclusive = Bool(False, config=True) | |
90 |
username = |
|
96 | username = Unicode(find_username(), config=True) | |
91 |
job_type = |
|
97 | job_type = Unicode('Batch', config=True) | |
92 | priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'), |
|
98 | priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'), | |
93 | default_value='Highest', config=True) |
|
99 | default_value='Highest', config=True) | |
94 |
requested_nodes = |
|
100 | requested_nodes = Unicode('', config=True) | |
95 |
project = |
|
101 | project = Unicode('IPython', config=True) | |
96 |
xmlns = |
|
102 | xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/') | |
97 |
version = |
|
103 | version = Unicode("2.000") | |
98 | tasks = List([]) |
|
104 | tasks = List([]) | |
99 |
|
105 | |||
100 | @property |
|
106 | @property | |
@@ -165,21 +171,21 b' class WinHPCJob(Configurable):' | |||||
165 |
|
171 | |||
166 | class WinHPCTask(Configurable): |
|
172 | class WinHPCTask(Configurable): | |
167 |
|
173 | |||
168 |
task_id = |
|
174 | task_id = Unicode('') | |
169 |
task_name = |
|
175 | task_name = Unicode('') | |
170 |
version = |
|
176 | version = Unicode("2.000") | |
171 | min_cores = Int(1, config=True) |
|
177 | min_cores = Int(1, config=True) | |
172 | max_cores = Int(1, config=True) |
|
178 | max_cores = Int(1, config=True) | |
173 | min_sockets = Int(1, config=True) |
|
179 | min_sockets = Int(1, config=True) | |
174 | max_sockets = Int(1, config=True) |
|
180 | max_sockets = Int(1, config=True) | |
175 | min_nodes = Int(1, config=True) |
|
181 | min_nodes = Int(1, config=True) | |
176 | max_nodes = Int(1, config=True) |
|
182 | max_nodes = Int(1, config=True) | |
177 |
unit_type = |
|
183 | unit_type = Unicode("Core", config=True) | |
178 |
command_line = |
|
184 | command_line = Unicode('', config=True) | |
179 |
work_directory = |
|
185 | work_directory = Unicode('', config=True) | |
180 | is_rerunnaable = Bool(True, config=True) |
|
186 | is_rerunnaable = Bool(True, config=True) | |
181 |
std_out_file_path = |
|
187 | std_out_file_path = Unicode('', config=True) | |
182 |
std_err_file_path = |
|
188 | std_err_file_path = Unicode('', config=True) | |
183 | is_parametric = Bool(False, config=True) |
|
189 | is_parametric = Bool(False, config=True) | |
184 | environment_variables = Instance(dict, args=(), config=True) |
|
190 | environment_variables = Instance(dict, args=(), config=True) | |
185 |
|
191 | |||
@@ -223,41 +229,41 b' class WinHPCTask(Configurable):' | |||||
223 | # By declaring these, we can configure the controller and engine separately! |
|
229 | # By declaring these, we can configure the controller and engine separately! | |
224 |
|
230 | |||
225 | class IPControllerJob(WinHPCJob): |
|
231 | class IPControllerJob(WinHPCJob): | |
226 |
job_name = |
|
232 | job_name = Unicode('IPController', config=False) | |
227 | is_exclusive = Bool(False, config=True) |
|
233 | is_exclusive = Bool(False, config=True) | |
228 |
username = |
|
234 | username = Unicode(find_username(), config=True) | |
229 | priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'), |
|
235 | priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'), | |
230 | default_value='Highest', config=True) |
|
236 | default_value='Highest', config=True) | |
231 |
requested_nodes = |
|
237 | requested_nodes = Unicode('', config=True) | |
232 |
project = |
|
238 | project = Unicode('IPython', config=True) | |
233 |
|
239 | |||
234 |
|
240 | |||
235 | class IPEngineSetJob(WinHPCJob): |
|
241 | class IPEngineSetJob(WinHPCJob): | |
236 |
job_name = |
|
242 | job_name = Unicode('IPEngineSet', config=False) | |
237 | is_exclusive = Bool(False, config=True) |
|
243 | is_exclusive = Bool(False, config=True) | |
238 |
username = |
|
244 | username = Unicode(find_username(), config=True) | |
239 | priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'), |
|
245 | priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'), | |
240 | default_value='Highest', config=True) |
|
246 | default_value='Highest', config=True) | |
241 |
requested_nodes = |
|
247 | requested_nodes = Unicode('', config=True) | |
242 |
project = |
|
248 | project = Unicode('IPython', config=True) | |
243 |
|
249 | |||
244 |
|
250 | |||
245 | class IPControllerTask(WinHPCTask): |
|
251 | class IPControllerTask(WinHPCTask): | |
246 |
|
252 | |||
247 |
task_name = |
|
253 | task_name = Unicode('IPController', config=True) | |
248 | controller_cmd = List(['ipcontroller.exe'], config=True) |
|
254 | controller_cmd = List(['ipcontroller.exe'], config=True) | |
249 |
controller_args = List(['--log-to-file', ' |
|
255 | controller_args = List(['--log-to-file', 'log-level=40'], config=True) | |
250 | # I don't want these to be configurable |
|
256 | # I don't want these to be configurable | |
251 |
std_out_file_path = |
|
257 | std_out_file_path = Unicode('', config=False) | |
252 |
std_err_file_path = |
|
258 | std_err_file_path = Unicode('', config=False) | |
253 | min_cores = Int(1, config=False) |
|
259 | min_cores = Int(1, config=False) | |
254 | max_cores = Int(1, config=False) |
|
260 | max_cores = Int(1, config=False) | |
255 | min_sockets = Int(1, config=False) |
|
261 | min_sockets = Int(1, config=False) | |
256 | max_sockets = Int(1, config=False) |
|
262 | max_sockets = Int(1, config=False) | |
257 | min_nodes = Int(1, config=False) |
|
263 | min_nodes = Int(1, config=False) | |
258 | max_nodes = Int(1, config=False) |
|
264 | max_nodes = Int(1, config=False) | |
259 |
unit_type = |
|
265 | unit_type = Unicode("Core", config=False) | |
260 |
work_directory = |
|
266 | work_directory = Unicode('', config=False) | |
261 |
|
267 | |||
262 | def __init__(self, config=None): |
|
268 | def __init__(self, config=None): | |
263 | super(IPControllerTask, self).__init__(config=config) |
|
269 | super(IPControllerTask, self).__init__(config=config) | |
@@ -272,20 +278,20 b' class IPControllerTask(WinHPCTask):' | |||||
272 |
|
278 | |||
273 | class IPEngineTask(WinHPCTask): |
|
279 | class IPEngineTask(WinHPCTask): | |
274 |
|
280 | |||
275 |
task_name = |
|
281 | task_name = Unicode('IPEngine', config=True) | |
276 | engine_cmd = List(['ipengine.exe'], config=True) |
|
282 | engine_cmd = List(['ipengine.exe'], config=True) | |
277 |
engine_args = List(['--log-to-file', ' |
|
283 | engine_args = List(['--log-to-file', 'log_level=40'], config=True) | |
278 | # I don't want these to be configurable |
|
284 | # I don't want these to be configurable | |
279 |
std_out_file_path = |
|
285 | std_out_file_path = Unicode('', config=False) | |
280 |
std_err_file_path = |
|
286 | std_err_file_path = Unicode('', config=False) | |
281 | min_cores = Int(1, config=False) |
|
287 | min_cores = Int(1, config=False) | |
282 | max_cores = Int(1, config=False) |
|
288 | max_cores = Int(1, config=False) | |
283 | min_sockets = Int(1, config=False) |
|
289 | min_sockets = Int(1, config=False) | |
284 | max_sockets = Int(1, config=False) |
|
290 | max_sockets = Int(1, config=False) | |
285 | min_nodes = Int(1, config=False) |
|
291 | min_nodes = Int(1, config=False) | |
286 | max_nodes = Int(1, config=False) |
|
292 | max_nodes = Int(1, config=False) | |
287 |
unit_type = |
|
293 | unit_type = Unicode("Core", config=False) | |
288 |
work_directory = |
|
294 | work_directory = Unicode('', config=False) | |
289 |
|
295 | |||
290 | def __init__(self, config=None): |
|
296 | def __init__(self, config=None): | |
291 | super(IPEngineTask,self).__init__(config=config) |
|
297 | super(IPEngineTask,self).__init__(config=config) |
@@ -1,4 +1,9 b'' | |||||
1 |
"""AsyncResult objects for the client |
|
1 | """AsyncResult objects for the client | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * MinRK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010-2011 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # |
@@ -1,6 +1,11 b'' | |||||
1 |
"""A semi-synchronous Client for the ZMQ cluster |
|
1 | """A semi-synchronous Client for the ZMQ cluster | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * MinRK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # | |
5 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
6 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
@@ -23,18 +28,20 b' pjoin = os.path.join' | |||||
23 | import zmq |
|
28 | import zmq | |
24 | # from zmq.eventloop import ioloop, zmqstream |
|
29 | # from zmq.eventloop import ioloop, zmqstream | |
25 |
|
30 | |||
|
31 | from IPython.utils.jsonutil import rekey | |||
26 | from IPython.utils.path import get_ipython_dir |
|
32 | from IPython.utils.path import get_ipython_dir | |
27 |
from IPython.utils.traitlets import (HasTraits, Int, Instance, |
|
33 | from IPython.utils.traitlets import (HasTraits, Int, Instance, Unicode, | |
28 |
Dict, List, Bool, |
|
34 | Dict, List, Bool, Set) | |
29 | from IPython.external.decorator import decorator |
|
35 | from IPython.external.decorator import decorator | |
30 | from IPython.external.ssh import tunnel |
|
36 | from IPython.external.ssh import tunnel | |
31 |
|
37 | |||
32 | from IPython.parallel import error |
|
38 | from IPython.parallel import error | |
33 | from IPython.parallel import streamsession as ss |
|
|||
34 | from IPython.parallel import util |
|
39 | from IPython.parallel import util | |
35 |
|
40 | |||
|
41 | from IPython.zmq.session import Session, Message | |||
|
42 | ||||
36 | from .asyncresult import AsyncResult, AsyncHubResult |
|
43 | from .asyncresult import AsyncResult, AsyncHubResult | |
37 |
from IPython. |
|
44 | from IPython.core.profiledir import ProfileDir, ProfileDirError | |
38 | from .view import DirectView, LoadBalancedView |
|
45 | from .view import DirectView, LoadBalancedView | |
39 |
|
46 | |||
40 | #-------------------------------------------------------------------------- |
|
47 | #-------------------------------------------------------------------------- | |
@@ -119,11 +126,27 b' class Client(HasTraits):' | |||||
119 | [Default: 'default'] |
|
126 | [Default: 'default'] | |
120 | context : zmq.Context |
|
127 | context : zmq.Context | |
121 | Pass an existing zmq.Context instance, otherwise the client will create its own. |
|
128 | Pass an existing zmq.Context instance, otherwise the client will create its own. | |
122 | username : bytes |
|
|||
123 | set username to be passed to the Session object |
|
|||
124 | debug : bool |
|
129 | debug : bool | |
125 | flag for lots of message printing for debug purposes |
|
130 | flag for lots of message printing for debug purposes | |
|
131 | timeout : int/float | |||
|
132 | time (in seconds) to wait for connection replies from the Hub | |||
|
133 | [Default: 10] | |||
126 |
|
134 | |||
|
135 | #-------------- session related args ---------------- | |||
|
136 | ||||
|
137 | config : Config object | |||
|
138 | If specified, this will be relayed to the Session for configuration | |||
|
139 | username : str | |||
|
140 | set username for the session object | |||
|
141 | packer : str (import_string) or callable | |||
|
142 | Can be either the simple keyword 'json' or 'pickle', or an import_string to a | |||
|
143 | function to serialize messages. Must support same input as | |||
|
144 | JSON, and output must be bytes. | |||
|
145 | You can pass a callable directly as `pack` | |||
|
146 | unpacker : str (import_string) or callable | |||
|
147 | The inverse of packer. Only necessary if packer is specified as *not* one | |||
|
148 | of 'json' or 'pickle'. | |||
|
149 | ||||
127 | #-------------- ssh related args ---------------- |
|
150 | #-------------- ssh related args ---------------- | |
128 | # These are args for configuring the ssh tunnel to be used |
|
151 | # These are args for configuring the ssh tunnel to be used | |
129 | # credentials are used to forward connections over ssh to the Controller |
|
152 | # credentials are used to forward connections over ssh to the Controller | |
@@ -149,9 +172,10 b' class Client(HasTraits):' | |||||
149 |
|
172 | |||
150 | ------- exec authentication args ------- |
|
173 | ------- exec authentication args ------- | |
151 | If even localhost is untrusted, you can have some protection against |
|
174 | If even localhost is untrusted, you can have some protection against | |
152 |
unauthorized execution by |
|
175 | unauthorized execution by signing messages with HMAC digests. | |
153 |
as cleartext, so if someone can snoop your |
|
176 | Messages are still sent as cleartext, so if someone can snoop your | |
154 | not help against malicious attacks. |
|
177 | loopback traffic this will not protect your privacy, but will prevent | |
|
178 | unauthorized execution. | |||
155 |
|
179 | |||
156 | exec_key : str |
|
180 | exec_key : str | |
157 | an authentication key or file containing a key |
|
181 | an authentication key or file containing a key | |
@@ -213,7 +237,7 b' class Client(HasTraits):' | |||||
213 | metadata = Instance('collections.defaultdict', (Metadata,)) |
|
237 | metadata = Instance('collections.defaultdict', (Metadata,)) | |
214 | history = List() |
|
238 | history = List() | |
215 | debug = Bool(False) |
|
239 | debug = Bool(False) | |
216 |
profile= |
|
240 | profile=Unicode('default') | |
217 |
|
241 | |||
218 | _outstanding_dict = Instance('collections.defaultdict', (set,)) |
|
242 | _outstanding_dict = Instance('collections.defaultdict', (set,)) | |
219 | _ids = List() |
|
243 | _ids = List() | |
@@ -229,15 +253,15 b' class Client(HasTraits):' | |||||
229 | _notification_socket=Instance('zmq.Socket') |
|
253 | _notification_socket=Instance('zmq.Socket') | |
230 | _mux_socket=Instance('zmq.Socket') |
|
254 | _mux_socket=Instance('zmq.Socket') | |
231 | _task_socket=Instance('zmq.Socket') |
|
255 | _task_socket=Instance('zmq.Socket') | |
232 |
_task_scheme= |
|
256 | _task_scheme=Unicode() | |
233 | _closed = False |
|
257 | _closed = False | |
234 | _ignored_control_replies=Int(0) |
|
258 | _ignored_control_replies=Int(0) | |
235 | _ignored_hub_replies=Int(0) |
|
259 | _ignored_hub_replies=Int(0) | |
236 |
|
260 | |||
237 |
def __init__(self, url_or_file=None, profile='default', |
|
261 | def __init__(self, url_or_file=None, profile='default', profile_dir=None, ipython_dir=None, | |
238 |
context |
|
262 | context=None, debug=False, exec_key=None, | |
239 | sshserver=None, sshkey=None, password=None, paramiko=None, |
|
263 | sshserver=None, sshkey=None, password=None, paramiko=None, | |
240 | timeout=10 |
|
264 | timeout=10, **extra_args | |
241 | ): |
|
265 | ): | |
242 | super(Client, self).__init__(debug=debug, profile=profile) |
|
266 | super(Client, self).__init__(debug=debug, profile=profile) | |
243 | if context is None: |
|
267 | if context is None: | |
@@ -245,7 +269,7 b' class Client(HasTraits):' | |||||
245 | self._context = context |
|
269 | self._context = context | |
246 |
|
270 | |||
247 |
|
271 | |||
248 |
self._setup_ |
|
272 | self._setup_profile_dir(profile, profile_dir, ipython_dir) | |
249 | if self._cd is not None: |
|
273 | if self._cd is not None: | |
250 | if url_or_file is None: |
|
274 | if url_or_file is None: | |
251 | url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json') |
|
275 | url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json') | |
@@ -288,15 +312,15 b' class Client(HasTraits):' | |||||
288 | else: |
|
312 | else: | |
289 | password = getpass("SSH Password for %s: "%sshserver) |
|
313 | password = getpass("SSH Password for %s: "%sshserver) | |
290 | ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko) |
|
314 | ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko) | |
291 | if exec_key is not None and os.path.isfile(exec_key): |
|
315 | ||
292 | arg = 'keyfile' |
|
316 | # configure and construct the session | |
293 | else: |
|
317 | if exec_key is not None: | |
294 | arg = 'key' |
|
318 | if os.path.isfile(exec_key): | |
295 |
|
|
319 | extra_args['keyfile'] = exec_key | |
296 | if username is None: |
|
320 | else: | |
297 | self.session = ss.StreamSession(**key_arg) |
|
321 | extra_args['key'] = exec_key | |
298 | else: |
|
322 | self.session = Session(**extra_args) | |
299 | self.session = ss.StreamSession(username, **key_arg) |
|
323 | ||
300 | self._query_socket = self._context.socket(zmq.XREQ) |
|
324 | self._query_socket = self._context.socket(zmq.XREQ) | |
301 | self._query_socket.setsockopt(zmq.IDENTITY, self.session.session) |
|
325 | self._query_socket.setsockopt(zmq.IDENTITY, self.session.session) | |
302 | if self._ssh: |
|
326 | if self._ssh: | |
@@ -318,21 +342,21 b' class Client(HasTraits):' | |||||
318 | """cleanup sockets, but _not_ context.""" |
|
342 | """cleanup sockets, but _not_ context.""" | |
319 | self.close() |
|
343 | self.close() | |
320 |
|
344 | |||
321 |
def _setup_ |
|
345 | def _setup_profile_dir(self, profile, profile_dir, ipython_dir): | |
322 | if ipython_dir is None: |
|
346 | if ipython_dir is None: | |
323 | ipython_dir = get_ipython_dir() |
|
347 | ipython_dir = get_ipython_dir() | |
324 |
if |
|
348 | if profile_dir is not None: | |
325 | try: |
|
349 | try: | |
326 |
self._cd = |
|
350 | self._cd = ProfileDir.find_profile_dir(profile_dir) | |
327 | return |
|
351 | return | |
328 |
except |
|
352 | except ProfileDirError: | |
329 | pass |
|
353 | pass | |
330 | elif profile is not None: |
|
354 | elif profile is not None: | |
331 | try: |
|
355 | try: | |
332 |
self._cd = |
|
356 | self._cd = ProfileDir.find_profile_dir_by_name( | |
333 | ipython_dir, profile) |
|
357 | ipython_dir, profile) | |
334 | return |
|
358 | return | |
335 |
except |
|
359 | except ProfileDirError: | |
336 | pass |
|
360 | pass | |
337 | self._cd = None |
|
361 | self._cd = None | |
338 |
|
362 | |||
@@ -416,7 +440,7 b' class Client(HasTraits):' | |||||
416 | idents,msg = self.session.recv(self._query_socket,mode=0) |
|
440 | idents,msg = self.session.recv(self._query_socket,mode=0) | |
417 | if self.debug: |
|
441 | if self.debug: | |
418 | pprint(msg) |
|
442 | pprint(msg) | |
419 |
msg = |
|
443 | msg = Message(msg) | |
420 | content = msg.content |
|
444 | content = msg.content | |
421 | self._config['registration'] = dict(content) |
|
445 | self._config['registration'] = dict(content) | |
422 | if content.status == 'ok': |
|
446 | if content.status == 'ok': | |
@@ -478,11 +502,11 b' class Client(HasTraits):' | |||||
478 | md['engine_id'] = self._engines.get(md['engine_uuid'], None) |
|
502 | md['engine_id'] = self._engines.get(md['engine_uuid'], None) | |
479 |
|
503 | |||
480 | if 'date' in parent: |
|
504 | if 'date' in parent: | |
481 |
md['submitted'] = |
|
505 | md['submitted'] = parent['date'] | |
482 | if 'started' in header: |
|
506 | if 'started' in header: | |
483 |
md['started'] = |
|
507 | md['started'] = header['started'] | |
484 | if 'date' in header: |
|
508 | if 'date' in header: | |
485 |
md['completed'] = |
|
509 | md['completed'] = header['date'] | |
486 | return md |
|
510 | return md | |
487 |
|
511 | |||
488 | def _register_engine(self, msg): |
|
512 | def _register_engine(self, msg): | |
@@ -528,7 +552,7 b' class Client(HasTraits):' | |||||
528 | header = {} |
|
552 | header = {} | |
529 | parent['msg_id'] = msg_id |
|
553 | parent['msg_id'] = msg_id | |
530 | header['engine'] = uuid |
|
554 | header['engine'] = uuid | |
531 |
header['date'] = datetime.now() |
|
555 | header['date'] = datetime.now() | |
532 | msg = dict(parent_header=parent, header=header, content=content) |
|
556 | msg = dict(parent_header=parent, header=header, content=content) | |
533 | self._handle_apply_reply(msg) |
|
557 | self._handle_apply_reply(msg) | |
534 |
|
558 | |||
@@ -589,33 +613,31 b' class Client(HasTraits):' | |||||
589 | def _flush_notifications(self): |
|
613 | def _flush_notifications(self): | |
590 | """Flush notifications of engine registrations waiting |
|
614 | """Flush notifications of engine registrations waiting | |
591 | in ZMQ queue.""" |
|
615 | in ZMQ queue.""" | |
592 | msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) |
|
616 | idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) | |
593 | while msg is not None: |
|
617 | while msg is not None: | |
594 | if self.debug: |
|
618 | if self.debug: | |
595 | pprint(msg) |
|
619 | pprint(msg) | |
596 | msg = msg[-1] |
|
|||
597 | msg_type = msg['msg_type'] |
|
620 | msg_type = msg['msg_type'] | |
598 | handler = self._notification_handlers.get(msg_type, None) |
|
621 | handler = self._notification_handlers.get(msg_type, None) | |
599 | if handler is None: |
|
622 | if handler is None: | |
600 | raise Exception("Unhandled message type: %s"%msg.msg_type) |
|
623 | raise Exception("Unhandled message type: %s"%msg.msg_type) | |
601 | else: |
|
624 | else: | |
602 | handler(msg) |
|
625 | handler(msg) | |
603 | msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) |
|
626 | idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) | |
604 |
|
627 | |||
605 | def _flush_results(self, sock): |
|
628 | def _flush_results(self, sock): | |
606 | """Flush task or queue results waiting in ZMQ queue.""" |
|
629 | """Flush task or queue results waiting in ZMQ queue.""" | |
607 | msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
630 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
608 | while msg is not None: |
|
631 | while msg is not None: | |
609 | if self.debug: |
|
632 | if self.debug: | |
610 | pprint(msg) |
|
633 | pprint(msg) | |
611 | msg = msg[-1] |
|
|||
612 | msg_type = msg['msg_type'] |
|
634 | msg_type = msg['msg_type'] | |
613 | handler = self._queue_handlers.get(msg_type, None) |
|
635 | handler = self._queue_handlers.get(msg_type, None) | |
614 | if handler is None: |
|
636 | if handler is None: | |
615 | raise Exception("Unhandled message type: %s"%msg.msg_type) |
|
637 | raise Exception("Unhandled message type: %s"%msg.msg_type) | |
616 | else: |
|
638 | else: | |
617 | handler(msg) |
|
639 | handler(msg) | |
618 | msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
640 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
619 |
|
641 | |||
620 | def _flush_control(self, sock): |
|
642 | def _flush_control(self, sock): | |
621 | """Flush replies from the control channel waiting |
|
643 | """Flush replies from the control channel waiting | |
@@ -624,12 +646,12 b' class Client(HasTraits):' | |||||
624 | Currently: ignore them.""" |
|
646 | Currently: ignore them.""" | |
625 | if self._ignored_control_replies <= 0: |
|
647 | if self._ignored_control_replies <= 0: | |
626 | return |
|
648 | return | |
627 | msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
649 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
628 | while msg is not None: |
|
650 | while msg is not None: | |
629 | self._ignored_control_replies -= 1 |
|
651 | self._ignored_control_replies -= 1 | |
630 | if self.debug: |
|
652 | if self.debug: | |
631 | pprint(msg) |
|
653 | pprint(msg) | |
632 | msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
654 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
633 |
|
655 | |||
634 | def _flush_ignored_control(self): |
|
656 | def _flush_ignored_control(self): | |
635 | """flush ignored control replies""" |
|
657 | """flush ignored control replies""" | |
@@ -638,19 +660,18 b' class Client(HasTraits):' | |||||
638 | self._ignored_control_replies -= 1 |
|
660 | self._ignored_control_replies -= 1 | |
639 |
|
661 | |||
640 | def _flush_ignored_hub_replies(self): |
|
662 | def _flush_ignored_hub_replies(self): | |
641 | msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) |
|
663 | ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) | |
642 | while msg is not None: |
|
664 | while msg is not None: | |
643 | msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) |
|
665 | ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) | |
644 |
|
666 | |||
645 | def _flush_iopub(self, sock): |
|
667 | def _flush_iopub(self, sock): | |
646 | """Flush replies from the iopub channel waiting |
|
668 | """Flush replies from the iopub channel waiting | |
647 | in the ZMQ queue. |
|
669 | in the ZMQ queue. | |
648 | """ |
|
670 | """ | |
649 | msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
671 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
650 | while msg is not None: |
|
672 | while msg is not None: | |
651 | if self.debug: |
|
673 | if self.debug: | |
652 | pprint(msg) |
|
674 | pprint(msg) | |
653 | msg = msg[-1] |
|
|||
654 | parent = msg['parent_header'] |
|
675 | parent = msg['parent_header'] | |
655 | msg_id = parent['msg_id'] |
|
676 | msg_id = parent['msg_id'] | |
656 | content = msg['content'] |
|
677 | content = msg['content'] | |
@@ -674,7 +695,7 b' class Client(HasTraits):' | |||||
674 | # reduntant? |
|
695 | # reduntant? | |
675 | self.metadata[msg_id] = md |
|
696 | self.metadata[msg_id] = md | |
676 |
|
697 | |||
677 | msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
698 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
678 |
|
699 | |||
679 | #-------------------------------------------------------------------------- |
|
700 | #-------------------------------------------------------------------------- | |
680 | # len, getitem |
|
701 | # len, getitem | |
@@ -1172,6 +1193,7 b' class Client(HasTraits):' | |||||
1172 | failures = [] |
|
1193 | failures = [] | |
1173 | # load cached results into result: |
|
1194 | # load cached results into result: | |
1174 | content.update(local_results) |
|
1195 | content.update(local_results) | |
|
1196 | ||||
1175 | # update cache with results: |
|
1197 | # update cache with results: | |
1176 | for msg_id in sorted(theids): |
|
1198 | for msg_id in sorted(theids): | |
1177 | if msg_id in content['completed']: |
|
1199 | if msg_id in content['completed']: | |
@@ -1226,7 +1248,7 b' class Client(HasTraits):' | |||||
1226 | status = content.pop('status') |
|
1248 | status = content.pop('status') | |
1227 | if status != 'ok': |
|
1249 | if status != 'ok': | |
1228 | raise self._unwrap_exception(content) |
|
1250 | raise self._unwrap_exception(content) | |
1229 |
content = |
|
1251 | content = rekey(content) | |
1230 | if isinstance(targets, int): |
|
1252 | if isinstance(targets, int): | |
1231 | return content[targets] |
|
1253 | return content[targets] | |
1232 | else: |
|
1254 | else: | |
@@ -1332,6 +1354,7 b' class Client(HasTraits):' | |||||
1332 | raise self._unwrap_exception(content) |
|
1354 | raise self._unwrap_exception(content) | |
1333 |
|
1355 | |||
1334 | records = content['records'] |
|
1356 | records = content['records'] | |
|
1357 | ||||
1335 | buffer_lens = content['buffer_lens'] |
|
1358 | buffer_lens = content['buffer_lens'] | |
1336 | result_buffer_lens = content['result_buffer_lens'] |
|
1359 | result_buffer_lens = content['result_buffer_lens'] | |
1337 | buffers = msg['buffers'] |
|
1360 | buffers = msg['buffers'] | |
@@ -1345,11 +1368,6 b' class Client(HasTraits):' | |||||
1345 | if has_rbufs: |
|
1368 | if has_rbufs: | |
1346 | blen = result_buffer_lens[i] |
|
1369 | blen = result_buffer_lens[i] | |
1347 | rec['result_buffers'], buffers = buffers[:blen],buffers[blen:] |
|
1370 | rec['result_buffers'], buffers = buffers[:blen],buffers[blen:] | |
1348 | # turn timestamps back into times |
|
|||
1349 | for key in 'submitted started completed resubmitted'.split(): |
|
|||
1350 | maybedate = rec.get(key, None) |
|
|||
1351 | if maybedate and util.ISO8601_RE.match(maybedate): |
|
|||
1352 | rec[key] = datetime.strptime(maybedate, util.ISO8601) |
|
|||
1353 |
|
1371 | |||
1354 | return records |
|
1372 | return records | |
1355 |
|
1373 |
@@ -4,12 +4,19 b'' | |||||
4 |
|
4 | |||
5 | Scattering consists of partitioning a sequence and sending the various |
|
5 | Scattering consists of partitioning a sequence and sending the various | |
6 | pieces to individual nodes in a cluster. |
|
6 | pieces to individual nodes in a cluster. | |
|
7 | ||||
|
8 | ||||
|
9 | Authors: | |||
|
10 | ||||
|
11 | * Brian Granger | |||
|
12 | * MinRK | |||
|
13 | ||||
7 | """ |
|
14 | """ | |
8 |
|
15 | |||
9 | __docformat__ = "restructuredtext en" |
|
16 | __docformat__ = "restructuredtext en" | |
10 |
|
17 | |||
11 | #------------------------------------------------------------------------------- |
|
18 | #------------------------------------------------------------------------------- | |
12 | # Copyright (C) 2008 The IPython Development Team |
|
19 | # Copyright (C) 2008-2011 The IPython Development Team | |
13 | # |
|
20 | # | |
14 | # Distributed under the terms of the BSD License. The full license is in |
|
21 | # Distributed under the terms of the BSD License. The full license is in | |
15 | # the file COPYING, distributed as part of this software. |
|
22 | # the file COPYING, distributed as part of this software. |
@@ -1,6 +1,12 b'' | |||||
1 |
"""Remote Functions and decorators for Views. |
|
1 | """Remote Functions and decorators for Views. | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Brian Granger | |||
|
6 | * Min RK | |||
|
7 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
8 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010 The IPython Development Team |
|
9 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
10 | # | |
5 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | # Distributed under the terms of the BSD License. The full license is in | |
6 | # the file COPYING, distributed as part of this software. |
|
12 | # the file COPYING, distributed as part of this software. |
@@ -1,6 +1,11 b'' | |||||
1 |
"""Views of remote engines. |
|
1 | """Views of remote engines. | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # | |
5 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
6 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. |
@@ -1,4 +1,9 b'' | |||||
1 |
"""Dependency utilities |
|
1 | """Dependency utilities | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010-2011 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # |
@@ -1,6 +1,11 b'' | |||||
1 | """A Task logger that presents our DB interface, |
|
1 | """A Task logger that presents our DB interface, | |
2 | but exists entirely in memory and implemented with dicts. |
|
2 | but exists entirely in memory and implemented with dicts. | |
3 |
|
3 | |||
|
4 | Authors: | |||
|
5 | ||||
|
6 | * Min RK | |||
|
7 | ||||
|
8 | ||||
4 | TaskRecords are dicts of the form: |
|
9 | TaskRecords are dicts of the form: | |
5 | { |
|
10 | { | |
6 | 'msg_id' : str(uuid), |
|
11 | 'msg_id' : str(uuid), | |
@@ -35,7 +40,7 b' We support a subset of mongodb operators:' | |||||
35 | $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists |
|
40 | $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists | |
36 | """ |
|
41 | """ | |
37 | #----------------------------------------------------------------------------- |
|
42 | #----------------------------------------------------------------------------- | |
38 | # Copyright (C) 2010 The IPython Development Team |
|
43 | # Copyright (C) 2010-2011 The IPython Development Team | |
39 | # |
|
44 | # | |
40 | # Distributed under the terms of the BSD License. The full license is in |
|
45 | # Distributed under the terms of the BSD License. The full license is in | |
41 | # the file COPYING, distributed as part of this software. |
|
46 | # the file COPYING, distributed as part of this software. | |
@@ -44,9 +49,9 b' We support a subset of mongodb operators:' | |||||
44 |
|
49 | |||
45 | from datetime import datetime |
|
50 | from datetime import datetime | |
46 |
|
51 | |||
47 | from IPython.config.configurable import Configurable |
|
52 | from IPython.config.configurable import LoggingConfigurable | |
48 |
|
53 | |||
49 |
from IPython.utils.traitlets import Dict, |
|
54 | from IPython.utils.traitlets import Dict, Unicode, Instance | |
50 |
|
55 | |||
51 | filters = { |
|
56 | filters = { | |
52 | '$lt' : lambda a,b: a < b, |
|
57 | '$lt' : lambda a,b: a < b, | |
@@ -79,10 +84,10 b' class CompositeFilter(object):' | |||||
79 | return False |
|
84 | return False | |
80 | return True |
|
85 | return True | |
81 |
|
86 | |||
82 | class BaseDB(Configurable): |
|
87 | class BaseDB(LoggingConfigurable): | |
83 | """Empty Parent class so traitlets work on DB.""" |
|
88 | """Empty Parent class so traitlets work on DB.""" | |
84 | # base configurable traits: |
|
89 | # base configurable traits: | |
85 |
session = |
|
90 | session = Unicode("") | |
86 |
|
91 | |||
87 | class DictDB(BaseDB): |
|
92 | class DictDB(BaseDB): | |
88 | """Basic in-memory dict-based object for saving Task Records. |
|
93 | """Basic in-memory dict-based object for saving Task Records. |
@@ -2,6 +2,10 b'' | |||||
2 | """ |
|
2 | """ | |
3 | A multi-heart Heartbeat system using PUB and XREP sockets. pings are sent out on the PUB, |
|
3 | A multi-heart Heartbeat system using PUB and XREP sockets. pings are sent out on the PUB, | |
4 | and hearts are tracked based on their XREQ identities. |
|
4 | and hearts are tracked based on their XREQ identities. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Min RK | |||
5 | """ |
|
9 | """ | |
6 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
7 | # Copyright (C) 2010-2011 The IPython Development Team |
|
11 | # Copyright (C) 2010-2011 The IPython Development Team | |
@@ -15,11 +19,11 b' import time' | |||||
15 | import uuid |
|
19 | import uuid | |
16 |
|
20 | |||
17 | import zmq |
|
21 | import zmq | |
18 |
from zmq.devices import |
|
22 | from zmq.devices import ThreadDevice | |
19 | from zmq.eventloop import ioloop, zmqstream |
|
23 | from zmq.eventloop import ioloop, zmqstream | |
20 |
|
24 | |||
21 | from IPython.utils.traitlets import Set, Instance, CFloat, Bool |
|
25 | from IPython.config.configurable import LoggingConfigurable | |
22 | from IPython.parallel.factory import LoggingFactory |
|
26 | from IPython.utils.traitlets import Set, Instance, CFloat | |
23 |
|
27 | |||
24 | class Heart(object): |
|
28 | class Heart(object): | |
25 | """A basic heart object for responding to a HeartMonitor. |
|
29 | """A basic heart object for responding to a HeartMonitor. | |
@@ -47,20 +51,22 b' class Heart(object):' | |||||
47 | def start(self): |
|
51 | def start(self): | |
48 | return self.device.start() |
|
52 | return self.device.start() | |
49 |
|
53 | |||
50 |
class HeartMonitor(Logging |
|
54 | class HeartMonitor(LoggingConfigurable): | |
51 | """A basic HeartMonitor class |
|
55 | """A basic HeartMonitor class | |
52 | pingstream: a PUB stream |
|
56 | pingstream: a PUB stream | |
53 | pongstream: an XREP stream |
|
57 | pongstream: an XREP stream | |
54 | period: the period of the heartbeat in milliseconds""" |
|
58 | period: the period of the heartbeat in milliseconds""" | |
55 |
|
59 | |||
56 |
period=CFloat(1000, config=True |
|
60 | period=CFloat(1000, config=True, | |
|
61 | help='The frequency at which the Hub pings the engines for heartbeats ' | |||
|
62 | ' (in ms) [default: 100]', | |||
|
63 | ) | |||
57 |
|
64 | |||
58 | pingstream=Instance('zmq.eventloop.zmqstream.ZMQStream') |
|
65 | pingstream=Instance('zmq.eventloop.zmqstream.ZMQStream') | |
59 | pongstream=Instance('zmq.eventloop.zmqstream.ZMQStream') |
|
66 | pongstream=Instance('zmq.eventloop.zmqstream.ZMQStream') | |
60 | loop = Instance('zmq.eventloop.ioloop.IOLoop') |
|
67 | loop = Instance('zmq.eventloop.ioloop.IOLoop') | |
61 | def _loop_default(self): |
|
68 | def _loop_default(self): | |
62 | return ioloop.IOLoop.instance() |
|
69 | return ioloop.IOLoop.instance() | |
63 | debug=Bool(False) |
|
|||
64 |
|
70 | |||
65 | # not settable: |
|
71 | # not settable: | |
66 | hearts=Set() |
|
72 | hearts=Set() |
@@ -2,6 +2,10 b'' | |||||
2 | """The IPython Controller Hub with 0MQ |
|
2 | """The IPython Controller Hub with 0MQ | |
3 | This is the master object that handles connections from engines and clients, |
|
3 | This is the master object that handles connections from engines and clients, | |
4 | and monitors traffic through the various queues. |
|
4 | and monitors traffic through the various queues. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Min RK | |||
5 | """ |
|
9 | """ | |
6 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
7 | # Copyright (C) 2010 The IPython Development Team |
|
11 | # Copyright (C) 2010 The IPython Development Team | |
@@ -25,10 +29,14 b' from zmq.eventloop.zmqstream import ZMQStream' | |||||
25 |
|
29 | |||
26 | # internal: |
|
30 | # internal: | |
27 | from IPython.utils.importstring import import_item |
|
31 | from IPython.utils.importstring import import_item | |
28 | from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool |
|
32 | from IPython.utils.traitlets import ( | |
|
33 | HasTraits, Instance, Int, Unicode, Dict, Set, Tuple, CStr | |||
|
34 | ) | |||
29 |
|
35 | |||
30 | from IPython.parallel import error, util |
|
36 | from IPython.parallel import error, util | |
31 |
from IPython.parallel.factory import RegistrationFactory |
|
37 | from IPython.parallel.factory import RegistrationFactory | |
|
38 | ||||
|
39 | from IPython.zmq.session import SessionFactory | |||
32 |
|
40 | |||
33 | from .heartmonitor import HeartMonitor |
|
41 | from .heartmonitor import HeartMonitor | |
34 |
|
42 | |||
@@ -75,7 +83,7 b' def init_record(msg):' | |||||
75 | 'header' : header, |
|
83 | 'header' : header, | |
76 | 'content': msg['content'], |
|
84 | 'content': msg['content'], | |
77 | 'buffers': msg['buffers'], |
|
85 | 'buffers': msg['buffers'], | |
78 |
'submitted': |
|
86 | 'submitted': header['date'], | |
79 | 'client_uuid' : None, |
|
87 | 'client_uuid' : None, | |
80 | 'engine_uuid' : None, |
|
88 | 'engine_uuid' : None, | |
81 | 'started': None, |
|
89 | 'started': None, | |
@@ -103,68 +111,80 b' class EngineConnector(HasTraits):' | |||||
103 | heartbeat (str): identity of heartbeat XREQ socket |
|
111 | heartbeat (str): identity of heartbeat XREQ socket | |
104 | """ |
|
112 | """ | |
105 | id=Int(0) |
|
113 | id=Int(0) | |
106 | queue=Str() |
|
114 | queue=CStr() | |
107 | control=Str() |
|
115 | control=CStr() | |
108 | registration=Str() |
|
116 | registration=CStr() | |
109 | heartbeat=Str() |
|
117 | heartbeat=CStr() | |
110 | pending=Set() |
|
118 | pending=Set() | |
111 |
|
119 | |||
112 | class HubFactory(RegistrationFactory): |
|
120 | class HubFactory(RegistrationFactory): | |
113 | """The Configurable for setting up a Hub.""" |
|
121 | """The Configurable for setting up a Hub.""" | |
114 |
|
122 | |||
115 | # name of a scheduler scheme |
|
|||
116 | scheme = Str('leastload', config=True) |
|
|||
117 |
|
||||
118 | # port-pairs for monitoredqueues: |
|
123 | # port-pairs for monitoredqueues: | |
119 |
hb = |
|
124 | hb = Tuple(Int,Int,config=True, | |
|
125 | help="""XREQ/SUB Port pair for Engine heartbeats""") | |||
120 | def _hb_default(self): |
|
126 | def _hb_default(self): | |
121 | return util.select_random_ports(2) |
|
127 | return tuple(util.select_random_ports(2)) | |
|
128 | ||||
|
129 | mux = Tuple(Int,Int,config=True, | |||
|
130 | help="""Engine/Client Port pair for MUX queue""") | |||
122 |
|
131 | |||
123 | mux = Instance(list, config=True) |
|
|||
124 | def _mux_default(self): |
|
132 | def _mux_default(self): | |
125 | return util.select_random_ports(2) |
|
133 | return tuple(util.select_random_ports(2)) | |
126 |
|
134 | |||
127 |
task = |
|
135 | task = Tuple(Int,Int,config=True, | |
|
136 | help="""Engine/Client Port pair for Task queue""") | |||
128 | def _task_default(self): |
|
137 | def _task_default(self): | |
129 | return util.select_random_ports(2) |
|
138 | return tuple(util.select_random_ports(2)) | |
|
139 | ||||
|
140 | control = Tuple(Int,Int,config=True, | |||
|
141 | help="""Engine/Client Port pair for Control queue""") | |||
130 |
|
142 | |||
131 | control = Instance(list, config=True) |
|
|||
132 | def _control_default(self): |
|
143 | def _control_default(self): | |
133 | return util.select_random_ports(2) |
|
144 | return tuple(util.select_random_ports(2)) | |
|
145 | ||||
|
146 | iopub = Tuple(Int,Int,config=True, | |||
|
147 | help="""Engine/Client Port pair for IOPub relay""") | |||
134 |
|
148 | |||
135 | iopub = Instance(list, config=True) |
|
|||
136 | def _iopub_default(self): |
|
149 | def _iopub_default(self): | |
137 | return util.select_random_ports(2) |
|
150 | return tuple(util.select_random_ports(2)) | |
138 |
|
151 | |||
139 | # single ports: |
|
152 | # single ports: | |
140 |
mon_port = In |
|
153 | mon_port = Int(config=True, | |
|
154 | help="""Monitor (SUB) port for queue traffic""") | |||
|
155 | ||||
141 | def _mon_port_default(self): |
|
156 | def _mon_port_default(self): | |
142 | return util.select_random_ports(1)[0] |
|
157 | return util.select_random_ports(1)[0] | |
143 |
|
158 | |||
144 |
notifier_port = In |
|
159 | notifier_port = Int(config=True, | |
|
160 | help="""PUB port for sending engine status notifications""") | |||
|
161 | ||||
145 | def _notifier_port_default(self): |
|
162 | def _notifier_port_default(self): | |
146 | return util.select_random_ports(1)[0] |
|
163 | return util.select_random_ports(1)[0] | |
147 |
|
164 | |||
148 | ping = Int(1000, config=True) # ping frequency |
|
165 | engine_ip = Unicode('127.0.0.1', config=True, | |
|
166 | help="IP on which to listen for engine connections. [default: loopback]") | |||
|
167 | engine_transport = Unicode('tcp', config=True, | |||
|
168 | help="0MQ transport for engine connections. [default: tcp]") | |||
149 |
|
169 | |||
150 |
|
|
170 | client_ip = Unicode('127.0.0.1', config=True, | |
151 | engine_transport = CStr('tcp', config=True) |
|
171 | help="IP on which to listen for client connections. [default: loopback]") | |
|
172 | client_transport = Unicode('tcp', config=True, | |||
|
173 | help="0MQ transport for client connections. [default : tcp]") | |||
152 |
|
174 | |||
153 |
|
|
175 | monitor_ip = Unicode('127.0.0.1', config=True, | |
154 | client_transport = CStr('tcp', config=True) |
|
176 | help="IP on which to listen for monitor messages. [default: loopback]") | |
|
177 | monitor_transport = Unicode('tcp', config=True, | |||
|
178 | help="0MQ transport for monitor messages. [default : tcp]") | |||
155 |
|
179 | |||
156 | monitor_ip = CStr('127.0.0.1', config=True) |
|
180 | monitor_url = Unicode('') | |
157 | monitor_transport = CStr('tcp', config=True) |
|
|||
158 |
|
181 | |||
159 | monitor_url = CStr('') |
|
182 | db_class = Unicode('IPython.parallel.controller.dictdb.DictDB', config=True, | |
160 |
|
183 | help="""The class to use for the DB backend""") | ||
161 | db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True) |
|
|||
162 |
|
184 | |||
163 | # not configurable |
|
185 | # not configurable | |
164 | db = Instance('IPython.parallel.controller.dictdb.BaseDB') |
|
186 | db = Instance('IPython.parallel.controller.dictdb.BaseDB') | |
165 | heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor') |
|
187 | heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor') | |
166 | subconstructors = List() |
|
|||
167 | _constructed = Bool(False) |
|
|||
168 |
|
188 | |||
169 | def _ip_changed(self, name, old, new): |
|
189 | def _ip_changed(self, name, old, new): | |
170 | self.engine_ip = new |
|
190 | self.engine_ip = new | |
@@ -184,26 +204,16 b' class HubFactory(RegistrationFactory):' | |||||
184 | def __init__(self, **kwargs): |
|
204 | def __init__(self, **kwargs): | |
185 | super(HubFactory, self).__init__(**kwargs) |
|
205 | super(HubFactory, self).__init__(**kwargs) | |
186 | self._update_monitor_url() |
|
206 | self._update_monitor_url() | |
187 | # self.on_trait_change(self._sync_ips, 'ip') |
|
|||
188 | # self.on_trait_change(self._sync_transports, 'transport') |
|
|||
189 | self.subconstructors.append(self.construct_hub) |
|
|||
190 |
|
207 | |||
191 |
|
208 | |||
192 | def construct(self): |
|
209 | def construct(self): | |
193 | assert not self._constructed, "already constructed!" |
|
210 | self.init_hub() | |
194 |
|
||||
195 | for subc in self.subconstructors: |
|
|||
196 | subc() |
|
|||
197 |
|
||||
198 | self._constructed = True |
|
|||
199 |
|
||||
200 |
|
211 | |||
201 | def start(self): |
|
212 | def start(self): | |
202 | assert self._constructed, "must be constructed by self.construct() first!" |
|
|||
203 | self.heartmonitor.start() |
|
213 | self.heartmonitor.start() | |
204 | self.log.info("Heartmonitor started") |
|
214 | self.log.info("Heartmonitor started") | |
205 |
|
215 | |||
206 |
def |
|
216 | def init_hub(self): | |
207 | """construct""" |
|
217 | """construct""" | |
208 | client_iface = "%s://%s:"%(self.client_transport, self.client_ip) + "%i" |
|
218 | client_iface = "%s://%s:"%(self.client_transport, self.client_ip) + "%i" | |
209 | engine_iface = "%s://%s:"%(self.engine_transport, self.engine_ip) + "%i" |
|
219 | engine_iface = "%s://%s:"%(self.engine_transport, self.engine_ip) + "%i" | |
@@ -226,8 +236,10 b' class HubFactory(RegistrationFactory):' | |||||
226 | hpub.bind(engine_iface % self.hb[0]) |
|
236 | hpub.bind(engine_iface % self.hb[0]) | |
227 | hrep = ctx.socket(zmq.XREP) |
|
237 | hrep = ctx.socket(zmq.XREP) | |
228 | hrep.bind(engine_iface % self.hb[1]) |
|
238 | hrep.bind(engine_iface % self.hb[1]) | |
229 |
self.heartmonitor = HeartMonitor(loop=loop, |
|
239 | self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log, | |
230 |
p |
|
240 | pingstream=ZMQStream(hpub,loop), | |
|
241 | pongstream=ZMQStream(hrep,loop) | |||
|
242 | ) | |||
231 |
|
243 | |||
232 | ### Client connections ### |
|
244 | ### Client connections ### | |
233 | # Notifier socket |
|
245 | # Notifier socket | |
@@ -246,9 +258,14 b' class HubFactory(RegistrationFactory):' | |||||
246 | # connect the db |
|
258 | # connect the db | |
247 | self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1])) |
|
259 | self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1])) | |
248 | # cdir = self.config.Global.cluster_dir |
|
260 | # cdir = self.config.Global.cluster_dir | |
249 |
self.db = import_item(self.db_class)(session=self.session.session, |
|
261 | self.db = import_item(str(self.db_class))(session=self.session.session, | |
|
262 | config=self.config, log=self.log) | |||
250 | time.sleep(.25) |
|
263 | time.sleep(.25) | |
251 |
|
264 | try: | ||
|
265 | scheme = self.config.TaskScheduler.scheme_name | |||
|
266 | except AttributeError: | |||
|
267 | from .scheduler import TaskScheduler | |||
|
268 | scheme = TaskScheduler.scheme_name.get_default_value() | |||
252 | # build connection dicts |
|
269 | # build connection dicts | |
253 | self.engine_info = { |
|
270 | self.engine_info = { | |
254 | 'control' : engine_iface%self.control[1], |
|
271 | 'control' : engine_iface%self.control[1], | |
@@ -262,7 +279,7 b' class HubFactory(RegistrationFactory):' | |||||
262 | self.client_info = { |
|
279 | self.client_info = { | |
263 | 'control' : client_iface%self.control[0], |
|
280 | 'control' : client_iface%self.control[0], | |
264 | 'mux': client_iface%self.mux[0], |
|
281 | 'mux': client_iface%self.mux[0], | |
265 |
'task' : ( |
|
282 | 'task' : (scheme, client_iface%self.task[0]), | |
266 | 'iopub' : client_iface%self.iopub[0], |
|
283 | 'iopub' : client_iface%self.iopub[0], | |
267 | 'notification': client_iface%self.notifier_port |
|
284 | 'notification': client_iface%self.notifier_port | |
268 | } |
|
285 | } | |
@@ -278,16 +295,16 b' class HubFactory(RegistrationFactory):' | |||||
278 | self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor, |
|
295 | self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor, | |
279 | query=q, notifier=n, resubmit=r, db=self.db, |
|
296 | query=q, notifier=n, resubmit=r, db=self.db, | |
280 | engine_info=self.engine_info, client_info=self.client_info, |
|
297 | engine_info=self.engine_info, client_info=self.client_info, | |
281 |
log |
|
298 | log=self.log) | |
282 |
|
299 | |||
283 |
|
300 | |||
284 |
class Hub( |
|
301 | class Hub(SessionFactory): | |
285 | """The IPython Controller Hub with 0MQ connections |
|
302 | """The IPython Controller Hub with 0MQ connections | |
286 |
|
303 | |||
287 | Parameters |
|
304 | Parameters | |
288 | ========== |
|
305 | ========== | |
289 | loop: zmq IOLoop instance |
|
306 | loop: zmq IOLoop instance | |
290 |
session: S |
|
307 | session: Session object | |
291 | <removed> context: zmq context for creating new connections (?) |
|
308 | <removed> context: zmq context for creating new connections (?) | |
292 | queue: ZMQStream for monitoring the command queue (SUB) |
|
309 | queue: ZMQStream for monitoring the command queue (SUB) | |
293 | query: ZMQStream for engine registration and client queries requests (XREP) |
|
310 | query: ZMQStream for engine registration and client queries requests (XREP) | |
@@ -319,7 +336,6 b' class Hub(LoggingFactory):' | |||||
319 | _idcounter=Int(0) |
|
336 | _idcounter=Int(0) | |
320 |
|
337 | |||
321 | # objects from constructor: |
|
338 | # objects from constructor: | |
322 | loop=Instance(ioloop.IOLoop) |
|
|||
323 | query=Instance(ZMQStream) |
|
339 | query=Instance(ZMQStream) | |
324 | monitor=Instance(ZMQStream) |
|
340 | monitor=Instance(ZMQStream) | |
325 | notifier=Instance(ZMQStream) |
|
341 | notifier=Instance(ZMQStream) | |
@@ -438,34 +454,16 b' class Hub(LoggingFactory):' | |||||
438 | # dispatch methods (1 per stream) |
|
454 | # dispatch methods (1 per stream) | |
439 | #----------------------------------------------------------------------------- |
|
455 | #----------------------------------------------------------------------------- | |
440 |
|
456 | |||
441 | # def dispatch_registration_request(self, msg): |
|
|||
442 | # """""" |
|
|||
443 | # self.log.debug("registration::dispatch_register_request(%s)"%msg) |
|
|||
444 | # idents,msg = self.session.feed_identities(msg) |
|
|||
445 | # if not idents: |
|
|||
446 | # self.log.error("Bad Query Message: %s"%msg, exc_info=True) |
|
|||
447 | # return |
|
|||
448 | # try: |
|
|||
449 | # msg = self.session.unpack_message(msg,content=True) |
|
|||
450 | # except: |
|
|||
451 | # self.log.error("registration::got bad registration message: %s"%msg, exc_info=True) |
|
|||
452 | # return |
|
|||
453 | # |
|
|||
454 | # msg_type = msg['msg_type'] |
|
|||
455 | # content = msg['content'] |
|
|||
456 | # |
|
|||
457 | # handler = self.query_handlers.get(msg_type, None) |
|
|||
458 | # if handler is None: |
|
|||
459 | # self.log.error("registration::got bad registration message: %s"%msg) |
|
|||
460 | # else: |
|
|||
461 | # handler(idents, msg) |
|
|||
462 |
|
457 | |||
463 | def dispatch_monitor_traffic(self, msg): |
|
458 | def dispatch_monitor_traffic(self, msg): | |
464 | """all ME and Task queue messages come through here, as well as |
|
459 | """all ME and Task queue messages come through here, as well as | |
465 | IOPub traffic.""" |
|
460 | IOPub traffic.""" | |
466 | self.log.debug("monitor traffic: %r"%msg[:2]) |
|
461 | self.log.debug("monitor traffic: %r"%msg[:2]) | |
467 | switch = msg[0] |
|
462 | switch = msg[0] | |
468 | idents, msg = self.session.feed_identities(msg[1:]) |
|
463 | try: | |
|
464 | idents, msg = self.session.feed_identities(msg[1:]) | |||
|
465 | except ValueError: | |||
|
466 | idents=[] | |||
469 | if not idents: |
|
467 | if not idents: | |
470 | self.log.error("Bad Monitor Message: %r"%msg) |
|
468 | self.log.error("Bad Monitor Message: %r"%msg) | |
471 | return |
|
469 | return | |
@@ -478,20 +476,22 b' class Hub(LoggingFactory):' | |||||
478 |
|
476 | |||
479 | def dispatch_query(self, msg): |
|
477 | def dispatch_query(self, msg): | |
480 | """Route registration requests and queries from clients.""" |
|
478 | """Route registration requests and queries from clients.""" | |
481 | idents, msg = self.session.feed_identities(msg) |
|
479 | try: | |
|
480 | idents, msg = self.session.feed_identities(msg) | |||
|
481 | except ValueError: | |||
|
482 | idents = [] | |||
482 | if not idents: |
|
483 | if not idents: | |
483 | self.log.error("Bad Query Message: %r"%msg) |
|
484 | self.log.error("Bad Query Message: %r"%msg) | |
484 | return |
|
485 | return | |
485 | client_id = idents[0] |
|
486 | client_id = idents[0] | |
486 | try: |
|
487 | try: | |
487 | msg = self.session.unpack_message(msg, content=True) |
|
488 | msg = self.session.unpack_message(msg, content=True) | |
488 | except: |
|
489 | except Exception: | |
489 | content = error.wrap_exception() |
|
490 | content = error.wrap_exception() | |
490 | self.log.error("Bad Query Message: %r"%msg, exc_info=True) |
|
491 | self.log.error("Bad Query Message: %r"%msg, exc_info=True) | |
491 | self.session.send(self.query, "hub_error", ident=client_id, |
|
492 | self.session.send(self.query, "hub_error", ident=client_id, | |
492 | content=content) |
|
493 | content=content) | |
493 | return |
|
494 | return | |
494 |
|
||||
495 | # print client_id, header, parent, content |
|
495 | # print client_id, header, parent, content | |
496 | #switch on message type: |
|
496 | #switch on message type: | |
497 | msg_type = msg['msg_type'] |
|
497 | msg_type = msg['msg_type'] | |
@@ -546,24 +546,22 b' class Hub(LoggingFactory):' | |||||
546 |
|
546 | |||
547 | def save_queue_request(self, idents, msg): |
|
547 | def save_queue_request(self, idents, msg): | |
548 | if len(idents) < 2: |
|
548 | if len(idents) < 2: | |
549 |
self.log.error("invalid identity prefix: % |
|
549 | self.log.error("invalid identity prefix: %r"%idents) | |
550 | return |
|
550 | return | |
551 | queue_id, client_id = idents[:2] |
|
551 | queue_id, client_id = idents[:2] | |
552 | try: |
|
552 | try: | |
553 |
msg = self.session.unpack_message(msg |
|
553 | msg = self.session.unpack_message(msg) | |
554 | except: |
|
554 | except Exception: | |
555 |
self.log.error("queue::client %r sent invalid message to %r: % |
|
555 | self.log.error("queue::client %r sent invalid message to %r: %r"%(client_id, queue_id, msg), exc_info=True) | |
556 | return |
|
556 | return | |
557 |
|
557 | |||
558 | eid = self.by_ident.get(queue_id, None) |
|
558 | eid = self.by_ident.get(queue_id, None) | |
559 | if eid is None: |
|
559 | if eid is None: | |
560 | self.log.error("queue::target %r not registered"%queue_id) |
|
560 | self.log.error("queue::target %r not registered"%queue_id) | |
561 |
self.log.debug("queue:: valid are: % |
|
561 | self.log.debug("queue:: valid are: %r"%(self.by_ident.keys())) | |
562 | return |
|
562 | return | |
563 |
|
||||
564 | header = msg['header'] |
|
|||
565 | msg_id = header['msg_id'] |
|
|||
566 | record = init_record(msg) |
|
563 | record = init_record(msg) | |
|
564 | msg_id = record['msg_id'] | |||
567 | record['engine_uuid'] = queue_id |
|
565 | record['engine_uuid'] = queue_id | |
568 | record['client_uuid'] = client_id |
|
566 | record['client_uuid'] = client_id | |
569 | record['queue'] = 'mux' |
|
567 | record['queue'] = 'mux' | |
@@ -577,30 +575,36 b' class Hub(LoggingFactory):' | |||||
577 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r"%(msg_id, rvalue, key, evalue)) |
|
575 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r"%(msg_id, rvalue, key, evalue)) | |
578 | elif evalue and not rvalue: |
|
576 | elif evalue and not rvalue: | |
579 | record[key] = evalue |
|
577 | record[key] = evalue | |
580 | self.db.update_record(msg_id, record) |
|
578 | try: | |
|
579 | self.db.update_record(msg_id, record) | |||
|
580 | except Exception: | |||
|
581 | self.log.error("DB Error updating record %r"%msg_id, exc_info=True) | |||
581 | except KeyError: |
|
582 | except KeyError: | |
582 | self.db.add_record(msg_id, record) |
|
583 | try: | |
|
584 | self.db.add_record(msg_id, record) | |||
|
585 | except Exception: | |||
|
586 | self.log.error("DB Error adding record %r"%msg_id, exc_info=True) | |||
|
587 | ||||
583 |
|
588 | |||
584 | self.pending.add(msg_id) |
|
589 | self.pending.add(msg_id) | |
585 | self.queues[eid].append(msg_id) |
|
590 | self.queues[eid].append(msg_id) | |
586 |
|
591 | |||
587 | def save_queue_result(self, idents, msg): |
|
592 | def save_queue_result(self, idents, msg): | |
588 | if len(idents) < 2: |
|
593 | if len(idents) < 2: | |
589 |
self.log.error("invalid identity prefix: % |
|
594 | self.log.error("invalid identity prefix: %r"%idents) | |
590 | return |
|
595 | return | |
591 |
|
596 | |||
592 | client_id, queue_id = idents[:2] |
|
597 | client_id, queue_id = idents[:2] | |
593 | try: |
|
598 | try: | |
594 |
msg = self.session.unpack_message(msg |
|
599 | msg = self.session.unpack_message(msg) | |
595 | except: |
|
600 | except Exception: | |
596 |
self.log.error("queue::engine %r sent invalid message to %r: % |
|
601 | self.log.error("queue::engine %r sent invalid message to %r: %r"%( | |
597 | queue_id,client_id, msg), exc_info=True) |
|
602 | queue_id,client_id, msg), exc_info=True) | |
598 | return |
|
603 | return | |
599 |
|
604 | |||
600 | eid = self.by_ident.get(queue_id, None) |
|
605 | eid = self.by_ident.get(queue_id, None) | |
601 | if eid is None: |
|
606 | if eid is None: | |
602 | self.log.error("queue::unknown engine %r is sending a reply: "%queue_id) |
|
607 | self.log.error("queue::unknown engine %r is sending a reply: "%queue_id) | |
603 | # self.log.debug("queue:: %s"%msg[2:]) |
|
|||
604 | return |
|
608 | return | |
605 |
|
609 | |||
606 | parent = msg['parent_header'] |
|
610 | parent = msg['parent_header'] | |
@@ -615,14 +619,12 b' class Hub(LoggingFactory):' | |||||
615 | elif msg_id not in self.all_completed: |
|
619 | elif msg_id not in self.all_completed: | |
616 | # it could be a result from a dead engine that died before delivering the |
|
620 | # it could be a result from a dead engine that died before delivering the | |
617 | # result |
|
621 | # result | |
618 |
self.log.warn("queue:: unknown msg finished % |
|
622 | self.log.warn("queue:: unknown msg finished %r"%msg_id) | |
619 | return |
|
623 | return | |
620 | # update record anyway, because the unregistration could have been premature |
|
624 | # update record anyway, because the unregistration could have been premature | |
621 | rheader = msg['header'] |
|
625 | rheader = msg['header'] | |
622 |
completed = |
|
626 | completed = rheader['date'] | |
623 | started = rheader.get('started', None) |
|
627 | started = rheader.get('started', None) | |
624 | if started is not None: |
|
|||
625 | started = datetime.strptime(started, util.ISO8601) |
|
|||
626 | result = { |
|
628 | result = { | |
627 | 'result_header' : rheader, |
|
629 | 'result_header' : rheader, | |
628 | 'result_content': msg['content'], |
|
630 | 'result_content': msg['content'], | |
@@ -644,9 +646,9 b' class Hub(LoggingFactory):' | |||||
644 | client_id = idents[0] |
|
646 | client_id = idents[0] | |
645 |
|
647 | |||
646 | try: |
|
648 | try: | |
647 |
msg = self.session.unpack_message(msg |
|
649 | msg = self.session.unpack_message(msg) | |
648 | except: |
|
650 | except Exception: | |
649 |
self.log.error("task::client %r sent invalid task message: % |
|
651 | self.log.error("task::client %r sent invalid task message: %r"%( | |
650 | client_id, msg), exc_info=True) |
|
652 | client_id, msg), exc_info=True) | |
651 | return |
|
653 | return | |
652 | record = init_record(msg) |
|
654 | record = init_record(msg) | |
@@ -678,9 +680,15 b' class Hub(LoggingFactory):' | |||||
678 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r"%(msg_id, rvalue, key, evalue)) |
|
680 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r"%(msg_id, rvalue, key, evalue)) | |
679 | elif evalue and not rvalue: |
|
681 | elif evalue and not rvalue: | |
680 | record[key] = evalue |
|
682 | record[key] = evalue | |
681 | self.db.update_record(msg_id, record) |
|
683 | try: | |
|
684 | self.db.update_record(msg_id, record) | |||
|
685 | except Exception: | |||
|
686 | self.log.error("DB Error updating record %r"%msg_id, exc_info=True) | |||
682 | except KeyError: |
|
687 | except KeyError: | |
683 | self.db.add_record(msg_id, record) |
|
688 | try: | |
|
689 | self.db.add_record(msg_id, record) | |||
|
690 | except Exception: | |||
|
691 | self.log.error("DB Error adding record %r"%msg_id, exc_info=True) | |||
684 | except Exception: |
|
692 | except Exception: | |
685 | self.log.error("DB Error saving task request %r"%msg_id, exc_info=True) |
|
693 | self.log.error("DB Error saving task request %r"%msg_id, exc_info=True) | |
686 |
|
694 | |||
@@ -688,11 +696,10 b' class Hub(LoggingFactory):' | |||||
688 | """save the result of a completed task.""" |
|
696 | """save the result of a completed task.""" | |
689 | client_id = idents[0] |
|
697 | client_id = idents[0] | |
690 | try: |
|
698 | try: | |
691 |
msg = self.session.unpack_message(msg |
|
699 | msg = self.session.unpack_message(msg) | |
692 | except: |
|
700 | except Exception: | |
693 |
self.log.error("task::invalid task result message send to %r: % |
|
701 | self.log.error("task::invalid task result message send to %r: %r"%( | |
694 | client_id, msg), exc_info=True) |
|
702 | client_id, msg), exc_info=True) | |
695 | raise |
|
|||
696 | return |
|
703 | return | |
697 |
|
704 | |||
698 | parent = msg['parent_header'] |
|
705 | parent = msg['parent_header'] | |
@@ -715,10 +722,8 b' class Hub(LoggingFactory):' | |||||
715 | self.completed[eid].append(msg_id) |
|
722 | self.completed[eid].append(msg_id) | |
716 | if msg_id in self.tasks[eid]: |
|
723 | if msg_id in self.tasks[eid]: | |
717 | self.tasks[eid].remove(msg_id) |
|
724 | self.tasks[eid].remove(msg_id) | |
718 |
completed = |
|
725 | completed = header['date'] | |
719 | started = header.get('started', None) |
|
726 | started = header.get('started', None) | |
720 | if started is not None: |
|
|||
721 | started = datetime.strptime(started, util.ISO8601) |
|
|||
722 | result = { |
|
727 | result = { | |
723 | 'result_header' : header, |
|
728 | 'result_header' : header, | |
724 | 'result_content': msg['content'], |
|
729 | 'result_content': msg['content'], | |
@@ -734,12 +739,12 b' class Hub(LoggingFactory):' | |||||
734 | self.log.error("DB Error saving task request %r"%msg_id, exc_info=True) |
|
739 | self.log.error("DB Error saving task request %r"%msg_id, exc_info=True) | |
735 |
|
740 | |||
736 | else: |
|
741 | else: | |
737 |
self.log.debug("task::unknown task % |
|
742 | self.log.debug("task::unknown task %r finished"%msg_id) | |
738 |
|
743 | |||
739 | def save_task_destination(self, idents, msg): |
|
744 | def save_task_destination(self, idents, msg): | |
740 | try: |
|
745 | try: | |
741 | msg = self.session.unpack_message(msg, content=True) |
|
746 | msg = self.session.unpack_message(msg, content=True) | |
742 | except: |
|
747 | except Exception: | |
743 | self.log.error("task::invalid task tracking message", exc_info=True) |
|
748 | self.log.error("task::invalid task tracking message", exc_info=True) | |
744 | return |
|
749 | return | |
745 | content = msg['content'] |
|
750 | content = msg['content'] | |
@@ -748,11 +753,11 b' class Hub(LoggingFactory):' | |||||
748 | engine_uuid = content['engine_id'] |
|
753 | engine_uuid = content['engine_id'] | |
749 | eid = self.by_ident[engine_uuid] |
|
754 | eid = self.by_ident[engine_uuid] | |
750 |
|
755 | |||
751 |
self.log.info("task::task % |
|
756 | self.log.info("task::task %r arrived on %r"%(msg_id, eid)) | |
752 | if msg_id in self.unassigned: |
|
757 | if msg_id in self.unassigned: | |
753 | self.unassigned.remove(msg_id) |
|
758 | self.unassigned.remove(msg_id) | |
754 | # else: |
|
759 | # else: | |
755 |
# self.log.debug("task::task % |
|
760 | # self.log.debug("task::task %r not listed as MIA?!"%(msg_id)) | |
756 |
|
761 | |||
757 | self.tasks[eid].append(msg_id) |
|
762 | self.tasks[eid].append(msg_id) | |
758 | # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid)) |
|
763 | # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid)) | |
@@ -776,13 +781,13 b' class Hub(LoggingFactory):' | |||||
776 | # print (topics) |
|
781 | # print (topics) | |
777 | try: |
|
782 | try: | |
778 | msg = self.session.unpack_message(msg, content=True) |
|
783 | msg = self.session.unpack_message(msg, content=True) | |
779 | except: |
|
784 | except Exception: | |
780 | self.log.error("iopub::invalid IOPub message", exc_info=True) |
|
785 | self.log.error("iopub::invalid IOPub message", exc_info=True) | |
781 | return |
|
786 | return | |
782 |
|
787 | |||
783 | parent = msg['parent_header'] |
|
788 | parent = msg['parent_header'] | |
784 | if not parent: |
|
789 | if not parent: | |
785 |
self.log.error("iopub::invalid IOPub message: % |
|
790 | self.log.error("iopub::invalid IOPub message: %r"%msg) | |
786 | return |
|
791 | return | |
787 | msg_id = parent['msg_id'] |
|
792 | msg_id = parent['msg_id'] | |
788 | msg_type = msg['msg_type'] |
|
793 | msg_type = msg['msg_type'] | |
@@ -822,7 +827,7 b' class Hub(LoggingFactory):' | |||||
822 |
|
827 | |||
823 | def connection_request(self, client_id, msg): |
|
828 | def connection_request(self, client_id, msg): | |
824 | """Reply with connection addresses for clients.""" |
|
829 | """Reply with connection addresses for clients.""" | |
825 |
self.log.info("client::client % |
|
830 | self.log.info("client::client %r connected"%client_id) | |
826 | content = dict(status='ok') |
|
831 | content = dict(status='ok') | |
827 | content.update(self.client_info) |
|
832 | content.update(self.client_info) | |
828 | jsonable = {} |
|
833 | jsonable = {} | |
@@ -894,7 +899,7 b' class Hub(LoggingFactory):' | |||||
894 | dc.start() |
|
899 | dc.start() | |
895 | self.incoming_registrations[heart] = (eid,queue,reg[0],dc) |
|
900 | self.incoming_registrations[heart] = (eid,queue,reg[0],dc) | |
896 | else: |
|
901 | else: | |
897 |
self.log.error("registration::registration %i failed: % |
|
902 | self.log.error("registration::registration %i failed: %r"%(eid, content['evalue'])) | |
898 | return eid |
|
903 | return eid | |
899 |
|
904 | |||
900 | def unregister_engine(self, ident, msg): |
|
905 | def unregister_engine(self, ident, msg): | |
@@ -902,9 +907,9 b' class Hub(LoggingFactory):' | |||||
902 | try: |
|
907 | try: | |
903 | eid = msg['content']['id'] |
|
908 | eid = msg['content']['id'] | |
904 | except: |
|
909 | except: | |
905 |
self.log.error("registration::bad engine id for unregistration: % |
|
910 | self.log.error("registration::bad engine id for unregistration: %r"%ident, exc_info=True) | |
906 | return |
|
911 | return | |
907 |
self.log.info("registration::unregister_engine(% |
|
912 | self.log.info("registration::unregister_engine(%r)"%eid) | |
908 | # print (eid) |
|
913 | # print (eid) | |
909 | uuid = self.keytable[eid] |
|
914 | uuid = self.keytable[eid] | |
910 | content=dict(id=eid, queue=uuid) |
|
915 | content=dict(id=eid, queue=uuid) | |
@@ -1124,7 +1129,7 b' class Hub(LoggingFactory):' | |||||
1124 | elif len(records) < len(msg_ids): |
|
1129 | elif len(records) < len(msg_ids): | |
1125 | missing = [ m for m in msg_ids if m not in found_ids ] |
|
1130 | missing = [ m for m in msg_ids if m not in found_ids ] | |
1126 | try: |
|
1131 | try: | |
1127 |
raise KeyError("No such msg(s): % |
|
1132 | raise KeyError("No such msg(s): %r"%missing) | |
1128 | except KeyError: |
|
1133 | except KeyError: | |
1129 | return finish(error.wrap_exception()) |
|
1134 | return finish(error.wrap_exception()) | |
1130 | elif invalid_ids: |
|
1135 | elif invalid_ids: | |
@@ -1135,9 +1140,10 b' class Hub(LoggingFactory):' | |||||
1135 | return finish(error.wrap_exception()) |
|
1140 | return finish(error.wrap_exception()) | |
1136 |
|
1141 | |||
1137 | # clear the existing records |
|
1142 | # clear the existing records | |
|
1143 | now = datetime.now() | |||
1138 | rec = empty_record() |
|
1144 | rec = empty_record() | |
1139 | map(rec.pop, ['msg_id', 'header', 'content', 'buffers', 'submitted']) |
|
1145 | map(rec.pop, ['msg_id', 'header', 'content', 'buffers', 'submitted']) | |
1140 |
rec['resubmitted'] = |
|
1146 | rec['resubmitted'] = now | |
1141 | rec['queue'] = 'task' |
|
1147 | rec['queue'] = 'task' | |
1142 | rec['client_uuid'] = client_id[0] |
|
1148 | rec['client_uuid'] = client_id[0] | |
1143 | try: |
|
1149 | try: | |
@@ -1151,6 +1157,8 b' class Hub(LoggingFactory):' | |||||
1151 | # send the messages |
|
1157 | # send the messages | |
1152 | for rec in records: |
|
1158 | for rec in records: | |
1153 | header = rec['header'] |
|
1159 | header = rec['header'] | |
|
1160 | # include resubmitted in header to prevent digest collision | |||
|
1161 | header['resubmitted'] = now | |||
1154 | msg = self.session.msg(header['msg_type']) |
|
1162 | msg = self.session.msg(header['msg_type']) | |
1155 | msg['content'] = rec['content'] |
|
1163 | msg['content'] = rec['content'] | |
1156 | msg['header'] = header |
|
1164 | msg['header'] = header | |
@@ -1246,10 +1254,8 b' class Hub(LoggingFactory):' | |||||
1246 | content = msg['content'] |
|
1254 | content = msg['content'] | |
1247 | query = content.get('query', {}) |
|
1255 | query = content.get('query', {}) | |
1248 | keys = content.get('keys', None) |
|
1256 | keys = content.get('keys', None) | |
1249 | query = util.extract_dates(query) |
|
|||
1250 | buffers = [] |
|
1257 | buffers = [] | |
1251 | empty = list() |
|
1258 | empty = list() | |
1252 |
|
||||
1253 | try: |
|
1259 | try: | |
1254 | records = self.db.find_records(query, keys) |
|
1260 | records = self.db.find_records(query, keys) | |
1255 | except Exception as e: |
|
1261 | except Exception as e: |
@@ -1,6 +1,11 b'' | |||||
1 |
"""A TaskRecord backend using mongodb |
|
1 | """A TaskRecord backend using mongodb | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # | |
5 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
6 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
@@ -9,7 +14,7 b'' | |||||
9 | from pymongo import Connection |
|
14 | from pymongo import Connection | |
10 | from pymongo.binary import Binary |
|
15 | from pymongo.binary import Binary | |
11 |
|
16 | |||
12 |
from IPython.utils.traitlets import Dict, List, |
|
17 | from IPython.utils.traitlets import Dict, List, Unicode, Instance | |
13 |
|
18 | |||
14 | from .dictdb import BaseDB |
|
19 | from .dictdb import BaseDB | |
15 |
|
20 | |||
@@ -20,9 +25,20 b' from .dictdb import BaseDB' | |||||
20 | class MongoDB(BaseDB): |
|
25 | class MongoDB(BaseDB): | |
21 | """MongoDB TaskRecord backend.""" |
|
26 | """MongoDB TaskRecord backend.""" | |
22 |
|
27 | |||
23 |
connection_args = List(config=True |
|
28 | connection_args = List(config=True, | |
24 | connection_kwargs = Dict(config=True) # kwargs passed to pymongo.Connection |
|
29 | help="""Positional arguments to be passed to pymongo.Connection. Only | |
25 | database = CUnicode(config=True) # name of the mongodb database |
|
30 | necessary if the default mongodb configuration does not point to your | |
|
31 | mongod instance.""") | |||
|
32 | connection_kwargs = Dict(config=True, | |||
|
33 | help="""Keyword arguments to be passed to pymongo.Connection. Only | |||
|
34 | necessary if the default mongodb configuration does not point to your | |||
|
35 | mongod instance.""" | |||
|
36 | ) | |||
|
37 | database = Unicode(config=True, | |||
|
38 | help="""The MongoDB database name to use for storing tasks for this session. If unspecified, | |||
|
39 | a new database will be created with the Hub's IDENT. Specifying the database will result | |||
|
40 | in tasks from previous sessions being available via Clients' db_query and | |||
|
41 | get_result methods.""") | |||
26 |
|
42 | |||
27 | _connection = Instance(Connection) # pymongo connection |
|
43 | _connection = Instance(Connection) # pymongo connection | |
28 |
|
44 |
@@ -3,6 +3,10 b'' | |||||
3 | The Pure ZMQ scheduler does not allow routing schemes other than LRU, |
|
3 | The Pure ZMQ scheduler does not allow routing schemes other than LRU, | |
4 | nor does it check msg_id DAG dependencies. For those, a slightly slower |
|
4 | nor does it check msg_id DAG dependencies. For those, a slightly slower | |
5 | Python Scheduler exists. |
|
5 | Python Scheduler exists. | |
|
6 | ||||
|
7 | Authors: | |||
|
8 | ||||
|
9 | * Min RK | |||
6 | """ |
|
10 | """ | |
7 | #----------------------------------------------------------------------------- |
|
11 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2010-2011 The IPython Development Team |
|
12 | # Copyright (C) 2010-2011 The IPython Development Team | |
@@ -35,7 +39,7 b' from zmq.eventloop import ioloop, zmqstream' | |||||
35 | # local imports |
|
39 | # local imports | |
36 | from IPython.external.decorator import decorator |
|
40 | from IPython.external.decorator import decorator | |
37 | from IPython.config.loader import Config |
|
41 | from IPython.config.loader import Config | |
38 | from IPython.utils.traitlets import Instance, Dict, List, Set, Int |
|
42 | from IPython.utils.traitlets import Instance, Dict, List, Set, Int, Str, Enum | |
39 |
|
43 | |||
40 | from IPython.parallel import error |
|
44 | from IPython.parallel import error | |
41 | from IPython.parallel.factory import SessionFactory |
|
45 | from IPython.parallel.factory import SessionFactory | |
@@ -126,10 +130,24 b' class TaskScheduler(SessionFactory):' | |||||
126 |
|
130 | |||
127 | """ |
|
131 | """ | |
128 |
|
132 | |||
129 | hwm = Int(0, config=True) # limit number of outstanding tasks |
|
133 | hwm = Int(0, config=True, shortname='hwm', | |
|
134 | help="""specify the High Water Mark (HWM) for the downstream | |||
|
135 | socket in the Task scheduler. This is the maximum number | |||
|
136 | of allowed outstanding tasks on each engine.""" | |||
|
137 | ) | |||
|
138 | scheme_name = Enum(('leastload', 'pure', 'lru', 'plainrandom', 'weighted', 'twobin'), | |||
|
139 | 'leastload', config=True, shortname='scheme', allow_none=False, | |||
|
140 | help="""select the task scheduler scheme [default: Python LRU] | |||
|
141 | Options are: 'pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'""" | |||
|
142 | ) | |||
|
143 | def _scheme_name_changed(self, old, new): | |||
|
144 | self.log.debug("Using scheme %r"%new) | |||
|
145 | self.scheme = globals()[new] | |||
130 |
|
146 | |||
131 | # input arguments: |
|
147 | # input arguments: | |
132 |
scheme = Instance(FunctionType |
|
148 | scheme = Instance(FunctionType) # function for determining the destination | |
|
149 | def _scheme_default(self): | |||
|
150 | return leastload | |||
133 | client_stream = Instance(zmqstream.ZMQStream) # client-facing stream |
|
151 | client_stream = Instance(zmqstream.ZMQStream) # client-facing stream | |
134 | engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream |
|
152 | engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream | |
135 | notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream |
|
153 | notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream | |
@@ -165,7 +183,7 b' class TaskScheduler(SessionFactory):' | |||||
165 | self.notifier_stream.on_recv(self.dispatch_notification) |
|
183 | self.notifier_stream.on_recv(self.dispatch_notification) | |
166 | self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz |
|
184 | self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz | |
167 | self.auditor.start() |
|
185 | self.auditor.start() | |
168 |
self.log.info("Scheduler started |
|
186 | self.log.info("Scheduler started [%s]"%self.scheme_name) | |
169 |
|
187 | |||
170 | def resume_receiving(self): |
|
188 | def resume_receiving(self): | |
171 | """Resume accepting jobs.""" |
|
189 | """Resume accepting jobs.""" | |
@@ -182,17 +200,27 b' class TaskScheduler(SessionFactory):' | |||||
182 |
|
200 | |||
183 | def dispatch_notification(self, msg): |
|
201 | def dispatch_notification(self, msg): | |
184 | """dispatch register/unregister events.""" |
|
202 | """dispatch register/unregister events.""" | |
185 | idents,msg = self.session.feed_identities(msg) |
|
203 | try: | |
186 |
msg = self.session. |
|
204 | idents,msg = self.session.feed_identities(msg) | |
|
205 | except ValueError: | |||
|
206 | self.log.warn("task::Invalid Message: %r"%msg) | |||
|
207 | return | |||
|
208 | try: | |||
|
209 | msg = self.session.unpack_message(msg) | |||
|
210 | except ValueError: | |||
|
211 | self.log.warn("task::Unauthorized message from: %r"%idents) | |||
|
212 | return | |||
|
213 | ||||
187 | msg_type = msg['msg_type'] |
|
214 | msg_type = msg['msg_type'] | |
|
215 | ||||
188 | handler = self._notification_handlers.get(msg_type, None) |
|
216 | handler = self._notification_handlers.get(msg_type, None) | |
189 | if handler is None: |
|
217 | if handler is None: | |
190 |
|
|
218 | self.log.error("Unhandled message type: %r"%msg_type) | |
191 | else: |
|
219 | else: | |
192 | try: |
|
220 | try: | |
193 | handler(str(msg['content']['queue'])) |
|
221 | handler(str(msg['content']['queue'])) | |
194 | except KeyError: |
|
222 | except KeyError: | |
195 |
self.log.error("task::Invalid notification msg: % |
|
223 | self.log.error("task::Invalid notification msg: %r"%msg) | |
196 |
|
224 | |||
197 | @logged |
|
225 | @logged | |
198 | def _register_engine(self, uid): |
|
226 | def _register_engine(self, uid): | |
@@ -247,10 +275,8 b' class TaskScheduler(SessionFactory):' | |||||
247 | continue |
|
275 | continue | |
248 |
|
276 | |||
249 | raw_msg = lost[msg_id][0] |
|
277 | raw_msg = lost[msg_id][0] | |
250 |
|
||||
251 | idents,msg = self.session.feed_identities(raw_msg, copy=False) |
|
278 | idents,msg = self.session.feed_identities(raw_msg, copy=False) | |
252 |
|
|
279 | parent = self.session.unpack(msg[1].bytes) | |
253 | parent = msg['header'] |
|
|||
254 | idents = [engine, idents[0]] |
|
280 | idents = [engine, idents[0]] | |
255 |
|
281 | |||
256 | # build fake error reply |
|
282 | # build fake error reply | |
@@ -280,9 +306,10 b' class TaskScheduler(SessionFactory):' | |||||
280 | idents, msg = self.session.feed_identities(raw_msg, copy=False) |
|
306 | idents, msg = self.session.feed_identities(raw_msg, copy=False) | |
281 | msg = self.session.unpack_message(msg, content=False, copy=False) |
|
307 | msg = self.session.unpack_message(msg, content=False, copy=False) | |
282 | except Exception: |
|
308 | except Exception: | |
283 |
self.log.error("task::Invaid task |
|
309 | self.log.error("task::Invaid task msg: %r"%raw_msg, exc_info=True) | |
284 | return |
|
310 | return | |
285 |
|
311 | |||
|
312 | ||||
286 | # send to monitor |
|
313 | # send to monitor | |
287 | self.mon_stream.send_multipart(['intask']+raw_msg, copy=False) |
|
314 | self.mon_stream.send_multipart(['intask']+raw_msg, copy=False) | |
288 |
|
315 | |||
@@ -363,8 +390,7 b' class TaskScheduler(SessionFactory):' | |||||
363 |
|
390 | |||
364 | # FIXME: unpacking a message I've already unpacked, but didn't save: |
|
391 | # FIXME: unpacking a message I've already unpacked, but didn't save: | |
365 | idents,msg = self.session.feed_identities(raw_msg, copy=False) |
|
392 | idents,msg = self.session.feed_identities(raw_msg, copy=False) | |
366 |
|
|
393 | header = self.session.unpack(msg[1].bytes) | |
367 | header = msg['header'] |
|
|||
368 |
|
394 | |||
369 | try: |
|
395 | try: | |
370 | raise why() |
|
396 | raise why() | |
@@ -483,7 +509,7 b' class TaskScheduler(SessionFactory):' | |||||
483 | else: |
|
509 | else: | |
484 | self.finish_job(idx) |
|
510 | self.finish_job(idx) | |
485 | except Exception: |
|
511 | except Exception: | |
486 |
self.log.error("task::Invaid result: % |
|
512 | self.log.error("task::Invaid result: %r"%raw_msg, exc_info=True) | |
487 | return |
|
513 | return | |
488 |
|
514 | |||
489 | header = msg['header'] |
|
515 | header = msg['header'] | |
@@ -590,7 +616,8 b' class TaskScheduler(SessionFactory):' | |||||
590 | for msg_id in jobs: |
|
616 | for msg_id in jobs: | |
591 | raw_msg, targets, after, follow, timeout = self.depending[msg_id] |
|
617 | raw_msg, targets, after, follow, timeout = self.depending[msg_id] | |
592 |
|
618 | |||
593 |
if after.unreachable(self.all_completed, self.all_failed) |
|
619 | if after.unreachable(self.all_completed, self.all_failed)\ | |
|
620 | or follow.unreachable(self.all_completed, self.all_failed): | |||
594 | self.fail_unreachable(msg_id) |
|
621 | self.fail_unreachable(msg_id) | |
595 |
|
622 | |||
596 | elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run |
|
623 | elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run | |
@@ -621,9 +648,9 b' class TaskScheduler(SessionFactory):' | |||||
621 |
|
648 | |||
622 |
|
649 | |||
623 |
|
650 | |||
624 |
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None, |
|
651 | def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None, | |
625 |
|
|
652 | logname='root', log_url=None, loglevel=logging.DEBUG, | |
626 |
|
|
653 | identity=b'task'): | |
627 | from zmq.eventloop import ioloop |
|
654 | from zmq.eventloop import ioloop | |
628 | from zmq.eventloop.zmqstream import ZMQStream |
|
655 | from zmq.eventloop.zmqstream import ZMQStream | |
629 |
|
656 | |||
@@ -646,16 +673,16 b' def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname=' | |||||
646 | nots.setsockopt(zmq.SUBSCRIBE, '') |
|
673 | nots.setsockopt(zmq.SUBSCRIBE, '') | |
647 | nots.connect(not_addr) |
|
674 | nots.connect(not_addr) | |
648 |
|
675 | |||
649 | scheme = globals().get(scheme, None) |
|
676 | # setup logging. Note that these will not work in-process, because they clobber | |
650 |
# |
|
677 | # existing loggers. | |
651 |
if log_ |
|
678 | if log_url: | |
652 |
connect_logger(logname, ctx, log_ |
|
679 | log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel) | |
653 | else: |
|
680 | else: | |
654 | local_logger(logname, loglevel) |
|
681 | log = local_logger(logname, loglevel) | |
655 |
|
682 | |||
656 | scheduler = TaskScheduler(client_stream=ins, engine_stream=outs, |
|
683 | scheduler = TaskScheduler(client_stream=ins, engine_stream=outs, | |
657 | mon_stream=mons, notifier_stream=nots, |
|
684 | mon_stream=mons, notifier_stream=nots, | |
658 |
|
|
685 | loop=loop, log=log, | |
659 | config=config) |
|
686 | config=config) | |
660 | scheduler.start() |
|
687 | scheduler.start() | |
661 | try: |
|
688 | try: |
@@ -1,4 +1,9 b'' | |||||
1 |
"""A TaskRecord backend using sqlite3 |
|
1 | """A TaskRecord backend using sqlite3 | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2011 The IPython Development Team |
|
8 | # Copyright (C) 2011 The IPython Development Team | |
4 | # |
|
9 | # | |
@@ -15,9 +20,9 b' import sqlite3' | |||||
15 |
|
20 | |||
16 | from zmq.eventloop import ioloop |
|
21 | from zmq.eventloop import ioloop | |
17 |
|
22 | |||
18 |
from IPython.utils.traitlets import |
|
23 | from IPython.utils.traitlets import Unicode, Instance, List, Dict | |
19 | from .dictdb import BaseDB |
|
24 | from .dictdb import BaseDB | |
20 | from IPython.parallel.util import ISO8601 |
|
25 | from IPython.utils.jsonutil import date_default, extract_dates, squash_dates | |
21 |
|
26 | |||
22 | #----------------------------------------------------------------------------- |
|
27 | #----------------------------------------------------------------------------- | |
23 | # SQLite operators, adapters, and converters |
|
28 | # SQLite operators, adapters, and converters | |
@@ -42,23 +47,14 b' null_operators = {' | |||||
42 | '!=' : "IS NOT NULL", |
|
47 | '!=' : "IS NOT NULL", | |
43 | } |
|
48 | } | |
44 |
|
49 | |||
45 | def _adapt_datetime(dt): |
|
|||
46 | return dt.strftime(ISO8601) |
|
|||
47 |
|
||||
48 | def _convert_datetime(ds): |
|
|||
49 | if ds is None: |
|
|||
50 | return ds |
|
|||
51 | else: |
|
|||
52 | return datetime.strptime(ds, ISO8601) |
|
|||
53 |
|
||||
54 | def _adapt_dict(d): |
|
50 | def _adapt_dict(d): | |
55 | return json.dumps(d) |
|
51 | return json.dumps(d, default=date_default) | |
56 |
|
52 | |||
57 | def _convert_dict(ds): |
|
53 | def _convert_dict(ds): | |
58 | if ds is None: |
|
54 | if ds is None: | |
59 | return ds |
|
55 | return ds | |
60 | else: |
|
56 | else: | |
61 | return json.loads(ds) |
|
57 | return extract_dates(json.loads(ds)) | |
62 |
|
58 | |||
63 | def _adapt_bufs(bufs): |
|
59 | def _adapt_bufs(bufs): | |
64 | # this is *horrible* |
|
60 | # this is *horrible* | |
@@ -83,11 +79,19 b' def _convert_bufs(bs):' | |||||
83 | class SQLiteDB(BaseDB): |
|
79 | class SQLiteDB(BaseDB): | |
84 | """SQLite3 TaskRecord backend.""" |
|
80 | """SQLite3 TaskRecord backend.""" | |
85 |
|
81 | |||
86 |
filename = |
|
82 | filename = Unicode('tasks.db', config=True, | |
87 | location = CUnicode('', config=True) |
|
83 | help="""The filename of the sqlite task database. [default: 'tasks.db']""") | |
88 |
|
|
84 | location = Unicode('', config=True, | |
|
85 | help="""The directory containing the sqlite task database. The default | |||
|
86 | is to use the cluster_dir location.""") | |||
|
87 | table = Unicode("", config=True, | |||
|
88 | help="""The SQLite Table to use for storing tasks for this session. If unspecified, | |||
|
89 | a new table will be created with the Hub's IDENT. Specifying the table will result | |||
|
90 | in tasks from previous sessions being available via Clients' db_query and | |||
|
91 | get_result methods.""") | |||
89 |
|
92 | |||
90 | _db = Instance('sqlite3.Connection') |
|
93 | _db = Instance('sqlite3.Connection') | |
|
94 | # the ordered list of column names | |||
91 | _keys = List(['msg_id' , |
|
95 | _keys = List(['msg_id' , | |
92 | 'header' , |
|
96 | 'header' , | |
93 | 'content', |
|
97 | 'content', | |
@@ -108,6 +112,27 b' class SQLiteDB(BaseDB):' | |||||
108 | 'stdout', |
|
112 | 'stdout', | |
109 | 'stderr', |
|
113 | 'stderr', | |
110 | ]) |
|
114 | ]) | |
|
115 | # sqlite datatypes for checking that db is current format | |||
|
116 | _types = Dict({'msg_id' : 'text' , | |||
|
117 | 'header' : 'dict text', | |||
|
118 | 'content' : 'dict text', | |||
|
119 | 'buffers' : 'bufs blob', | |||
|
120 | 'submitted' : 'timestamp', | |||
|
121 | 'client_uuid' : 'text', | |||
|
122 | 'engine_uuid' : 'text', | |||
|
123 | 'started' : 'timestamp', | |||
|
124 | 'completed' : 'timestamp', | |||
|
125 | 'resubmitted' : 'timestamp', | |||
|
126 | 'result_header' : 'dict text', | |||
|
127 | 'result_content' : 'dict text', | |||
|
128 | 'result_buffers' : 'bufs blob', | |||
|
129 | 'queue' : 'text', | |||
|
130 | 'pyin' : 'text', | |||
|
131 | 'pyout' : 'text', | |||
|
132 | 'pyerr' : 'text', | |||
|
133 | 'stdout' : 'text', | |||
|
134 | 'stderr' : 'text', | |||
|
135 | }) | |||
111 |
|
136 | |||
112 | def __init__(self, **kwargs): |
|
137 | def __init__(self, **kwargs): | |
113 | super(SQLiteDB, self).__init__(**kwargs) |
|
138 | super(SQLiteDB, self).__init__(**kwargs) | |
@@ -115,10 +140,16 b' class SQLiteDB(BaseDB):' | |||||
115 | # use session, and prefix _, since starting with # is illegal |
|
140 | # use session, and prefix _, since starting with # is illegal | |
116 | self.table = '_'+self.session.replace('-','_') |
|
141 | self.table = '_'+self.session.replace('-','_') | |
117 | if not self.location: |
|
142 | if not self.location: | |
118 | if hasattr(self.config.Global, 'cluster_dir'): |
|
143 | # get current profile | |
119 | self.location = self.config.Global.cluster_dir |
|
144 | from IPython.core.application import BaseIPythonApplication | |
|
145 | if BaseIPythonApplication.initialized(): | |||
|
146 | app = BaseIPythonApplication.instance() | |||
|
147 | if app.profile_dir is not None: | |||
|
148 | self.location = app.profile_dir.location | |||
|
149 | else: | |||
|
150 | self.location = u'.' | |||
120 | else: |
|
151 | else: | |
121 | self.location = '.' |
|
152 | self.location = u'.' | |
122 | self._init_db() |
|
153 | self._init_db() | |
123 |
|
154 | |||
124 | # register db commit as 2s periodic callback |
|
155 | # register db commit as 2s periodic callback | |
@@ -136,11 +167,36 b' class SQLiteDB(BaseDB):' | |||||
136 | d[key] = None |
|
167 | d[key] = None | |
137 | return d |
|
168 | return d | |
138 |
|
169 | |||
|
170 | def _check_table(self): | |||
|
171 | """Ensure that an incorrect table doesn't exist | |||
|
172 | ||||
|
173 | If a bad (old) table does exist, return False | |||
|
174 | """ | |||
|
175 | cursor = self._db.execute("PRAGMA table_info(%s)"%self.table) | |||
|
176 | lines = cursor.fetchall() | |||
|
177 | if not lines: | |||
|
178 | # table does not exist | |||
|
179 | return True | |||
|
180 | types = {} | |||
|
181 | keys = [] | |||
|
182 | for line in lines: | |||
|
183 | keys.append(line[1]) | |||
|
184 | types[line[1]] = line[2] | |||
|
185 | if self._keys != keys: | |||
|
186 | # key mismatch | |||
|
187 | self.log.warn('keys mismatch') | |||
|
188 | return False | |||
|
189 | for key in self._keys: | |||
|
190 | if types[key] != self._types[key]: | |||
|
191 | self.log.warn( | |||
|
192 | 'type mismatch: %s: %s != %s'%(key,types[key],self._types[key]) | |||
|
193 | ) | |||
|
194 | return False | |||
|
195 | return True | |||
|
196 | ||||
139 | def _init_db(self): |
|
197 | def _init_db(self): | |
140 | """Connect to the database and get new session number.""" |
|
198 | """Connect to the database and get new session number.""" | |
141 | # register adapters |
|
199 | # register adapters | |
142 | sqlite3.register_adapter(datetime, _adapt_datetime) |
|
|||
143 | sqlite3.register_converter('datetime', _convert_datetime) |
|
|||
144 | sqlite3.register_adapter(dict, _adapt_dict) |
|
200 | sqlite3.register_adapter(dict, _adapt_dict) | |
145 | sqlite3.register_converter('dict', _convert_dict) |
|
201 | sqlite3.register_converter('dict', _convert_dict) | |
146 | sqlite3.register_adapter(list, _adapt_bufs) |
|
202 | sqlite3.register_adapter(list, _adapt_bufs) | |
@@ -151,18 +207,27 b' class SQLiteDB(BaseDB):' | |||||
151 | # isolation_level = None)#, |
|
207 | # isolation_level = None)#, | |
152 | cached_statements=64) |
|
208 | cached_statements=64) | |
153 | # print dir(self._db) |
|
209 | # print dir(self._db) | |
|
210 | first_table = self.table | |||
|
211 | i=0 | |||
|
212 | while not self._check_table(): | |||
|
213 | i+=1 | |||
|
214 | self.table = first_table+'_%i'%i | |||
|
215 | self.log.warn( | |||
|
216 | "Table %s exists and doesn't match db format, trying %s"% | |||
|
217 | (first_table,self.table) | |||
|
218 | ) | |||
154 |
|
219 | |||
155 | self._db.execute("""CREATE TABLE IF NOT EXISTS %s |
|
220 | self._db.execute("""CREATE TABLE IF NOT EXISTS %s | |
156 | (msg_id text PRIMARY KEY, |
|
221 | (msg_id text PRIMARY KEY, | |
157 | header dict text, |
|
222 | header dict text, | |
158 | content dict text, |
|
223 | content dict text, | |
159 | buffers bufs blob, |
|
224 | buffers bufs blob, | |
160 |
submitted |
|
225 | submitted timestamp, | |
161 | client_uuid text, |
|
226 | client_uuid text, | |
162 | engine_uuid text, |
|
227 | engine_uuid text, | |
163 |
started |
|
228 | started timestamp, | |
164 |
completed |
|
229 | completed timestamp, | |
165 |
resubmitted |
|
230 | resubmitted timestamp, | |
166 | result_header dict text, |
|
231 | result_header dict text, | |
167 | result_content dict text, |
|
232 | result_content dict text, | |
168 | result_buffers bufs blob, |
|
233 | result_buffers bufs blob, |
@@ -2,6 +2,10 b'' | |||||
2 | """A simple engine that talks to a controller over 0MQ. |
|
2 | """A simple engine that talks to a controller over 0MQ. | |
3 | it handles registration, etc. and launches a kernel |
|
3 | it handles registration, etc. and launches a kernel | |
4 | connected to the Controller's Schedulers. |
|
4 | connected to the Controller's Schedulers. | |
|
5 | ||||
|
6 | Authors: | |||
|
7 | ||||
|
8 | * Min RK | |||
5 | """ |
|
9 | """ | |
6 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
7 | # Copyright (C) 2010-2011 The IPython Development Team |
|
11 | # Copyright (C) 2010-2011 The IPython Development Team | |
@@ -19,27 +23,37 b' import zmq' | |||||
19 | from zmq.eventloop import ioloop, zmqstream |
|
23 | from zmq.eventloop import ioloop, zmqstream | |
20 |
|
24 | |||
21 | # internal |
|
25 | # internal | |
22 |
from IPython.utils.traitlets import Instance, |
|
26 | from IPython.utils.traitlets import Instance, Dict, Int, Type, CFloat, Unicode | |
23 | # from IPython.utils.localinterfaces import LOCALHOST |
|
27 | # from IPython.utils.localinterfaces import LOCALHOST | |
24 |
|
28 | |||
25 | from IPython.parallel.controller.heartmonitor import Heart |
|
29 | from IPython.parallel.controller.heartmonitor import Heart | |
26 | from IPython.parallel.factory import RegistrationFactory |
|
30 | from IPython.parallel.factory import RegistrationFactory | |
27 | from IPython.parallel.streamsession import Message |
|
|||
28 | from IPython.parallel.util import disambiguate_url |
|
31 | from IPython.parallel.util import disambiguate_url | |
29 |
|
32 | |||
|
33 | from IPython.zmq.session import Message | |||
|
34 | ||||
30 | from .streamkernel import Kernel |
|
35 | from .streamkernel import Kernel | |
31 |
|
36 | |||
32 | class EngineFactory(RegistrationFactory): |
|
37 | class EngineFactory(RegistrationFactory): | |
33 | """IPython engine""" |
|
38 | """IPython engine""" | |
34 |
|
39 | |||
35 | # configurables: |
|
40 | # configurables: | |
36 | user_ns=Dict(config=True) |
|
41 | out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True, | |
37 | out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True) |
|
42 | help="""The OutStream for handling stdout/err. | |
38 | display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True) |
|
43 | Typically 'IPython.zmq.iostream.OutStream'""") | |
39 | location=Str(config=True) |
|
44 | display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True, | |
40 | timeout=CFloat(2,config=True) |
|
45 | help="""The class for handling displayhook. | |
|
46 | Typically 'IPython.zmq.displayhook.DisplayHook'""") | |||
|
47 | location=Unicode(config=True, | |||
|
48 | help="""The location (an IP address) of the controller. This is | |||
|
49 | used for disambiguating URLs, to determine whether | |||
|
50 | loopback should be used to connect or the public address.""") | |||
|
51 | timeout=CFloat(2,config=True, | |||
|
52 | help="""The time (in seconds) to wait for the Controller to respond | |||
|
53 | to registration requests before giving up.""") | |||
41 |
|
54 | |||
42 | # not configurable: |
|
55 | # not configurable: | |
|
56 | user_ns=Dict() | |||
43 | id=Int(allow_none=True) |
|
57 | id=Int(allow_none=True) | |
44 | registrar=Instance('zmq.eventloop.zmqstream.ZMQStream') |
|
58 | registrar=Instance('zmq.eventloop.zmqstream.ZMQStream') | |
45 | kernel=Instance(Kernel) |
|
59 | kernel=Instance(Kernel) | |
@@ -47,6 +61,7 b' class EngineFactory(RegistrationFactory):' | |||||
47 |
|
61 | |||
48 | def __init__(self, **kwargs): |
|
62 | def __init__(self, **kwargs): | |
49 | super(EngineFactory, self).__init__(**kwargs) |
|
63 | super(EngineFactory, self).__init__(**kwargs) | |
|
64 | self.ident = self.session.session | |||
50 | ctx = self.context |
|
65 | ctx = self.context | |
51 |
|
66 | |||
52 | reg = ctx.socket(zmq.XREQ) |
|
67 | reg = ctx.socket(zmq.XREQ) | |
@@ -127,11 +142,10 b' class EngineFactory(RegistrationFactory):' | |||||
127 |
|
142 | |||
128 | self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, |
|
143 | self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, | |
129 | control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, |
|
144 | control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, | |
130 |
loop=loop, user_ns = self.user_ns, log |
|
145 | loop=loop, user_ns = self.user_ns, log=self.log) | |
131 | self.kernel.start() |
|
146 | self.kernel.start() | |
132 | hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ] |
|
147 | hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ] | |
133 | heart = Heart(*map(str, hb_addrs), heart_id=identity) |
|
148 | heart = Heart(*map(str, hb_addrs), heart_id=identity) | |
134 | # ioloop.DelayedCallback(heart.start, 1000, self.loop).start() |
|
|||
135 | heart.start() |
|
149 | heart.start() | |
136 |
|
150 | |||
137 |
|
151 | |||
@@ -143,7 +157,7 b' class EngineFactory(RegistrationFactory):' | |||||
143 |
|
157 | |||
144 |
|
158 | |||
145 | def abort(self): |
|
159 | def abort(self): | |
146 | self.log.fatal("Registration timed out") |
|
160 | self.log.fatal("Registration timed out after %.1f seconds"%self.timeout) | |
147 | self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) |
|
161 | self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) | |
148 | time.sleep(1) |
|
162 | time.sleep(1) | |
149 | sys.exit(255) |
|
163 | sys.exit(255) |
@@ -1,4 +1,9 b'' | |||||
1 |
"""KernelStarter class that intercepts Control Queue messages, and handles process management. |
|
1 | """KernelStarter class that intercepts Control Queue messages, and handles process management. | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010-2011 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # | |
@@ -8,7 +13,7 b'' | |||||
8 |
|
13 | |||
9 | from zmq.eventloop import ioloop |
|
14 | from zmq.eventloop import ioloop | |
10 |
|
15 | |||
11 |
from IPython. |
|
16 | from IPython.zmq.session import Session | |
12 |
|
17 | |||
13 | class KernelStarter(object): |
|
18 | class KernelStarter(object): | |
14 | """Object for resetting/killing the Kernel.""" |
|
19 | """Object for resetting/killing the Kernel.""" | |
@@ -213,7 +218,7 b' def make_starter(up_addr, down_addr, *args, **kwargs):' | |||||
213 | """entry point function for launching a kernelstarter in a subprocess""" |
|
218 | """entry point function for launching a kernelstarter in a subprocess""" | |
214 | loop = ioloop.IOLoop.instance() |
|
219 | loop = ioloop.IOLoop.instance() | |
215 | ctx = zmq.Context() |
|
220 | ctx = zmq.Context() | |
216 |
session = |
|
221 | session = Session() | |
217 | upstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop) |
|
222 | upstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop) | |
218 | upstream.connect(up_addr) |
|
223 | upstream.connect(up_addr) | |
219 | downstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop) |
|
224 | downstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop) |
@@ -1,6 +1,13 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | """ |
|
2 | """ | |
3 | Kernel adapted from kernel.py to use ZMQ Streams |
|
3 | Kernel adapted from kernel.py to use ZMQ Streams | |
|
4 | ||||
|
5 | Authors: | |||
|
6 | ||||
|
7 | * Min RK | |||
|
8 | * Brian Granger | |||
|
9 | * Fernando Perez | |||
|
10 | * Evan Patterson | |||
4 | """ |
|
11 | """ | |
5 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
6 | # Copyright (C) 2010-2011 The IPython Development Team |
|
13 | # Copyright (C) 2010-2011 The IPython Development Team | |
@@ -28,12 +35,12 b' import zmq' | |||||
28 | from zmq.eventloop import ioloop, zmqstream |
|
35 | from zmq.eventloop import ioloop, zmqstream | |
29 |
|
36 | |||
30 | # Local imports. |
|
37 | # Local imports. | |
31 |
from IPython.utils.traitlets import Instance, List, Int, Dict, Set, |
|
38 | from IPython.utils.traitlets import Instance, List, Int, Dict, Set, Unicode | |
32 | from IPython.zmq.completer import KernelCompleter |
|
39 | from IPython.zmq.completer import KernelCompleter | |
33 |
|
40 | |||
34 | from IPython.parallel.error import wrap_exception |
|
41 | from IPython.parallel.error import wrap_exception | |
35 | from IPython.parallel.factory import SessionFactory |
|
42 | from IPython.parallel.factory import SessionFactory | |
36 |
from IPython.parallel.util import serialize_object, unpack_apply_message |
|
43 | from IPython.parallel.util import serialize_object, unpack_apply_message | |
37 |
|
44 | |||
38 | def printer(*args): |
|
45 | def printer(*args): | |
39 | pprint(args, stream=sys.__stdout__) |
|
46 | pprint(args, stream=sys.__stdout__) | |
@@ -42,7 +49,7 b' def printer(*args):' | |||||
42 | class _Passer(zmqstream.ZMQStream): |
|
49 | class _Passer(zmqstream.ZMQStream): | |
43 | """Empty class that implements `send()` that does nothing. |
|
50 | """Empty class that implements `send()` that does nothing. | |
44 |
|
51 | |||
45 |
Subclass ZMQStream for S |
|
52 | Subclass ZMQStream for Session typechecking | |
46 |
|
53 | |||
47 | """ |
|
54 | """ | |
48 | def __init__(self, *args, **kwargs): |
|
55 | def __init__(self, *args, **kwargs): | |
@@ -64,9 +71,11 b' class Kernel(SessionFactory):' | |||||
64 | #--------------------------------------------------------------------------- |
|
71 | #--------------------------------------------------------------------------- | |
65 |
|
72 | |||
66 | # kwargs: |
|
73 | # kwargs: | |
67 |
|
|
74 | exec_lines = List(Unicode, config=True, | |
68 | user_ns = Dict(config=True) |
|
75 | help="List of lines to execute") | |
69 | exec_lines = List(config=True) |
|
76 | ||
|
77 | int_id = Int(-1) | |||
|
78 | user_ns = Dict(config=True, help="""Set the user's namespace of the Kernel""") | |||
70 |
|
79 | |||
71 | control_stream = Instance(zmqstream.ZMQStream) |
|
80 | control_stream = Instance(zmqstream.ZMQStream) | |
72 | task_stream = Instance(zmqstream.ZMQStream) |
|
81 | task_stream = Instance(zmqstream.ZMQStream) | |
@@ -129,21 +138,10 b' class Kernel(SessionFactory):' | |||||
129 |
|
138 | |||
130 | def abort_queue(self, stream): |
|
139 | def abort_queue(self, stream): | |
131 | while True: |
|
140 | while True: | |
132 | try: |
|
141 | idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) | |
133 | msg = self.session.recv(stream, zmq.NOBLOCK,content=True) |
|
142 | if msg is None: | |
134 | except zmq.ZMQError as e: |
|
143 | return | |
135 | if e.errno == zmq.EAGAIN: |
|
|||
136 | break |
|
|||
137 | else: |
|
|||
138 | return |
|
|||
139 | else: |
|
|||
140 | if msg is None: |
|
|||
141 | return |
|
|||
142 | else: |
|
|||
143 | idents,msg = msg |
|
|||
144 |
|
144 | |||
145 | # assert self.reply_socketly_socket.rcvmore(), "Unexpected missing message part." |
|
|||
146 | # msg = self.reply_socket.recv_json() |
|
|||
147 | self.log.info("Aborting:") |
|
145 | self.log.info("Aborting:") | |
148 | self.log.info(str(msg)) |
|
146 | self.log.info(str(msg)) | |
149 | msg_type = msg['msg_type'] |
|
147 | msg_type = msg['msg_type'] | |
@@ -250,7 +248,7 b' class Kernel(SessionFactory):' | |||||
250 | return |
|
248 | return | |
251 | self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent, |
|
249 | self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent, | |
252 | ident='%s.pyin'%self.prefix) |
|
250 | ident='%s.pyin'%self.prefix) | |
253 |
started = datetime.now() |
|
251 | started = datetime.now() | |
254 | try: |
|
252 | try: | |
255 | comp_code = self.compiler(code, '<zmq-kernel>') |
|
253 | comp_code = self.compiler(code, '<zmq-kernel>') | |
256 | # allow for not overriding displayhook |
|
254 | # allow for not overriding displayhook | |
@@ -300,7 +298,7 b' class Kernel(SessionFactory):' | |||||
300 | # self.iopub_stream.send(pyin_msg) |
|
298 | # self.iopub_stream.send(pyin_msg) | |
301 | # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent) |
|
299 | # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent) | |
302 | sub = {'dependencies_met' : True, 'engine' : self.ident, |
|
300 | sub = {'dependencies_met' : True, 'engine' : self.ident, | |
303 |
'started': datetime.now() |
|
301 | 'started': datetime.now()} | |
304 | try: |
|
302 | try: | |
305 | # allow for not overriding displayhook |
|
303 | # allow for not overriding displayhook | |
306 | if hasattr(sys.displayhook, 'set_parent'): |
|
304 | if hasattr(sys.displayhook, 'set_parent'): |
@@ -1,6 +1,12 b'' | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 |
|
2 | |||
3 |
"""Classes and functions for kernel related errors and exceptions. |
|
3 | """Classes and functions for kernel related errors and exceptions. | |
|
4 | ||||
|
5 | Authors: | |||
|
6 | ||||
|
7 | * Brian Granger | |||
|
8 | * Min RK | |||
|
9 | """ | |||
4 | from __future__ import print_function |
|
10 | from __future__ import print_function | |
5 |
|
11 | |||
6 | import sys |
|
12 | import sys | |
@@ -12,7 +18,7 b' __docformat__ = "restructuredtext en"' | |||||
12 | __test__ = {} |
|
18 | __test__ = {} | |
13 |
|
19 | |||
14 | #------------------------------------------------------------------------------- |
|
20 | #------------------------------------------------------------------------------- | |
15 | # Copyright (C) 2008 The IPython Development Team |
|
21 | # Copyright (C) 2008-2011 The IPython Development Team | |
16 | # |
|
22 | # | |
17 | # Distributed under the terms of the BSD License. The full license is in |
|
23 | # Distributed under the terms of the BSD License. The full license is in | |
18 | # the file COPYING, distributed as part of this software. |
|
24 | # the file COPYING, distributed as part of this software. |
@@ -1,7 +1,12 b'' | |||||
1 |
"""Base config factories. |
|
1 | """Base config factories. | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #----------------------------------------------------------------------------- |
|
8 | #----------------------------------------------------------------------------- | |
4 |
# Copyright (C) 20 |
|
9 | # Copyright (C) 2010-2011 The IPython Development Team | |
5 | # |
|
10 | # | |
6 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | # Distributed under the terms of the BSD License. The full license is in | |
7 | # the file COPYING, distributed as part of this software. |
|
12 | # the file COPYING, distributed as part of this software. | |
@@ -14,75 +19,38 b'' | |||||
14 |
|
19 | |||
15 | import logging |
|
20 | import logging | |
16 | import os |
|
21 | import os | |
17 | import uuid |
|
|||
18 |
|
22 | |||
|
23 | import zmq | |||
19 | from zmq.eventloop.ioloop import IOLoop |
|
24 | from zmq.eventloop.ioloop import IOLoop | |
20 |
|
25 | |||
21 | from IPython.config.configurable import Configurable |
|
26 | from IPython.config.configurable import Configurable | |
22 |
from IPython.utils. |
|
27 | from IPython.utils.traitlets import Int, Instance, Unicode | |
23 | from IPython.utils.traitlets import Str,Int,Instance, CUnicode, CStr |
|
|||
24 |
|
28 | |||
25 | import IPython.parallel.streamsession as ss |
|
|||
26 | from IPython.parallel.util import select_random_ports |
|
29 | from IPython.parallel.util import select_random_ports | |
|
30 | from IPython.zmq.session import Session, SessionFactory | |||
27 |
|
31 | |||
28 | #----------------------------------------------------------------------------- |
|
32 | #----------------------------------------------------------------------------- | |
29 | # Classes |
|
33 | # Classes | |
30 | #----------------------------------------------------------------------------- |
|
34 | #----------------------------------------------------------------------------- | |
31 | class LoggingFactory(Configurable): |
|
|||
32 | """A most basic class, that has a `log` (type:`Logger`) attribute, set via a `logname` Trait.""" |
|
|||
33 | log = Instance('logging.Logger', ('ZMQ', logging.WARN)) |
|
|||
34 | logname = CUnicode('ZMQ') |
|
|||
35 | def _logname_changed(self, name, old, new): |
|
|||
36 | self.log = logging.getLogger(new) |
|
|||
37 |
|
||||
38 |
|
35 | |||
39 | class SessionFactory(LoggingFactory): |
|
|||
40 | """The Base factory from which every factory in IPython.parallel inherits""" |
|
|||
41 |
|
||||
42 | packer = Str('',config=True) |
|
|||
43 | unpacker = Str('',config=True) |
|
|||
44 | ident = CStr('',config=True) |
|
|||
45 | def _ident_default(self): |
|
|||
46 | return str(uuid.uuid4()) |
|
|||
47 | username = CUnicode(os.environ.get('USER','username'),config=True) |
|
|||
48 | exec_key = CUnicode('',config=True) |
|
|||
49 | # not configurable: |
|
|||
50 | context = Instance('zmq.Context', (), {}) |
|
|||
51 | session = Instance('IPython.parallel.streamsession.StreamSession') |
|
|||
52 | loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False) |
|
|||
53 | def _loop_default(self): |
|
|||
54 | return IOLoop.instance() |
|
|||
55 |
|
||||
56 |
|
||||
57 | def __init__(self, **kwargs): |
|
|||
58 | super(SessionFactory, self).__init__(**kwargs) |
|
|||
59 | exec_key = self.exec_key or None |
|
|||
60 | # set the packers: |
|
|||
61 | if not self.packer: |
|
|||
62 | packer_f = unpacker_f = None |
|
|||
63 | elif self.packer.lower() == 'json': |
|
|||
64 | packer_f = ss.json_packer |
|
|||
65 | unpacker_f = ss.json_unpacker |
|
|||
66 | elif self.packer.lower() == 'pickle': |
|
|||
67 | packer_f = ss.pickle_packer |
|
|||
68 | unpacker_f = ss.pickle_unpacker |
|
|||
69 | else: |
|
|||
70 | packer_f = import_item(self.packer) |
|
|||
71 | unpacker_f = import_item(self.unpacker) |
|
|||
72 |
|
||||
73 | # construct the session |
|
|||
74 | self.session = ss.StreamSession(self.username, self.ident, packer=packer_f, unpacker=unpacker_f, key=exec_key) |
|
|||
75 |
|
||||
76 |
|
36 | |||
77 | class RegistrationFactory(SessionFactory): |
|
37 | class RegistrationFactory(SessionFactory): | |
78 | """The Base Configurable for objects that involve registration.""" |
|
38 | """The Base Configurable for objects that involve registration.""" | |
79 |
|
39 | |||
80 | url = Str('', config=True) # url takes precedence over ip,regport,transport |
|
40 | url = Unicode('', config=True, | |
81 | transport = Str('tcp', config=True) |
|
41 | help="""The 0MQ url used for registration. This sets transport, ip, and port | |
82 | ip = Str('127.0.0.1', config=True) |
|
42 | in one variable. For example: url='tcp://127.0.0.1:12345' or | |
83 | regport = Instance(int, config=True) |
|
43 | url='epgm://*:90210'""") # url takes precedence over ip,regport,transport | |
|
44 | transport = Unicode('tcp', config=True, | |||
|
45 | help="""The 0MQ transport for communications. This will likely be | |||
|
46 | the default of 'tcp', but other values include 'ipc', 'epgm', 'inproc'.""") | |||
|
47 | ip = Unicode('127.0.0.1', config=True, | |||
|
48 | help="""The IP address for registration. This is generally either | |||
|
49 | '127.0.0.1' for loopback only or '*' for all interfaces. | |||
|
50 | [default: '127.0.0.1']""") | |||
|
51 | regport = Int(config=True, | |||
|
52 | help="""The port on which the Hub listens for registration.""") | |||
84 | def _regport_default(self): |
|
53 | def _regport_default(self): | |
85 | # return 10101 |
|
|||
86 | return select_random_ports(1)[0] |
|
54 | return select_random_ports(1)[0] | |
87 |
|
55 | |||
88 | def __init__(self, **kwargs): |
|
56 | def __init__(self, **kwargs): | |
@@ -107,46 +75,3 b' class RegistrationFactory(SessionFactory):' | |||||
107 | self.ip = iface[0] |
|
75 | self.ip = iface[0] | |
108 | if iface[1]: |
|
76 | if iface[1]: | |
109 | self.regport = int(iface[1]) |
|
77 | self.regport = int(iface[1]) | |
110 |
|
||||
111 | #----------------------------------------------------------------------------- |
|
|||
112 | # argparse argument extenders |
|
|||
113 | #----------------------------------------------------------------------------- |
|
|||
114 |
|
||||
115 |
|
||||
116 | def add_session_arguments(parser): |
|
|||
117 | paa = parser.add_argument |
|
|||
118 | paa('--ident', |
|
|||
119 | type=str, dest='SessionFactory.ident', |
|
|||
120 | help='set the ZMQ and session identity [default: random uuid]', |
|
|||
121 | metavar='identity') |
|
|||
122 | # paa('--execkey', |
|
|||
123 | # type=str, dest='SessionFactory.exec_key', |
|
|||
124 | # help='path to a file containing an execution key.', |
|
|||
125 | # metavar='execkey') |
|
|||
126 | paa('--packer', |
|
|||
127 | type=str, dest='SessionFactory.packer', |
|
|||
128 | help='method to serialize messages: {json,pickle} [default: json]', |
|
|||
129 | metavar='packer') |
|
|||
130 | paa('--unpacker', |
|
|||
131 | type=str, dest='SessionFactory.unpacker', |
|
|||
132 | help='inverse function of `packer`. Only necessary when using something other than json|pickle', |
|
|||
133 | metavar='packer') |
|
|||
134 |
|
||||
135 | def add_registration_arguments(parser): |
|
|||
136 | paa = parser.add_argument |
|
|||
137 | paa('--ip', |
|
|||
138 | type=str, dest='RegistrationFactory.ip', |
|
|||
139 | help="The IP used for registration [default: localhost]", |
|
|||
140 | metavar='ip') |
|
|||
141 | paa('--transport', |
|
|||
142 | type=str, dest='RegistrationFactory.transport', |
|
|||
143 | help="The ZeroMQ transport used for registration [default: tcp]", |
|
|||
144 | metavar='transport') |
|
|||
145 | paa('--url', |
|
|||
146 | type=str, dest='RegistrationFactory.url', |
|
|||
147 | help='set transport,ip,regport in one go, e.g. tcp://127.0.0.1:10101', |
|
|||
148 | metavar='url') |
|
|||
149 | paa('--regport', |
|
|||
150 | type=int, dest='RegistrationFactory.regport', |
|
|||
151 | help="The port used for registration [default: 10101]", |
|
|||
152 | metavar='ip') |
|
@@ -48,10 +48,10 b' class TestProcessLauncher(LocalProcessLauncher):' | |||||
48 | def setup(): |
|
48 | def setup(): | |
49 | cp = TestProcessLauncher() |
|
49 | cp = TestProcessLauncher() | |
50 | cp.cmd_and_args = ipcontroller_cmd_argv + \ |
|
50 | cp.cmd_and_args = ipcontroller_cmd_argv + \ | |
51 |
[' |
|
51 | ['profile=iptest', 'log_level=50', '--reuse'] | |
52 | cp.start() |
|
52 | cp.start() | |
53 | launchers.append(cp) |
|
53 | launchers.append(cp) | |
54 |
cluster_dir = os.path.join(get_ipython_dir(), ' |
|
54 | cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest') | |
55 | engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json') |
|
55 | engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json') | |
56 | client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json') |
|
56 | client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json') | |
57 | tic = time.time() |
|
57 | tic = time.time() | |
@@ -70,7 +70,7 b" def add_engines(n=1, profile='iptest'):" | |||||
70 | eps = [] |
|
70 | eps = [] | |
71 | for i in range(n): |
|
71 | for i in range(n): | |
72 | ep = TestProcessLauncher() |
|
72 | ep = TestProcessLauncher() | |
73 |
ep.cmd_and_args = ipengine_cmd_argv + [' |
|
73 | ep.cmd_and_args = ipengine_cmd_argv + ['profile=%s'%profile, 'log_level=50'] | |
74 | ep.start() |
|
74 | ep.start() | |
75 | launchers.append(ep) |
|
75 | launchers.append(ep) | |
76 | eps.append(ep) |
|
76 | eps.append(ep) |
@@ -1,4 +1,9 b'' | |||||
1 |
"""base class for parallel client tests |
|
1 | """base class for parallel client tests | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team |
@@ -1,4 +1,9 b'' | |||||
1 |
"""Tests for asyncresult.py |
|
1 | """Tests for asyncresult.py | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team |
@@ -1,4 +1,9 b'' | |||||
1 |
"""Tests for parallel client.py |
|
1 | """Tests for parallel client.py | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team |
@@ -1,4 +1,9 b'' | |||||
1 |
"""Tests for db backends |
|
1 | """Tests for db backends | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team | |
@@ -20,18 +25,21 b' from unittest import TestCase' | |||||
20 |
|
25 | |||
21 | from nose import SkipTest |
|
26 | from nose import SkipTest | |
22 |
|
27 | |||
23 |
from IPython.parallel import error |
|
28 | from IPython.parallel import error | |
24 | from IPython.parallel.controller.dictdb import DictDB |
|
29 | from IPython.parallel.controller.dictdb import DictDB | |
25 | from IPython.parallel.controller.sqlitedb import SQLiteDB |
|
30 | from IPython.parallel.controller.sqlitedb import SQLiteDB | |
26 | from IPython.parallel.controller.hub import init_record, empty_record |
|
31 | from IPython.parallel.controller.hub import init_record, empty_record | |
27 |
|
32 | |||
|
33 | from IPython.zmq.session import Session | |||
|
34 | ||||
|
35 | ||||
28 | #------------------------------------------------------------------------------- |
|
36 | #------------------------------------------------------------------------------- | |
29 | # TestCases |
|
37 | # TestCases | |
30 | #------------------------------------------------------------------------------- |
|
38 | #------------------------------------------------------------------------------- | |
31 |
|
39 | |||
32 | class TestDictBackend(TestCase): |
|
40 | class TestDictBackend(TestCase): | |
33 | def setUp(self): |
|
41 | def setUp(self): | |
34 |
self.session = |
|
42 | self.session = Session() | |
35 | self.db = self.create_db() |
|
43 | self.db = self.create_db() | |
36 | self.load_records(16) |
|
44 | self.load_records(16) | |
37 |
|
45 |
@@ -1,4 +1,9 b'' | |||||
1 |
"""Tests for dependency.py |
|
1 | """Tests for dependency.py | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | __docformat__ = "restructuredtext en" |
|
8 | __docformat__ = "restructuredtext en" | |
4 |
|
9 |
@@ -1,5 +1,10 b'' | |||||
1 | """test LoadBalancedView objects""" |
|
|||
2 |
|
|
1 | # -*- coding: utf-8 -*- | |
|
2 | """test LoadBalancedView objects | |||
|
3 | ||||
|
4 | Authors: | |||
|
5 | ||||
|
6 | * Min RK | |||
|
7 | """ | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team | |
5 | # |
|
10 | # |
@@ -1,4 +1,9 b'' | |||||
1 |
"""Tests for mongodb backend |
|
1 | """Tests for mongodb backend | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team |
@@ -1,4 +1,9 b'' | |||||
1 |
"""test serialization with newserialized |
|
1 | """test serialization with newserialized | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 |
|
7 | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team |
@@ -1,5 +1,10 b'' | |||||
1 | """test View objects""" |
|
|||
2 |
|
|
1 | # -*- coding: utf-8 -*- | |
|
2 | """test View objects | |||
|
3 | ||||
|
4 | Authors: | |||
|
5 | ||||
|
6 | * Min RK | |||
|
7 | """ | |||
3 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
4 | # Copyright (C) 2011 The IPython Development Team |
|
9 | # Copyright (C) 2011 The IPython Development Team | |
5 | # |
|
10 | # | |
@@ -315,6 +320,7 b' class TestView(ClusterTestCase):' | |||||
315 | sys.stdout = savestdout |
|
320 | sys.stdout = savestdout | |
316 | sio.read() |
|
321 | sio.read() | |
317 | self.assertTrue('[stdout:%i]'%v.targets in sio.buf) |
|
322 | self.assertTrue('[stdout:%i]'%v.targets in sio.buf) | |
|
323 | self.assertTrue(sio.buf.rstrip().endswith('10')) | |||
318 | self.assertRaisesRemote(ZeroDivisionError, ip.magic_px, '1/0') |
|
324 | self.assertRaisesRemote(ZeroDivisionError, ip.magic_px, '1/0') | |
319 |
|
325 | |||
320 | def test_magic_px_nonblocking(self): |
|
326 | def test_magic_px_nonblocking(self): |
@@ -1,4 +1,9 b'' | |||||
1 |
"""some generic utilities for dealing with classes, urls, and serialization |
|
1 | """some generic utilities for dealing with classes, urls, and serialization | |
|
2 | ||||
|
3 | Authors: | |||
|
4 | ||||
|
5 | * Min RK | |||
|
6 | """ | |||
2 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
3 | # Copyright (C) 2010-2011 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
4 | # |
|
9 | # | |
@@ -17,7 +22,6 b' import re' | |||||
17 | import stat |
|
22 | import stat | |
18 | import socket |
|
23 | import socket | |
19 | import sys |
|
24 | import sys | |
20 | from datetime import datetime |
|
|||
21 | from signal import signal, SIGINT, SIGABRT, SIGTERM |
|
25 | from signal import signal, SIGINT, SIGABRT, SIGTERM | |
22 | try: |
|
26 | try: | |
23 | from signal import SIGKILL |
|
27 | from signal import SIGKILL | |
@@ -40,10 +44,6 b' from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence' | |||||
40 | from IPython.utils.newserialized import serialize, unserialize |
|
44 | from IPython.utils.newserialized import serialize, unserialize | |
41 | from IPython.zmq.log import EnginePUBHandler |
|
45 | from IPython.zmq.log import EnginePUBHandler | |
42 |
|
46 | |||
43 | # globals |
|
|||
44 | ISO8601="%Y-%m-%dT%H:%M:%S.%f" |
|
|||
45 | ISO8601_RE=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$") |
|
|||
46 |
|
||||
47 | #----------------------------------------------------------------------------- |
|
47 | #----------------------------------------------------------------------------- | |
48 | # Classes |
|
48 | # Classes | |
49 | #----------------------------------------------------------------------------- |
|
49 | #----------------------------------------------------------------------------- | |
@@ -101,18 +101,6 b' class ReverseDict(dict):' | |||||
101 | # Functions |
|
101 | # Functions | |
102 | #----------------------------------------------------------------------------- |
|
102 | #----------------------------------------------------------------------------- | |
103 |
|
103 | |||
104 | def extract_dates(obj): |
|
|||
105 | """extract ISO8601 dates from unpacked JSON""" |
|
|||
106 | if isinstance(obj, dict): |
|
|||
107 | for k,v in obj.iteritems(): |
|
|||
108 | obj[k] = extract_dates(v) |
|
|||
109 | elif isinstance(obj, list): |
|
|||
110 | obj = [ extract_dates(o) for o in obj ] |
|
|||
111 | elif isinstance(obj, basestring): |
|
|||
112 | if ISO8601_RE.match(obj): |
|
|||
113 | obj = datetime.strptime(obj, ISO8601) |
|
|||
114 | return obj |
|
|||
115 |
|
||||
116 | def validate_url(url): |
|
104 | def validate_url(url): | |
117 | """validate a url for zeromq""" |
|
105 | """validate a url for zeromq""" | |
118 | if not isinstance(url, basestring): |
|
106 | if not isinstance(url, basestring): | |
@@ -194,29 +182,6 b' def disambiguate_url(url, location=None):' | |||||
194 |
|
182 | |||
195 | return "%s://%s:%s"%(proto,ip,port) |
|
183 | return "%s://%s:%s"%(proto,ip,port) | |
196 |
|
184 | |||
197 |
|
||||
198 | def rekey(dikt): |
|
|||
199 | """Rekey a dict that has been forced to use str keys where there should be |
|
|||
200 | ints by json. This belongs in the jsonutil added by fperez.""" |
|
|||
201 | for k in dikt.iterkeys(): |
|
|||
202 | if isinstance(k, str): |
|
|||
203 | ik=fk=None |
|
|||
204 | try: |
|
|||
205 | ik = int(k) |
|
|||
206 | except ValueError: |
|
|||
207 | try: |
|
|||
208 | fk = float(k) |
|
|||
209 | except ValueError: |
|
|||
210 | continue |
|
|||
211 | if ik is not None: |
|
|||
212 | nk = ik |
|
|||
213 | else: |
|
|||
214 | nk = fk |
|
|||
215 | if nk in dikt: |
|
|||
216 | raise KeyError("already have key %r"%nk) |
|
|||
217 | dikt[nk] = dikt.pop(k) |
|
|||
218 | return dikt |
|
|||
219 |
|
||||
220 | def serialize_object(obj, threshold=64e-6): |
|
185 | def serialize_object(obj, threshold=64e-6): | |
221 | """Serialize an object into a list of sendable buffers. |
|
186 | """Serialize an object into a list of sendable buffers. | |
222 |
|
187 | |||
@@ -469,6 +434,7 b' def connect_engine_logger(context, iface, engine, loglevel=logging.DEBUG):' | |||||
469 | handler.setLevel(loglevel) |
|
434 | handler.setLevel(loglevel) | |
470 | logger.addHandler(handler) |
|
435 | logger.addHandler(handler) | |
471 | logger.setLevel(loglevel) |
|
436 | logger.setLevel(loglevel) | |
|
437 | return logger | |||
472 |
|
438 | |||
473 | def local_logger(logname, loglevel=logging.DEBUG): |
|
439 | def local_logger(logname, loglevel=logging.DEBUG): | |
474 | loglevel = integer_loglevel(loglevel) |
|
440 | loglevel = integer_loglevel(loglevel) | |
@@ -480,4 +446,5 b' def local_logger(logname, loglevel=logging.DEBUG):' | |||||
480 | handler.setLevel(loglevel) |
|
446 | handler.setLevel(loglevel) | |
481 | logger.addHandler(handler) |
|
447 | logger.addHandler(handler) | |
482 | logger.setLevel(loglevel) |
|
448 | logger.setLevel(loglevel) | |
|
449 | return logger | |||
483 |
|
450 |
@@ -158,8 +158,8 b' def default_argv():' | |||||
158 |
|
158 | |||
159 | return ['--quick', # so no config file is loaded |
|
159 | return ['--quick', # so no config file is loaded | |
160 | # Other defaults to minimize side effects on stdout |
|
160 | # Other defaults to minimize side effects on stdout | |
161 |
' |
|
161 | 'colors=NoColor', '--no-term-title','--no-banner', | |
162 |
' |
|
162 | 'autocall=0'] | |
163 |
|
163 | |||
164 |
|
164 | |||
165 | def default_config(): |
|
165 | def default_config(): | |
@@ -197,7 +197,10 b' def ipexec(fname, options=None):' | |||||
197 |
|
197 | |||
198 | # For these subprocess calls, eliminate all prompt printing so we only see |
|
198 | # For these subprocess calls, eliminate all prompt printing so we only see | |
199 | # output from script execution |
|
199 | # output from script execution | |
200 | prompt_opts = ['--prompt-in1=""', '--prompt-in2=""', '--prompt-out=""'] |
|
200 | prompt_opts = [ 'InteractiveShell.prompt_in1=""', | |
|
201 | 'InteractiveShell.prompt_in2=""', | |||
|
202 | 'InteractiveShell.prompt_out=""' | |||
|
203 | ] | |||
201 | cmdargs = ' '.join(default_argv() + prompt_opts + options) |
|
204 | cmdargs = ' '.join(default_argv() + prompt_opts + options) | |
202 |
|
205 | |||
203 | _ip = get_ipython() |
|
206 | _ip = get_ipython() |
@@ -11,12 +11,79 b'' | |||||
11 | # Imports |
|
11 | # Imports | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 | # stdlib |
|
13 | # stdlib | |
|
14 | import re | |||
14 | import types |
|
15 | import types | |
|
16 | from datetime import datetime | |||
|
17 | ||||
|
18 | #----------------------------------------------------------------------------- | |||
|
19 | # Globals and constants | |||
|
20 | #----------------------------------------------------------------------------- | |||
|
21 | ||||
|
22 | # timestamp formats | |||
|
23 | ISO8601="%Y-%m-%dT%H:%M:%S.%f" | |||
|
24 | ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$") | |||
15 |
|
25 | |||
16 | #----------------------------------------------------------------------------- |
|
26 | #----------------------------------------------------------------------------- | |
17 | # Classes and functions |
|
27 | # Classes and functions | |
18 | #----------------------------------------------------------------------------- |
|
28 | #----------------------------------------------------------------------------- | |
19 |
|
29 | |||
|
30 | def rekey(dikt): | |||
|
31 | """Rekey a dict that has been forced to use str keys where there should be | |||
|
32 | ints by json.""" | |||
|
33 | for k in dikt.iterkeys(): | |||
|
34 | if isinstance(k, basestring): | |||
|
35 | ik=fk=None | |||
|
36 | try: | |||
|
37 | ik = int(k) | |||
|
38 | except ValueError: | |||
|
39 | try: | |||
|
40 | fk = float(k) | |||
|
41 | except ValueError: | |||
|
42 | continue | |||
|
43 | if ik is not None: | |||
|
44 | nk = ik | |||
|
45 | else: | |||
|
46 | nk = fk | |||
|
47 | if nk in dikt: | |||
|
48 | raise KeyError("already have key %r"%nk) | |||
|
49 | dikt[nk] = dikt.pop(k) | |||
|
50 | return dikt | |||
|
51 | ||||
|
52 | ||||
|
53 | def extract_dates(obj): | |||
|
54 | """extract ISO8601 dates from unpacked JSON""" | |||
|
55 | if isinstance(obj, dict): | |||
|
56 | obj = dict(obj) # don't clobber | |||
|
57 | for k,v in obj.iteritems(): | |||
|
58 | obj[k] = extract_dates(v) | |||
|
59 | elif isinstance(obj, (list, tuple)): | |||
|
60 | obj = [ extract_dates(o) for o in obj ] | |||
|
61 | elif isinstance(obj, basestring): | |||
|
62 | if ISO8601_PAT.match(obj): | |||
|
63 | obj = datetime.strptime(obj, ISO8601) | |||
|
64 | return obj | |||
|
65 | ||||
|
66 | def squash_dates(obj): | |||
|
67 | """squash datetime objects into ISO8601 strings""" | |||
|
68 | if isinstance(obj, dict): | |||
|
69 | obj = dict(obj) # don't clobber | |||
|
70 | for k,v in obj.iteritems(): | |||
|
71 | obj[k] = squash_dates(v) | |||
|
72 | elif isinstance(obj, (list, tuple)): | |||
|
73 | obj = [ squash_dates(o) for o in obj ] | |||
|
74 | elif isinstance(obj, datetime): | |||
|
75 | obj = obj.strftime(ISO8601) | |||
|
76 | return obj | |||
|
77 | ||||
|
78 | def date_default(obj): | |||
|
79 | """default function for packing datetime objects in JSON.""" | |||
|
80 | if isinstance(obj, datetime): | |||
|
81 | return obj.strftime(ISO8601) | |||
|
82 | else: | |||
|
83 | raise TypeError("%r is not JSON serializable"%obj) | |||
|
84 | ||||
|
85 | ||||
|
86 | ||||
20 | def json_clean(obj): |
|
87 | def json_clean(obj): | |
21 | """Clean an object to ensure it's safe to encode in JSON. |
|
88 | """Clean an object to ensure it's safe to encode in JSON. | |
22 |
|
89 |
@@ -413,13 +413,20 b' def check_for_old_config(ipython_dir=None):' | |||||
413 | if ipython_dir is None: |
|
413 | if ipython_dir is None: | |
414 | ipython_dir = get_ipython_dir() |
|
414 | ipython_dir = get_ipython_dir() | |
415 |
|
415 | |||
416 | old_configs = ['ipy_user_conf.py', 'ipythonrc'] |
|
416 | old_configs = ['ipy_user_conf.py', 'ipythonrc', 'ipython_config.py'] | |
|
417 | warned = False | |||
417 | for cfg in old_configs: |
|
418 | for cfg in old_configs: | |
418 | f = os.path.join(ipython_dir, cfg) |
|
419 | f = os.path.join(ipython_dir, cfg) | |
419 | if os.path.exists(f): |
|
420 | if os.path.exists(f): | |
420 | warn.warn("""Found old IPython config file %r. |
|
421 | warned = True | |
421 | The IPython configuration system has changed as of 0.11, and this file will be ignored. |
|
422 | warn.warn("Found old IPython config file %r"%f) | |
422 | See http://ipython.github.com/ipython-doc/dev/config for details on the new config system. |
|
423 | ||
423 | The current default config file is 'ipython_config.py', where you can suppress these |
|
424 | if warned: | |
424 | warnings with `Global.ignore_old_config = True`."""%f) |
|
425 | warn.warn(""" | |
|
426 | The IPython configuration system has changed as of 0.11, and these files | |||
|
427 | will be ignored. See http://ipython.github.com/ipython-doc/dev/config for | |||
|
428 | details on the new config system. To start configuring IPython, do | |||
|
429 | `ipython profile create`, and edit `ipython_config.py` in | |||
|
430 | <ipython_dir>/profile_default.""") | |||
|
431 | ||||
425 |
|
432 |
@@ -19,6 +19,8 b' import __main__' | |||||
19 | import os |
|
19 | import os | |
20 | import re |
|
20 | import re | |
21 | import shutil |
|
21 | import shutil | |
|
22 | import textwrap | |||
|
23 | from string import Formatter | |||
22 |
|
24 | |||
23 | from IPython.external.path import path |
|
25 | from IPython.external.path import path | |
24 |
|
26 | |||
@@ -391,15 +393,39 b' def igrep(pat,list):' | |||||
391 | return grep(pat,list,case=0) |
|
393 | return grep(pat,list,case=0) | |
392 |
|
394 | |||
393 |
|
395 | |||
394 | def indent(str,nspaces=4,ntabs=0): |
|
396 | def indent(instr,nspaces=4, ntabs=0, flatten=False): | |
395 | """Indent a string a given number of spaces or tabstops. |
|
397 | """Indent a string a given number of spaces or tabstops. | |
396 |
|
398 | |||
397 | indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces. |
|
399 | indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces. | |
|
400 | ||||
|
401 | Parameters | |||
|
402 | ---------- | |||
|
403 | ||||
|
404 | instr : basestring | |||
|
405 | The string to be indented. | |||
|
406 | nspaces : int (default: 4) | |||
|
407 | The number of spaces to be indented. | |||
|
408 | ntabs : int (default: 0) | |||
|
409 | The number of tabs to be indented. | |||
|
410 | flatten : bool (default: False) | |||
|
411 | Whether to scrub existing indentation. If True, all lines will be | |||
|
412 | aligned to the same indentation. If False, existing indentation will | |||
|
413 | be strictly increased. | |||
|
414 | ||||
|
415 | Returns | |||
|
416 | ------- | |||
|
417 | ||||
|
418 | str|unicode : string indented by ntabs and nspaces. | |||
|
419 | ||||
398 | """ |
|
420 | """ | |
399 | if str is None: |
|
421 | if instr is None: | |
400 | return |
|
422 | return | |
401 | ind = '\t'*ntabs+' '*nspaces |
|
423 | ind = '\t'*ntabs+' '*nspaces | |
402 | outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind)) |
|
424 | if flatten: | |
|
425 | pat = re.compile(r'^\s*', re.MULTILINE) | |||
|
426 | else: | |||
|
427 | pat = re.compile(r'^', re.MULTILINE) | |||
|
428 | outstr = re.sub(pat, ind, instr) | |||
403 | if outstr.endswith(os.linesep+ind): |
|
429 | if outstr.endswith(os.linesep+ind): | |
404 | return outstr[:-len(ind)] |
|
430 | return outstr[:-len(ind)] | |
405 | else: |
|
431 | else: | |
@@ -495,3 +521,93 b' def format_screen(strng):' | |||||
495 | strng = par_re.sub('',strng) |
|
521 | strng = par_re.sub('',strng) | |
496 | return strng |
|
522 | return strng | |
497 |
|
523 | |||
|
524 | def dedent(text): | |||
|
525 | """Equivalent of textwrap.dedent that ignores unindented first line. | |||
|
526 | ||||
|
527 | This means it will still dedent strings like: | |||
|
528 | '''foo | |||
|
529 | is a bar | |||
|
530 | ''' | |||
|
531 | ||||
|
532 | For use in wrap_paragraphs. | |||
|
533 | """ | |||
|
534 | ||||
|
535 | if text.startswith('\n'): | |||
|
536 | # text starts with blank line, don't ignore the first line | |||
|
537 | return textwrap.dedent(text) | |||
|
538 | ||||
|
539 | # split first line | |||
|
540 | splits = text.split('\n',1) | |||
|
541 | if len(splits) == 1: | |||
|
542 | # only one line | |||
|
543 | return textwrap.dedent(text) | |||
|
544 | ||||
|
545 | first, rest = splits | |||
|
546 | # dedent everything but the first line | |||
|
547 | rest = textwrap.dedent(rest) | |||
|
548 | return '\n'.join([first, rest]) | |||
|
549 | ||||
|
550 | def wrap_paragraphs(text, ncols=80): | |||
|
551 | """Wrap multiple paragraphs to fit a specified width. | |||
|
552 | ||||
|
553 | This is equivalent to textwrap.wrap, but with support for multiple | |||
|
554 | paragraphs, as separated by empty lines. | |||
|
555 | ||||
|
556 | Returns | |||
|
557 | ------- | |||
|
558 | ||||
|
559 | list of complete paragraphs, wrapped to fill `ncols` columns. | |||
|
560 | """ | |||
|
561 | paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE) | |||
|
562 | text = dedent(text).strip() | |||
|
563 | paragraphs = paragraph_re.split(text)[::2] # every other entry is space | |||
|
564 | out_ps = [] | |||
|
565 | indent_re = re.compile(r'\n\s+', re.MULTILINE) | |||
|
566 | for p in paragraphs: | |||
|
567 | # presume indentation that survives dedent is meaningful formatting, | |||
|
568 | # so don't fill unless text is flush. | |||
|
569 | if indent_re.search(p) is None: | |||
|
570 | # wrap paragraph | |||
|
571 | p = textwrap.fill(p, ncols) | |||
|
572 | out_ps.append(p) | |||
|
573 | return out_ps | |||
|
574 | ||||
|
575 | ||||
|
576 | ||||
|
577 | class EvalFormatter(Formatter): | |||
|
578 | """A String Formatter that allows evaluation of simple expressions. | |||
|
579 | ||||
|
580 | Any time a format key is not found in the kwargs, | |||
|
581 | it will be tried as an expression in the kwargs namespace. | |||
|
582 | ||||
|
583 | This is to be used in templating cases, such as the parallel batch | |||
|
584 | script templates, where simple arithmetic on arguments is useful. | |||
|
585 | ||||
|
586 | Examples | |||
|
587 | -------- | |||
|
588 | ||||
|
589 | In [1]: f = EvalFormatter() | |||
|
590 | In [2]: f.format('{n/4}', n=8) | |||
|
591 | Out[2]: '2' | |||
|
592 | ||||
|
593 | In [3]: f.format('{range(3)}') | |||
|
594 | Out[3]: '[0, 1, 2]' | |||
|
595 | ||||
|
596 | In [4]: f.format('{3*2}') | |||
|
597 | Out[4]: '6' | |||
|
598 | """ | |||
|
599 | ||||
|
600 | def get_value(self, key, args, kwargs): | |||
|
601 | if isinstance(key, (int, long)): | |||
|
602 | return args[key] | |||
|
603 | elif key in kwargs: | |||
|
604 | return kwargs[key] | |||
|
605 | else: | |||
|
606 | # evaluate the expression using kwargs as namespace | |||
|
607 | try: | |||
|
608 | return eval(key, kwargs) | |||
|
609 | except Exception: | |||
|
610 | # classify all bad expressions as key errors | |||
|
611 | raise KeyError(key) | |||
|
612 | ||||
|
613 |
@@ -6,10 +6,10 b'' | |||||
6 | #----------------------------------------------------------------------------- |
|
6 | #----------------------------------------------------------------------------- | |
7 |
|
7 | |||
8 | #----------------------------------------------------------------------------- |
|
8 | #----------------------------------------------------------------------------- | |
9 |
# Verify zmq version dependency >= 2. |
|
9 | # Verify zmq version dependency >= 2.1.4 | |
10 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
11 |
|
11 | |||
12 |
minimum_pyzmq_version = "2. |
|
12 | minimum_pyzmq_version = "2.1.4" | |
13 |
|
13 | |||
14 | try: |
|
14 | try: | |
15 | import zmq |
|
15 | import zmq |
@@ -21,8 +21,8 b' from Queue import Queue, Empty' | |||||
21 | from IPython.utils import io |
|
21 | from IPython.utils import io | |
22 | from IPython.utils.traitlets import Type |
|
22 | from IPython.utils.traitlets import Type | |
23 |
|
23 | |||
24 | from .kernelmanager import (KernelManager, SubSocketChannel, |
|
24 | from .kernelmanager import (KernelManager, SubSocketChannel, HBSocketChannel, | |
25 |
|
|
25 | ShellSocketChannel, StdInSocketChannel) | |
26 |
|
26 | |||
27 | #----------------------------------------------------------------------------- |
|
27 | #----------------------------------------------------------------------------- | |
28 | # Functions and classes |
|
28 | # Functions and classes | |
@@ -61,15 +61,15 b' class BlockingSubSocketChannel(SubSocketChannel):' | |||||
61 | return msgs |
|
61 | return msgs | |
62 |
|
62 | |||
63 |
|
63 | |||
64 |
class Blocking |
|
64 | class BlockingShellSocketChannel(ShellSocketChannel): | |
65 |
|
65 | |||
66 | def __init__(self, context, session, address=None): |
|
66 | def __init__(self, context, session, address=None): | |
67 |
super(Blocking |
|
67 | super(BlockingShellSocketChannel, self).__init__(context, session, | |
68 | address) |
|
68 | address) | |
69 | self._in_queue = Queue() |
|
69 | self._in_queue = Queue() | |
70 |
|
70 | |||
71 | def call_handlers(self, msg): |
|
71 | def call_handlers(self, msg): | |
72 |
#io.rprint('[[ |
|
72 | #io.rprint('[[Shell]]', msg) # dbg | |
73 | self._in_queue.put(msg) |
|
73 | self._in_queue.put(msg) | |
74 |
|
74 | |||
75 | def msg_ready(self): |
|
75 | def msg_ready(self): | |
@@ -94,7 +94,7 b' class BlockingXReqSocketChannel(XReqSocketChannel):' | |||||
94 | return msgs |
|
94 | return msgs | |
95 |
|
95 | |||
96 |
|
96 | |||
97 |
class Blocking |
|
97 | class BlockingStdInSocketChannel(StdInSocketChannel): | |
98 |
|
98 | |||
99 | def call_handlers(self, msg): |
|
99 | def call_handlers(self, msg): | |
100 | #io.rprint('[[Rep]]', msg) # dbg |
|
100 | #io.rprint('[[Rep]]', msg) # dbg | |
@@ -114,8 +114,8 b' class BlockingHBSocketChannel(HBSocketChannel):' | |||||
114 | class BlockingKernelManager(KernelManager): |
|
114 | class BlockingKernelManager(KernelManager): | |
115 |
|
115 | |||
116 | # The classes to use for the various channels. |
|
116 | # The classes to use for the various channels. | |
117 |
|
|
117 | shell_channel_class = Type(BlockingShellSocketChannel) | |
118 | sub_channel_class = Type(BlockingSubSocketChannel) |
|
118 | sub_channel_class = Type(BlockingSubSocketChannel) | |
119 |
|
|
119 | stdin_channel_class = Type(BlockingStdInSocketChannel) | |
120 | hb_channel_class = Type(BlockingHBSocketChannel) |
|
120 | hb_channel_class = Type(BlockingHBSocketChannel) | |
121 |
|
121 |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: file renamed from IPython/parallel/tests/test_streamsession.py to IPython/zmq/tests/test_session.py |
|
NO CONTENT: file renamed from IPython/parallel/tests/test_streamsession.py to IPython/zmq/tests/test_session.py | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: modified file |
|
NO CONTENT: modified file | ||
The requested commit or file is too big and content was truncated. Show full diff |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed | ||
This diff has been collapsed as it changes many lines, (566 lines changed) Show them Hide them |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
1 | NO CONTENT: file was removed |
|
NO CONTENT: file was removed |
General Comments 0
You need to be logged in to leave comments.
Login now