Show More
@@ -0,0 +1,257 b'' | |||
|
1 | #!/usr/bin/env python | |
|
2 | # encoding: utf-8 | |
|
3 | """ | |
|
4 | The IPython controller application | |
|
5 | """ | |
|
6 | ||
|
7 | #----------------------------------------------------------------------------- | |
|
8 | # Copyright (C) 2008-2009 The IPython Development Team | |
|
9 | # | |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
|
11 | # the file COPYING, distributed as part of this software. | |
|
12 | #----------------------------------------------------------------------------- | |
|
13 | ||
|
14 | #----------------------------------------------------------------------------- | |
|
15 | # Imports | |
|
16 | #----------------------------------------------------------------------------- | |
|
17 | ||
|
18 | import os | |
|
19 | import sys | |
|
20 | ||
|
21 | from twisted.application import service | |
|
22 | from twisted.internet import reactor | |
|
23 | from twisted.python import log | |
|
24 | ||
|
25 | from IPython.config.loader import NoConfigDefault | |
|
26 | ||
|
27 | from IPython.core.application import ( | |
|
28 | ApplicationWithDir, | |
|
29 | AppWithDirArgParseConfigLoader | |
|
30 | ) | |
|
31 | from IPython.core import release | |
|
32 | ||
|
33 | from IPython.utils.importstring import import_item | |
|
34 | ||
|
35 | from IPython.kernel.engineservice import EngineService | |
|
36 | from IPython.kernel.fcutil import Tub | |
|
37 | from IPython.kernel.engineconnector import EngineConnector | |
|
38 | ||
|
39 | #----------------------------------------------------------------------------- | |
|
40 | # The main application | |
|
41 | #----------------------------------------------------------------------------- | |
|
42 | ||
|
43 | ||
|
44 | cl_args = ( | |
|
45 | # Controller config | |
|
46 | (('--furl-file',), dict( | |
|
47 | type=str, dest='Global.furl_file', default=NoConfigDefault, | |
|
48 | help='The full location of the file containing the FURL of the ' | |
|
49 | 'controller. If this is not given, the FURL file must be in the ' | |
|
50 | 'security directory of the cluster directory. This location is ' | |
|
51 | 'resolved using the --profile and --app-dir options.', | |
|
52 | metavar='Global.furl_file') | |
|
53 | ), | |
|
54 | # MPI | |
|
55 | (('--mpi',), dict( | |
|
56 | type=str, dest='MPI.use', default=NoConfigDefault, | |
|
57 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', | |
|
58 | metavar='MPI.use') | |
|
59 | ), | |
|
60 | # Global config | |
|
61 | (('--log-to-file',), dict( | |
|
62 | action='store_true', dest='Global.log_to_file', default=NoConfigDefault, | |
|
63 | help='Log to a file in the log directory (default is stdout)') | |
|
64 | ) | |
|
65 | ) | |
|
66 | ||
|
67 | ||
|
68 | class IPEngineAppCLConfigLoader(AppWithDirArgParseConfigLoader): | |
|
69 | ||
|
70 | arguments = cl_args | |
|
71 | ||
|
72 | ||
|
73 | mpi4py_init = """from mpi4py import MPI as mpi | |
|
74 | mpi.size = mpi.COMM_WORLD.Get_size() | |
|
75 | mpi.rank = mpi.COMM_WORLD.Get_rank() | |
|
76 | """ | |
|
77 | ||
|
78 | pytrilinos_init = """from PyTrilinos import Epetra | |
|
79 | class SimpleStruct: | |
|
80 | pass | |
|
81 | mpi = SimpleStruct() | |
|
82 | mpi.rank = 0 | |
|
83 | mpi.size = 0 | |
|
84 | """ | |
|
85 | ||
|
86 | ||
|
87 | default_config_file_name = 'ipengine_config.py' | |
|
88 | ||
|
89 | ||
|
90 | class IPEngineApp(ApplicationWithDir): | |
|
91 | ||
|
92 | name = 'ipengine' | |
|
93 | app_dir_basename = 'cluster' | |
|
94 | description = 'Start the IPython engine for parallel computing.' | |
|
95 | config_file_name = default_config_file_name | |
|
96 | ||
|
97 | def create_default_config(self): | |
|
98 | super(IPEngineApp, self).create_default_config() | |
|
99 | ||
|
100 | # Global config attributes | |
|
101 | self.default_config.Global.log_to_file = False | |
|
102 | self.default_config.Global.exec_lines = [] | |
|
103 | # The log and security dir names must match that of the controller | |
|
104 | self.default_config.Global.log_dir_name = 'log' | |
|
105 | self.default_config.Global.security_dir_name = 'security' | |
|
106 | self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' | |
|
107 | ||
|
108 | # Configuration related to the controller | |
|
109 | # This must match the filename (path not included) that the controller | |
|
110 | # used for the FURL file. | |
|
111 | self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl' | |
|
112 | # If given, this is the actual location of the controller's FURL file. | |
|
113 | # If not, this is computed using the profile, app_dir and furl_file_name | |
|
114 | self.default_config.Global.furl_file = '' | |
|
115 | ||
|
116 | # MPI related config attributes | |
|
117 | self.default_config.MPI.use = '' | |
|
118 | self.default_config.MPI.mpi4py = mpi4py_init | |
|
119 | self.default_config.MPI.pytrilinos = pytrilinos_init | |
|
120 | ||
|
121 | def create_command_line_config(self): | |
|
122 | """Create and return a command line config loader.""" | |
|
123 | return IPEngineAppCLConfigLoader( | |
|
124 | description=self.description, | |
|
125 | version=release.version | |
|
126 | ) | |
|
127 | ||
|
128 | def post_load_command_line_config(self): | |
|
129 | pass | |
|
130 | ||
|
131 | def pre_construct(self): | |
|
132 | config = self.master_config | |
|
133 | # Now set the security_dir and log_dir and create them. We use | |
|
134 | # the names an construct the absolute paths. | |
|
135 | security_dir = os.path.join(config.Global.app_dir, | |
|
136 | config.Global.security_dir_name) | |
|
137 | log_dir = os.path.join(config.Global.app_dir, | |
|
138 | config.Global.log_dir_name) | |
|
139 | if not os.path.isdir(security_dir): | |
|
140 | os.mkdir(security_dir, 0700) | |
|
141 | else: | |
|
142 | os.chmod(security_dir, 0700) | |
|
143 | if not os.path.isdir(log_dir): | |
|
144 | os.mkdir(log_dir, 0777) | |
|
145 | ||
|
146 | self.security_dir = config.Global.security_dir = security_dir | |
|
147 | self.log_dir = config.Global.log_dir = log_dir | |
|
148 | self.log.info("Log directory set to: %s" % self.log_dir) | |
|
149 | self.log.info("Security directory set to: %s" % self.security_dir) | |
|
150 | ||
|
151 | self.find_cont_furl_file() | |
|
152 | ||
|
153 | def find_cont_furl_file(self): | |
|
154 | config = self.master_config | |
|
155 | # Find the actual controller FURL file | |
|
156 | if os.path.isfile(config.Global.furl_file): | |
|
157 | return | |
|
158 | else: | |
|
159 | # We should know what the app dir is | |
|
160 | try_this = os.path.join( | |
|
161 | config.Global.app_dir, | |
|
162 | config.Global.security_dir, | |
|
163 | config.Global.furl_file_name | |
|
164 | ) | |
|
165 | if os.path.isfile(try_this): | |
|
166 | config.Global.furl_file = try_this | |
|
167 | return | |
|
168 | else: | |
|
169 | self.log.critical("Could not find a valid controller FURL file.") | |
|
170 | self.abort() | |
|
171 | ||
|
172 | def construct(self): | |
|
173 | # I am a little hesitant to put these into InteractiveShell itself. | |
|
174 | # But that might be the place for them | |
|
175 | sys.path.insert(0, '') | |
|
176 | ||
|
177 | self.start_mpi() | |
|
178 | self.start_logging() | |
|
179 | ||
|
180 | # Create the underlying shell class and EngineService | |
|
181 | shell_class = import_item(self.master_config.Global.shell_class) | |
|
182 | self.engine_service = EngineService(shell_class, mpi=mpi) | |
|
183 | ||
|
184 | self.exec_lines() | |
|
185 | ||
|
186 | # Create the service hierarchy | |
|
187 | self.main_service = service.MultiService() | |
|
188 | self.engine_service.setServiceParent(self.main_service) | |
|
189 | self.tub_service = Tub() | |
|
190 | self.tub_service.setServiceParent(self.main_service) | |
|
191 | # This needs to be called before the connection is initiated | |
|
192 | self.main_service.startService() | |
|
193 | ||
|
194 | # This initiates the connection to the controller and calls | |
|
195 | # register_engine to tell the controller we are ready to do work | |
|
196 | self.engine_connector = EngineConnector(self.tub_service) | |
|
197 | ||
|
198 | log.msg("Using furl file: %s" % self.master_config.Global.furl_file) | |
|
199 | ||
|
200 | reactor.callWhenRunning(self.call_connect) | |
|
201 | ||
|
202 | def call_connect(self): | |
|
203 | d = self.engine_connector.connect_to_controller( | |
|
204 | self.engine_service, | |
|
205 | self.master_config.Global.furl_file | |
|
206 | ) | |
|
207 | ||
|
208 | def handle_error(f): | |
|
209 | # If this print statement is replaced by a log.err(f) I get | |
|
210 | # an unhandled error, which makes no sense. I shouldn't have | |
|
211 | # to use a print statement here. My only thought is that | |
|
212 | # at the beginning of the process the logging is still starting up | |
|
213 | print "Error connecting to controller:", f.getErrorMessage() | |
|
214 | reactor.callLater(0.1, reactor.stop) | |
|
215 | ||
|
216 | d.addErrback(handle_error) | |
|
217 | ||
|
218 | def start_mpi(self): | |
|
219 | global mpi | |
|
220 | mpikey = self.master_config.MPI.use | |
|
221 | mpi_import_statement = self.master_config.MPI.get(mpikey, None) | |
|
222 | if mpi_import_statement is not None: | |
|
223 | try: | |
|
224 | self.log.info("Initializing MPI:") | |
|
225 | self.log.info(mpi_import_statement) | |
|
226 | exec mpi_import_statement in globals() | |
|
227 | except: | |
|
228 | mpi = None | |
|
229 | else: | |
|
230 | mpi = None | |
|
231 | ||
|
232 | def start_logging(self): | |
|
233 | if self.master_config.Global.log_to_file: | |
|
234 | log_filename = self.name + '-' + str(os.getpid()) + '.log' | |
|
235 | logfile = os.path.join(self.log_dir, log_filename) | |
|
236 | open_log_file = open(logfile, 'w') | |
|
237 | else: | |
|
238 | open_log_file = sys.stdout | |
|
239 | log.startLogging(open_log_file) | |
|
240 | ||
|
241 | def exec_lines(self): | |
|
242 | for line in self.master_config.Global.exec_lines: | |
|
243 | try: | |
|
244 | log.msg("Executing statement: '%s'" % line) | |
|
245 | self.engine_service.execute(line) | |
|
246 | except: | |
|
247 | log.msg("Error executing statement: %s" % line) | |
|
248 | ||
|
249 | def start_app(self): | |
|
250 | # Start the controller service and set things running | |
|
251 | reactor.run() | |
|
252 | ||
|
253 | ||
|
254 | def launch_new_instance(): | |
|
255 | """Create and run the IPython controller""" | |
|
256 | app = IPEngineApp() | |
|
257 | app.start() |
@@ -1,6 +1,6 b'' | |||
|
1 | 1 | c = get_config() |
|
2 | 2 | |
|
3 |
c.MPI. |
|
|
3 | c.MPI.use = 'mpi4py' | |
|
4 | 4 | |
|
5 | 5 | c.MPI.mpi4py = """from mpi4py import MPI as mpi |
|
6 | 6 | mpi.size = mpi.COMM_WORLD.Get_size() |
@@ -15,5 +15,11 b' mpi.rank = 0' | |||
|
15 | 15 | mpi.size = 0 |
|
16 | 16 | """ |
|
17 | 17 | |
|
18 |
c.Global.logfile = |
|
|
19 | c.Global.furl_file = 'ipcontroller-engine.furl' | |
|
18 | c.Global.log_to_file = False | |
|
19 | c.Global.exec_lines = [] | |
|
20 | c.Global.log_dir_name = 'log' | |
|
21 | c.Global.security_dir_name = 'security' | |
|
22 | c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' | |
|
23 | self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl' | |
|
24 | self.default_config.Global.furl_file = '' | |
|
25 |
@@ -61,7 +61,8 b' class BaseAppArgParseConfigLoader(ArgParseConfigLoader):' | |||
|
61 | 61 | self.parser.add_argument('-log_level', '--log-level', |
|
62 | 62 | dest="Global.log_level",type=int, |
|
63 | 63 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
64 |
default=NoConfigDefault |
|
|
64 | default=NoConfigDefault, | |
|
65 | metavar='Global.log_level') | |
|
65 | 66 | self.parser.add_argument('-config_file', '--config-file', |
|
66 | 67 | dest='Global.config_file',type=str, |
|
67 | 68 | help='Set the config file name to override default.', |
@@ -251,7 +252,7 b' class Application(object):' | |||
|
251 | 252 | ``CONFIG_FILE`` config variable is set to the resolved config file |
|
252 | 253 | location. If not successful, an empty config is used. |
|
253 | 254 | """ |
|
254 |
self.log.debug("Attempting to load config file: |
|
|
255 | self.log.debug("Attempting to load config file: %s" % self.config_file_name) | |
|
255 | 256 | loader = PyFileConfigLoader(self.config_file_name, |
|
256 | 257 | path=self.config_file_paths) |
|
257 | 258 | try: |
@@ -260,11 +261,11 b' class Application(object):' | |||
|
260 | 261 | except IOError: |
|
261 | 262 | # Only warn if the default config file was NOT being used. |
|
262 | 263 | if not self.config_file_name==self.default_config_file_name: |
|
263 |
self.log.warn("Config file not found, skipping: |
|
|
264 | self.log.warn("Config file not found, skipping: %s" % \ | |
|
264 | 265 | self.config_file_name, exc_info=True) |
|
265 | 266 | self.file_config = Config() |
|
266 | 267 | except: |
|
267 |
self.log.warn("Error loading config file: |
|
|
268 | self.log.warn("Error loading config file: %s" % \ | |
|
268 | 269 | self.config_file_name, exc_info=True) |
|
269 | 270 | self.file_config = Config() |
|
270 | 271 | |
@@ -284,7 +285,7 b' class Application(object):' | |||
|
284 | 285 | |
|
285 | 286 | def log_file_config(self): |
|
286 | 287 | if hasattr(self.file_config.Global, 'config_file'): |
|
287 |
self.log.debug("Config file loaded: |
|
|
288 | self.log.debug("Config file loaded: %s" % self.file_config.Global.config_file) | |
|
288 | 289 | self.log.debug(repr(self.file_config)) |
|
289 | 290 | |
|
290 | 291 | def merge_configs(self): |
@@ -441,6 +442,7 b' class ApplicationWithDir(Application):' | |||
|
441 | 442 | # priority, this will always end up in the master_config. |
|
442 | 443 | self.default_config.Global.app_dir = self.app_dir |
|
443 | 444 | self.command_line_config.Global.app_dir = self.app_dir |
|
445 | self.log.info("Application directory set to: %s" % self.app_dir) | |
|
444 | 446 | |
|
445 | 447 | def create_app_dir(self): |
|
446 | 448 | """Make sure that the app dir exists.""" |
@@ -171,7 +171,6 b' class IPControllerApp(ApplicationWithDir):' | |||
|
171 | 171 | app_dir_basename = 'cluster' |
|
172 | 172 | description = 'Start the IPython controller for parallel computing.' |
|
173 | 173 | config_file_name = default_config_file_name |
|
174 | default_log_level = logging.WARN | |
|
175 | 174 | |
|
176 | 175 | def create_default_config(self): |
|
177 | 176 | super(IPControllerApp, self).create_default_config() |
@@ -205,12 +204,13 b' class IPControllerApp(ApplicationWithDir):' | |||
|
205 | 204 | del self.command_line_config.Global.secure |
|
206 | 205 | |
|
207 | 206 | def pre_construct(self): |
|
207 | config = self.master_config | |
|
208 | 208 | # Now set the security_dir and log_dir and create them. We use |
|
209 | 209 | # the names an construct the absolute paths. |
|
210 |
security_dir = os.path.join( |
|
|
211 |
|
|
|
212 |
log_dir = os.path.join( |
|
|
213 |
|
|
|
210 | security_dir = os.path.join(config.Global.app_dir, | |
|
211 | config.Global.security_dir_name) | |
|
212 | log_dir = os.path.join(config.Global.app_dir, | |
|
213 | config.Global.log_dir_name) | |
|
214 | 214 | if not os.path.isdir(security_dir): |
|
215 | 215 | os.mkdir(security_dir, 0700) |
|
216 | 216 | else: |
@@ -218,8 +218,10 b' class IPControllerApp(ApplicationWithDir):' | |||
|
218 | 218 | if not os.path.isdir(log_dir): |
|
219 | 219 | os.mkdir(log_dir, 0777) |
|
220 | 220 | |
|
221 |
self.security_dir = |
|
|
222 |
self.log_dir = |
|
|
221 | self.security_dir = config.Global.security_dir = security_dir | |
|
222 | self.log_dir = config.Global.log_dir = log_dir | |
|
223 | self.log.info("Log directory set to: %s" % self.log_dir) | |
|
224 | self.log.info("Security directory set to: %s" % self.security_dir) | |
|
223 | 225 | |
|
224 | 226 | def construct(self): |
|
225 | 227 | # I am a little hesitant to put these into InteractiveShell itself. |
@@ -1,20 +1,20 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | # encoding: utf-8 |
|
3 | 3 | |
|
4 | __docformat__ = "restructuredtext en" | |
|
5 | ||
|
6 | #------------------------------------------------------------------------------- | |
|
7 | # Copyright (C) 2008 The IPython Development Team | |
|
4 | #----------------------------------------------------------------------------- | |
|
5 | # Copyright (C) 2008-2009 The IPython Development Team | |
|
8 | 6 | # |
|
9 | 7 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | 8 | # the file COPYING, distributed as part of this software. |
|
11 |
#----------------------------------------------------------------------------- |
|
|
9 | #----------------------------------------------------------------------------- | |
|
12 | 10 | |
|
13 |
#----------------------------------------------------------------------------- |
|
|
11 | #----------------------------------------------------------------------------- | |
|
14 | 12 | # Imports |
|
15 |
#----------------------------------------------------------------------------- |
|
|
13 | #----------------------------------------------------------------------------- | |
|
14 | ||
|
15 | ||
|
16 | from IPython.kernel.ipengineapp import launch_new_instance | |
|
17 | ||
|
18 | launch_new_instance() | |
|
16 | 19 | |
|
17 | if __name__ == '__main__': | |
|
18 | from IPython.kernel.scripts import ipengine | |
|
19 | ipengine.main() | |
|
20 | 20 |
@@ -170,7 +170,7 b" if 'setuptools' in sys.modules:" | |||
|
170 | 170 | 'ipython = IPython.core.ipapp:launch_new_instance', |
|
171 | 171 | 'pycolor = IPython.utils.PyColorize:main', |
|
172 | 172 | 'ipcontroller = IPython.kernel.ipcontrollerapp:launch_new_instance', |
|
173 |
'ipengine = IPython.kernel. |
|
|
173 | 'ipengine = IPython.kernel.ipengineapp:launch_new_instance', | |
|
174 | 174 | 'ipcluster = IPython.kernel.scripts.ipcluster:main', |
|
175 | 175 | 'ipythonx = IPython.frontend.wx.ipythonx:main', |
|
176 | 176 | 'iptest = IPython.testing.iptest:main', |
|
1 | NO CONTENT: file was removed |
General Comments 0
You need to be logged in to leave comments.
Login now