##// END OF EJS Templates
General work on the kernel config.
Brian Granger -
Show More
@@ -6,10 +6,14 b' c = get_config()'
6 6 # Global configuration
7 7 #-----------------------------------------------------------------------------
8 8
9 c.Global.logfile = ''
10 c.Global.import_statement = ''
9 c.Global.log_to_file = False
10 c.Global.import_statements = []
11 11 c.Global.reuse_furls = False
12 12
13 # You shouldn't have to edit these
14 c.Global.log_dir_name = 'log'
15 c.Global.security_dir_name = 'security'
16
13 17
14 18 #-----------------------------------------------------------------------------
15 19 # Configure the client services
@@ -19,18 +23,23 b" c.FCClientServiceFactory.ip = ''"
19 23 c.FCClientServiceFactory.port = 0
20 24 c.FCClientServiceFactory.location = ''
21 25 c.FCClientServiceFactory.secure = True
26 c.FCClientServiceFactory.reuse_furls = False
22 27 c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem'
23 28
24 29 c.FCClientServiceFactory.Interfaces.Task.interface_chain = [
25 30 'IPython.kernel.task.ITaskController',
26 31 'IPython.kernel.taskfc.IFCTaskController'
27 32 ]
33 # This is just the filename of the furl file. The path is always the
34 # security dir of the cluster directory.
28 35 c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl'
29 36
30 37 c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [
31 38 'IPython.kernel.multiengine.IMultiEngine',
32 39 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
33 40 ]
41 # This is just the filename of the furl file. The path is always the
42 # security dir of the cluster directory.
34 43 c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
35 44
36 45
@@ -42,19 +51,17 b" c.FCEngineServiceFactory.ip = ''"
42 51 c.FCEngineServiceFactory.port = 0
43 52 c.FCEngineServiceFactory.location = ''
44 53 c.FCEngineServiceFactory.secure = True
54 c.FCEngineServiceFactory.reuse_furls = False
45 55 c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem'
46 56
47 engine_config = Config()
48 engine_config.furl_file =
49 c.Global.engine_furl_file = 'ipcontroller-engine.furl'
50 c.Global.engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase'
51
57 c.FCEngineServiceFactory.Intefaces.Default.interface_chain = [
58 'IPython.kernel.enginefc.IFCControllerBase'
59 ]
52 60
61 # This is just the filename of the furl file. The path is always the
62 # security dir of the cluster directory.
63 c.FCEngineServiceFactory.Intefaces.Default.furl_file = 'ipcontroller-engine.furl'
53 64
54 65
55 66
56 CLIENT_INTERFACES = dict(
57 TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'),
58 MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl')
59 )
60 67
@@ -46,7 +46,7 b' class IPythonArgParseConfigLoader(ArgParseConfigLoader):'
46 46 """Default command line options for IPython based applications."""
47 47
48 48 def _add_other_arguments(self):
49 self.parser.add_argument('-ipythondir', '--ipythondir',
49 self.parser.add_argument('-ipythondir', '--ipython-dir',
50 50 dest='Global.ipythondir',type=str,
51 51 help='Set to override default location of Global.ipythondir.',
52 52 default=NoConfigDefault,
@@ -77,6 +77,8 b' class Application(object):'
77 77
78 78 config_file_name = 'ipython_config.py'
79 79 name = 'ipython'
80 default_log_level = logging.WARN
81
80 82
81 83 def __init__(self):
82 84 self.init_logger()
@@ -85,7 +87,7 b' class Application(object):'
85 87 def init_logger(self):
86 88 self.log = logging.getLogger(self.__class__.__name__)
87 89 # This is used as the default until the command line arguments are read.
88 self.log.setLevel(logging.WARN)
90 self.log.setLevel(self.default_log_level)
89 91 self._log_handler = logging.StreamHandler()
90 92 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
91 93 self._log_handler.setFormatter(self._log_formatter)
@@ -102,16 +104,23 b' class Application(object):'
102 104 def start(self):
103 105 """Start the application."""
104 106 self.attempt(self.create_default_config)
107 self.log_default_config()
108 self.set_default_config_log_level()
105 109 self.attempt(self.pre_load_command_line_config)
106 110 self.attempt(self.load_command_line_config, action='abort')
111 self.set_command_line_config_log_level()
107 112 self.attempt(self.post_load_command_line_config)
113 self.log_command_line_config()
108 114 self.attempt(self.find_ipythondir)
109 115 self.attempt(self.find_config_file_name)
110 116 self.attempt(self.find_config_file_paths)
111 117 self.attempt(self.pre_load_file_config)
112 118 self.attempt(self.load_file_config)
119 self.set_file_config_log_level()
113 120 self.attempt(self.post_load_file_config)
121 self.log_file_config()
114 122 self.attempt(self.merge_configs)
123 self.log_master_config()
115 124 self.attempt(self.pre_construct)
116 125 self.attempt(self.construct)
117 126 self.attempt(self.post_construct)
@@ -132,9 +141,18 b' class Application(object):'
132 141 """
133 142 self.default_config = Config()
134 143 self.default_config.Global.ipythondir = get_ipython_dir()
144
145 def log_default_config(self):
135 146 self.log.debug('Default config loaded:')
136 147 self.log.debug(repr(self.default_config))
137 148
149 def set_default_config_log_level(self):
150 try:
151 self.log_level = self.default_config.Global.log_level
152 except AttributeError:
153 # Fallback to the default_log_level class attribute
154 pass
155
138 156 def create_command_line_config(self):
139 157 """Create and return a command line config loader."""
140 158 return IPythonArgParseConfigLoader(description=self.name)
@@ -144,26 +162,25 b' class Application(object):'
144 162 pass
145 163
146 164 def load_command_line_config(self):
147 """Load the command line config.
148
149 This method also sets ``self.debug``.
150 """
151
165 """Load the command line config."""
152 166 loader = self.create_command_line_config()
153 167 self.command_line_config = loader.load_config()
154 168 self.extra_args = loader.get_extra_args()
155 169
170 def set_command_line_config_log_level(self):
156 171 try:
157 172 self.log_level = self.command_line_config.Global.log_level
158 173 except AttributeError:
159 pass # Use existing value which is set in Application.init_logger.
160 self.log.debug("Command line config loaded:")
161 self.log.debug(repr(self.command_line_config))
174 pass
162 175
163 176 def post_load_command_line_config(self):
164 177 """Do actions just after loading the command line config."""
165 178 pass
166 179
180 def log_command_line_config(self):
181 self.log.debug("Command line config loaded:")
182 self.log.debug(repr(self.command_line_config))
183
167 184 def find_ipythondir(self):
168 185 """Set the IPython directory.
169 186
@@ -186,10 +203,13 b' class Application(object):'
186 203 def find_config_file_name(self):
187 204 """Find the config file name for this application.
188 205
206 This must set ``self.config_file_name`` to the filename of the
207 config file to use (just the filename). The search paths for the
208 config file are set in :meth:`find_config_file_paths` and then passed
209 to the config file loader where they are resolved to an absolute path.
210
189 211 If a profile has been set at the command line, this will resolve
190 it. The search paths for the config file are set in
191 :meth:`find_config_file_paths` and then passed to the config file
192 loader where they are resolved to an absolute path.
212 it.
193 213 """
194 214
195 215 try:
@@ -206,7 +226,11 b' class Application(object):'
206 226 pass
207 227
208 228 def find_config_file_paths(self):
209 """Set the search paths for resolving the config file."""
229 """Set the search paths for resolving the config file.
230
231 This must set ``self.config_file_paths`` to a sequence of search
232 paths to pass to the config file loader.
233 """
210 234 self.config_file_paths = (os.getcwd(), self.ipythondir)
211 235
212 236 def pre_load_file_config(self):
@@ -236,12 +260,11 b' class Application(object):'
236 260 self.log.warn("Error loading config file: <%s>" % \
237 261 self.config_file_name, exc_info=True)
238 262 self.file_config = Config()
239 else:
240 self.log.debug("Config file loaded: <%s>" % loader.full_filename)
241 self.log.debug(repr(self.file_config))
263
264 def set_file_config_log_level(self):
242 265 # We need to keeep self.log_level updated. But we only use the value
243 266 # of the file_config if a value was not specified at the command
244 # line.
267 # line, because the command line overrides everything.
245 268 if not hasattr(self.command_line_config.Global, 'log_level'):
246 269 try:
247 270 self.log_level = self.file_config.Global.log_level
@@ -252,6 +275,11 b' class Application(object):'
252 275 """Do actions after the config file is loaded."""
253 276 pass
254 277
278 def log_file_config(self):
279 if hasattr(self.file_config.Global, 'config_file'):
280 self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file)
281 self.log.debug(repr(self.file_config))
282
255 283 def merge_configs(self):
256 284 """Merge the default, command line and file config objects."""
257 285 config = Config()
@@ -259,6 +287,8 b' class Application(object):'
259 287 config._merge(self.file_config)
260 288 config._merge(self.command_line_config)
261 289 self.master_config = config
290
291 def log_master_config(self):
262 292 self.log.debug("Master config created:")
263 293 self.log.debug(repr(self.master_config))
264 294
@@ -41,7 +41,7 b' from IPython.kernel.twistedutil import blockingCallFromThread'
41 41
42 42 # These enable various things
43 43 from IPython.kernel import codeutil
44 import IPython.kernel.magic
44 # import IPython.kernel.magic
45 45
46 46 # Other things that the user will need
47 47 from IPython.kernel.task import MapTask, StringTask
@@ -159,17 +159,35 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
159 159 secure = Bool(True, config=True)
160 160 cert_file = Str('', config=True)
161 161 location = Str('', config=True)
162 reuse_furls = Bool(False, config=True)
162 163 Interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
163 164
165 def __init__(self, config, adaptee):
166 super(FCServiceFactory, self).__init__(config, adaptee)
167 self._check_reuse_furls()
168
164 169 def _ip_changed(self, name, old, new):
165 170 if new == 'localhost' or new == '127.0.0.1':
166 171 self.location = '127.0.0.1'
167 172
173 def _check_reuse_furls(self):
174 if not self.reuse_furls:
175 furl_files = [i.furl_file for i in self.Interfaces.values()]
176 for ff in furl_files:
177 fullfile = self._get_security_file(ff)
178 if os.path.isfile(fullfile):
179 os.remove(fullfile)
180
181 def _get_security_file(self, filename):
182 return os.path.join(self.config.Global.security_dir, filename)
183
168 184 def create(self):
169 185 """Create and return the Foolscap tub with everything running."""
170 186
171 187 self.tub, self.listener = make_tub(
172 self.ip, self.port, self.secure, self.cert_file)
188 self.ip, self.port, self.secure, self._get_security_file(self.cert_file))
189 log.msg("Created a tub and listener [%r]: %r, %r" % (self.__class__, self.tub, self.listener))
190 log.msg("Interfaces to register [%r]: %r" % (self.__class__, self.Interfaces))
173 191 if not self.secure:
174 192 log.msg("WARNING: running with no security: %s" % self.__class__.__name__)
175 193 reactor.callWhenRunning(self.set_location_and_register)
@@ -189,13 +207,14 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
189 207 """Run through the interfaces, adapt and register."""
190 208
191 209 for ifname, ifconfig in self.Interfaces.iteritems():
210 ff = self._get_security_file(ifconfig.furl_file)
192 211 log.msg("Adapting %r to interface: %s" % (self.adaptee, ifname))
193 log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ifconfig.furl_file))
194 check_furl_file_security(ifconfig.furl_file, self.secure)
212 log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ff))
213 check_furl_file_security(ff, self.secure)
195 214 adaptee = self.adaptee
196 215 for i in ifconfig.interface_chain:
197 216 adaptee = import_item(i)(adaptee)
198 d.addCallback(self.register, adaptee, furl_file=ifconfig.furl_file)
217 d.addCallback(self.register, adaptee, furl_file=ff)
199 218
200 219 def register(self, empty, ref, furl_file):
201 220 """Register the reference with the FURL file.
@@ -16,6 +16,7 b' The IPython controller application'
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import copy
19 import logging
19 20 import os
20 21 import sys
21 22
@@ -40,7 +41,7 b' from IPython.kernel.configobjfactory import ('
40 41 from IPython.kernel.fcutil import FCServiceFactory
41 42
42 43 #-----------------------------------------------------------------------------
43 # Components for creating services
44 # Default interfaces
44 45 #-----------------------------------------------------------------------------
45 46
46 47
@@ -50,11 +51,13 b' default_client_interfaces.Task.interface_chain = ['
50 51 'IPython.kernel.task.ITaskController',
51 52 'IPython.kernel.taskfc.IFCTaskController'
52 53 ]
54
53 55 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
54 56 default_client_interfaces.MultiEngine.interface_chain = [
55 57 'IPython.kernel.multiengine.IMultiEngine',
56 58 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
57 59 ]
60
58 61 default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
59 62
60 63 # Make this a dict we can pass to Config.__init__ for the default
@@ -67,12 +70,17 b' default_engine_interfaces = Config()'
67 70 default_engine_interfaces.Default.interface_chain = [
68 71 'IPython.kernel.enginefc.IFCControllerBase'
69 72 ]
73
70 74 default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
71 75
72 76 # Make this a dict we can pass to Config.__init__ for the default
73 77 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
74 78
75 79
80 #-----------------------------------------------------------------------------
81 # Service factories
82 #-----------------------------------------------------------------------------
83
76 84
77 85 class FCClientServiceFactory(FCServiceFactory):
78 86 """A Foolscap implementation of the client services."""
@@ -86,7 +94,7 b' class FCEngineServiceFactory(FCServiceFactory):'
86 94 """A Foolscap implementation of the engine services."""
87 95
88 96 cert_file = Str('ipcontroller-engine.pem', config=True)
89 interfaces = Instance(klass=dict, kw=default_engine_interfaces,
97 Interfaces = Instance(klass=dict, kw=default_engine_interfaces,
90 98 allow_none=False, config=True)
91 99
92 100
@@ -116,21 +124,6 b' cl_args = ('
116 124 action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault,
117 125 help='Turn off all client security.')
118 126 ),
119 (('--client-cert-file',), dict(
120 type=str, dest='FCClientServiceFactory.cert_file', default=NoConfigDefault,
121 help='File to store the client SSL certificate in.',
122 metavar='FCClientServiceFactory.cert_file')
123 ),
124 (('--task-furl-file',), dict(
125 type=str, dest='FCClientServiceFactory.Interfaces.Task.furl_file', default=NoConfigDefault,
126 help='File to store the FURL in for task clients to connect with.',
127 metavar='FCClientServiceFactory.Interfaces.Task.furl_file')
128 ),
129 (('--multiengine-furl-file',), dict(
130 type=str, dest='FCClientServiceFactory.Interfaces.MultiEngine.furl_file', default=NoConfigDefault,
131 help='File to store the FURL in for multiengine clients to connect with.',
132 metavar='FCClientServiceFactory.Interfaces.MultiEngine.furl_file')
133 ),
134 127 # Engine config
135 128 (('--engine-ip',), dict(
136 129 type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
@@ -151,26 +144,20 b' cl_args = ('
151 144 action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault,
152 145 help='Turn off all engine security.')
153 146 ),
154 (('--engine-cert-file',), dict(
155 type=str, dest='FCEngineServiceFactory.cert_file', default=NoConfigDefault,
156 help='File to store the client SSL certificate in.',
157 metavar='FCEngineServiceFactory.cert_file')
158 ),
159 (('--engine-furl-file',), dict(
160 type=str, dest='FCEngineServiceFactory.Interfaces.Default.furl_file', default=NoConfigDefault,
161 help='File to store the FURL in for engines to connect with.',
162 metavar='FCEngineServiceFactory.Interfaces.Default.furl_file')
163 ),
164 147 # Global config
165 (('-l','--logfile'), dict(
166 type=str, dest='Global.logfile', default=NoConfigDefault,
167 help='Log file name (default is stdout)',
168 metavar='Global.logfile')
148 (('--log-to-file',), dict(
149 action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
150 help='Log to a file in the log directory (default is stdout)')
169 151 ),
170 (('-r',), dict(
152 (('-r','--reuse-furls'), dict(
171 153 action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
172 154 help='Try to reuse all FURL files.')
173 )
155 ),
156 (('-cluster_dir', '--cluster-dir',), dict(
157 type=str, dest='Global.cluster_dir', default=NoConfigDefault,
158 help='Absolute or relative path to the cluster directory.',
159 metavar='Global.cluster_dir')
160 ),
174 161 )
175 162
176 163
@@ -185,12 +172,18 b' class IPControllerApp(Application):'
185 172
186 173 name = 'ipcontroller'
187 174 config_file_name = _default_config_file_name
175 default_log_level = logging.DEBUG
188 176
189 177 def create_default_config(self):
190 178 super(IPControllerApp, self).create_default_config()
191 self.default_config.Global.logfile = ''
192 179 self.default_config.Global.reuse_furls = False
193 180 self.default_config.Global.import_statements = []
181 self.default_config.Global.profile = 'default'
182 self.default_config.Global.log_dir_name = 'log'
183 self.default_config.Global.security_dir_name = 'security'
184 self.default_config.Global.log_to_file = False
185 # Resolve the default cluster_dir using the default profile
186 self.default_config.Global.cluster_dir = ''
194 187
195 188 def create_command_line_config(self):
196 189 """Create and return a command line config loader."""
@@ -199,6 +192,75 b' class IPControllerApp(Application):'
199 192 description="Start an IPython controller",
200 193 version=release.version)
201 194
195 def find_config_file_name(self):
196 """Find the config file name for this application."""
197 self.find_cluster_dir()
198 self.create_cluster_dir()
199
200 def find_cluster_dir(self):
201 """This resolves into full paths, the various cluster directories.
202
203 This method must set ``self.cluster_dir`` to the full paths of
204 the directory.
205 """
206 # Ignore self.command_line_config.Global.config_file
207 # Instead, first look for an explicit cluster_dir
208 try:
209 self.cluster_dir = self.command_line_config.Global.cluster_dir
210 except AttributeError:
211 self.cluster_dir = self.default_config.Global.cluster_dir
212 self.cluster_dir = os.path.expandvars(os.path.expanduser(self.cluster_dir))
213 if not self.cluster_dir:
214 # Then look for a profile
215 try:
216 self.profile = self.command_line_config.Global.profile
217 except AttributeError:
218 self.profile = self.default_config.Global.profile
219 cluster_dir_name = 'cluster_' + self.profile
220 try_this = os.path.join(os.getcwd(), cluster_dir_name)
221 if os.path.isdir(try_this):
222 self.cluster_dir = try_this
223 else:
224 self.cluster_dir = os.path.join(self.ipythondir, cluster_dir_name)
225 # These have to be set because they could be different from the one
226 # that we just computed. Because command line has the highest
227 # priority, this will always end up in the master_config.
228 self.default_config.Global.cluster_dir = self.cluster_dir
229 self.command_line_config.Global.cluster_dir = self.cluster_dir
230
231 def create_cluster_dir(self):
232 """Make sure that the cluster, security and log dirs exist."""
233 if not os.path.isdir(self.cluster_dir):
234 os.makedirs(self.cluster_dir, mode=0777)
235
236 def find_config_file_paths(self):
237 """Set the search paths for resolving the config file."""
238 self.config_file_paths = (self.cluster_dir,)
239
240 def pre_construct(self):
241 # Now set the security_dir and log_dir and create them. We use
242 # the names an construct the absolute paths.
243 security_dir = os.path.join(self.master_config.Global.cluster_dir,
244 self.master_config.Global.security_dir_name)
245 log_dir = os.path.join(self.master_config.Global.cluster_dir,
246 self.master_config.Global.log_dir_name)
247 if not os.path.isdir(security_dir):
248 os.mkdir(security_dir, 0700)
249 else:
250 os.chmod(security_dir, 0700)
251 if not os.path.isdir(log_dir):
252 os.mkdir(log_dir, 0777)
253
254 self.security_dir = self.master_config.Global.security_dir = security_dir
255 self.log_dir = self.master_config.Global.log_dir = log_dir
256
257 # Now setup reuse_furls
258 if hasattr(self.master_config.Global.reuse_furls):
259 self.master_config.FCClientServiceFactory.reuse_furls = \
260 self.master_config.Global.reuse_furls
261 self.master_config.FCEngineServiceFactory.reuse_furls = \
262 self.master_config.Global.reuse_furls
263
202 264 def construct(self):
203 265 # I am a little hesitant to put these into InteractiveShell itself.
204 266 # But that might be the place for them
@@ -206,7 +268,6 b' class IPControllerApp(Application):'
206 268
207 269 self.start_logging()
208 270 self.import_statements()
209 self.reuse_furls()
210 271
211 272 # Create the service hierarchy
212 273 self.main_service = service.MultiService()
@@ -223,16 +284,13 b' class IPControllerApp(Application):'
223 284 engine_service.setServiceParent(self.main_service)
224 285
225 286 def start_logging(self):
226 logfile = self.master_config.Global.logfile
227 if logfile:
228 logfile = logfile + str(os.getpid()) + '.log'
229 try:
230 openLogFile = open(logfile, 'w')
231 except:
232 openLogFile = sys.stdout
287 if self.master_config.Global.log_to_file:
288 log_filename = self.name + '-' + str(os.getpid()) + '.log'
289 logfile = os.path.join(self.log_dir, log_filename)
290 open_log_file = open(logfile, 'w')
233 291 else:
234 openLogFile = sys.stdout
235 log.startLogging(openLogFile)
292 open_log_file = sys.stdout
293 log.startLogging(open_log_file)
236 294
237 295 def import_statements(self):
238 296 statements = self.master_config.Global.import_statements
@@ -242,20 +300,6 b' class IPControllerApp(Application):'
242 300 except:
243 301 log.msg("Error running import statement: %s" % s)
244 302
245 def reuse_furls(self):
246 # This logic might need to be moved into the components
247 # Delete old furl files unless the reuse_furls is set
248 reuse = self.master_config.Global.reuse_furls
249 # if not reuse:
250 # paths = (
251 # self.master_config.FCEngineServiceFactory.Interfaces.Default.furl_file,
252 # self.master_config.FCClientServiceFactory.Interfaces.Task.furl_file,
253 # self.master_config.FCClientServiceFactory.Interfaces.MultiEngine.furl_file
254 # )
255 # for p in paths:
256 # if os.path.isfile(p):
257 # os.remove(p)
258
259 303 def start_app(self):
260 304 # Start the controller service and set things running
261 305 self.main_service.startService()
General Comments 0
You need to be logged in to leave comments. Login now