##// END OF EJS Templates
disallow no-prefix `ipython foo=bar` argument style....
MinRK -
Show More
@@ -1,562 +1,569 b''
1 """A simple configuration system.
1 """A simple configuration system.
2
2
3 Authors
3 Authors
4 -------
4 -------
5 * Brian Granger
5 * Brian Granger
6 * Fernando Perez
6 * Fernando Perez
7 * Min RK
7 * Min RK
8 """
8 """
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Copyright (C) 2008-2011 The IPython Development Team
11 # Copyright (C) 2008-2011 The IPython Development Team
12 #
12 #
13 # Distributed under the terms of the BSD License. The full license is in
13 # Distributed under the terms of the BSD License. The full license is in
14 # the file COPYING, distributed as part of this software.
14 # the file COPYING, distributed as part of this software.
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 import __builtin__
21 import __builtin__
22 import re
22 import re
23 import sys
23 import sys
24
24
25 from IPython.external import argparse
25 from IPython.external import argparse
26 from IPython.utils.path import filefind, get_ipython_dir
26 from IPython.utils.path import filefind, get_ipython_dir
27
27
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29 # Exceptions
29 # Exceptions
30 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
31
31
32
32
33 class ConfigError(Exception):
33 class ConfigError(Exception):
34 pass
34 pass
35
35
36
36
37 class ConfigLoaderError(ConfigError):
37 class ConfigLoaderError(ConfigError):
38 pass
38 pass
39
39
40 class ArgumentError(ConfigLoaderError):
40 class ArgumentError(ConfigLoaderError):
41 pass
41 pass
42
42
43 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
44 # Argparse fix
44 # Argparse fix
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46
46
47 # Unfortunately argparse by default prints help messages to stderr instead of
47 # Unfortunately argparse by default prints help messages to stderr instead of
48 # stdout. This makes it annoying to capture long help screens at the command
48 # stdout. This makes it annoying to capture long help screens at the command
49 # line, since one must know how to pipe stderr, which many users don't know how
49 # line, since one must know how to pipe stderr, which many users don't know how
50 # to do. So we override the print_help method with one that defaults to
50 # to do. So we override the print_help method with one that defaults to
51 # stdout and use our class instead.
51 # stdout and use our class instead.
52
52
53 class ArgumentParser(argparse.ArgumentParser):
53 class ArgumentParser(argparse.ArgumentParser):
54 """Simple argparse subclass that prints help to stdout by default."""
54 """Simple argparse subclass that prints help to stdout by default."""
55
55
56 def print_help(self, file=None):
56 def print_help(self, file=None):
57 if file is None:
57 if file is None:
58 file = sys.stdout
58 file = sys.stdout
59 return super(ArgumentParser, self).print_help(file)
59 return super(ArgumentParser, self).print_help(file)
60
60
61 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
61 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
62
62
63 #-----------------------------------------------------------------------------
63 #-----------------------------------------------------------------------------
64 # Config class for holding config information
64 # Config class for holding config information
65 #-----------------------------------------------------------------------------
65 #-----------------------------------------------------------------------------
66
66
67
67
68 class Config(dict):
68 class Config(dict):
69 """An attribute based dict that can do smart merges."""
69 """An attribute based dict that can do smart merges."""
70
70
71 def __init__(self, *args, **kwds):
71 def __init__(self, *args, **kwds):
72 dict.__init__(self, *args, **kwds)
72 dict.__init__(self, *args, **kwds)
73 # This sets self.__dict__ = self, but it has to be done this way
73 # This sets self.__dict__ = self, but it has to be done this way
74 # because we are also overriding __setattr__.
74 # because we are also overriding __setattr__.
75 dict.__setattr__(self, '__dict__', self)
75 dict.__setattr__(self, '__dict__', self)
76
76
77 def _merge(self, other):
77 def _merge(self, other):
78 to_update = {}
78 to_update = {}
79 for k, v in other.iteritems():
79 for k, v in other.iteritems():
80 if not self.has_key(k):
80 if not self.has_key(k):
81 to_update[k] = v
81 to_update[k] = v
82 else: # I have this key
82 else: # I have this key
83 if isinstance(v, Config):
83 if isinstance(v, Config):
84 # Recursively merge common sub Configs
84 # Recursively merge common sub Configs
85 self[k]._merge(v)
85 self[k]._merge(v)
86 else:
86 else:
87 # Plain updates for non-Configs
87 # Plain updates for non-Configs
88 to_update[k] = v
88 to_update[k] = v
89
89
90 self.update(to_update)
90 self.update(to_update)
91
91
92 def _is_section_key(self, key):
92 def _is_section_key(self, key):
93 if key[0].upper()==key[0] and not key.startswith('_'):
93 if key[0].upper()==key[0] and not key.startswith('_'):
94 return True
94 return True
95 else:
95 else:
96 return False
96 return False
97
97
98 def __contains__(self, key):
98 def __contains__(self, key):
99 if self._is_section_key(key):
99 if self._is_section_key(key):
100 return True
100 return True
101 else:
101 else:
102 return super(Config, self).__contains__(key)
102 return super(Config, self).__contains__(key)
103 # .has_key is deprecated for dictionaries.
103 # .has_key is deprecated for dictionaries.
104 has_key = __contains__
104 has_key = __contains__
105
105
106 def _has_section(self, key):
106 def _has_section(self, key):
107 if self._is_section_key(key):
107 if self._is_section_key(key):
108 if super(Config, self).__contains__(key):
108 if super(Config, self).__contains__(key):
109 return True
109 return True
110 return False
110 return False
111
111
112 def copy(self):
112 def copy(self):
113 return type(self)(dict.copy(self))
113 return type(self)(dict.copy(self))
114
114
115 def __copy__(self):
115 def __copy__(self):
116 return self.copy()
116 return self.copy()
117
117
118 def __deepcopy__(self, memo):
118 def __deepcopy__(self, memo):
119 import copy
119 import copy
120 return type(self)(copy.deepcopy(self.items()))
120 return type(self)(copy.deepcopy(self.items()))
121
121
122 def __getitem__(self, key):
122 def __getitem__(self, key):
123 # We cannot use directly self._is_section_key, because it triggers
123 # We cannot use directly self._is_section_key, because it triggers
124 # infinite recursion on top of PyPy. Instead, we manually fish the
124 # infinite recursion on top of PyPy. Instead, we manually fish the
125 # bound method.
125 # bound method.
126 is_section_key = self.__class__._is_section_key.__get__(self)
126 is_section_key = self.__class__._is_section_key.__get__(self)
127
127
128 # Because we use this for an exec namespace, we need to delegate
128 # Because we use this for an exec namespace, we need to delegate
129 # the lookup of names in __builtin__ to itself. This means
129 # the lookup of names in __builtin__ to itself. This means
130 # that you can't have section or attribute names that are
130 # that you can't have section or attribute names that are
131 # builtins.
131 # builtins.
132 try:
132 try:
133 return getattr(__builtin__, key)
133 return getattr(__builtin__, key)
134 except AttributeError:
134 except AttributeError:
135 pass
135 pass
136 if is_section_key(key):
136 if is_section_key(key):
137 try:
137 try:
138 return dict.__getitem__(self, key)
138 return dict.__getitem__(self, key)
139 except KeyError:
139 except KeyError:
140 c = Config()
140 c = Config()
141 dict.__setitem__(self, key, c)
141 dict.__setitem__(self, key, c)
142 return c
142 return c
143 else:
143 else:
144 return dict.__getitem__(self, key)
144 return dict.__getitem__(self, key)
145
145
146 def __setitem__(self, key, value):
146 def __setitem__(self, key, value):
147 # Don't allow names in __builtin__ to be modified.
147 # Don't allow names in __builtin__ to be modified.
148 if hasattr(__builtin__, key):
148 if hasattr(__builtin__, key):
149 raise ConfigError('Config variable names cannot have the same name '
149 raise ConfigError('Config variable names cannot have the same name '
150 'as a Python builtin: %s' % key)
150 'as a Python builtin: %s' % key)
151 if self._is_section_key(key):
151 if self._is_section_key(key):
152 if not isinstance(value, Config):
152 if not isinstance(value, Config):
153 raise ValueError('values whose keys begin with an uppercase '
153 raise ValueError('values whose keys begin with an uppercase '
154 'char must be Config instances: %r, %r' % (key, value))
154 'char must be Config instances: %r, %r' % (key, value))
155 else:
155 else:
156 dict.__setitem__(self, key, value)
156 dict.__setitem__(self, key, value)
157
157
158 def __getattr__(self, key):
158 def __getattr__(self, key):
159 try:
159 try:
160 return self.__getitem__(key)
160 return self.__getitem__(key)
161 except KeyError, e:
161 except KeyError, e:
162 raise AttributeError(e)
162 raise AttributeError(e)
163
163
164 def __setattr__(self, key, value):
164 def __setattr__(self, key, value):
165 try:
165 try:
166 self.__setitem__(key, value)
166 self.__setitem__(key, value)
167 except KeyError, e:
167 except KeyError, e:
168 raise AttributeError(e)
168 raise AttributeError(e)
169
169
170 def __delattr__(self, key):
170 def __delattr__(self, key):
171 try:
171 try:
172 dict.__delitem__(self, key)
172 dict.__delitem__(self, key)
173 except KeyError, e:
173 except KeyError, e:
174 raise AttributeError(e)
174 raise AttributeError(e)
175
175
176
176
177 #-----------------------------------------------------------------------------
177 #-----------------------------------------------------------------------------
178 # Config loading classes
178 # Config loading classes
179 #-----------------------------------------------------------------------------
179 #-----------------------------------------------------------------------------
180
180
181
181
182 class ConfigLoader(object):
182 class ConfigLoader(object):
183 """A object for loading configurations from just about anywhere.
183 """A object for loading configurations from just about anywhere.
184
184
185 The resulting configuration is packaged as a :class:`Struct`.
185 The resulting configuration is packaged as a :class:`Struct`.
186
186
187 Notes
187 Notes
188 -----
188 -----
189 A :class:`ConfigLoader` does one thing: load a config from a source
189 A :class:`ConfigLoader` does one thing: load a config from a source
190 (file, command line arguments) and returns the data as a :class:`Struct`.
190 (file, command line arguments) and returns the data as a :class:`Struct`.
191 There are lots of things that :class:`ConfigLoader` does not do. It does
191 There are lots of things that :class:`ConfigLoader` does not do. It does
192 not implement complex logic for finding config files. It does not handle
192 not implement complex logic for finding config files. It does not handle
193 default values or merge multiple configs. These things need to be
193 default values or merge multiple configs. These things need to be
194 handled elsewhere.
194 handled elsewhere.
195 """
195 """
196
196
197 def __init__(self):
197 def __init__(self):
198 """A base class for config loaders.
198 """A base class for config loaders.
199
199
200 Examples
200 Examples
201 --------
201 --------
202
202
203 >>> cl = ConfigLoader()
203 >>> cl = ConfigLoader()
204 >>> config = cl.load_config()
204 >>> config = cl.load_config()
205 >>> config
205 >>> config
206 {}
206 {}
207 """
207 """
208 self.clear()
208 self.clear()
209
209
210 def clear(self):
210 def clear(self):
211 self.config = Config()
211 self.config = Config()
212
212
213 def load_config(self):
213 def load_config(self):
214 """Load a config from somewhere, return a :class:`Config` instance.
214 """Load a config from somewhere, return a :class:`Config` instance.
215
215
216 Usually, this will cause self.config to be set and then returned.
216 Usually, this will cause self.config to be set and then returned.
217 However, in most cases, :meth:`ConfigLoader.clear` should be called
217 However, in most cases, :meth:`ConfigLoader.clear` should be called
218 to erase any previous state.
218 to erase any previous state.
219 """
219 """
220 self.clear()
220 self.clear()
221 return self.config
221 return self.config
222
222
223
223
224 class FileConfigLoader(ConfigLoader):
224 class FileConfigLoader(ConfigLoader):
225 """A base class for file based configurations.
225 """A base class for file based configurations.
226
226
227 As we add more file based config loaders, the common logic should go
227 As we add more file based config loaders, the common logic should go
228 here.
228 here.
229 """
229 """
230 pass
230 pass
231
231
232
232
233 class PyFileConfigLoader(FileConfigLoader):
233 class PyFileConfigLoader(FileConfigLoader):
234 """A config loader for pure python files.
234 """A config loader for pure python files.
235
235
236 This calls execfile on a plain python file and looks for attributes
236 This calls execfile on a plain python file and looks for attributes
237 that are all caps. These attribute are added to the config Struct.
237 that are all caps. These attribute are added to the config Struct.
238 """
238 """
239
239
240 def __init__(self, filename, path=None):
240 def __init__(self, filename, path=None):
241 """Build a config loader for a filename and path.
241 """Build a config loader for a filename and path.
242
242
243 Parameters
243 Parameters
244 ----------
244 ----------
245 filename : str
245 filename : str
246 The file name of the config file.
246 The file name of the config file.
247 path : str, list, tuple
247 path : str, list, tuple
248 The path to search for the config file on, or a sequence of
248 The path to search for the config file on, or a sequence of
249 paths to try in order.
249 paths to try in order.
250 """
250 """
251 super(PyFileConfigLoader, self).__init__()
251 super(PyFileConfigLoader, self).__init__()
252 self.filename = filename
252 self.filename = filename
253 self.path = path
253 self.path = path
254 self.full_filename = ''
254 self.full_filename = ''
255 self.data = None
255 self.data = None
256
256
257 def load_config(self):
257 def load_config(self):
258 """Load the config from a file and return it as a Struct."""
258 """Load the config from a file and return it as a Struct."""
259 self.clear()
259 self.clear()
260 self._find_file()
260 self._find_file()
261 self._read_file_as_dict()
261 self._read_file_as_dict()
262 self._convert_to_config()
262 self._convert_to_config()
263 return self.config
263 return self.config
264
264
265 def _find_file(self):
265 def _find_file(self):
266 """Try to find the file by searching the paths."""
266 """Try to find the file by searching the paths."""
267 self.full_filename = filefind(self.filename, self.path)
267 self.full_filename = filefind(self.filename, self.path)
268
268
269 def _read_file_as_dict(self):
269 def _read_file_as_dict(self):
270 """Load the config file into self.config, with recursive loading."""
270 """Load the config file into self.config, with recursive loading."""
271 # This closure is made available in the namespace that is used
271 # This closure is made available in the namespace that is used
272 # to exec the config file. It allows users to call
272 # to exec the config file. It allows users to call
273 # load_subconfig('myconfig.py') to load config files recursively.
273 # load_subconfig('myconfig.py') to load config files recursively.
274 # It needs to be a closure because it has references to self.path
274 # It needs to be a closure because it has references to self.path
275 # and self.config. The sub-config is loaded with the same path
275 # and self.config. The sub-config is loaded with the same path
276 # as the parent, but it uses an empty config which is then merged
276 # as the parent, but it uses an empty config which is then merged
277 # with the parents.
277 # with the parents.
278
278
279 # If a profile is specified, the config file will be loaded
279 # If a profile is specified, the config file will be loaded
280 # from that profile
280 # from that profile
281
281
282 def load_subconfig(fname, profile=None):
282 def load_subconfig(fname, profile=None):
283 # import here to prevent circular imports
283 # import here to prevent circular imports
284 from IPython.core.profiledir import ProfileDir, ProfileDirError
284 from IPython.core.profiledir import ProfileDir, ProfileDirError
285 if profile is not None:
285 if profile is not None:
286 try:
286 try:
287 profile_dir = ProfileDir.find_profile_dir_by_name(
287 profile_dir = ProfileDir.find_profile_dir_by_name(
288 get_ipython_dir(),
288 get_ipython_dir(),
289 profile,
289 profile,
290 )
290 )
291 except ProfileDirError:
291 except ProfileDirError:
292 return
292 return
293 path = profile_dir.location
293 path = profile_dir.location
294 else:
294 else:
295 path = self.path
295 path = self.path
296 loader = PyFileConfigLoader(fname, path)
296 loader = PyFileConfigLoader(fname, path)
297 try:
297 try:
298 sub_config = loader.load_config()
298 sub_config = loader.load_config()
299 except IOError:
299 except IOError:
300 # Pass silently if the sub config is not there. This happens
300 # Pass silently if the sub config is not there. This happens
301 # when a user s using a profile, but not the default config.
301 # when a user s using a profile, but not the default config.
302 pass
302 pass
303 else:
303 else:
304 self.config._merge(sub_config)
304 self.config._merge(sub_config)
305
305
306 # Again, this needs to be a closure and should be used in config
306 # Again, this needs to be a closure and should be used in config
307 # files to get the config being loaded.
307 # files to get the config being loaded.
308 def get_config():
308 def get_config():
309 return self.config
309 return self.config
310
310
311 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
311 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
312 fs_encoding = sys.getfilesystemencoding() or 'ascii'
312 fs_encoding = sys.getfilesystemencoding() or 'ascii'
313 conf_filename = self.full_filename.encode(fs_encoding)
313 conf_filename = self.full_filename.encode(fs_encoding)
314 execfile(conf_filename, namespace)
314 execfile(conf_filename, namespace)
315
315
316 def _convert_to_config(self):
316 def _convert_to_config(self):
317 if self.data is None:
317 if self.data is None:
318 ConfigLoaderError('self.data does not exist')
318 ConfigLoaderError('self.data does not exist')
319
319
320
320
321 class CommandLineConfigLoader(ConfigLoader):
321 class CommandLineConfigLoader(ConfigLoader):
322 """A config loader for command line arguments.
322 """A config loader for command line arguments.
323
323
324 As we add more command line based loaders, the common logic should go
324 As we add more command line based loaders, the common logic should go
325 here.
325 here.
326 """
326 """
327
327
328 kv_pattern = re.compile(r'[A-Za-z]\w*(\.\w+)*\=.*')
328 kv_pattern = re.compile(r'\-\-[A-Za-z]\w*(\.\w+)*\=.*')
329 flag_pattern = re.compile(r'\w+(\-\w)*')
329 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
330
330
331 class KeyValueConfigLoader(CommandLineConfigLoader):
331 class KeyValueConfigLoader(CommandLineConfigLoader):
332 """A config loader that loads key value pairs from the command line.
332 """A config loader that loads key value pairs from the command line.
333
333
334 This allows command line options to be gives in the following form::
334 This allows command line options to be gives in the following form::
335
335
336 ipython Global.profile="foo" InteractiveShell.autocall=False
336 ipython --profile="foo" --InteractiveShell.autocall=False
337 """
337 """
338
338
339 def __init__(self, argv=None, aliases=None, flags=None):
339 def __init__(self, argv=None, aliases=None, flags=None):
340 """Create a key value pair config loader.
340 """Create a key value pair config loader.
341
341
342 Parameters
342 Parameters
343 ----------
343 ----------
344 argv : list
344 argv : list
345 A list that has the form of sys.argv[1:] which has unicode
345 A list that has the form of sys.argv[1:] which has unicode
346 elements of the form u"key=value". If this is None (default),
346 elements of the form u"key=value". If this is None (default),
347 then sys.argv[1:] will be used.
347 then sys.argv[1:] will be used.
348 aliases : dict
348 aliases : dict
349 A dict of aliases for configurable traits.
349 A dict of aliases for configurable traits.
350 Keys are the short aliases, Values are the resolved trait.
350 Keys are the short aliases, Values are the resolved trait.
351 Of the form: `{'alias' : 'Configurable.trait'}`
351 Of the form: `{'alias' : 'Configurable.trait'}`
352 flags : dict
352 flags : dict
353 A dict of flags, keyed by str name. Vaues can be Config objects,
353 A dict of flags, keyed by str name. Vaues can be Config objects,
354 dicts, or "key=value" strings. If Config or dict, when the flag
354 dicts, or "key=value" strings. If Config or dict, when the flag
355 is triggered, The flag is loaded as `self.config.update(m)`.
355 is triggered, The flag is loaded as `self.config.update(m)`.
356
356
357 Returns
357 Returns
358 -------
358 -------
359 config : Config
359 config : Config
360 The resulting Config object.
360 The resulting Config object.
361
361
362 Examples
362 Examples
363 --------
363 --------
364
364
365 >>> from IPython.config.loader import KeyValueConfigLoader
365 >>> from IPython.config.loader import KeyValueConfigLoader
366 >>> cl = KeyValueConfigLoader()
366 >>> cl = KeyValueConfigLoader()
367 >>> cl.load_config(["foo='bar'","A.name='brian'","B.number=0"])
367 >>> cl.load_config(["--foo='bar'","--A.name='brian'","--B.number=0"])
368 {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'}
368 {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'}
369 """
369 """
370 self.clear()
370 self.clear()
371 if argv is None:
371 if argv is None:
372 argv = sys.argv[1:]
372 argv = sys.argv[1:]
373 self.argv = argv
373 self.argv = argv
374 self.aliases = aliases or {}
374 self.aliases = aliases or {}
375 self.flags = flags or {}
375 self.flags = flags or {}
376
376
377
377
378 def clear(self):
378 def clear(self):
379 super(KeyValueConfigLoader, self).clear()
379 super(KeyValueConfigLoader, self).clear()
380 self.extra_args = []
380 self.extra_args = []
381
381
382
382
383 def _decode_argv(self, argv, enc=None):
383 def _decode_argv(self, argv, enc=None):
384 """decode argv if bytes, using stin.encoding, falling back on default enc"""
384 """decode argv if bytes, using stin.encoding, falling back on default enc"""
385 uargv = []
385 uargv = []
386 if enc is None:
386 if enc is None:
387 enc = sys.stdin.encoding or sys.getdefaultencoding()
387 enc = sys.stdin.encoding or sys.getdefaultencoding()
388 for arg in argv:
388 for arg in argv:
389 if not isinstance(arg, unicode):
389 if not isinstance(arg, unicode):
390 # only decode if not already decoded
390 # only decode if not already decoded
391 arg = arg.decode(enc)
391 arg = arg.decode(enc)
392 uargv.append(arg)
392 uargv.append(arg)
393 return uargv
393 return uargv
394
394
395
395
396 def load_config(self, argv=None, aliases=None, flags=None):
396 def load_config(self, argv=None, aliases=None, flags=None):
397 """Parse the configuration and generate the Config object.
397 """Parse the configuration and generate the Config object.
398
398
399 After loading, any arguments that are not key-value or
399 After loading, any arguments that are not key-value or
400 flags will be stored in self.extra_args - a list of
400 flags will be stored in self.extra_args - a list of
401 unparsed command-line arguments. This is used for
401 unparsed command-line arguments. This is used for
402 arguments such as input files or subcommands.
402 arguments such as input files or subcommands.
403
403
404 Parameters
404 Parameters
405 ----------
405 ----------
406 argv : list, optional
406 argv : list, optional
407 A list that has the form of sys.argv[1:] which has unicode
407 A list that has the form of sys.argv[1:] which has unicode
408 elements of the form u"key=value". If this is None (default),
408 elements of the form u"key=value". If this is None (default),
409 then self.argv will be used.
409 then self.argv will be used.
410 aliases : dict
410 aliases : dict
411 A dict of aliases for configurable traits.
411 A dict of aliases for configurable traits.
412 Keys are the short aliases, Values are the resolved trait.
412 Keys are the short aliases, Values are the resolved trait.
413 Of the form: `{'alias' : 'Configurable.trait'}`
413 Of the form: `{'alias' : 'Configurable.trait'}`
414 flags : dict
414 flags : dict
415 A dict of flags, keyed by str name. Values can be Config objects
415 A dict of flags, keyed by str name. Values can be Config objects
416 or dicts. When the flag is triggered, The config is loaded as
416 or dicts. When the flag is triggered, The config is loaded as
417 `self.config.update(cfg)`.
417 `self.config.update(cfg)`.
418 """
418 """
419 from IPython.config.configurable import Configurable
419 from IPython.config.configurable import Configurable
420
420
421 self.clear()
421 self.clear()
422 if argv is None:
422 if argv is None:
423 argv = self.argv
423 argv = self.argv
424 if aliases is None:
424 if aliases is None:
425 aliases = self.aliases
425 aliases = self.aliases
426 if flags is None:
426 if flags is None:
427 flags = self.flags
427 flags = self.flags
428
428
429 # ensure argv is a list of unicode strings:
429 # ensure argv is a list of unicode strings:
430 uargv = self._decode_argv(argv)
430 uargv = self._decode_argv(argv)
431 for idx,raw in enumerate(uargv):
431 for idx,raw in enumerate(uargv):
432 # strip leading '-'
432 # strip leading '-'
433 item = raw.lstrip('-')
433 item = raw.lstrip('-')
434
434
435 if raw == '--':
435 if raw == '--':
436 # don't parse arguments after '--'
436 # don't parse arguments after '--'
437 self.extra_args.extend(uargv[idx+1:])
437 self.extra_args.extend(uargv[idx+1:])
438 break
438 break
439
439
440 if kv_pattern.match(item):
440 if kv_pattern.match(raw):
441 lhs,rhs = item.split('=',1)
441 lhs,rhs = item.split('=',1)
442 # Substitute longnames for aliases.
442 # Substitute longnames for aliases.
443 if lhs in aliases:
443 if lhs in aliases:
444 lhs = aliases[lhs]
444 lhs = aliases[lhs]
445 exec_str = 'self.config.' + lhs + '=' + rhs
445 exec_str = 'self.config.' + lhs + '=' + rhs
446 try:
446 try:
447 # Try to see if regular Python syntax will work. This
447 # Try to see if regular Python syntax will work. This
448 # won't handle strings as the quote marks are removed
448 # won't handle strings as the quote marks are removed
449 # by the system shell.
449 # by the system shell.
450 exec exec_str in locals(), globals()
450 exec exec_str in locals(), globals()
451 except (NameError, SyntaxError):
451 except (NameError, SyntaxError):
452 # This case happens if the rhs is a string but without
452 # This case happens if the rhs is a string but without
453 # the quote marks. Use repr, to get quote marks, and
453 # the quote marks. Use repr, to get quote marks, and
454 # 'u' prefix and see if
454 # 'u' prefix and see if
455 # it succeeds. If it still fails, we let it raise.
455 # it succeeds. If it still fails, we let it raise.
456 exec_str = u'self.config.' + lhs + '=' + repr(rhs)
456 exec_str = u'self.config.' + lhs + '=' + repr(rhs)
457 exec exec_str in locals(), globals()
457 exec exec_str in locals(), globals()
458 elif item in flags:
458 elif flag_pattern.match(raw):
459 if item in flags:
459 cfg,help = flags[item]
460 cfg,help = flags[item]
460 if isinstance(cfg, (dict, Config)):
461 if isinstance(cfg, (dict, Config)):
461 # don't clobber whole config sections, update
462 # don't clobber whole config sections, update
462 # each section from config:
463 # each section from config:
463 for sec,c in cfg.iteritems():
464 for sec,c in cfg.iteritems():
464 self.config[sec].update(c)
465 self.config[sec].update(c)
465 else:
466 else:
466 raise ValueError("Invalid flag: '%s'"%raw)
467 raise ValueError("Invalid flag: '%s'"%raw)
468 else:
469 raise ArgumentError("Unrecognized flag: '%s'"%raw)
467 elif raw.startswith('-'):
470 elif raw.startswith('-'):
468 raise ArgumentError("invalid argument: '%s'"%raw)
471 kv = '--'+item
472 if kv_pattern.match(kv):
473 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
474 else:
475 raise ArgumentError("Invalid argument: '%s'"%raw)
469 else:
476 else:
470 # keep all args that aren't valid in a list,
477 # keep all args that aren't valid in a list,
471 # in case our parent knows what to do with them.
478 # in case our parent knows what to do with them.
472 # self.extra_args.append(item)
479 # self.extra_args.append(item)
473 self.extra_args.extend(uargv[idx:])
480 self.extra_args.extend(uargv[idx:])
474 break
481 break
475 return self.config
482 return self.config
476
483
477 class ArgParseConfigLoader(CommandLineConfigLoader):
484 class ArgParseConfigLoader(CommandLineConfigLoader):
478 """A loader that uses the argparse module to load from the command line."""
485 """A loader that uses the argparse module to load from the command line."""
479
486
480 def __init__(self, argv=None, *parser_args, **parser_kw):
487 def __init__(self, argv=None, *parser_args, **parser_kw):
481 """Create a config loader for use with argparse.
488 """Create a config loader for use with argparse.
482
489
483 Parameters
490 Parameters
484 ----------
491 ----------
485
492
486 argv : optional, list
493 argv : optional, list
487 If given, used to read command-line arguments from, otherwise
494 If given, used to read command-line arguments from, otherwise
488 sys.argv[1:] is used.
495 sys.argv[1:] is used.
489
496
490 parser_args : tuple
497 parser_args : tuple
491 A tuple of positional arguments that will be passed to the
498 A tuple of positional arguments that will be passed to the
492 constructor of :class:`argparse.ArgumentParser`.
499 constructor of :class:`argparse.ArgumentParser`.
493
500
494 parser_kw : dict
501 parser_kw : dict
495 A tuple of keyword arguments that will be passed to the
502 A tuple of keyword arguments that will be passed to the
496 constructor of :class:`argparse.ArgumentParser`.
503 constructor of :class:`argparse.ArgumentParser`.
497
504
498 Returns
505 Returns
499 -------
506 -------
500 config : Config
507 config : Config
501 The resulting Config object.
508 The resulting Config object.
502 """
509 """
503 super(CommandLineConfigLoader, self).__init__()
510 super(CommandLineConfigLoader, self).__init__()
504 if argv == None:
511 if argv == None:
505 argv = sys.argv[1:]
512 argv = sys.argv[1:]
506 self.argv = argv
513 self.argv = argv
507 self.parser_args = parser_args
514 self.parser_args = parser_args
508 self.version = parser_kw.pop("version", None)
515 self.version = parser_kw.pop("version", None)
509 kwargs = dict(argument_default=argparse.SUPPRESS)
516 kwargs = dict(argument_default=argparse.SUPPRESS)
510 kwargs.update(parser_kw)
517 kwargs.update(parser_kw)
511 self.parser_kw = kwargs
518 self.parser_kw = kwargs
512
519
513 def load_config(self, argv=None):
520 def load_config(self, argv=None):
514 """Parse command line arguments and return as a Config object.
521 """Parse command line arguments and return as a Config object.
515
522
516 Parameters
523 Parameters
517 ----------
524 ----------
518
525
519 args : optional, list
526 args : optional, list
520 If given, a list with the structure of sys.argv[1:] to parse
527 If given, a list with the structure of sys.argv[1:] to parse
521 arguments from. If not given, the instance's self.argv attribute
528 arguments from. If not given, the instance's self.argv attribute
522 (given at construction time) is used."""
529 (given at construction time) is used."""
523 self.clear()
530 self.clear()
524 if argv is None:
531 if argv is None:
525 argv = self.argv
532 argv = self.argv
526 self._create_parser()
533 self._create_parser()
527 self._parse_args(argv)
534 self._parse_args(argv)
528 self._convert_to_config()
535 self._convert_to_config()
529 return self.config
536 return self.config
530
537
531 def get_extra_args(self):
538 def get_extra_args(self):
532 if hasattr(self, 'extra_args'):
539 if hasattr(self, 'extra_args'):
533 return self.extra_args
540 return self.extra_args
534 else:
541 else:
535 return []
542 return []
536
543
537 def _create_parser(self):
544 def _create_parser(self):
538 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
545 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
539 self._add_arguments()
546 self._add_arguments()
540
547
541 def _add_arguments(self):
548 def _add_arguments(self):
542 raise NotImplementedError("subclasses must implement _add_arguments")
549 raise NotImplementedError("subclasses must implement _add_arguments")
543
550
544 def _parse_args(self, args):
551 def _parse_args(self, args):
545 """self.parser->self.parsed_data"""
552 """self.parser->self.parsed_data"""
546 # decode sys.argv to support unicode command-line options
553 # decode sys.argv to support unicode command-line options
547 uargs = []
554 uargs = []
548 for a in args:
555 for a in args:
549 if isinstance(a, str):
556 if isinstance(a, str):
550 # don't decode if we already got unicode
557 # don't decode if we already got unicode
551 a = a.decode(sys.stdin.encoding or
558 a = a.decode(sys.stdin.encoding or
552 sys.getdefaultencoding())
559 sys.getdefaultencoding())
553 uargs.append(a)
560 uargs.append(a)
554 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
561 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
555
562
556 def _convert_to_config(self):
563 def _convert_to_config(self):
557 """self.parsed_data->self.config"""
564 """self.parsed_data->self.config"""
558 for k, v in vars(self.parsed_data).iteritems():
565 for k, v in vars(self.parsed_data).iteritems():
559 exec_str = 'self.config.' + k + '= v'
566 exec_str = 'self.config.' + k + '= v'
560 exec exec_str in locals(), globals()
567 exec exec_str in locals(), globals()
561
568
562
569
@@ -1,135 +1,135 b''
1 """
1 """
2 Tests for IPython.config.application.Application
2 Tests for IPython.config.application.Application
3
3
4 Authors:
4 Authors:
5
5
6 * Brian Granger
6 * Brian Granger
7 """
7 """
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (C) 2008-2011 The IPython Development Team
10 # Copyright (C) 2008-2011 The IPython Development Team
11 #
11 #
12 # Distributed under the terms of the BSD License. The full license is in
12 # Distributed under the terms of the BSD License. The full license is in
13 # the file COPYING, distributed as part of this software.
13 # the file COPYING, distributed as part of this software.
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Imports
17 # Imports
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 from unittest import TestCase
20 from unittest import TestCase
21
21
22 from IPython.config.configurable import Configurable
22 from IPython.config.configurable import Configurable
23
23
24 from IPython.config.application import (
24 from IPython.config.application import (
25 Application
25 Application
26 )
26 )
27
27
28 from IPython.utils.traitlets import (
28 from IPython.utils.traitlets import (
29 Bool, Unicode, Int, Float, List, Dict
29 Bool, Unicode, Int, Float, List, Dict
30 )
30 )
31
31
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33 # Code
33 # Code
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35
35
36 class Foo(Configurable):
36 class Foo(Configurable):
37
37
38 i = Int(0, config=True, help="The integer i.")
38 i = Int(0, config=True, help="The integer i.")
39 j = Int(1, config=True, help="The integer j.")
39 j = Int(1, config=True, help="The integer j.")
40 name = Unicode(u'Brian', config=True, help="First name.")
40 name = Unicode(u'Brian', config=True, help="First name.")
41
41
42
42
43 class Bar(Configurable):
43 class Bar(Configurable):
44
44
45 b = Int(0, config=True, help="The integer b.")
45 b = Int(0, config=True, help="The integer b.")
46 enabled = Bool(True, config=True, help="Enable bar.")
46 enabled = Bool(True, config=True, help="Enable bar.")
47
47
48
48
49 class MyApp(Application):
49 class MyApp(Application):
50
50
51 name = Unicode(u'myapp')
51 name = Unicode(u'myapp')
52 running = Bool(False, config=True,
52 running = Bool(False, config=True,
53 help="Is the app running?")
53 help="Is the app running?")
54 classes = List([Bar, Foo])
54 classes = List([Bar, Foo])
55 config_file = Unicode(u'', config=True,
55 config_file = Unicode(u'', config=True,
56 help="Load this config file")
56 help="Load this config file")
57
57
58 aliases = Dict(dict(i='Foo.i',j='Foo.j',name='Foo.name',
58 aliases = Dict(dict(i='Foo.i',j='Foo.j',name='Foo.name',
59 enabled='Bar.enabled', log_level='MyApp.log_level'))
59 enabled='Bar.enabled', log_level='MyApp.log_level'))
60
60
61 flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
61 flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
62 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
62 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
63
63
64 def init_foo(self):
64 def init_foo(self):
65 self.foo = Foo(config=self.config)
65 self.foo = Foo(config=self.config)
66
66
67 def init_bar(self):
67 def init_bar(self):
68 self.bar = Bar(config=self.config)
68 self.bar = Bar(config=self.config)
69
69
70
70
71 class TestApplication(TestCase):
71 class TestApplication(TestCase):
72
72
73 def test_basic(self):
73 def test_basic(self):
74 app = MyApp()
74 app = MyApp()
75 self.assertEquals(app.name, u'myapp')
75 self.assertEquals(app.name, u'myapp')
76 self.assertEquals(app.running, False)
76 self.assertEquals(app.running, False)
77 self.assertEquals(app.classes, [MyApp,Bar,Foo])
77 self.assertEquals(app.classes, [MyApp,Bar,Foo])
78 self.assertEquals(app.config_file, u'')
78 self.assertEquals(app.config_file, u'')
79
79
80 def test_config(self):
80 def test_config(self):
81 app = MyApp()
81 app = MyApp()
82 app.parse_command_line(["--i=10","Foo.j=10","--enabled=False","-log_level=50"])
82 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log_level=50"])
83 config = app.config
83 config = app.config
84 self.assertEquals(config.Foo.i, 10)
84 self.assertEquals(config.Foo.i, 10)
85 self.assertEquals(config.Foo.j, 10)
85 self.assertEquals(config.Foo.j, 10)
86 self.assertEquals(config.Bar.enabled, False)
86 self.assertEquals(config.Bar.enabled, False)
87 self.assertEquals(config.MyApp.log_level,50)
87 self.assertEquals(config.MyApp.log_level,50)
88
88
89 def test_config_propagation(self):
89 def test_config_propagation(self):
90 app = MyApp()
90 app = MyApp()
91 app.parse_command_line(["i=10","--Foo.j=10","enabled=False","log_level=50"])
91 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log_level=50"])
92 app.init_foo()
92 app.init_foo()
93 app.init_bar()
93 app.init_bar()
94 self.assertEquals(app.foo.i, 10)
94 self.assertEquals(app.foo.i, 10)
95 self.assertEquals(app.foo.j, 10)
95 self.assertEquals(app.foo.j, 10)
96 self.assertEquals(app.bar.enabled, False)
96 self.assertEquals(app.bar.enabled, False)
97
97
98 def test_flags(self):
98 def test_flags(self):
99 app = MyApp()
99 app = MyApp()
100 app.parse_command_line(["-disable"])
100 app.parse_command_line(["--disable"])
101 app.init_bar()
101 app.init_bar()
102 self.assertEquals(app.bar.enabled, False)
102 self.assertEquals(app.bar.enabled, False)
103 app.parse_command_line(["--enable"])
103 app.parse_command_line(["--enable"])
104 app.init_bar()
104 app.init_bar()
105 self.assertEquals(app.bar.enabled, True)
105 self.assertEquals(app.bar.enabled, True)
106
106
107 def test_aliases(self):
107 def test_aliases(self):
108 app = MyApp()
108 app = MyApp()
109 app.parse_command_line(["i=5", "j=10"])
109 app.parse_command_line(["--i=5", "--j=10"])
110 app.init_foo()
110 app.init_foo()
111 self.assertEquals(app.foo.i, 5)
111 self.assertEquals(app.foo.i, 5)
112 app.init_foo()
112 app.init_foo()
113 self.assertEquals(app.foo.j, 10)
113 self.assertEquals(app.foo.j, 10)
114
114
115 def test_flag_clobber(self):
115 def test_flag_clobber(self):
116 """test that setting flags doesn't clobber existing settings"""
116 """test that setting flags doesn't clobber existing settings"""
117 app = MyApp()
117 app = MyApp()
118 app.parse_command_line(["Bar.b=5", "--disable"])
118 app.parse_command_line(["--Bar.b=5", "--disable"])
119 app.init_bar()
119 app.init_bar()
120 self.assertEquals(app.bar.enabled, False)
120 self.assertEquals(app.bar.enabled, False)
121 self.assertEquals(app.bar.b, 5)
121 self.assertEquals(app.bar.b, 5)
122 app.parse_command_line(["--enable", "Bar.b=10"])
122 app.parse_command_line(["--enable", "--Bar.b=10"])
123 app.init_bar()
123 app.init_bar()
124 self.assertEquals(app.bar.enabled, True)
124 self.assertEquals(app.bar.enabled, True)
125 self.assertEquals(app.bar.b, 10)
125 self.assertEquals(app.bar.b, 10)
126
126
127 def test_extra_args(self):
127 def test_extra_args(self):
128 app = MyApp()
128 app = MyApp()
129 app.parse_command_line(["Bar.b=5", 'extra', "--disable", 'args'])
129 app.parse_command_line(["--Bar.b=5", 'extra', "--disable", 'args'])
130 app.init_bar()
130 app.init_bar()
131 self.assertEquals(app.bar.enabled, True)
131 self.assertEquals(app.bar.enabled, True)
132 self.assertEquals(app.bar.b, 5)
132 self.assertEquals(app.bar.b, 5)
133 self.assertEquals(app.extra_args, ['extra', "--disable", 'args'])
133 self.assertEquals(app.extra_args, ['extra', "--disable", 'args'])
134
134
135
135
@@ -1,219 +1,219 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 Tests for IPython.config.loader
4 Tests for IPython.config.loader
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * Fernando Perez (design help)
9 * Fernando Perez (design help)
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2009 The IPython Development Team
13 # Copyright (C) 2008-2009 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import sys
24 import sys
25 from tempfile import mkstemp
25 from tempfile import mkstemp
26 from unittest import TestCase
26 from unittest import TestCase
27
27
28 from nose import SkipTest
28 from nose import SkipTest
29
29
30 from IPython.utils.traitlets import Int, Unicode
30 from IPython.utils.traitlets import Int, Unicode
31 from IPython.config.configurable import Configurable
31 from IPython.config.configurable import Configurable
32 from IPython.config.loader import (
32 from IPython.config.loader import (
33 Config,
33 Config,
34 PyFileConfigLoader,
34 PyFileConfigLoader,
35 KeyValueConfigLoader,
35 KeyValueConfigLoader,
36 ArgParseConfigLoader,
36 ArgParseConfigLoader,
37 ConfigError
37 ConfigError
38 )
38 )
39
39
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41 # Actual tests
41 # Actual tests
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43
43
44
44
45 pyfile = """
45 pyfile = """
46 c = get_config()
46 c = get_config()
47 c.a=10
47 c.a=10
48 c.b=20
48 c.b=20
49 c.Foo.Bar.value=10
49 c.Foo.Bar.value=10
50 c.Foo.Bam.value=range(10)
50 c.Foo.Bam.value=range(10)
51 c.D.C.value='hi there'
51 c.D.C.value='hi there'
52 """
52 """
53
53
54 class TestPyFileCL(TestCase):
54 class TestPyFileCL(TestCase):
55
55
56 def test_basic(self):
56 def test_basic(self):
57 fd, fname = mkstemp('.py')
57 fd, fname = mkstemp('.py')
58 f = os.fdopen(fd, 'w')
58 f = os.fdopen(fd, 'w')
59 f.write(pyfile)
59 f.write(pyfile)
60 f.close()
60 f.close()
61 # Unlink the file
61 # Unlink the file
62 cl = PyFileConfigLoader(fname)
62 cl = PyFileConfigLoader(fname)
63 config = cl.load_config()
63 config = cl.load_config()
64 self.assertEquals(config.a, 10)
64 self.assertEquals(config.a, 10)
65 self.assertEquals(config.b, 20)
65 self.assertEquals(config.b, 20)
66 self.assertEquals(config.Foo.Bar.value, 10)
66 self.assertEquals(config.Foo.Bar.value, 10)
67 self.assertEquals(config.Foo.Bam.value, range(10))
67 self.assertEquals(config.Foo.Bam.value, range(10))
68 self.assertEquals(config.D.C.value, 'hi there')
68 self.assertEquals(config.D.C.value, 'hi there')
69
69
70 class MyLoader1(ArgParseConfigLoader):
70 class MyLoader1(ArgParseConfigLoader):
71 def _add_arguments(self):
71 def _add_arguments(self):
72 p = self.parser
72 p = self.parser
73 p.add_argument('-f', '--foo', dest='Global.foo', type=str)
73 p.add_argument('-f', '--foo', dest='Global.foo', type=str)
74 p.add_argument('-b', dest='MyClass.bar', type=int)
74 p.add_argument('-b', dest='MyClass.bar', type=int)
75 p.add_argument('-n', dest='n', action='store_true')
75 p.add_argument('-n', dest='n', action='store_true')
76 p.add_argument('Global.bam', type=str)
76 p.add_argument('Global.bam', type=str)
77
77
78 class MyLoader2(ArgParseConfigLoader):
78 class MyLoader2(ArgParseConfigLoader):
79 def _add_arguments(self):
79 def _add_arguments(self):
80 subparsers = self.parser.add_subparsers(dest='subparser_name')
80 subparsers = self.parser.add_subparsers(dest='subparser_name')
81 subparser1 = subparsers.add_parser('1')
81 subparser1 = subparsers.add_parser('1')
82 subparser1.add_argument('-x',dest='Global.x')
82 subparser1.add_argument('-x',dest='Global.x')
83 subparser2 = subparsers.add_parser('2')
83 subparser2 = subparsers.add_parser('2')
84 subparser2.add_argument('y')
84 subparser2.add_argument('y')
85
85
86 class TestArgParseCL(TestCase):
86 class TestArgParseCL(TestCase):
87
87
88 def test_basic(self):
88 def test_basic(self):
89 cl = MyLoader1()
89 cl = MyLoader1()
90 config = cl.load_config('-f hi -b 10 -n wow'.split())
90 config = cl.load_config('-f hi -b 10 -n wow'.split())
91 self.assertEquals(config.Global.foo, 'hi')
91 self.assertEquals(config.Global.foo, 'hi')
92 self.assertEquals(config.MyClass.bar, 10)
92 self.assertEquals(config.MyClass.bar, 10)
93 self.assertEquals(config.n, True)
93 self.assertEquals(config.n, True)
94 self.assertEquals(config.Global.bam, 'wow')
94 self.assertEquals(config.Global.bam, 'wow')
95 config = cl.load_config(['wow'])
95 config = cl.load_config(['wow'])
96 self.assertEquals(config.keys(), ['Global'])
96 self.assertEquals(config.keys(), ['Global'])
97 self.assertEquals(config.Global.keys(), ['bam'])
97 self.assertEquals(config.Global.keys(), ['bam'])
98 self.assertEquals(config.Global.bam, 'wow')
98 self.assertEquals(config.Global.bam, 'wow')
99
99
100 def test_add_arguments(self):
100 def test_add_arguments(self):
101 cl = MyLoader2()
101 cl = MyLoader2()
102 config = cl.load_config('2 frobble'.split())
102 config = cl.load_config('2 frobble'.split())
103 self.assertEquals(config.subparser_name, '2')
103 self.assertEquals(config.subparser_name, '2')
104 self.assertEquals(config.y, 'frobble')
104 self.assertEquals(config.y, 'frobble')
105 config = cl.load_config('1 -x frobble'.split())
105 config = cl.load_config('1 -x frobble'.split())
106 self.assertEquals(config.subparser_name, '1')
106 self.assertEquals(config.subparser_name, '1')
107 self.assertEquals(config.Global.x, 'frobble')
107 self.assertEquals(config.Global.x, 'frobble')
108
108
109 def test_argv(self):
109 def test_argv(self):
110 cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
110 cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
111 config = cl.load_config()
111 config = cl.load_config()
112 self.assertEquals(config.Global.foo, 'hi')
112 self.assertEquals(config.Global.foo, 'hi')
113 self.assertEquals(config.MyClass.bar, 10)
113 self.assertEquals(config.MyClass.bar, 10)
114 self.assertEquals(config.n, True)
114 self.assertEquals(config.n, True)
115 self.assertEquals(config.Global.bam, 'wow')
115 self.assertEquals(config.Global.bam, 'wow')
116
116
117
117
118 class TestKeyValueCL(TestCase):
118 class TestKeyValueCL(TestCase):
119
119
120 def test_basic(self):
120 def test_basic(self):
121 cl = KeyValueConfigLoader()
121 cl = KeyValueConfigLoader()
122 argv = [s.strip('c.') for s in pyfile.split('\n')[2:-1]]
122 argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
123 config = cl.load_config(argv)
123 config = cl.load_config(argv)
124 self.assertEquals(config.a, 10)
124 self.assertEquals(config.a, 10)
125 self.assertEquals(config.b, 20)
125 self.assertEquals(config.b, 20)
126 self.assertEquals(config.Foo.Bar.value, 10)
126 self.assertEquals(config.Foo.Bar.value, 10)
127 self.assertEquals(config.Foo.Bam.value, range(10))
127 self.assertEquals(config.Foo.Bam.value, range(10))
128 self.assertEquals(config.D.C.value, 'hi there')
128 self.assertEquals(config.D.C.value, 'hi there')
129
129
130 def test_extra_args(self):
130 def test_extra_args(self):
131 cl = KeyValueConfigLoader()
131 cl = KeyValueConfigLoader()
132 config = cl.load_config(['a=5', 'b', 'c=10', 'd'])
132 config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
133 self.assertEquals(cl.extra_args, ['b', 'c=10' , 'd'])
133 self.assertEquals(cl.extra_args, ['b', '--c=10' , 'd'])
134 self.assertEquals(config.a, 5)
134 self.assertEquals(config.a, 5)
135 self.assertRaises(AttributeError, getattr, config, 'c')
135 self.assertRaises(AttributeError, getattr, config, 'c')
136 config = cl.load_config(['--', 'a=5', 'c=10'])
136 config = cl.load_config(['--', '--a=5', '--c=10'])
137 self.assertEquals(cl.extra_args, ['a=5', 'c=10'])
137 self.assertEquals(cl.extra_args, ['--a=5', '--c=10'])
138
138
139 def test_unicode_args(self):
139 def test_unicode_args(self):
140 cl = KeyValueConfigLoader()
140 cl = KeyValueConfigLoader()
141 argv = [u'a=épsîlön']
141 argv = [u'--a=épsîlön']
142 config = cl.load_config(argv)
142 config = cl.load_config(argv)
143 self.assertEquals(config.a, u'épsîlön')
143 self.assertEquals(config.a, u'épsîlön')
144
144
145 def test_unicode_bytes_args(self):
145 def test_unicode_bytes_args(self):
146 uarg = u'a=é'
146 uarg = u'--a=é'
147 try:
147 try:
148 barg = uarg.encode(sys.stdin.encoding)
148 barg = uarg.encode(sys.stdin.encoding)
149 except (TypeError, UnicodeEncodeError):
149 except (TypeError, UnicodeEncodeError):
150 raise SkipTest("sys.stdin.encoding can't handle 'é'")
150 raise SkipTest("sys.stdin.encoding can't handle 'é'")
151
151
152 cl = KeyValueConfigLoader()
152 cl = KeyValueConfigLoader()
153 config = cl.load_config([barg])
153 config = cl.load_config([barg])
154 self.assertEquals(config.a, u'é')
154 self.assertEquals(config.a, u'é')
155
155
156
156
157 class TestConfig(TestCase):
157 class TestConfig(TestCase):
158
158
159 def test_setget(self):
159 def test_setget(self):
160 c = Config()
160 c = Config()
161 c.a = 10
161 c.a = 10
162 self.assertEquals(c.a, 10)
162 self.assertEquals(c.a, 10)
163 self.assertEquals(c.has_key('b'), False)
163 self.assertEquals(c.has_key('b'), False)
164
164
165 def test_auto_section(self):
165 def test_auto_section(self):
166 c = Config()
166 c = Config()
167 self.assertEquals(c.has_key('A'), True)
167 self.assertEquals(c.has_key('A'), True)
168 self.assertEquals(c._has_section('A'), False)
168 self.assertEquals(c._has_section('A'), False)
169 A = c.A
169 A = c.A
170 A.foo = 'hi there'
170 A.foo = 'hi there'
171 self.assertEquals(c._has_section('A'), True)
171 self.assertEquals(c._has_section('A'), True)
172 self.assertEquals(c.A.foo, 'hi there')
172 self.assertEquals(c.A.foo, 'hi there')
173 del c.A
173 del c.A
174 self.assertEquals(len(c.A.keys()),0)
174 self.assertEquals(len(c.A.keys()),0)
175
175
176 def test_merge_doesnt_exist(self):
176 def test_merge_doesnt_exist(self):
177 c1 = Config()
177 c1 = Config()
178 c2 = Config()
178 c2 = Config()
179 c2.bar = 10
179 c2.bar = 10
180 c2.Foo.bar = 10
180 c2.Foo.bar = 10
181 c1._merge(c2)
181 c1._merge(c2)
182 self.assertEquals(c1.Foo.bar, 10)
182 self.assertEquals(c1.Foo.bar, 10)
183 self.assertEquals(c1.bar, 10)
183 self.assertEquals(c1.bar, 10)
184 c2.Bar.bar = 10
184 c2.Bar.bar = 10
185 c1._merge(c2)
185 c1._merge(c2)
186 self.assertEquals(c1.Bar.bar, 10)
186 self.assertEquals(c1.Bar.bar, 10)
187
187
188 def test_merge_exists(self):
188 def test_merge_exists(self):
189 c1 = Config()
189 c1 = Config()
190 c2 = Config()
190 c2 = Config()
191 c1.Foo.bar = 10
191 c1.Foo.bar = 10
192 c1.Foo.bam = 30
192 c1.Foo.bam = 30
193 c2.Foo.bar = 20
193 c2.Foo.bar = 20
194 c2.Foo.wow = 40
194 c2.Foo.wow = 40
195 c1._merge(c2)
195 c1._merge(c2)
196 self.assertEquals(c1.Foo.bam, 30)
196 self.assertEquals(c1.Foo.bam, 30)
197 self.assertEquals(c1.Foo.bar, 20)
197 self.assertEquals(c1.Foo.bar, 20)
198 self.assertEquals(c1.Foo.wow, 40)
198 self.assertEquals(c1.Foo.wow, 40)
199 c2.Foo.Bam.bam = 10
199 c2.Foo.Bam.bam = 10
200 c1._merge(c2)
200 c1._merge(c2)
201 self.assertEquals(c1.Foo.Bam.bam, 10)
201 self.assertEquals(c1.Foo.Bam.bam, 10)
202
202
203 def test_deepcopy(self):
203 def test_deepcopy(self):
204 c1 = Config()
204 c1 = Config()
205 c1.Foo.bar = 10
205 c1.Foo.bar = 10
206 c1.Foo.bam = 30
206 c1.Foo.bam = 30
207 c1.a = 'asdf'
207 c1.a = 'asdf'
208 c1.b = range(10)
208 c1.b = range(10)
209 import copy
209 import copy
210 c2 = copy.deepcopy(c1)
210 c2 = copy.deepcopy(c1)
211 self.assertEquals(c1, c2)
211 self.assertEquals(c1, c2)
212 self.assert_(c1 is not c2)
212 self.assert_(c1 is not c2)
213 self.assert_(c1.Foo is not c2.Foo)
213 self.assert_(c1.Foo is not c2.Foo)
214
214
215 def test_builtin(self):
215 def test_builtin(self):
216 c1 = Config()
216 c1 = Config()
217 exec 'foo = True' in c1
217 exec 'foo = True' in c1
218 self.assertEquals(c1.foo, True)
218 self.assertEquals(c1.foo, True)
219 self.assertRaises(ConfigError, setattr, c1, 'ValueError', 10)
219 self.assertRaises(ConfigError, setattr, c1, 'ValueError', 10)
@@ -1,422 +1,422 b''
1 """ A minimal application using the Qt console-style IPython frontend.
1 """ A minimal application using the Qt console-style IPython frontend.
2
2
3 This is not a complete console app, as subprocess will not be able to receive
3 This is not a complete console app, as subprocess will not be able to receive
4 input, there is no real readline support, among other limitations.
4 input, there is no real readline support, among other limitations.
5
5
6 Authors:
6 Authors:
7
7
8 * Evan Patterson
8 * Evan Patterson
9 * Min RK
9 * Min RK
10 * Erik Tollerud
10 * Erik Tollerud
11 * Fernando Perez
11 * Fernando Perez
12
12
13 """
13 """
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 # stdlib imports
19 # stdlib imports
20 import os
20 import os
21 import signal
21 import signal
22 import sys
22 import sys
23
23
24 # System library imports
24 # System library imports
25 from IPython.external.qt import QtGui
25 from IPython.external.qt import QtGui
26 from pygments.styles import get_all_styles
26 from pygments.styles import get_all_styles
27
27
28 # Local imports
28 # Local imports
29 from IPython.config.application import boolean_flag
29 from IPython.config.application import boolean_flag
30 from IPython.core.application import BaseIPythonApplication
30 from IPython.core.application import BaseIPythonApplication
31 from IPython.core.profiledir import ProfileDir
31 from IPython.core.profiledir import ProfileDir
32 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
32 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
33 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
33 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
34 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
34 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
35 from IPython.frontend.qt.console import styles
35 from IPython.frontend.qt.console import styles
36 from IPython.frontend.qt.kernelmanager import QtKernelManager
36 from IPython.frontend.qt.kernelmanager import QtKernelManager
37 from IPython.utils.traitlets import (
37 from IPython.utils.traitlets import (
38 Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
38 Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
39 )
39 )
40 from IPython.zmq.ipkernel import (
40 from IPython.zmq.ipkernel import (
41 flags as ipkernel_flags,
41 flags as ipkernel_flags,
42 aliases as ipkernel_aliases,
42 aliases as ipkernel_aliases,
43 IPKernelApp
43 IPKernelApp
44 )
44 )
45 from IPython.zmq.session import Session
45 from IPython.zmq.session import Session
46 from IPython.zmq.zmqshell import ZMQInteractiveShell
46 from IPython.zmq.zmqshell import ZMQInteractiveShell
47
47
48
48
49 #-----------------------------------------------------------------------------
49 #-----------------------------------------------------------------------------
50 # Network Constants
50 # Network Constants
51 #-----------------------------------------------------------------------------
51 #-----------------------------------------------------------------------------
52
52
53 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
53 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Classes
56 # Classes
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 class MainWindow(QtGui.QMainWindow):
59 class MainWindow(QtGui.QMainWindow):
60
60
61 #---------------------------------------------------------------------------
61 #---------------------------------------------------------------------------
62 # 'object' interface
62 # 'object' interface
63 #---------------------------------------------------------------------------
63 #---------------------------------------------------------------------------
64
64
65 def __init__(self, app, frontend, existing=False, may_close=True,
65 def __init__(self, app, frontend, existing=False, may_close=True,
66 confirm_exit=True):
66 confirm_exit=True):
67 """ Create a MainWindow for the specified FrontendWidget.
67 """ Create a MainWindow for the specified FrontendWidget.
68
68
69 The app is passed as an argument to allow for different
69 The app is passed as an argument to allow for different
70 closing behavior depending on whether we are the Kernel's parent.
70 closing behavior depending on whether we are the Kernel's parent.
71
71
72 If existing is True, then this Console does not own the Kernel.
72 If existing is True, then this Console does not own the Kernel.
73
73
74 If may_close is True, then this Console is permitted to close the kernel
74 If may_close is True, then this Console is permitted to close the kernel
75 """
75 """
76 super(MainWindow, self).__init__()
76 super(MainWindow, self).__init__()
77 self._app = app
77 self._app = app
78 self._frontend = frontend
78 self._frontend = frontend
79 self._existing = existing
79 self._existing = existing
80 if existing:
80 if existing:
81 self._may_close = may_close
81 self._may_close = may_close
82 else:
82 else:
83 self._may_close = True
83 self._may_close = True
84 self._frontend.exit_requested.connect(self.close)
84 self._frontend.exit_requested.connect(self.close)
85 self._confirm_exit = confirm_exit
85 self._confirm_exit = confirm_exit
86 self.setCentralWidget(frontend)
86 self.setCentralWidget(frontend)
87
87
88 #---------------------------------------------------------------------------
88 #---------------------------------------------------------------------------
89 # QWidget interface
89 # QWidget interface
90 #---------------------------------------------------------------------------
90 #---------------------------------------------------------------------------
91
91
92 def closeEvent(self, event):
92 def closeEvent(self, event):
93 """ Close the window and the kernel (if necessary).
93 """ Close the window and the kernel (if necessary).
94
94
95 This will prompt the user if they are finished with the kernel, and if
95 This will prompt the user if they are finished with the kernel, and if
96 so, closes the kernel cleanly. Alternatively, if the exit magic is used,
96 so, closes the kernel cleanly. Alternatively, if the exit magic is used,
97 it closes without prompt.
97 it closes without prompt.
98 """
98 """
99 keepkernel = None #Use the prompt by default
99 keepkernel = None #Use the prompt by default
100 if hasattr(self._frontend,'_keep_kernel_on_exit'): #set by exit magic
100 if hasattr(self._frontend,'_keep_kernel_on_exit'): #set by exit magic
101 keepkernel = self._frontend._keep_kernel_on_exit
101 keepkernel = self._frontend._keep_kernel_on_exit
102
102
103 kernel_manager = self._frontend.kernel_manager
103 kernel_manager = self._frontend.kernel_manager
104
104
105 if keepkernel is None and not self._confirm_exit:
105 if keepkernel is None and not self._confirm_exit:
106 # don't prompt, just terminate the kernel if we own it
106 # don't prompt, just terminate the kernel if we own it
107 # or leave it alone if we don't
107 # or leave it alone if we don't
108 keepkernel = not self._existing
108 keepkernel = not self._existing
109
109
110 if keepkernel is None: #show prompt
110 if keepkernel is None: #show prompt
111 if kernel_manager and kernel_manager.channels_running:
111 if kernel_manager and kernel_manager.channels_running:
112 title = self.window().windowTitle()
112 title = self.window().windowTitle()
113 cancel = QtGui.QMessageBox.Cancel
113 cancel = QtGui.QMessageBox.Cancel
114 okay = QtGui.QMessageBox.Ok
114 okay = QtGui.QMessageBox.Ok
115 if self._may_close:
115 if self._may_close:
116 msg = "You are closing this Console window."
116 msg = "You are closing this Console window."
117 info = "Would you like to quit the Kernel and all attached Consoles as well?"
117 info = "Would you like to quit the Kernel and all attached Consoles as well?"
118 justthis = QtGui.QPushButton("&No, just this Console", self)
118 justthis = QtGui.QPushButton("&No, just this Console", self)
119 justthis.setShortcut('N')
119 justthis.setShortcut('N')
120 closeall = QtGui.QPushButton("&Yes, quit everything", self)
120 closeall = QtGui.QPushButton("&Yes, quit everything", self)
121 closeall.setShortcut('Y')
121 closeall.setShortcut('Y')
122 box = QtGui.QMessageBox(QtGui.QMessageBox.Question,
122 box = QtGui.QMessageBox(QtGui.QMessageBox.Question,
123 title, msg)
123 title, msg)
124 box.setInformativeText(info)
124 box.setInformativeText(info)
125 box.addButton(cancel)
125 box.addButton(cancel)
126 box.addButton(justthis, QtGui.QMessageBox.NoRole)
126 box.addButton(justthis, QtGui.QMessageBox.NoRole)
127 box.addButton(closeall, QtGui.QMessageBox.YesRole)
127 box.addButton(closeall, QtGui.QMessageBox.YesRole)
128 box.setDefaultButton(closeall)
128 box.setDefaultButton(closeall)
129 box.setEscapeButton(cancel)
129 box.setEscapeButton(cancel)
130 reply = box.exec_()
130 reply = box.exec_()
131 if reply == 1: # close All
131 if reply == 1: # close All
132 kernel_manager.shutdown_kernel()
132 kernel_manager.shutdown_kernel()
133 #kernel_manager.stop_channels()
133 #kernel_manager.stop_channels()
134 event.accept()
134 event.accept()
135 elif reply == 0: # close Console
135 elif reply == 0: # close Console
136 if not self._existing:
136 if not self._existing:
137 # Have kernel: don't quit, just close the window
137 # Have kernel: don't quit, just close the window
138 self._app.setQuitOnLastWindowClosed(False)
138 self._app.setQuitOnLastWindowClosed(False)
139 self.deleteLater()
139 self.deleteLater()
140 event.accept()
140 event.accept()
141 else:
141 else:
142 event.ignore()
142 event.ignore()
143 else:
143 else:
144 reply = QtGui.QMessageBox.question(self, title,
144 reply = QtGui.QMessageBox.question(self, title,
145 "Are you sure you want to close this Console?"+
145 "Are you sure you want to close this Console?"+
146 "\nThe Kernel and other Consoles will remain active.",
146 "\nThe Kernel and other Consoles will remain active.",
147 okay|cancel,
147 okay|cancel,
148 defaultButton=okay
148 defaultButton=okay
149 )
149 )
150 if reply == okay:
150 if reply == okay:
151 event.accept()
151 event.accept()
152 else:
152 else:
153 event.ignore()
153 event.ignore()
154 elif keepkernel: #close console but leave kernel running (no prompt)
154 elif keepkernel: #close console but leave kernel running (no prompt)
155 if kernel_manager and kernel_manager.channels_running:
155 if kernel_manager and kernel_manager.channels_running:
156 if not self._existing:
156 if not self._existing:
157 # I have the kernel: don't quit, just close the window
157 # I have the kernel: don't quit, just close the window
158 self._app.setQuitOnLastWindowClosed(False)
158 self._app.setQuitOnLastWindowClosed(False)
159 event.accept()
159 event.accept()
160 else: #close console and kernel (no prompt)
160 else: #close console and kernel (no prompt)
161 if kernel_manager and kernel_manager.channels_running:
161 if kernel_manager and kernel_manager.channels_running:
162 kernel_manager.shutdown_kernel()
162 kernel_manager.shutdown_kernel()
163 event.accept()
163 event.accept()
164
164
165 #-----------------------------------------------------------------------------
165 #-----------------------------------------------------------------------------
166 # Aliases and Flags
166 # Aliases and Flags
167 #-----------------------------------------------------------------------------
167 #-----------------------------------------------------------------------------
168
168
169 flags = dict(ipkernel_flags)
169 flags = dict(ipkernel_flags)
170
170
171 flags.update({
171 flags.update({
172 'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
172 'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
173 "Connect to an existing kernel."),
173 "Connect to an existing kernel."),
174 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
174 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
175 "Use a pure Python kernel instead of an IPython kernel."),
175 "Use a pure Python kernel instead of an IPython kernel."),
176 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
176 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
177 "Disable rich text support."),
177 "Disable rich text support."),
178 })
178 })
179 flags.update(boolean_flag(
179 flags.update(boolean_flag(
180 'gui-completion', 'ConsoleWidget.gui_completion',
180 'gui-completion', 'ConsoleWidget.gui_completion',
181 "use a GUI widget for tab completion",
181 "use a GUI widget for tab completion",
182 "use plaintext output for completion"
182 "use plaintext output for completion"
183 ))
183 ))
184 flags.update(boolean_flag(
184 flags.update(boolean_flag(
185 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
185 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
186 """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
186 """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
187 to force a direct exit without any confirmation.
187 to force a direct exit without any confirmation.
188 """,
188 """,
189 """Don't prompt the user when exiting. This will terminate the kernel
189 """Don't prompt the user when exiting. This will terminate the kernel
190 if it is owned by the frontend, and leave it alive if it is external.
190 if it is owned by the frontend, and leave it alive if it is external.
191 """
191 """
192 ))
192 ))
193 # the flags that are specific to the frontend
193 # the flags that are specific to the frontend
194 # these must be scrubbed before being passed to the kernel,
194 # these must be scrubbed before being passed to the kernel,
195 # or it will raise an error on unrecognized flags
195 # or it will raise an error on unrecognized flags
196 qt_flags = ['existing', 'pure', 'plain', 'gui-completion', 'no-gui-completion',
196 qt_flags = ['existing', 'pure', 'plain', 'gui-completion', 'no-gui-completion',
197 'confirm-exit', 'no-confirm-exit']
197 'confirm-exit', 'no-confirm-exit']
198
198
199 aliases = dict(ipkernel_aliases)
199 aliases = dict(ipkernel_aliases)
200
200
201 aliases.update(dict(
201 aliases.update(dict(
202 hb = 'IPythonQtConsoleApp.hb_port',
202 hb = 'IPythonQtConsoleApp.hb_port',
203 shell = 'IPythonQtConsoleApp.shell_port',
203 shell = 'IPythonQtConsoleApp.shell_port',
204 iopub = 'IPythonQtConsoleApp.iopub_port',
204 iopub = 'IPythonQtConsoleApp.iopub_port',
205 stdin = 'IPythonQtConsoleApp.stdin_port',
205 stdin = 'IPythonQtConsoleApp.stdin_port',
206 ip = 'IPythonQtConsoleApp.ip',
206 ip = 'IPythonQtConsoleApp.ip',
207
207
208 plain = 'IPythonQtConsoleApp.plain',
208 plain = 'IPythonQtConsoleApp.plain',
209 pure = 'IPythonQtConsoleApp.pure',
209 pure = 'IPythonQtConsoleApp.pure',
210 gui_completion = 'ConsoleWidget.gui_completion',
210 gui_completion = 'ConsoleWidget.gui_completion',
211 style = 'IPythonWidget.syntax_style',
211 style = 'IPythonWidget.syntax_style',
212 stylesheet = 'IPythonQtConsoleApp.stylesheet',
212 stylesheet = 'IPythonQtConsoleApp.stylesheet',
213 colors = 'ZMQInteractiveShell.colors',
213 colors = 'ZMQInteractiveShell.colors',
214
214
215 editor = 'IPythonWidget.editor',
215 editor = 'IPythonWidget.editor',
216 ))
216 ))
217
217
218 #-----------------------------------------------------------------------------
218 #-----------------------------------------------------------------------------
219 # IPythonQtConsole
219 # IPythonQtConsole
220 #-----------------------------------------------------------------------------
220 #-----------------------------------------------------------------------------
221 class IPythonQtConsoleApp(BaseIPythonApplication):
221 class IPythonQtConsoleApp(BaseIPythonApplication):
222 name = 'ipython-qtconsole'
222 name = 'ipython-qtconsole'
223 default_config_file_name='ipython_config.py'
223 default_config_file_name='ipython_config.py'
224
224
225 description = """
225 description = """
226 The IPython QtConsole.
226 The IPython QtConsole.
227
227
228 This launches a Console-style application using Qt. It is not a full
228 This launches a Console-style application using Qt. It is not a full
229 console, in that launched terminal subprocesses will not.
229 console, in that launched terminal subprocesses will not.
230
230
231 The QtConsole supports various extra features beyond the
231 The QtConsole supports various extra features beyond the
232
232
233 """
233 """
234
234
235 classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
235 classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
236 flags = Dict(flags)
236 flags = Dict(flags)
237 aliases = Dict(aliases)
237 aliases = Dict(aliases)
238
238
239 kernel_argv = List(Unicode)
239 kernel_argv = List(Unicode)
240
240
241 # connection info:
241 # connection info:
242 ip = Unicode(LOCALHOST, config=True,
242 ip = Unicode(LOCALHOST, config=True,
243 help="""Set the kernel\'s IP address [default localhost].
243 help="""Set the kernel\'s IP address [default localhost].
244 If the IP address is something other than localhost, then
244 If the IP address is something other than localhost, then
245 Consoles on other machines will be able to connect
245 Consoles on other machines will be able to connect
246 to the Kernel, so be careful!"""
246 to the Kernel, so be careful!"""
247 )
247 )
248 hb_port = Int(0, config=True,
248 hb_port = Int(0, config=True,
249 help="set the heartbeat port [default: random]")
249 help="set the heartbeat port [default: random]")
250 shell_port = Int(0, config=True,
250 shell_port = Int(0, config=True,
251 help="set the shell (XREP) port [default: random]")
251 help="set the shell (XREP) port [default: random]")
252 iopub_port = Int(0, config=True,
252 iopub_port = Int(0, config=True,
253 help="set the iopub (PUB) port [default: random]")
253 help="set the iopub (PUB) port [default: random]")
254 stdin_port = Int(0, config=True,
254 stdin_port = Int(0, config=True,
255 help="set the stdin (XREQ) port [default: random]")
255 help="set the stdin (XREQ) port [default: random]")
256
256
257 existing = CBool(False, config=True,
257 existing = CBool(False, config=True,
258 help="Whether to connect to an already running Kernel.")
258 help="Whether to connect to an already running Kernel.")
259
259
260 stylesheet = Unicode('', config=True,
260 stylesheet = Unicode('', config=True,
261 help="path to a custom CSS stylesheet")
261 help="path to a custom CSS stylesheet")
262
262
263 pure = CBool(False, config=True,
263 pure = CBool(False, config=True,
264 help="Use a pure Python kernel instead of an IPython kernel.")
264 help="Use a pure Python kernel instead of an IPython kernel.")
265 plain = CBool(False, config=True,
265 plain = CBool(False, config=True,
266 help="Use a plaintext widget instead of rich text (plain can't print/save).")
266 help="Use a plaintext widget instead of rich text (plain can't print/save).")
267
267
268 def _pure_changed(self, name, old, new):
268 def _pure_changed(self, name, old, new):
269 kind = 'plain' if self.plain else 'rich'
269 kind = 'plain' if self.plain else 'rich'
270 self.config.ConsoleWidget.kind = kind
270 self.config.ConsoleWidget.kind = kind
271 if self.pure:
271 if self.pure:
272 self.widget_factory = FrontendWidget
272 self.widget_factory = FrontendWidget
273 elif self.plain:
273 elif self.plain:
274 self.widget_factory = IPythonWidget
274 self.widget_factory = IPythonWidget
275 else:
275 else:
276 self.widget_factory = RichIPythonWidget
276 self.widget_factory = RichIPythonWidget
277
277
278 _plain_changed = _pure_changed
278 _plain_changed = _pure_changed
279
279
280 confirm_exit = CBool(True, config=True,
280 confirm_exit = CBool(True, config=True,
281 help="""
281 help="""
282 Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
282 Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
283 to force a direct exit without any confirmation.""",
283 to force a direct exit without any confirmation.""",
284 )
284 )
285
285
286 # the factory for creating a widget
286 # the factory for creating a widget
287 widget_factory = Any(RichIPythonWidget)
287 widget_factory = Any(RichIPythonWidget)
288
288
289 def parse_command_line(self, argv=None):
289 def parse_command_line(self, argv=None):
290 super(IPythonQtConsoleApp, self).parse_command_line(argv)
290 super(IPythonQtConsoleApp, self).parse_command_line(argv)
291 if argv is None:
291 if argv is None:
292 argv = sys.argv[1:]
292 argv = sys.argv[1:]
293
293
294 self.kernel_argv = list(argv) # copy
294 self.kernel_argv = list(argv) # copy
295 # kernel should inherit default config file from frontend
295 # kernel should inherit default config file from frontend
296 self.kernel_argv.append("KernelApp.parent_appname='%s'"%self.name)
296 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
297 # scrub frontend-specific flags
297 # scrub frontend-specific flags
298 for a in argv:
298 for a in argv:
299 if a.startswith('--') and a[2:] in qt_flags:
299 if a.startswith('-') and a.lstrip('-') in qt_flags:
300 self.kernel_argv.remove(a)
300 self.kernel_argv.remove(a)
301
301
302 def init_kernel_manager(self):
302 def init_kernel_manager(self):
303 # Don't let Qt or ZMQ swallow KeyboardInterupts.
303 # Don't let Qt or ZMQ swallow KeyboardInterupts.
304 signal.signal(signal.SIGINT, signal.SIG_DFL)
304 signal.signal(signal.SIGINT, signal.SIG_DFL)
305
305
306 # Create a KernelManager and start a kernel.
306 # Create a KernelManager and start a kernel.
307 self.kernel_manager = QtKernelManager(
307 self.kernel_manager = QtKernelManager(
308 shell_address=(self.ip, self.shell_port),
308 shell_address=(self.ip, self.shell_port),
309 sub_address=(self.ip, self.iopub_port),
309 sub_address=(self.ip, self.iopub_port),
310 stdin_address=(self.ip, self.stdin_port),
310 stdin_address=(self.ip, self.stdin_port),
311 hb_address=(self.ip, self.hb_port),
311 hb_address=(self.ip, self.hb_port),
312 config=self.config
312 config=self.config
313 )
313 )
314 # start the kernel
314 # start the kernel
315 if not self.existing:
315 if not self.existing:
316 kwargs = dict(ip=self.ip, ipython=not self.pure)
316 kwargs = dict(ip=self.ip, ipython=not self.pure)
317 kwargs['extra_arguments'] = self.kernel_argv
317 kwargs['extra_arguments'] = self.kernel_argv
318 self.kernel_manager.start_kernel(**kwargs)
318 self.kernel_manager.start_kernel(**kwargs)
319 self.kernel_manager.start_channels()
319 self.kernel_manager.start_channels()
320
320
321
321
322 def init_qt_elements(self):
322 def init_qt_elements(self):
323 # Create the widget.
323 # Create the widget.
324 self.app = QtGui.QApplication([])
324 self.app = QtGui.QApplication([])
325 local_kernel = (not self.existing) or self.ip in LOCAL_IPS
325 local_kernel = (not self.existing) or self.ip in LOCAL_IPS
326 self.widget = self.widget_factory(config=self.config,
326 self.widget = self.widget_factory(config=self.config,
327 local_kernel=local_kernel)
327 local_kernel=local_kernel)
328 self.widget.kernel_manager = self.kernel_manager
328 self.widget.kernel_manager = self.kernel_manager
329 self.window = MainWindow(self.app, self.widget, self.existing,
329 self.window = MainWindow(self.app, self.widget, self.existing,
330 may_close=local_kernel,
330 may_close=local_kernel,
331 confirm_exit=self.confirm_exit)
331 confirm_exit=self.confirm_exit)
332 self.window.setWindowTitle('Python' if self.pure else 'IPython')
332 self.window.setWindowTitle('Python' if self.pure else 'IPython')
333
333
334 def init_colors(self):
334 def init_colors(self):
335 """Configure the coloring of the widget"""
335 """Configure the coloring of the widget"""
336 # Note: This will be dramatically simplified when colors
336 # Note: This will be dramatically simplified when colors
337 # are removed from the backend.
337 # are removed from the backend.
338
338
339 if self.pure:
339 if self.pure:
340 # only IPythonWidget supports styling
340 # only IPythonWidget supports styling
341 return
341 return
342
342
343 # parse the colors arg down to current known labels
343 # parse the colors arg down to current known labels
344 try:
344 try:
345 colors = self.config.ZMQInteractiveShell.colors
345 colors = self.config.ZMQInteractiveShell.colors
346 except AttributeError:
346 except AttributeError:
347 colors = None
347 colors = None
348 try:
348 try:
349 style = self.config.IPythonWidget.colors
349 style = self.config.IPythonWidget.colors
350 except AttributeError:
350 except AttributeError:
351 style = None
351 style = None
352
352
353 # find the value for colors:
353 # find the value for colors:
354 if colors:
354 if colors:
355 colors=colors.lower()
355 colors=colors.lower()
356 if colors in ('lightbg', 'light'):
356 if colors in ('lightbg', 'light'):
357 colors='lightbg'
357 colors='lightbg'
358 elif colors in ('dark', 'linux'):
358 elif colors in ('dark', 'linux'):
359 colors='linux'
359 colors='linux'
360 else:
360 else:
361 colors='nocolor'
361 colors='nocolor'
362 elif style:
362 elif style:
363 if style=='bw':
363 if style=='bw':
364 colors='nocolor'
364 colors='nocolor'
365 elif styles.dark_style(style):
365 elif styles.dark_style(style):
366 colors='linux'
366 colors='linux'
367 else:
367 else:
368 colors='lightbg'
368 colors='lightbg'
369 else:
369 else:
370 colors=None
370 colors=None
371
371
372 # Configure the style.
372 # Configure the style.
373 widget = self.widget
373 widget = self.widget
374 if style:
374 if style:
375 widget.style_sheet = styles.sheet_from_template(style, colors)
375 widget.style_sheet = styles.sheet_from_template(style, colors)
376 widget.syntax_style = style
376 widget.syntax_style = style
377 widget._syntax_style_changed()
377 widget._syntax_style_changed()
378 widget._style_sheet_changed()
378 widget._style_sheet_changed()
379 elif colors:
379 elif colors:
380 # use a default style
380 # use a default style
381 widget.set_default_style(colors=colors)
381 widget.set_default_style(colors=colors)
382 else:
382 else:
383 # this is redundant for now, but allows the widget's
383 # this is redundant for now, but allows the widget's
384 # defaults to change
384 # defaults to change
385 widget.set_default_style()
385 widget.set_default_style()
386
386
387 if self.stylesheet:
387 if self.stylesheet:
388 # we got an expicit stylesheet
388 # we got an expicit stylesheet
389 if os.path.isfile(self.stylesheet):
389 if os.path.isfile(self.stylesheet):
390 with open(self.stylesheet) as f:
390 with open(self.stylesheet) as f:
391 sheet = f.read()
391 sheet = f.read()
392 widget.style_sheet = sheet
392 widget.style_sheet = sheet
393 widget._style_sheet_changed()
393 widget._style_sheet_changed()
394 else:
394 else:
395 raise IOError("Stylesheet %r not found."%self.stylesheet)
395 raise IOError("Stylesheet %r not found."%self.stylesheet)
396
396
397 def initialize(self, argv=None):
397 def initialize(self, argv=None):
398 super(IPythonQtConsoleApp, self).initialize(argv)
398 super(IPythonQtConsoleApp, self).initialize(argv)
399 self.init_kernel_manager()
399 self.init_kernel_manager()
400 self.init_qt_elements()
400 self.init_qt_elements()
401 self.init_colors()
401 self.init_colors()
402
402
403 def start(self):
403 def start(self):
404
404
405 # draw the window
405 # draw the window
406 self.window.show()
406 self.window.show()
407
407
408 # Start the application main loop.
408 # Start the application main loop.
409 self.app.exec_()
409 self.app.exec_()
410
410
411 #-----------------------------------------------------------------------------
411 #-----------------------------------------------------------------------------
412 # Main entry point
412 # Main entry point
413 #-----------------------------------------------------------------------------
413 #-----------------------------------------------------------------------------
414
414
415 def main():
415 def main():
416 app = IPythonQtConsoleApp()
416 app = IPythonQtConsoleApp()
417 app.initialize()
417 app.initialize()
418 app.start()
418 app.start()
419
419
420
420
421 if __name__ == '__main__':
421 if __name__ == '__main__':
422 main()
422 main()
@@ -1,363 +1,364 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The :class:`~IPython.core.application.Application` object for the command
4 The :class:`~IPython.core.application.Application` object for the command
5 line :command:`ipython` program.
5 line :command:`ipython` program.
6
6
7 Authors
7 Authors
8 -------
8 -------
9
9
10 * Brian Granger
10 * Brian Granger
11 * Fernando Perez
11 * Fernando Perez
12 * Min Ragan-Kelley
12 * Min Ragan-Kelley
13 """
13 """
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Copyright (C) 2008-2010 The IPython Development Team
16 # Copyright (C) 2008-2010 The IPython Development Team
17 #
17 #
18 # Distributed under the terms of the BSD License. The full license is in
18 # Distributed under the terms of the BSD License. The full license is in
19 # the file COPYING, distributed as part of this software.
19 # the file COPYING, distributed as part of this software.
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23 # Imports
23 # Imports
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25
25
26 from __future__ import absolute_import
26 from __future__ import absolute_import
27
27
28 import logging
28 import logging
29 import os
29 import os
30 import sys
30 import sys
31
31
32 from IPython.config.loader import (
32 from IPython.config.loader import (
33 Config, PyFileConfigLoader
33 Config, PyFileConfigLoader
34 )
34 )
35 from IPython.config.application import boolean_flag
35 from IPython.config.application import boolean_flag
36 from IPython.core import release
36 from IPython.core import release
37 from IPython.core import usage
37 from IPython.core import usage
38 from IPython.core.crashhandler import CrashHandler
38 from IPython.core.crashhandler import CrashHandler
39 from IPython.core.formatters import PlainTextFormatter
39 from IPython.core.formatters import PlainTextFormatter
40 from IPython.core.application import (
40 from IPython.core.application import (
41 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
41 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
42 )
42 )
43 from IPython.core.shellapp import (
43 from IPython.core.shellapp import (
44 InteractiveShellApp, shell_flags, shell_aliases
44 InteractiveShellApp, shell_flags, shell_aliases
45 )
45 )
46 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
46 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
47 from IPython.lib import inputhook
47 from IPython.lib import inputhook
48 from IPython.utils import warn
48 from IPython.utils import warn
49 from IPython.utils.path import get_ipython_dir, check_for_old_config
49 from IPython.utils.path import get_ipython_dir, check_for_old_config
50 from IPython.utils.traitlets import (
50 from IPython.utils.traitlets import (
51 Bool, Dict, CaselessStrEnum
51 Bool, Dict, CaselessStrEnum
52 )
52 )
53
53
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55 # Globals, utilities and helpers
55 # Globals, utilities and helpers
56 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
57
57
58 #: The default config file name for this application.
58 #: The default config file name for this application.
59 default_config_file_name = u'ipython_config.py'
59 default_config_file_name = u'ipython_config.py'
60
60
61
61
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63 # Crash handler for this application
63 # Crash handler for this application
64 #-----------------------------------------------------------------------------
64 #-----------------------------------------------------------------------------
65
65
66 class IPAppCrashHandler(CrashHandler):
66 class IPAppCrashHandler(CrashHandler):
67 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
67 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
68
68
69 def __init__(self, app):
69 def __init__(self, app):
70 contact_name = release.authors['Fernando'][0]
70 contact_name = release.authors['Fernando'][0]
71 contact_email = release.authors['Fernando'][1]
71 contact_email = release.authors['Fernando'][1]
72 bug_tracker = 'http://github.com/ipython/ipython/issues'
72 bug_tracker = 'http://github.com/ipython/ipython/issues'
73 super(IPAppCrashHandler,self).__init__(
73 super(IPAppCrashHandler,self).__init__(
74 app, contact_name, contact_email, bug_tracker
74 app, contact_name, contact_email, bug_tracker
75 )
75 )
76
76
77 def make_report(self,traceback):
77 def make_report(self,traceback):
78 """Return a string containing a crash report."""
78 """Return a string containing a crash report."""
79
79
80 sec_sep = self.section_sep
80 sec_sep = self.section_sep
81 # Start with parent report
81 # Start with parent report
82 report = [super(IPAppCrashHandler, self).make_report(traceback)]
82 report = [super(IPAppCrashHandler, self).make_report(traceback)]
83 # Add interactive-specific info we may have
83 # Add interactive-specific info we may have
84 rpt_add = report.append
84 rpt_add = report.append
85 try:
85 try:
86 rpt_add(sec_sep+"History of session input:")
86 rpt_add(sec_sep+"History of session input:")
87 for line in self.app.shell.user_ns['_ih']:
87 for line in self.app.shell.user_ns['_ih']:
88 rpt_add(line)
88 rpt_add(line)
89 rpt_add('\n*** Last line of input (may not be in above history):\n')
89 rpt_add('\n*** Last line of input (may not be in above history):\n')
90 rpt_add(self.app.shell._last_input_line+'\n')
90 rpt_add(self.app.shell._last_input_line+'\n')
91 except:
91 except:
92 pass
92 pass
93
93
94 return ''.join(report)
94 return ''.join(report)
95
95
96 #-----------------------------------------------------------------------------
96 #-----------------------------------------------------------------------------
97 # Aliases and Flags
97 # Aliases and Flags
98 #-----------------------------------------------------------------------------
98 #-----------------------------------------------------------------------------
99 flags = dict(base_flags)
99 flags = dict(base_flags)
100 flags.update(shell_flags)
100 flags.update(shell_flags)
101 addflag = lambda *args: flags.update(boolean_flag(*args))
101 addflag = lambda *args: flags.update(boolean_flag(*args))
102 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
102 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
103 'Turn on auto editing of files with syntax errors.',
103 'Turn on auto editing of files with syntax errors.',
104 'Turn off auto editing of files with syntax errors.'
104 'Turn off auto editing of files with syntax errors.'
105 )
105 )
106 addflag('banner', 'TerminalIPythonApp.display_banner',
106 addflag('banner', 'TerminalIPythonApp.display_banner',
107 "Display a banner upon starting IPython.",
107 "Display a banner upon starting IPython.",
108 "Don't display a banner upon starting IPython."
108 "Don't display a banner upon starting IPython."
109 )
109 )
110 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
110 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
111 """Set to confirm when you try to exit IPython with an EOF (Control-D
111 """Set to confirm when you try to exit IPython with an EOF (Control-D
112 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
112 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
113 you can force a direct exit without any confirmation.""",
113 you can force a direct exit without any confirmation.""",
114 "Don't prompt the user when exiting."
114 "Don't prompt the user when exiting."
115 )
115 )
116 addflag('term-title', 'TerminalInteractiveShell.term_title',
116 addflag('term-title', 'TerminalInteractiveShell.term_title',
117 "Enable auto setting the terminal title.",
117 "Enable auto setting the terminal title.",
118 "Disable auto setting the terminal title."
118 "Disable auto setting the terminal title."
119 )
119 )
120 classic_config = Config()
120 classic_config = Config()
121 classic_config.InteractiveShell.cache_size = 0
121 classic_config.InteractiveShell.cache_size = 0
122 classic_config.PlainTextFormatter.pprint = False
122 classic_config.PlainTextFormatter.pprint = False
123 classic_config.InteractiveShell.prompt_in1 = '>>> '
123 classic_config.InteractiveShell.prompt_in1 = '>>> '
124 classic_config.InteractiveShell.prompt_in2 = '... '
124 classic_config.InteractiveShell.prompt_in2 = '... '
125 classic_config.InteractiveShell.prompt_out = ''
125 classic_config.InteractiveShell.prompt_out = ''
126 classic_config.InteractiveShell.separate_in = ''
126 classic_config.InteractiveShell.separate_in = ''
127 classic_config.InteractiveShell.separate_out = ''
127 classic_config.InteractiveShell.separate_out = ''
128 classic_config.InteractiveShell.separate_out2 = ''
128 classic_config.InteractiveShell.separate_out2 = ''
129 classic_config.InteractiveShell.colors = 'NoColor'
129 classic_config.InteractiveShell.colors = 'NoColor'
130 classic_config.InteractiveShell.xmode = 'Plain'
130 classic_config.InteractiveShell.xmode = 'Plain'
131
131
132 flags['classic']=(
132 flags['classic']=(
133 classic_config,
133 classic_config,
134 "Gives IPython a similar feel to the classic Python prompt."
134 "Gives IPython a similar feel to the classic Python prompt."
135 )
135 )
136 # # log doesn't make so much sense this way anymore
136 # # log doesn't make so much sense this way anymore
137 # paa('--log','-l',
137 # paa('--log','-l',
138 # action='store_true', dest='InteractiveShell.logstart',
138 # action='store_true', dest='InteractiveShell.logstart',
139 # help="Start logging to the default log file (./ipython_log.py).")
139 # help="Start logging to the default log file (./ipython_log.py).")
140 #
140 #
141 # # quick is harder to implement
141 # # quick is harder to implement
142 flags['quick']=(
142 flags['quick']=(
143 {'TerminalIPythonApp' : {'quick' : True}},
143 {'TerminalIPythonApp' : {'quick' : True}},
144 "Enable quick startup with no config files."
144 "Enable quick startup with no config files."
145 )
145 )
146
146
147 flags['i'] = (
147 flags['i'] = (
148 {'TerminalIPythonApp' : {'force_interact' : True}},
148 {'TerminalIPythonApp' : {'force_interact' : True}},
149 "If running code from the command line, become interactive afterwards."
149 """also works as '-i'
150 If running code from the command line, become interactive afterwards."""
150 )
151 )
151 flags['pylab'] = (
152 flags['pylab'] = (
152 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
153 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
153 """Pre-load matplotlib and numpy for interactive use with
154 """Pre-load matplotlib and numpy for interactive use with
154 the default matplotlib backend."""
155 the default matplotlib backend."""
155 )
156 )
156
157
157 aliases = dict(base_aliases)
158 aliases = dict(base_aliases)
158 aliases.update(shell_aliases)
159 aliases.update(shell_aliases)
159
160
160 # it's possible we don't want short aliases for *all* of these:
161 # it's possible we don't want short aliases for *all* of these:
161 aliases.update(dict(
162 aliases.update(dict(
162 gui='TerminalIPythonApp.gui',
163 gui='TerminalIPythonApp.gui',
163 pylab='TerminalIPythonApp.pylab',
164 pylab='TerminalIPythonApp.pylab',
164 ))
165 ))
165
166
166 #-----------------------------------------------------------------------------
167 #-----------------------------------------------------------------------------
167 # Main classes and functions
168 # Main classes and functions
168 #-----------------------------------------------------------------------------
169 #-----------------------------------------------------------------------------
169
170
170 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
171 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
171 name = u'ipython'
172 name = u'ipython'
172 description = usage.cl_usage
173 description = usage.cl_usage
173 default_config_file_name = default_config_file_name
174 default_config_file_name = default_config_file_name
174 crash_handler_class = IPAppCrashHandler
175 crash_handler_class = IPAppCrashHandler
175
176
176 flags = Dict(flags)
177 flags = Dict(flags)
177 aliases = Dict(aliases)
178 aliases = Dict(aliases)
178 classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
179 classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
179 subcommands = Dict(dict(
180 subcommands = Dict(dict(
180 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
181 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
181 """Launch the IPython Qt Console."""
182 """Launch the IPython Qt Console."""
182 ),
183 ),
183 profile = ("IPython.core.profileapp.ProfileApp",
184 profile = ("IPython.core.profileapp.ProfileApp",
184 "Create and manage IPython profiles.")
185 "Create and manage IPython profiles.")
185 ))
186 ))
186
187
187 # *do* autocreate requested profile, but don't create the config file.
188 # *do* autocreate requested profile, but don't create the config file.
188 auto_create=Bool(True)
189 auto_create=Bool(True)
189 # configurables
190 # configurables
190 ignore_old_config=Bool(False, config=True,
191 ignore_old_config=Bool(False, config=True,
191 help="Suppress warning messages about legacy config files"
192 help="Suppress warning messages about legacy config files"
192 )
193 )
193 quick = Bool(False, config=True,
194 quick = Bool(False, config=True,
194 help="""Start IPython quickly by skipping the loading of config files."""
195 help="""Start IPython quickly by skipping the loading of config files."""
195 )
196 )
196 def _quick_changed(self, name, old, new):
197 def _quick_changed(self, name, old, new):
197 if new:
198 if new:
198 self.load_config_file = lambda *a, **kw: None
199 self.load_config_file = lambda *a, **kw: None
199 self.ignore_old_config=True
200 self.ignore_old_config=True
200
201
201 gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
202 gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
202 help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
203 help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
203 )
204 )
204 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
205 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
205 config=True,
206 config=True,
206 help="""Pre-load matplotlib and numpy for interactive use,
207 help="""Pre-load matplotlib and numpy for interactive use,
207 selecting a particular matplotlib backend and loop integration.
208 selecting a particular matplotlib backend and loop integration.
208 """
209 """
209 )
210 )
210 display_banner = Bool(True, config=True,
211 display_banner = Bool(True, config=True,
211 help="Whether to display a banner upon starting IPython."
212 help="Whether to display a banner upon starting IPython."
212 )
213 )
213
214
214 # if there is code of files to run from the cmd line, don't interact
215 # if there is code of files to run from the cmd line, don't interact
215 # unless the --i flag (App.force_interact) is true.
216 # unless the --i flag (App.force_interact) is true.
216 force_interact = Bool(False, config=True,
217 force_interact = Bool(False, config=True,
217 help="""If a command or file is given via the command-line,
218 help="""If a command or file is given via the command-line,
218 e.g. 'ipython foo.py"""
219 e.g. 'ipython foo.py"""
219 )
220 )
220 def _force_interact_changed(self, name, old, new):
221 def _force_interact_changed(self, name, old, new):
221 if new:
222 if new:
222 self.interact = True
223 self.interact = True
223
224
224 def _file_to_run_changed(self, name, old, new):
225 def _file_to_run_changed(self, name, old, new):
225 if new and not self.force_interact:
226 if new and not self.force_interact:
226 self.interact = False
227 self.interact = False
227 _code_to_run_changed = _file_to_run_changed
228 _code_to_run_changed = _file_to_run_changed
228
229
229 # internal, not-configurable
230 # internal, not-configurable
230 interact=Bool(True)
231 interact=Bool(True)
231
232
232
233
233 def parse_command_line(self, argv=None):
234 def parse_command_line(self, argv=None):
234 """override to allow old '-pylab' flag with deprecation warning"""
235 """override to allow old '-pylab' flag with deprecation warning"""
235 argv = sys.argv[1:] if argv is None else argv
236 argv = sys.argv[1:] if argv is None else argv
236
237
237 try:
238 try:
238 idx = argv.index('-pylab')
239 idx = argv.index('-pylab')
239 except ValueError:
240 except ValueError:
240 # `-pylab` not given, proceed as normal
241 # `-pylab` not given, proceed as normal
241 pass
242 pass
242 else:
243 else:
243 # deprecated `-pylab` given,
244 # deprecated `-pylab` given,
244 # warn and transform into current syntax
245 # warn and transform into current syntax
245 argv = list(argv) # copy, don't clobber
246 argv = list(argv) # copy, don't clobber
246 warn.warn("`-pylab` flag has been deprecated.\n"
247 warn.warn("`-pylab` flag has been deprecated.\n"
247 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
248 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
248 sub = '--pylab'
249 sub = '--pylab'
249 if len(argv) > idx+1:
250 if len(argv) > idx+1:
250 # check for gui arg, as in '-pylab qt'
251 # check for gui arg, as in '-pylab qt'
251 gui = argv[idx+1]
252 gui = argv[idx+1]
252 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
253 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
253 sub = '--pylab='+gui
254 sub = '--pylab='+gui
254 argv.pop(idx+1)
255 argv.pop(idx+1)
255 argv[idx] = sub
256 argv[idx] = sub
256
257
257 return super(TerminalIPythonApp, self).parse_command_line(argv)
258 return super(TerminalIPythonApp, self).parse_command_line(argv)
258
259
259 def initialize(self, argv=None):
260 def initialize(self, argv=None):
260 """Do actions after construct, but before starting the app."""
261 """Do actions after construct, but before starting the app."""
261 super(TerminalIPythonApp, self).initialize(argv)
262 super(TerminalIPythonApp, self).initialize(argv)
262 if self.subapp is not None:
263 if self.subapp is not None:
263 # don't bother initializing further, starting subapp
264 # don't bother initializing further, starting subapp
264 return
265 return
265 if not self.ignore_old_config:
266 if not self.ignore_old_config:
266 check_for_old_config(self.ipython_dir)
267 check_for_old_config(self.ipython_dir)
267 # print self.extra_args
268 # print self.extra_args
268 if self.extra_args:
269 if self.extra_args:
269 self.file_to_run = self.extra_args[0]
270 self.file_to_run = self.extra_args[0]
270 # create the shell
271 # create the shell
271 self.init_shell()
272 self.init_shell()
272 # and draw the banner
273 # and draw the banner
273 self.init_banner()
274 self.init_banner()
274 # Now a variety of things that happen after the banner is printed.
275 # Now a variety of things that happen after the banner is printed.
275 self.init_gui_pylab()
276 self.init_gui_pylab()
276 self.init_extensions()
277 self.init_extensions()
277 self.init_code()
278 self.init_code()
278
279
279 def init_shell(self):
280 def init_shell(self):
280 """initialize the InteractiveShell instance"""
281 """initialize the InteractiveShell instance"""
281 # I am a little hesitant to put these into InteractiveShell itself.
282 # I am a little hesitant to put these into InteractiveShell itself.
282 # But that might be the place for them
283 # But that might be the place for them
283 sys.path.insert(0, '')
284 sys.path.insert(0, '')
284
285
285 # Create an InteractiveShell instance.
286 # Create an InteractiveShell instance.
286 # shell.display_banner should always be False for the terminal
287 # shell.display_banner should always be False for the terminal
287 # based app, because we call shell.show_banner() by hand below
288 # based app, because we call shell.show_banner() by hand below
288 # so the banner shows *before* all extension loading stuff.
289 # so the banner shows *before* all extension loading stuff.
289 self.shell = TerminalInteractiveShell.instance(config=self.config,
290 self.shell = TerminalInteractiveShell.instance(config=self.config,
290 display_banner=False, profile_dir=self.profile_dir,
291 display_banner=False, profile_dir=self.profile_dir,
291 ipython_dir=self.ipython_dir)
292 ipython_dir=self.ipython_dir)
292
293
293 def init_banner(self):
294 def init_banner(self):
294 """optionally display the banner"""
295 """optionally display the banner"""
295 if self.display_banner and self.interact:
296 if self.display_banner and self.interact:
296 self.shell.show_banner()
297 self.shell.show_banner()
297 # Make sure there is a space below the banner.
298 # Make sure there is a space below the banner.
298 if self.log_level <= logging.INFO: print
299 if self.log_level <= logging.INFO: print
299
300
300
301
301 def init_gui_pylab(self):
302 def init_gui_pylab(self):
302 """Enable GUI event loop integration, taking pylab into account."""
303 """Enable GUI event loop integration, taking pylab into account."""
303 gui = self.gui
304 gui = self.gui
304
305
305 # Using `pylab` will also require gui activation, though which toolkit
306 # Using `pylab` will also require gui activation, though which toolkit
306 # to use may be chosen automatically based on mpl configuration.
307 # to use may be chosen automatically based on mpl configuration.
307 if self.pylab:
308 if self.pylab:
308 activate = self.shell.enable_pylab
309 activate = self.shell.enable_pylab
309 if self.pylab == 'auto':
310 if self.pylab == 'auto':
310 gui = None
311 gui = None
311 else:
312 else:
312 gui = self.pylab
313 gui = self.pylab
313 else:
314 else:
314 # Enable only GUI integration, no pylab
315 # Enable only GUI integration, no pylab
315 activate = inputhook.enable_gui
316 activate = inputhook.enable_gui
316
317
317 if gui or self.pylab:
318 if gui or self.pylab:
318 try:
319 try:
319 self.log.info("Enabling GUI event loop integration, "
320 self.log.info("Enabling GUI event loop integration, "
320 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
321 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
321 activate(gui)
322 activate(gui)
322 except:
323 except:
323 self.log.warn("Error in enabling GUI event loop integration:")
324 self.log.warn("Error in enabling GUI event loop integration:")
324 self.shell.showtraceback()
325 self.shell.showtraceback()
325
326
326 def start(self):
327 def start(self):
327 if self.subapp is not None:
328 if self.subapp is not None:
328 return self.subapp.start()
329 return self.subapp.start()
329 # perform any prexec steps:
330 # perform any prexec steps:
330 if self.interact:
331 if self.interact:
331 self.log.debug("Starting IPython's mainloop...")
332 self.log.debug("Starting IPython's mainloop...")
332 self.shell.mainloop()
333 self.shell.mainloop()
333 else:
334 else:
334 self.log.debug("IPython not interactive...")
335 self.log.debug("IPython not interactive...")
335
336
336
337
337 def load_default_config(ipython_dir=None):
338 def load_default_config(ipython_dir=None):
338 """Load the default config file from the default ipython_dir.
339 """Load the default config file from the default ipython_dir.
339
340
340 This is useful for embedded shells.
341 This is useful for embedded shells.
341 """
342 """
342 if ipython_dir is None:
343 if ipython_dir is None:
343 ipython_dir = get_ipython_dir()
344 ipython_dir = get_ipython_dir()
344 profile_dir = os.path.join(ipython_dir, 'profile_default')
345 profile_dir = os.path.join(ipython_dir, 'profile_default')
345 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
346 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
346 try:
347 try:
347 config = cl.load_config()
348 config = cl.load_config()
348 except IOError:
349 except IOError:
349 # no config found
350 # no config found
350 config = Config()
351 config = Config()
351 return config
352 return config
352
353
353
354
354 def launch_new_instance():
355 def launch_new_instance():
355 """Create and run a full blown IPython instance"""
356 """Create and run a full blown IPython instance"""
356 app = TerminalIPythonApp.instance()
357 app = TerminalIPythonApp.instance()
357 app.initialize()
358 app.initialize()
358 app.start()
359 app.start()
359
360
360
361
361 if __name__ == '__main__':
362 if __name__ == '__main__':
362 launch_new_instance()
363 launch_new_instance()
363
364
@@ -1,440 +1,440 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """Module for interactively running scripts.
2 """Module for interactively running scripts.
3
3
4 This module implements classes for interactively running scripts written for
4 This module implements classes for interactively running scripts written for
5 any system with a prompt which can be matched by a regexp suitable for
5 any system with a prompt which can be matched by a regexp suitable for
6 pexpect. It can be used to run as if they had been typed up interactively, an
6 pexpect. It can be used to run as if they had been typed up interactively, an
7 arbitrary series of commands for the target system.
7 arbitrary series of commands for the target system.
8
8
9 The module includes classes ready for IPython (with the default prompts),
9 The module includes classes ready for IPython (with the default prompts),
10 plain Python and SAGE, but making a new one is trivial. To see how to use it,
10 plain Python and SAGE, but making a new one is trivial. To see how to use it,
11 simply run the module as a script:
11 simply run the module as a script:
12
12
13 ./irunner.py --help
13 ./irunner.py --help
14
14
15
15
16 This is an extension of Ken Schutte <kschutte-AT-csail.mit.edu>'s script
16 This is an extension of Ken Schutte <kschutte-AT-csail.mit.edu>'s script
17 contributed on the ipython-user list:
17 contributed on the ipython-user list:
18
18
19 http://scipy.net/pipermail/ipython-user/2006-May/001705.html
19 http://scipy.net/pipermail/ipython-user/2006-May/001705.html
20
20
21
21
22 NOTES:
22 NOTES:
23
23
24 - This module requires pexpect, available in most linux distros, or which can
24 - This module requires pexpect, available in most linux distros, or which can
25 be downloaded from
25 be downloaded from
26
26
27 http://pexpect.sourceforge.net
27 http://pexpect.sourceforge.net
28
28
29 - Because pexpect only works under Unix or Windows-Cygwin, this has the same
29 - Because pexpect only works under Unix or Windows-Cygwin, this has the same
30 limitations. This means that it will NOT work under native windows Python.
30 limitations. This means that it will NOT work under native windows Python.
31 """
31 """
32
32
33 # Stdlib imports
33 # Stdlib imports
34 import optparse
34 import optparse
35 import os
35 import os
36 import sys
36 import sys
37
37
38 # Third-party modules.
38 # Third-party modules.
39 import pexpect
39 import pexpect
40
40
41 # Global usage strings, to avoid indentation issues when typing it below.
41 # Global usage strings, to avoid indentation issues when typing it below.
42 USAGE = """
42 USAGE = """
43 Interactive script runner, type: %s
43 Interactive script runner, type: %s
44
44
45 runner [opts] script_name
45 runner [opts] script_name
46 """
46 """
47
47
48 def pexpect_monkeypatch():
48 def pexpect_monkeypatch():
49 """Patch pexpect to prevent unhandled exceptions at VM teardown.
49 """Patch pexpect to prevent unhandled exceptions at VM teardown.
50
50
51 Calling this function will monkeypatch the pexpect.spawn class and modify
51 Calling this function will monkeypatch the pexpect.spawn class and modify
52 its __del__ method to make it more robust in the face of failures that can
52 its __del__ method to make it more robust in the face of failures that can
53 occur if it is called when the Python VM is shutting down.
53 occur if it is called when the Python VM is shutting down.
54
54
55 Since Python may fire __del__ methods arbitrarily late, it's possible for
55 Since Python may fire __del__ methods arbitrarily late, it's possible for
56 them to execute during the teardown of the Python VM itself. At this
56 them to execute during the teardown of the Python VM itself. At this
57 point, various builtin modules have been reset to None. Thus, the call to
57 point, various builtin modules have been reset to None. Thus, the call to
58 self.close() will trigger an exception because it tries to call os.close(),
58 self.close() will trigger an exception because it tries to call os.close(),
59 and os is now None.
59 and os is now None.
60 """
60 """
61
61
62 if pexpect.__version__[:3] >= '2.2':
62 if pexpect.__version__[:3] >= '2.2':
63 # No need to patch, fix is already the upstream version.
63 # No need to patch, fix is already the upstream version.
64 return
64 return
65
65
66 def __del__(self):
66 def __del__(self):
67 """This makes sure that no system resources are left open.
67 """This makes sure that no system resources are left open.
68 Python only garbage collects Python objects. OS file descriptors
68 Python only garbage collects Python objects. OS file descriptors
69 are not Python objects, so they must be handled explicitly.
69 are not Python objects, so they must be handled explicitly.
70 If the child file descriptor was opened outside of this class
70 If the child file descriptor was opened outside of this class
71 (passed to the constructor) then this does not close it.
71 (passed to the constructor) then this does not close it.
72 """
72 """
73 if not self.closed:
73 if not self.closed:
74 try:
74 try:
75 self.close()
75 self.close()
76 except AttributeError:
76 except AttributeError:
77 pass
77 pass
78
78
79 pexpect.spawn.__del__ = __del__
79 pexpect.spawn.__del__ = __del__
80
80
81 pexpect_monkeypatch()
81 pexpect_monkeypatch()
82
82
83 # The generic runner class
83 # The generic runner class
84 class InteractiveRunner(object):
84 class InteractiveRunner(object):
85 """Class to run a sequence of commands through an interactive program."""
85 """Class to run a sequence of commands through an interactive program."""
86
86
87 def __init__(self,program,prompts,args=None,out=sys.stdout,echo=True):
87 def __init__(self,program,prompts,args=None,out=sys.stdout,echo=True):
88 """Construct a runner.
88 """Construct a runner.
89
89
90 Inputs:
90 Inputs:
91
91
92 - program: command to execute the given program.
92 - program: command to execute the given program.
93
93
94 - prompts: a list of patterns to match as valid prompts, in the
94 - prompts: a list of patterns to match as valid prompts, in the
95 format used by pexpect. This basically means that it can be either
95 format used by pexpect. This basically means that it can be either
96 a string (to be compiled as a regular expression) or a list of such
96 a string (to be compiled as a regular expression) or a list of such
97 (it must be a true list, as pexpect does type checks).
97 (it must be a true list, as pexpect does type checks).
98
98
99 If more than one prompt is given, the first is treated as the main
99 If more than one prompt is given, the first is treated as the main
100 program prompt and the others as 'continuation' prompts, like
100 program prompt and the others as 'continuation' prompts, like
101 python's. This means that blank lines in the input source are
101 python's. This means that blank lines in the input source are
102 ommitted when the first prompt is matched, but are NOT ommitted when
102 ommitted when the first prompt is matched, but are NOT ommitted when
103 the continuation one matches, since this is how python signals the
103 the continuation one matches, since this is how python signals the
104 end of multiline input interactively.
104 end of multiline input interactively.
105
105
106 Optional inputs:
106 Optional inputs:
107
107
108 - args(None): optional list of strings to pass as arguments to the
108 - args(None): optional list of strings to pass as arguments to the
109 child program.
109 child program.
110
110
111 - out(sys.stdout): if given, an output stream to be used when writing
111 - out(sys.stdout): if given, an output stream to be used when writing
112 output. The only requirement is that it must have a .write() method.
112 output. The only requirement is that it must have a .write() method.
113
113
114 Public members not parameterized in the constructor:
114 Public members not parameterized in the constructor:
115
115
116 - delaybeforesend(0): Newer versions of pexpect have a delay before
116 - delaybeforesend(0): Newer versions of pexpect have a delay before
117 sending each new input. For our purposes here, it's typically best
117 sending each new input. For our purposes here, it's typically best
118 to just set this to zero, but if you encounter reliability problems
118 to just set this to zero, but if you encounter reliability problems
119 or want an interactive run to pause briefly at each prompt, just
119 or want an interactive run to pause briefly at each prompt, just
120 increase this value (it is measured in seconds). Note that this
120 increase this value (it is measured in seconds). Note that this
121 variable is not honored at all by older versions of pexpect.
121 variable is not honored at all by older versions of pexpect.
122 """
122 """
123
123
124 self.program = program
124 self.program = program
125 self.prompts = prompts
125 self.prompts = prompts
126 if args is None: args = []
126 if args is None: args = []
127 self.args = args
127 self.args = args
128 self.out = out
128 self.out = out
129 self.echo = echo
129 self.echo = echo
130 # Other public members which we don't make as parameters, but which
130 # Other public members which we don't make as parameters, but which
131 # users may occasionally want to tweak
131 # users may occasionally want to tweak
132 self.delaybeforesend = 0
132 self.delaybeforesend = 0
133
133
134 # Create child process and hold on to it so we don't have to re-create
134 # Create child process and hold on to it so we don't have to re-create
135 # for every single execution call
135 # for every single execution call
136 c = self.child = pexpect.spawn(self.program,self.args,timeout=None)
136 c = self.child = pexpect.spawn(self.program,self.args,timeout=None)
137 c.delaybeforesend = self.delaybeforesend
137 c.delaybeforesend = self.delaybeforesend
138 # pexpect hard-codes the terminal size as (24,80) (rows,columns).
138 # pexpect hard-codes the terminal size as (24,80) (rows,columns).
139 # This causes problems because any line longer than 80 characters gets
139 # This causes problems because any line longer than 80 characters gets
140 # completely overwrapped on the printed outptut (even though
140 # completely overwrapped on the printed outptut (even though
141 # internally the code runs fine). We reset this to 99 rows X 200
141 # internally the code runs fine). We reset this to 99 rows X 200
142 # columns (arbitrarily chosen), which should avoid problems in all
142 # columns (arbitrarily chosen), which should avoid problems in all
143 # reasonable cases.
143 # reasonable cases.
144 c.setwinsize(99,200)
144 c.setwinsize(99,200)
145
145
146 def close(self):
146 def close(self):
147 """close child process"""
147 """close child process"""
148
148
149 self.child.close()
149 self.child.close()
150
150
151 def run_file(self,fname,interact=False,get_output=False):
151 def run_file(self,fname,interact=False,get_output=False):
152 """Run the given file interactively.
152 """Run the given file interactively.
153
153
154 Inputs:
154 Inputs:
155
155
156 -fname: name of the file to execute.
156 -fname: name of the file to execute.
157
157
158 See the run_source docstring for the meaning of the optional
158 See the run_source docstring for the meaning of the optional
159 arguments."""
159 arguments."""
160
160
161 fobj = open(fname,'r')
161 fobj = open(fname,'r')
162 try:
162 try:
163 out = self.run_source(fobj,interact,get_output)
163 out = self.run_source(fobj,interact,get_output)
164 finally:
164 finally:
165 fobj.close()
165 fobj.close()
166 if get_output:
166 if get_output:
167 return out
167 return out
168
168
169 def run_source(self,source,interact=False,get_output=False):
169 def run_source(self,source,interact=False,get_output=False):
170 """Run the given source code interactively.
170 """Run the given source code interactively.
171
171
172 Inputs:
172 Inputs:
173
173
174 - source: a string of code to be executed, or an open file object we
174 - source: a string of code to be executed, or an open file object we
175 can iterate over.
175 can iterate over.
176
176
177 Optional inputs:
177 Optional inputs:
178
178
179 - interact(False): if true, start to interact with the running
179 - interact(False): if true, start to interact with the running
180 program at the end of the script. Otherwise, just exit.
180 program at the end of the script. Otherwise, just exit.
181
181
182 - get_output(False): if true, capture the output of the child process
182 - get_output(False): if true, capture the output of the child process
183 (filtering the input commands out) and return it as a string.
183 (filtering the input commands out) and return it as a string.
184
184
185 Returns:
185 Returns:
186 A string containing the process output, but only if requested.
186 A string containing the process output, but only if requested.
187 """
187 """
188
188
189 # if the source is a string, chop it up in lines so we can iterate
189 # if the source is a string, chop it up in lines so we can iterate
190 # over it just as if it were an open file.
190 # over it just as if it were an open file.
191 if not isinstance(source,file):
191 if not isinstance(source,file):
192 source = source.splitlines(True)
192 source = source.splitlines(True)
193
193
194 if self.echo:
194 if self.echo:
195 # normalize all strings we write to use the native OS line
195 # normalize all strings we write to use the native OS line
196 # separators.
196 # separators.
197 linesep = os.linesep
197 linesep = os.linesep
198 stdwrite = self.out.write
198 stdwrite = self.out.write
199 write = lambda s: stdwrite(s.replace('\r\n',linesep))
199 write = lambda s: stdwrite(s.replace('\r\n',linesep))
200 else:
200 else:
201 # Quiet mode, all writes are no-ops
201 # Quiet mode, all writes are no-ops
202 write = lambda s: None
202 write = lambda s: None
203
203
204 c = self.child
204 c = self.child
205 prompts = c.compile_pattern_list(self.prompts)
205 prompts = c.compile_pattern_list(self.prompts)
206 prompt_idx = c.expect_list(prompts)
206 prompt_idx = c.expect_list(prompts)
207
207
208 # Flag whether the script ends normally or not, to know whether we can
208 # Flag whether the script ends normally or not, to know whether we can
209 # do anything further with the underlying process.
209 # do anything further with the underlying process.
210 end_normal = True
210 end_normal = True
211
211
212 # If the output was requested, store it in a list for return at the end
212 # If the output was requested, store it in a list for return at the end
213 if get_output:
213 if get_output:
214 output = []
214 output = []
215 store_output = output.append
215 store_output = output.append
216
216
217 for cmd in source:
217 for cmd in source:
218 # skip blank lines for all matches to the 'main' prompt, while the
218 # skip blank lines for all matches to the 'main' prompt, while the
219 # secondary prompts do not
219 # secondary prompts do not
220 if prompt_idx==0 and \
220 if prompt_idx==0 and \
221 (cmd.isspace() or cmd.lstrip().startswith('#')):
221 (cmd.isspace() or cmd.lstrip().startswith('#')):
222 write(cmd)
222 write(cmd)
223 continue
223 continue
224
224
225 # write('AFTER: '+c.after) # dbg
225 # write('AFTER: '+c.after) # dbg
226 write(c.after)
226 write(c.after)
227 c.send(cmd)
227 c.send(cmd)
228 try:
228 try:
229 prompt_idx = c.expect_list(prompts)
229 prompt_idx = c.expect_list(prompts)
230 except pexpect.EOF:
230 except pexpect.EOF:
231 # this will happen if the child dies unexpectedly
231 # this will happen if the child dies unexpectedly
232 write(c.before)
232 write(c.before)
233 end_normal = False
233 end_normal = False
234 break
234 break
235
235
236 write(c.before)
236 write(c.before)
237
237
238 # With an echoing process, the output we get in c.before contains
238 # With an echoing process, the output we get in c.before contains
239 # the command sent, a newline, and then the actual process output
239 # the command sent, a newline, and then the actual process output
240 if get_output:
240 if get_output:
241 store_output(c.before[len(cmd+'\n'):])
241 store_output(c.before[len(cmd+'\n'):])
242 #write('CMD: <<%s>>' % cmd) # dbg
242 #write('CMD: <<%s>>' % cmd) # dbg
243 #write('OUTPUT: <<%s>>' % output[-1]) # dbg
243 #write('OUTPUT: <<%s>>' % output[-1]) # dbg
244
244
245 self.out.flush()
245 self.out.flush()
246 if end_normal:
246 if end_normal:
247 if interact:
247 if interact:
248 c.send('\n')
248 c.send('\n')
249 print '<< Starting interactive mode >>',
249 print '<< Starting interactive mode >>',
250 try:
250 try:
251 c.interact()
251 c.interact()
252 except OSError:
252 except OSError:
253 # This is what fires when the child stops. Simply print a
253 # This is what fires when the child stops. Simply print a
254 # newline so the system prompt is aligned. The extra
254 # newline so the system prompt is aligned. The extra
255 # space is there to make sure it gets printed, otherwise
255 # space is there to make sure it gets printed, otherwise
256 # OS buffering sometimes just suppresses it.
256 # OS buffering sometimes just suppresses it.
257 write(' \n')
257 write(' \n')
258 self.out.flush()
258 self.out.flush()
259 else:
259 else:
260 if interact:
260 if interact:
261 e="Further interaction is not possible: child process is dead."
261 e="Further interaction is not possible: child process is dead."
262 print >> sys.stderr, e
262 print >> sys.stderr, e
263
263
264 # Leave the child ready for more input later on, otherwise select just
264 # Leave the child ready for more input later on, otherwise select just
265 # hangs on the second invocation.
265 # hangs on the second invocation.
266 if c.isalive():
266 if c.isalive():
267 c.send('\n')
267 c.send('\n')
268
268
269 # Return any requested output
269 # Return any requested output
270 if get_output:
270 if get_output:
271 return ''.join(output)
271 return ''.join(output)
272
272
273 def main(self,argv=None):
273 def main(self,argv=None):
274 """Run as a command-line script."""
274 """Run as a command-line script."""
275
275
276 parser = optparse.OptionParser(usage=USAGE % self.__class__.__name__)
276 parser = optparse.OptionParser(usage=USAGE % self.__class__.__name__)
277 newopt = parser.add_option
277 newopt = parser.add_option
278 newopt('-i','--interact',action='store_true',default=False,
278 newopt('-i','--interact',action='store_true',default=False,
279 help='Interact with the program after the script is run.')
279 help='Interact with the program after the script is run.')
280
280
281 opts,args = parser.parse_args(argv)
281 opts,args = parser.parse_args(argv)
282
282
283 if len(args) != 1:
283 if len(args) != 1:
284 print >> sys.stderr,"You must supply exactly one file to run."
284 print >> sys.stderr,"You must supply exactly one file to run."
285 sys.exit(1)
285 sys.exit(1)
286
286
287 self.run_file(args[0],opts.interact)
287 self.run_file(args[0],opts.interact)
288
288
289
289
290 # Specific runners for particular programs
290 # Specific runners for particular programs
291 class IPythonRunner(InteractiveRunner):
291 class IPythonRunner(InteractiveRunner):
292 """Interactive IPython runner.
292 """Interactive IPython runner.
293
293
294 This initalizes IPython in 'nocolor' mode for simplicity. This lets us
294 This initalizes IPython in 'nocolor' mode for simplicity. This lets us
295 avoid having to write a regexp that matches ANSI sequences, though pexpect
295 avoid having to write a regexp that matches ANSI sequences, though pexpect
296 does support them. If anyone contributes patches for ANSI color support,
296 does support them. If anyone contributes patches for ANSI color support,
297 they will be welcome.
297 they will be welcome.
298
298
299 It also sets the prompts manually, since the prompt regexps for
299 It also sets the prompts manually, since the prompt regexps for
300 pexpect need to be matched to the actual prompts, so user-customized
300 pexpect need to be matched to the actual prompts, so user-customized
301 prompts would break this.
301 prompts would break this.
302 """
302 """
303
303
304 def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True):
304 def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True):
305 """New runner, optionally passing the ipython command to use."""
305 """New runner, optionally passing the ipython command to use."""
306
306
307 args0 = ['colors=NoColor',
307 args0 = ['--colors=NoColor',
308 '--no-term-title',
308 '--no-term-title',
309 '--no-autoindent']
309 '--no-autoindent']
310 if args is None: args = args0
310 if args is None: args = args0
311 else: args = args0 + args
311 else: args = args0 + args
312 prompts = [r'In \[\d+\]: ',r' \.*: ']
312 prompts = [r'In \[\d+\]: ',r' \.*: ']
313 InteractiveRunner.__init__(self,program,prompts,args,out,echo)
313 InteractiveRunner.__init__(self,program,prompts,args,out,echo)
314
314
315
315
316 class PythonRunner(InteractiveRunner):
316 class PythonRunner(InteractiveRunner):
317 """Interactive Python runner."""
317 """Interactive Python runner."""
318
318
319 def __init__(self,program='python',args=None,out=sys.stdout,echo=True):
319 def __init__(self,program='python',args=None,out=sys.stdout,echo=True):
320 """New runner, optionally passing the python command to use."""
320 """New runner, optionally passing the python command to use."""
321
321
322 prompts = [r'>>> ',r'\.\.\. ']
322 prompts = [r'>>> ',r'\.\.\. ']
323 InteractiveRunner.__init__(self,program,prompts,args,out,echo)
323 InteractiveRunner.__init__(self,program,prompts,args,out,echo)
324
324
325
325
326 class SAGERunner(InteractiveRunner):
326 class SAGERunner(InteractiveRunner):
327 """Interactive SAGE runner.
327 """Interactive SAGE runner.
328
328
329 WARNING: this runner only works if you manually configure your SAGE copy
329 WARNING: this runner only works if you manually configure your SAGE copy
330 to use 'colors NoColor' in the ipythonrc config file, since currently the
330 to use 'colors NoColor' in the ipythonrc config file, since currently the
331 prompt matching regexp does not identify color sequences."""
331 prompt matching regexp does not identify color sequences."""
332
332
333 def __init__(self,program='sage',args=None,out=sys.stdout,echo=True):
333 def __init__(self,program='sage',args=None,out=sys.stdout,echo=True):
334 """New runner, optionally passing the sage command to use."""
334 """New runner, optionally passing the sage command to use."""
335
335
336 prompts = ['sage: ',r'\s*\.\.\. ']
336 prompts = ['sage: ',r'\s*\.\.\. ']
337 InteractiveRunner.__init__(self,program,prompts,args,out,echo)
337 InteractiveRunner.__init__(self,program,prompts,args,out,echo)
338
338
339
339
340 class RunnerFactory(object):
340 class RunnerFactory(object):
341 """Code runner factory.
341 """Code runner factory.
342
342
343 This class provides an IPython code runner, but enforces that only one
343 This class provides an IPython code runner, but enforces that only one
344 runner is ever instantiated. The runner is created based on the extension
344 runner is ever instantiated. The runner is created based on the extension
345 of the first file to run, and it raises an exception if a runner is later
345 of the first file to run, and it raises an exception if a runner is later
346 requested for a different extension type.
346 requested for a different extension type.
347
347
348 This ensures that we don't generate example files for doctest with a mix of
348 This ensures that we don't generate example files for doctest with a mix of
349 python and ipython syntax.
349 python and ipython syntax.
350 """
350 """
351
351
352 def __init__(self,out=sys.stdout):
352 def __init__(self,out=sys.stdout):
353 """Instantiate a code runner."""
353 """Instantiate a code runner."""
354
354
355 self.out = out
355 self.out = out
356 self.runner = None
356 self.runner = None
357 self.runnerClass = None
357 self.runnerClass = None
358
358
359 def _makeRunner(self,runnerClass):
359 def _makeRunner(self,runnerClass):
360 self.runnerClass = runnerClass
360 self.runnerClass = runnerClass
361 self.runner = runnerClass(out=self.out)
361 self.runner = runnerClass(out=self.out)
362 return self.runner
362 return self.runner
363
363
364 def __call__(self,fname):
364 def __call__(self,fname):
365 """Return a runner for the given filename."""
365 """Return a runner for the given filename."""
366
366
367 if fname.endswith('.py'):
367 if fname.endswith('.py'):
368 runnerClass = PythonRunner
368 runnerClass = PythonRunner
369 elif fname.endswith('.ipy'):
369 elif fname.endswith('.ipy'):
370 runnerClass = IPythonRunner
370 runnerClass = IPythonRunner
371 else:
371 else:
372 raise ValueError('Unknown file type for Runner: %r' % fname)
372 raise ValueError('Unknown file type for Runner: %r' % fname)
373
373
374 if self.runner is None:
374 if self.runner is None:
375 return self._makeRunner(runnerClass)
375 return self._makeRunner(runnerClass)
376 else:
376 else:
377 if runnerClass==self.runnerClass:
377 if runnerClass==self.runnerClass:
378 return self.runner
378 return self.runner
379 else:
379 else:
380 e='A runner of type %r can not run file %r' % \
380 e='A runner of type %r can not run file %r' % \
381 (self.runnerClass,fname)
381 (self.runnerClass,fname)
382 raise ValueError(e)
382 raise ValueError(e)
383
383
384
384
385 # Global usage string, to avoid indentation issues if typed in a function def.
385 # Global usage string, to avoid indentation issues if typed in a function def.
386 MAIN_USAGE = """
386 MAIN_USAGE = """
387 %prog [options] file_to_run
387 %prog [options] file_to_run
388
388
389 This is an interface to the various interactive runners available in this
389 This is an interface to the various interactive runners available in this
390 module. If you want to pass specific options to one of the runners, you need
390 module. If you want to pass specific options to one of the runners, you need
391 to first terminate the main options with a '--', and then provide the runner's
391 to first terminate the main options with a '--', and then provide the runner's
392 options. For example:
392 options. For example:
393
393
394 irunner.py --python -- --help
394 irunner.py --python -- --help
395
395
396 will pass --help to the python runner. Similarly,
396 will pass --help to the python runner. Similarly,
397
397
398 irunner.py --ipython -- --interact script.ipy
398 irunner.py --ipython -- --interact script.ipy
399
399
400 will run the script.ipy file under the IPython runner, and then will start to
400 will run the script.ipy file under the IPython runner, and then will start to
401 interact with IPython at the end of the script (instead of exiting).
401 interact with IPython at the end of the script (instead of exiting).
402
402
403 The already implemented runners are listed below; adding one for a new program
403 The already implemented runners are listed below; adding one for a new program
404 is a trivial task, see the source for examples.
404 is a trivial task, see the source for examples.
405
405
406 WARNING: the SAGE runner only works if you manually configure your SAGE copy
406 WARNING: the SAGE runner only works if you manually configure your SAGE copy
407 to use 'colors NoColor' in the ipythonrc config file, since currently the
407 to use 'colors NoColor' in the ipythonrc config file, since currently the
408 prompt matching regexp does not identify color sequences.
408 prompt matching regexp does not identify color sequences.
409 """
409 """
410
410
411 def main():
411 def main():
412 """Run as a command-line script."""
412 """Run as a command-line script."""
413
413
414 parser = optparse.OptionParser(usage=MAIN_USAGE)
414 parser = optparse.OptionParser(usage=MAIN_USAGE)
415 newopt = parser.add_option
415 newopt = parser.add_option
416 parser.set_defaults(mode='ipython')
416 parser.set_defaults(mode='ipython')
417 newopt('--ipython',action='store_const',dest='mode',const='ipython',
417 newopt('--ipython',action='store_const',dest='mode',const='ipython',
418 help='IPython interactive runner (default).')
418 help='IPython interactive runner (default).')
419 newopt('--python',action='store_const',dest='mode',const='python',
419 newopt('--python',action='store_const',dest='mode',const='python',
420 help='Python interactive runner.')
420 help='Python interactive runner.')
421 newopt('--sage',action='store_const',dest='mode',const='sage',
421 newopt('--sage',action='store_const',dest='mode',const='sage',
422 help='SAGE interactive runner.')
422 help='SAGE interactive runner.')
423
423
424 opts,args = parser.parse_args()
424 opts,args = parser.parse_args()
425 runners = dict(ipython=IPythonRunner,
425 runners = dict(ipython=IPythonRunner,
426 python=PythonRunner,
426 python=PythonRunner,
427 sage=SAGERunner)
427 sage=SAGERunner)
428
428
429 try:
429 try:
430 ext = os.path.splitext(args[0])[-1]
430 ext = os.path.splitext(args[0])[-1]
431 except IndexError:
431 except IndexError:
432 ext = ''
432 ext = ''
433 modes = {'.ipy':'ipython',
433 modes = {'.ipy':'ipython',
434 '.py':'python',
434 '.py':'python',
435 '.sage':'sage'}
435 '.sage':'sage'}
436 mode = modes.get(ext,opts.mode)
436 mode = modes.get(ext,opts.mode)
437 runners[mode]().main(args)
437 runners[mode]().main(args)
438
438
439 if __name__ == '__main__':
439 if __name__ == '__main__':
440 main()
440 main()
@@ -1,1065 +1,1065 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 Facilities for launching IPython processes asynchronously.
4 Facilities for launching IPython processes asynchronously.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2011 The IPython Development Team
13 # Copyright (C) 2008-2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import copy
23 import copy
24 import logging
24 import logging
25 import os
25 import os
26 import re
26 import re
27 import stat
27 import stat
28
28
29 # signal imports, handling various platforms, versions
29 # signal imports, handling various platforms, versions
30
30
31 from signal import SIGINT, SIGTERM
31 from signal import SIGINT, SIGTERM
32 try:
32 try:
33 from signal import SIGKILL
33 from signal import SIGKILL
34 except ImportError:
34 except ImportError:
35 # Windows
35 # Windows
36 SIGKILL=SIGTERM
36 SIGKILL=SIGTERM
37
37
38 try:
38 try:
39 # Windows >= 2.7, 3.2
39 # Windows >= 2.7, 3.2
40 from signal import CTRL_C_EVENT as SIGINT
40 from signal import CTRL_C_EVENT as SIGINT
41 except ImportError:
41 except ImportError:
42 pass
42 pass
43
43
44 from subprocess import Popen, PIPE, STDOUT
44 from subprocess import Popen, PIPE, STDOUT
45 try:
45 try:
46 from subprocess import check_output
46 from subprocess import check_output
47 except ImportError:
47 except ImportError:
48 # pre-2.7, define check_output with Popen
48 # pre-2.7, define check_output with Popen
49 def check_output(*args, **kwargs):
49 def check_output(*args, **kwargs):
50 kwargs.update(dict(stdout=PIPE))
50 kwargs.update(dict(stdout=PIPE))
51 p = Popen(*args, **kwargs)
51 p = Popen(*args, **kwargs)
52 out,err = p.communicate()
52 out,err = p.communicate()
53 return out
53 return out
54
54
55 from zmq.eventloop import ioloop
55 from zmq.eventloop import ioloop
56
56
57 from IPython.config.application import Application
57 from IPython.config.application import Application
58 from IPython.config.configurable import LoggingConfigurable
58 from IPython.config.configurable import LoggingConfigurable
59 from IPython.utils.text import EvalFormatter
59 from IPython.utils.text import EvalFormatter
60 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
60 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
61 from IPython.utils.path import get_ipython_module_path
61 from IPython.utils.path import get_ipython_module_path
62 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
62 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
63
63
64 from .win32support import forward_read_events
64 from .win32support import forward_read_events
65
65
66 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
66 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
67
67
68 WINDOWS = os.name == 'nt'
68 WINDOWS = os.name == 'nt'
69
69
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71 # Paths to the kernel apps
71 # Paths to the kernel apps
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73
73
74
74
75 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
75 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
76 'IPython.parallel.apps.ipclusterapp'
76 'IPython.parallel.apps.ipclusterapp'
77 ))
77 ))
78
78
79 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
79 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
80 'IPython.parallel.apps.ipengineapp'
80 'IPython.parallel.apps.ipengineapp'
81 ))
81 ))
82
82
83 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
83 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
84 'IPython.parallel.apps.ipcontrollerapp'
84 'IPython.parallel.apps.ipcontrollerapp'
85 ))
85 ))
86
86
87 #-----------------------------------------------------------------------------
87 #-----------------------------------------------------------------------------
88 # Base launchers and errors
88 # Base launchers and errors
89 #-----------------------------------------------------------------------------
89 #-----------------------------------------------------------------------------
90
90
91
91
92 class LauncherError(Exception):
92 class LauncherError(Exception):
93 pass
93 pass
94
94
95
95
96 class ProcessStateError(LauncherError):
96 class ProcessStateError(LauncherError):
97 pass
97 pass
98
98
99
99
100 class UnknownStatus(LauncherError):
100 class UnknownStatus(LauncherError):
101 pass
101 pass
102
102
103
103
104 class BaseLauncher(LoggingConfigurable):
104 class BaseLauncher(LoggingConfigurable):
105 """An asbtraction for starting, stopping and signaling a process."""
105 """An asbtraction for starting, stopping and signaling a process."""
106
106
107 # In all of the launchers, the work_dir is where child processes will be
107 # In all of the launchers, the work_dir is where child processes will be
108 # run. This will usually be the profile_dir, but may not be. any work_dir
108 # run. This will usually be the profile_dir, but may not be. any work_dir
109 # passed into the __init__ method will override the config value.
109 # passed into the __init__ method will override the config value.
110 # This should not be used to set the work_dir for the actual engine
110 # This should not be used to set the work_dir for the actual engine
111 # and controller. Instead, use their own config files or the
111 # and controller. Instead, use their own config files or the
112 # controller_args, engine_args attributes of the launchers to add
112 # controller_args, engine_args attributes of the launchers to add
113 # the work_dir option.
113 # the work_dir option.
114 work_dir = Unicode(u'.')
114 work_dir = Unicode(u'.')
115 loop = Instance('zmq.eventloop.ioloop.IOLoop')
115 loop = Instance('zmq.eventloop.ioloop.IOLoop')
116
116
117 start_data = Any()
117 start_data = Any()
118 stop_data = Any()
118 stop_data = Any()
119
119
120 def _loop_default(self):
120 def _loop_default(self):
121 return ioloop.IOLoop.instance()
121 return ioloop.IOLoop.instance()
122
122
123 def __init__(self, work_dir=u'.', config=None, **kwargs):
123 def __init__(self, work_dir=u'.', config=None, **kwargs):
124 super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
124 super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
125 self.state = 'before' # can be before, running, after
125 self.state = 'before' # can be before, running, after
126 self.stop_callbacks = []
126 self.stop_callbacks = []
127 self.start_data = None
127 self.start_data = None
128 self.stop_data = None
128 self.stop_data = None
129
129
130 @property
130 @property
131 def args(self):
131 def args(self):
132 """A list of cmd and args that will be used to start the process.
132 """A list of cmd and args that will be used to start the process.
133
133
134 This is what is passed to :func:`spawnProcess` and the first element
134 This is what is passed to :func:`spawnProcess` and the first element
135 will be the process name.
135 will be the process name.
136 """
136 """
137 return self.find_args()
137 return self.find_args()
138
138
139 def find_args(self):
139 def find_args(self):
140 """The ``.args`` property calls this to find the args list.
140 """The ``.args`` property calls this to find the args list.
141
141
142 Subcommand should implement this to construct the cmd and args.
142 Subcommand should implement this to construct the cmd and args.
143 """
143 """
144 raise NotImplementedError('find_args must be implemented in a subclass')
144 raise NotImplementedError('find_args must be implemented in a subclass')
145
145
146 @property
146 @property
147 def arg_str(self):
147 def arg_str(self):
148 """The string form of the program arguments."""
148 """The string form of the program arguments."""
149 return ' '.join(self.args)
149 return ' '.join(self.args)
150
150
151 @property
151 @property
152 def running(self):
152 def running(self):
153 """Am I running."""
153 """Am I running."""
154 if self.state == 'running':
154 if self.state == 'running':
155 return True
155 return True
156 else:
156 else:
157 return False
157 return False
158
158
159 def start(self):
159 def start(self):
160 """Start the process."""
160 """Start the process."""
161 raise NotImplementedError('start must be implemented in a subclass')
161 raise NotImplementedError('start must be implemented in a subclass')
162
162
163 def stop(self):
163 def stop(self):
164 """Stop the process and notify observers of stopping.
164 """Stop the process and notify observers of stopping.
165
165
166 This method will return None immediately.
166 This method will return None immediately.
167 To observe the actual process stopping, see :meth:`on_stop`.
167 To observe the actual process stopping, see :meth:`on_stop`.
168 """
168 """
169 raise NotImplementedError('stop must be implemented in a subclass')
169 raise NotImplementedError('stop must be implemented in a subclass')
170
170
171 def on_stop(self, f):
171 def on_stop(self, f):
172 """Register a callback to be called with this Launcher's stop_data
172 """Register a callback to be called with this Launcher's stop_data
173 when the process actually finishes.
173 when the process actually finishes.
174 """
174 """
175 if self.state=='after':
175 if self.state=='after':
176 return f(self.stop_data)
176 return f(self.stop_data)
177 else:
177 else:
178 self.stop_callbacks.append(f)
178 self.stop_callbacks.append(f)
179
179
180 def notify_start(self, data):
180 def notify_start(self, data):
181 """Call this to trigger startup actions.
181 """Call this to trigger startup actions.
182
182
183 This logs the process startup and sets the state to 'running'. It is
183 This logs the process startup and sets the state to 'running'. It is
184 a pass-through so it can be used as a callback.
184 a pass-through so it can be used as a callback.
185 """
185 """
186
186
187 self.log.info('Process %r started: %r' % (self.args[0], data))
187 self.log.info('Process %r started: %r' % (self.args[0], data))
188 self.start_data = data
188 self.start_data = data
189 self.state = 'running'
189 self.state = 'running'
190 return data
190 return data
191
191
192 def notify_stop(self, data):
192 def notify_stop(self, data):
193 """Call this to trigger process stop actions.
193 """Call this to trigger process stop actions.
194
194
195 This logs the process stopping and sets the state to 'after'. Call
195 This logs the process stopping and sets the state to 'after'. Call
196 this to trigger callbacks registered via :meth:`on_stop`."""
196 this to trigger callbacks registered via :meth:`on_stop`."""
197
197
198 self.log.info('Process %r stopped: %r' % (self.args[0], data))
198 self.log.info('Process %r stopped: %r' % (self.args[0], data))
199 self.stop_data = data
199 self.stop_data = data
200 self.state = 'after'
200 self.state = 'after'
201 for i in range(len(self.stop_callbacks)):
201 for i in range(len(self.stop_callbacks)):
202 d = self.stop_callbacks.pop()
202 d = self.stop_callbacks.pop()
203 d(data)
203 d(data)
204 return data
204 return data
205
205
206 def signal(self, sig):
206 def signal(self, sig):
207 """Signal the process.
207 """Signal the process.
208
208
209 Parameters
209 Parameters
210 ----------
210 ----------
211 sig : str or int
211 sig : str or int
212 'KILL', 'INT', etc., or any signal number
212 'KILL', 'INT', etc., or any signal number
213 """
213 """
214 raise NotImplementedError('signal must be implemented in a subclass')
214 raise NotImplementedError('signal must be implemented in a subclass')
215
215
216
216
217 #-----------------------------------------------------------------------------
217 #-----------------------------------------------------------------------------
218 # Local process launchers
218 # Local process launchers
219 #-----------------------------------------------------------------------------
219 #-----------------------------------------------------------------------------
220
220
221
221
222 class LocalProcessLauncher(BaseLauncher):
222 class LocalProcessLauncher(BaseLauncher):
223 """Start and stop an external process in an asynchronous manner.
223 """Start and stop an external process in an asynchronous manner.
224
224
225 This will launch the external process with a working directory of
225 This will launch the external process with a working directory of
226 ``self.work_dir``.
226 ``self.work_dir``.
227 """
227 """
228
228
229 # This is used to to construct self.args, which is passed to
229 # This is used to to construct self.args, which is passed to
230 # spawnProcess.
230 # spawnProcess.
231 cmd_and_args = List([])
231 cmd_and_args = List([])
232 poll_frequency = Int(100) # in ms
232 poll_frequency = Int(100) # in ms
233
233
234 def __init__(self, work_dir=u'.', config=None, **kwargs):
234 def __init__(self, work_dir=u'.', config=None, **kwargs):
235 super(LocalProcessLauncher, self).__init__(
235 super(LocalProcessLauncher, self).__init__(
236 work_dir=work_dir, config=config, **kwargs
236 work_dir=work_dir, config=config, **kwargs
237 )
237 )
238 self.process = None
238 self.process = None
239 self.poller = None
239 self.poller = None
240
240
241 def find_args(self):
241 def find_args(self):
242 return self.cmd_and_args
242 return self.cmd_and_args
243
243
244 def start(self):
244 def start(self):
245 if self.state == 'before':
245 if self.state == 'before':
246 self.process = Popen(self.args,
246 self.process = Popen(self.args,
247 stdout=PIPE,stderr=PIPE,stdin=PIPE,
247 stdout=PIPE,stderr=PIPE,stdin=PIPE,
248 env=os.environ,
248 env=os.environ,
249 cwd=self.work_dir
249 cwd=self.work_dir
250 )
250 )
251 if WINDOWS:
251 if WINDOWS:
252 self.stdout = forward_read_events(self.process.stdout)
252 self.stdout = forward_read_events(self.process.stdout)
253 self.stderr = forward_read_events(self.process.stderr)
253 self.stderr = forward_read_events(self.process.stderr)
254 else:
254 else:
255 self.stdout = self.process.stdout.fileno()
255 self.stdout = self.process.stdout.fileno()
256 self.stderr = self.process.stderr.fileno()
256 self.stderr = self.process.stderr.fileno()
257 self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
257 self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
258 self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
258 self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
259 self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
259 self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
260 self.poller.start()
260 self.poller.start()
261 self.notify_start(self.process.pid)
261 self.notify_start(self.process.pid)
262 else:
262 else:
263 s = 'The process was already started and has state: %r' % self.state
263 s = 'The process was already started and has state: %r' % self.state
264 raise ProcessStateError(s)
264 raise ProcessStateError(s)
265
265
266 def stop(self):
266 def stop(self):
267 return self.interrupt_then_kill()
267 return self.interrupt_then_kill()
268
268
269 def signal(self, sig):
269 def signal(self, sig):
270 if self.state == 'running':
270 if self.state == 'running':
271 if WINDOWS and sig != SIGINT:
271 if WINDOWS and sig != SIGINT:
272 # use Windows tree-kill for better child cleanup
272 # use Windows tree-kill for better child cleanup
273 check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
273 check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
274 else:
274 else:
275 self.process.send_signal(sig)
275 self.process.send_signal(sig)
276
276
277 def interrupt_then_kill(self, delay=2.0):
277 def interrupt_then_kill(self, delay=2.0):
278 """Send INT, wait a delay and then send KILL."""
278 """Send INT, wait a delay and then send KILL."""
279 try:
279 try:
280 self.signal(SIGINT)
280 self.signal(SIGINT)
281 except Exception:
281 except Exception:
282 self.log.debug("interrupt failed")
282 self.log.debug("interrupt failed")
283 pass
283 pass
284 self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
284 self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
285 self.killer.start()
285 self.killer.start()
286
286
287 # callbacks, etc:
287 # callbacks, etc:
288
288
289 def handle_stdout(self, fd, events):
289 def handle_stdout(self, fd, events):
290 if WINDOWS:
290 if WINDOWS:
291 line = self.stdout.recv()
291 line = self.stdout.recv()
292 else:
292 else:
293 line = self.process.stdout.readline()
293 line = self.process.stdout.readline()
294 # a stopped process will be readable but return empty strings
294 # a stopped process will be readable but return empty strings
295 if line:
295 if line:
296 self.log.info(line[:-1])
296 self.log.info(line[:-1])
297 else:
297 else:
298 self.poll()
298 self.poll()
299
299
300 def handle_stderr(self, fd, events):
300 def handle_stderr(self, fd, events):
301 if WINDOWS:
301 if WINDOWS:
302 line = self.stderr.recv()
302 line = self.stderr.recv()
303 else:
303 else:
304 line = self.process.stderr.readline()
304 line = self.process.stderr.readline()
305 # a stopped process will be readable but return empty strings
305 # a stopped process will be readable but return empty strings
306 if line:
306 if line:
307 self.log.error(line[:-1])
307 self.log.error(line[:-1])
308 else:
308 else:
309 self.poll()
309 self.poll()
310
310
311 def poll(self):
311 def poll(self):
312 status = self.process.poll()
312 status = self.process.poll()
313 if status is not None:
313 if status is not None:
314 self.poller.stop()
314 self.poller.stop()
315 self.loop.remove_handler(self.stdout)
315 self.loop.remove_handler(self.stdout)
316 self.loop.remove_handler(self.stderr)
316 self.loop.remove_handler(self.stderr)
317 self.notify_stop(dict(exit_code=status, pid=self.process.pid))
317 self.notify_stop(dict(exit_code=status, pid=self.process.pid))
318 return status
318 return status
319
319
320 class LocalControllerLauncher(LocalProcessLauncher):
320 class LocalControllerLauncher(LocalProcessLauncher):
321 """Launch a controller as a regular external process."""
321 """Launch a controller as a regular external process."""
322
322
323 controller_cmd = List(ipcontroller_cmd_argv, config=True,
323 controller_cmd = List(ipcontroller_cmd_argv, config=True,
324 help="""Popen command to launch ipcontroller.""")
324 help="""Popen command to launch ipcontroller.""")
325 # Command line arguments to ipcontroller.
325 # Command line arguments to ipcontroller.
326 controller_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True,
326 controller_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
327 help="""command-line args to pass to ipcontroller""")
327 help="""command-line args to pass to ipcontroller""")
328
328
329 def find_args(self):
329 def find_args(self):
330 return self.controller_cmd + self.controller_args
330 return self.controller_cmd + self.controller_args
331
331
332 def start(self, profile_dir):
332 def start(self, profile_dir):
333 """Start the controller by profile_dir."""
333 """Start the controller by profile_dir."""
334 self.controller_args.extend(['profile_dir=%s'%profile_dir])
334 self.controller_args.extend(['--profile_dir=%s'%profile_dir])
335 self.profile_dir = unicode(profile_dir)
335 self.profile_dir = unicode(profile_dir)
336 self.log.info("Starting LocalControllerLauncher: %r" % self.args)
336 self.log.info("Starting LocalControllerLauncher: %r" % self.args)
337 return super(LocalControllerLauncher, self).start()
337 return super(LocalControllerLauncher, self).start()
338
338
339
339
340 class LocalEngineLauncher(LocalProcessLauncher):
340 class LocalEngineLauncher(LocalProcessLauncher):
341 """Launch a single engine as a regular externall process."""
341 """Launch a single engine as a regular externall process."""
342
342
343 engine_cmd = List(ipengine_cmd_argv, config=True,
343 engine_cmd = List(ipengine_cmd_argv, config=True,
344 help="""command to launch the Engine.""")
344 help="""command to launch the Engine.""")
345 # Command line arguments for ipengine.
345 # Command line arguments for ipengine.
346 engine_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True,
346 engine_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
347 help="command-line arguments to pass to ipengine"
347 help="command-line arguments to pass to ipengine"
348 )
348 )
349
349
350 def find_args(self):
350 def find_args(self):
351 return self.engine_cmd + self.engine_args
351 return self.engine_cmd + self.engine_args
352
352
353 def start(self, profile_dir):
353 def start(self, profile_dir):
354 """Start the engine by profile_dir."""
354 """Start the engine by profile_dir."""
355 self.engine_args.extend(['profile_dir=%s'%profile_dir])
355 self.engine_args.extend(['--profile_dir=%s'%profile_dir])
356 self.profile_dir = unicode(profile_dir)
356 self.profile_dir = unicode(profile_dir)
357 return super(LocalEngineLauncher, self).start()
357 return super(LocalEngineLauncher, self).start()
358
358
359
359
360 class LocalEngineSetLauncher(BaseLauncher):
360 class LocalEngineSetLauncher(BaseLauncher):
361 """Launch a set of engines as regular external processes."""
361 """Launch a set of engines as regular external processes."""
362
362
363 # Command line arguments for ipengine.
363 # Command line arguments for ipengine.
364 engine_args = List(
364 engine_args = List(
365 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
365 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
366 help="command-line arguments to pass to ipengine"
366 help="command-line arguments to pass to ipengine"
367 )
367 )
368 # launcher class
368 # launcher class
369 launcher_class = LocalEngineLauncher
369 launcher_class = LocalEngineLauncher
370
370
371 launchers = Dict()
371 launchers = Dict()
372 stop_data = Dict()
372 stop_data = Dict()
373
373
374 def __init__(self, work_dir=u'.', config=None, **kwargs):
374 def __init__(self, work_dir=u'.', config=None, **kwargs):
375 super(LocalEngineSetLauncher, self).__init__(
375 super(LocalEngineSetLauncher, self).__init__(
376 work_dir=work_dir, config=config, **kwargs
376 work_dir=work_dir, config=config, **kwargs
377 )
377 )
378 self.stop_data = {}
378 self.stop_data = {}
379
379
380 def start(self, n, profile_dir):
380 def start(self, n, profile_dir):
381 """Start n engines by profile or profile_dir."""
381 """Start n engines by profile or profile_dir."""
382 self.profile_dir = unicode(profile_dir)
382 self.profile_dir = unicode(profile_dir)
383 dlist = []
383 dlist = []
384 for i in range(n):
384 for i in range(n):
385 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
385 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
386 # Copy the engine args over to each engine launcher.
386 # Copy the engine args over to each engine launcher.
387 el.engine_args = copy.deepcopy(self.engine_args)
387 el.engine_args = copy.deepcopy(self.engine_args)
388 el.on_stop(self._notice_engine_stopped)
388 el.on_stop(self._notice_engine_stopped)
389 d = el.start(profile_dir)
389 d = el.start(profile_dir)
390 if i==0:
390 if i==0:
391 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
391 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
392 self.launchers[i] = el
392 self.launchers[i] = el
393 dlist.append(d)
393 dlist.append(d)
394 self.notify_start(dlist)
394 self.notify_start(dlist)
395 # The consumeErrors here could be dangerous
395 # The consumeErrors here could be dangerous
396 # dfinal = gatherBoth(dlist, consumeErrors=True)
396 # dfinal = gatherBoth(dlist, consumeErrors=True)
397 # dfinal.addCallback(self.notify_start)
397 # dfinal.addCallback(self.notify_start)
398 return dlist
398 return dlist
399
399
400 def find_args(self):
400 def find_args(self):
401 return ['engine set']
401 return ['engine set']
402
402
403 def signal(self, sig):
403 def signal(self, sig):
404 dlist = []
404 dlist = []
405 for el in self.launchers.itervalues():
405 for el in self.launchers.itervalues():
406 d = el.signal(sig)
406 d = el.signal(sig)
407 dlist.append(d)
407 dlist.append(d)
408 # dfinal = gatherBoth(dlist, consumeErrors=True)
408 # dfinal = gatherBoth(dlist, consumeErrors=True)
409 return dlist
409 return dlist
410
410
411 def interrupt_then_kill(self, delay=1.0):
411 def interrupt_then_kill(self, delay=1.0):
412 dlist = []
412 dlist = []
413 for el in self.launchers.itervalues():
413 for el in self.launchers.itervalues():
414 d = el.interrupt_then_kill(delay)
414 d = el.interrupt_then_kill(delay)
415 dlist.append(d)
415 dlist.append(d)
416 # dfinal = gatherBoth(dlist, consumeErrors=True)
416 # dfinal = gatherBoth(dlist, consumeErrors=True)
417 return dlist
417 return dlist
418
418
419 def stop(self):
419 def stop(self):
420 return self.interrupt_then_kill()
420 return self.interrupt_then_kill()
421
421
422 def _notice_engine_stopped(self, data):
422 def _notice_engine_stopped(self, data):
423 pid = data['pid']
423 pid = data['pid']
424 for idx,el in self.launchers.iteritems():
424 for idx,el in self.launchers.iteritems():
425 if el.process.pid == pid:
425 if el.process.pid == pid:
426 break
426 break
427 self.launchers.pop(idx)
427 self.launchers.pop(idx)
428 self.stop_data[idx] = data
428 self.stop_data[idx] = data
429 if not self.launchers:
429 if not self.launchers:
430 self.notify_stop(self.stop_data)
430 self.notify_stop(self.stop_data)
431
431
432
432
433 #-----------------------------------------------------------------------------
433 #-----------------------------------------------------------------------------
434 # MPIExec launchers
434 # MPIExec launchers
435 #-----------------------------------------------------------------------------
435 #-----------------------------------------------------------------------------
436
436
437
437
438 class MPIExecLauncher(LocalProcessLauncher):
438 class MPIExecLauncher(LocalProcessLauncher):
439 """Launch an external process using mpiexec."""
439 """Launch an external process using mpiexec."""
440
440
441 mpi_cmd = List(['mpiexec'], config=True,
441 mpi_cmd = List(['mpiexec'], config=True,
442 help="The mpiexec command to use in starting the process."
442 help="The mpiexec command to use in starting the process."
443 )
443 )
444 mpi_args = List([], config=True,
444 mpi_args = List([], config=True,
445 help="The command line arguments to pass to mpiexec."
445 help="The command line arguments to pass to mpiexec."
446 )
446 )
447 program = List(['date'], config=True,
447 program = List(['date'], config=True,
448 help="The program to start via mpiexec.")
448 help="The program to start via mpiexec.")
449 program_args = List([], config=True,
449 program_args = List([], config=True,
450 help="The command line argument to the program."
450 help="The command line argument to the program."
451 )
451 )
452 n = Int(1)
452 n = Int(1)
453
453
454 def find_args(self):
454 def find_args(self):
455 """Build self.args using all the fields."""
455 """Build self.args using all the fields."""
456 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
456 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
457 self.program + self.program_args
457 self.program + self.program_args
458
458
459 def start(self, n):
459 def start(self, n):
460 """Start n instances of the program using mpiexec."""
460 """Start n instances of the program using mpiexec."""
461 self.n = n
461 self.n = n
462 return super(MPIExecLauncher, self).start()
462 return super(MPIExecLauncher, self).start()
463
463
464
464
465 class MPIExecControllerLauncher(MPIExecLauncher):
465 class MPIExecControllerLauncher(MPIExecLauncher):
466 """Launch a controller using mpiexec."""
466 """Launch a controller using mpiexec."""
467
467
468 controller_cmd = List(ipcontroller_cmd_argv, config=True,
468 controller_cmd = List(ipcontroller_cmd_argv, config=True,
469 help="Popen command to launch the Contropper"
469 help="Popen command to launch the Contropper"
470 )
470 )
471 controller_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True,
471 controller_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
472 help="Command line arguments to pass to ipcontroller."
472 help="Command line arguments to pass to ipcontroller."
473 )
473 )
474 n = Int(1)
474 n = Int(1)
475
475
476 def start(self, profile_dir):
476 def start(self, profile_dir):
477 """Start the controller by profile_dir."""
477 """Start the controller by profile_dir."""
478 self.controller_args.extend(['profile_dir=%s'%profile_dir])
478 self.controller_args.extend(['--profile_dir=%s'%profile_dir])
479 self.profile_dir = unicode(profile_dir)
479 self.profile_dir = unicode(profile_dir)
480 self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
480 self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
481 return super(MPIExecControllerLauncher, self).start(1)
481 return super(MPIExecControllerLauncher, self).start(1)
482
482
483 def find_args(self):
483 def find_args(self):
484 return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
484 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
485 self.controller_cmd + self.controller_args
485 self.controller_cmd + self.controller_args
486
486
487
487
488 class MPIExecEngineSetLauncher(MPIExecLauncher):
488 class MPIExecEngineSetLauncher(MPIExecLauncher):
489
489
490 program = List(ipengine_cmd_argv, config=True,
490 program = List(ipengine_cmd_argv, config=True,
491 help="Popen command for ipengine"
491 help="Popen command for ipengine"
492 )
492 )
493 program_args = List(
493 program_args = List(
494 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
494 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
495 help="Command line arguments for ipengine."
495 help="Command line arguments for ipengine."
496 )
496 )
497 n = Int(1)
497 n = Int(1)
498
498
499 def start(self, n, profile_dir):
499 def start(self, n, profile_dir):
500 """Start n engines by profile or profile_dir."""
500 """Start n engines by profile or profile_dir."""
501 self.program_args.extend(['profile_dir=%s'%profile_dir])
501 self.program_args.extend(['--profile_dir=%s'%profile_dir])
502 self.profile_dir = unicode(profile_dir)
502 self.profile_dir = unicode(profile_dir)
503 self.n = n
503 self.n = n
504 self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
504 self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
505 return super(MPIExecEngineSetLauncher, self).start(n)
505 return super(MPIExecEngineSetLauncher, self).start(n)
506
506
507 #-----------------------------------------------------------------------------
507 #-----------------------------------------------------------------------------
508 # SSH launchers
508 # SSH launchers
509 #-----------------------------------------------------------------------------
509 #-----------------------------------------------------------------------------
510
510
511 # TODO: Get SSH Launcher back to level of sshx in 0.10.2
511 # TODO: Get SSH Launcher back to level of sshx in 0.10.2
512
512
513 class SSHLauncher(LocalProcessLauncher):
513 class SSHLauncher(LocalProcessLauncher):
514 """A minimal launcher for ssh.
514 """A minimal launcher for ssh.
515
515
516 To be useful this will probably have to be extended to use the ``sshx``
516 To be useful this will probably have to be extended to use the ``sshx``
517 idea for environment variables. There could be other things this needs
517 idea for environment variables. There could be other things this needs
518 as well.
518 as well.
519 """
519 """
520
520
521 ssh_cmd = List(['ssh'], config=True,
521 ssh_cmd = List(['ssh'], config=True,
522 help="command for starting ssh")
522 help="command for starting ssh")
523 ssh_args = List(['-tt'], config=True,
523 ssh_args = List(['-tt'], config=True,
524 help="args to pass to ssh")
524 help="args to pass to ssh")
525 program = List(['date'], config=True,
525 program = List(['date'], config=True,
526 help="Program to launch via ssh")
526 help="Program to launch via ssh")
527 program_args = List([], config=True,
527 program_args = List([], config=True,
528 help="args to pass to remote program")
528 help="args to pass to remote program")
529 hostname = Unicode('', config=True,
529 hostname = Unicode('', config=True,
530 help="hostname on which to launch the program")
530 help="hostname on which to launch the program")
531 user = Unicode('', config=True,
531 user = Unicode('', config=True,
532 help="username for ssh")
532 help="username for ssh")
533 location = Unicode('', config=True,
533 location = Unicode('', config=True,
534 help="user@hostname location for ssh in one setting")
534 help="user@hostname location for ssh in one setting")
535
535
536 def _hostname_changed(self, name, old, new):
536 def _hostname_changed(self, name, old, new):
537 if self.user:
537 if self.user:
538 self.location = u'%s@%s' % (self.user, new)
538 self.location = u'%s@%s' % (self.user, new)
539 else:
539 else:
540 self.location = new
540 self.location = new
541
541
542 def _user_changed(self, name, old, new):
542 def _user_changed(self, name, old, new):
543 self.location = u'%s@%s' % (new, self.hostname)
543 self.location = u'%s@%s' % (new, self.hostname)
544
544
545 def find_args(self):
545 def find_args(self):
546 return self.ssh_cmd + self.ssh_args + [self.location] + \
546 return self.ssh_cmd + self.ssh_args + [self.location] + \
547 self.program + self.program_args
547 self.program + self.program_args
548
548
549 def start(self, profile_dir, hostname=None, user=None):
549 def start(self, profile_dir, hostname=None, user=None):
550 self.profile_dir = unicode(profile_dir)
550 self.profile_dir = unicode(profile_dir)
551 if hostname is not None:
551 if hostname is not None:
552 self.hostname = hostname
552 self.hostname = hostname
553 if user is not None:
553 if user is not None:
554 self.user = user
554 self.user = user
555
555
556 return super(SSHLauncher, self).start()
556 return super(SSHLauncher, self).start()
557
557
558 def signal(self, sig):
558 def signal(self, sig):
559 if self.state == 'running':
559 if self.state == 'running':
560 # send escaped ssh connection-closer
560 # send escaped ssh connection-closer
561 self.process.stdin.write('~.')
561 self.process.stdin.write('~.')
562 self.process.stdin.flush()
562 self.process.stdin.flush()
563
563
564
564
565
565
566 class SSHControllerLauncher(SSHLauncher):
566 class SSHControllerLauncher(SSHLauncher):
567
567
568 program = List(ipcontroller_cmd_argv, config=True,
568 program = List(ipcontroller_cmd_argv, config=True,
569 help="remote ipcontroller command.")
569 help="remote ipcontroller command.")
570 program_args = List(['--reuse-files', '--log-to-file','log_level=%i'%logging.INFO], config=True,
570 program_args = List(['--reuse-files', '--log-to-file','--log_level=%i'%logging.INFO], config=True,
571 help="Command line arguments to ipcontroller.")
571 help="Command line arguments to ipcontroller.")
572
572
573
573
574 class SSHEngineLauncher(SSHLauncher):
574 class SSHEngineLauncher(SSHLauncher):
575 program = List(ipengine_cmd_argv, config=True,
575 program = List(ipengine_cmd_argv, config=True,
576 help="remote ipengine command.")
576 help="remote ipengine command.")
577 # Command line arguments for ipengine.
577 # Command line arguments for ipengine.
578 program_args = List(
578 program_args = List(
579 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
579 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
580 help="Command line arguments to ipengine."
580 help="Command line arguments to ipengine."
581 )
581 )
582
582
583 class SSHEngineSetLauncher(LocalEngineSetLauncher):
583 class SSHEngineSetLauncher(LocalEngineSetLauncher):
584 launcher_class = SSHEngineLauncher
584 launcher_class = SSHEngineLauncher
585 engines = Dict(config=True,
585 engines = Dict(config=True,
586 help="""dict of engines to launch. This is a dict by hostname of ints,
586 help="""dict of engines to launch. This is a dict by hostname of ints,
587 corresponding to the number of engines to start on that host.""")
587 corresponding to the number of engines to start on that host.""")
588
588
589 def start(self, n, profile_dir):
589 def start(self, n, profile_dir):
590 """Start engines by profile or profile_dir.
590 """Start engines by profile or profile_dir.
591 `n` is ignored, and the `engines` config property is used instead.
591 `n` is ignored, and the `engines` config property is used instead.
592 """
592 """
593
593
594 self.profile_dir = unicode(profile_dir)
594 self.profile_dir = unicode(profile_dir)
595 dlist = []
595 dlist = []
596 for host, n in self.engines.iteritems():
596 for host, n in self.engines.iteritems():
597 if isinstance(n, (tuple, list)):
597 if isinstance(n, (tuple, list)):
598 n, args = n
598 n, args = n
599 else:
599 else:
600 args = copy.deepcopy(self.engine_args)
600 args = copy.deepcopy(self.engine_args)
601
601
602 if '@' in host:
602 if '@' in host:
603 user,host = host.split('@',1)
603 user,host = host.split('@',1)
604 else:
604 else:
605 user=None
605 user=None
606 for i in range(n):
606 for i in range(n):
607 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
607 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
608
608
609 # Copy the engine args over to each engine launcher.
609 # Copy the engine args over to each engine launcher.
610 i
610 i
611 el.program_args = args
611 el.program_args = args
612 el.on_stop(self._notice_engine_stopped)
612 el.on_stop(self._notice_engine_stopped)
613 d = el.start(profile_dir, user=user, hostname=host)
613 d = el.start(profile_dir, user=user, hostname=host)
614 if i==0:
614 if i==0:
615 self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
615 self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
616 self.launchers[host+str(i)] = el
616 self.launchers[host+str(i)] = el
617 dlist.append(d)
617 dlist.append(d)
618 self.notify_start(dlist)
618 self.notify_start(dlist)
619 return dlist
619 return dlist
620
620
621
621
622
622
623 #-----------------------------------------------------------------------------
623 #-----------------------------------------------------------------------------
624 # Windows HPC Server 2008 scheduler launchers
624 # Windows HPC Server 2008 scheduler launchers
625 #-----------------------------------------------------------------------------
625 #-----------------------------------------------------------------------------
626
626
627
627
628 # This is only used on Windows.
628 # This is only used on Windows.
629 def find_job_cmd():
629 def find_job_cmd():
630 if WINDOWS:
630 if WINDOWS:
631 try:
631 try:
632 return find_cmd('job')
632 return find_cmd('job')
633 except (FindCmdError, ImportError):
633 except (FindCmdError, ImportError):
634 # ImportError will be raised if win32api is not installed
634 # ImportError will be raised if win32api is not installed
635 return 'job'
635 return 'job'
636 else:
636 else:
637 return 'job'
637 return 'job'
638
638
639
639
640 class WindowsHPCLauncher(BaseLauncher):
640 class WindowsHPCLauncher(BaseLauncher):
641
641
642 job_id_regexp = Unicode(r'\d+', config=True,
642 job_id_regexp = Unicode(r'\d+', config=True,
643 help="""A regular expression used to get the job id from the output of the
643 help="""A regular expression used to get the job id from the output of the
644 submit_command. """
644 submit_command. """
645 )
645 )
646 job_file_name = Unicode(u'ipython_job.xml', config=True,
646 job_file_name = Unicode(u'ipython_job.xml', config=True,
647 help="The filename of the instantiated job script.")
647 help="The filename of the instantiated job script.")
648 # The full path to the instantiated job script. This gets made dynamically
648 # The full path to the instantiated job script. This gets made dynamically
649 # by combining the work_dir with the job_file_name.
649 # by combining the work_dir with the job_file_name.
650 job_file = Unicode(u'')
650 job_file = Unicode(u'')
651 scheduler = Unicode('', config=True,
651 scheduler = Unicode('', config=True,
652 help="The hostname of the scheduler to submit the job to.")
652 help="The hostname of the scheduler to submit the job to.")
653 job_cmd = Unicode(find_job_cmd(), config=True,
653 job_cmd = Unicode(find_job_cmd(), config=True,
654 help="The command for submitting jobs.")
654 help="The command for submitting jobs.")
655
655
656 def __init__(self, work_dir=u'.', config=None, **kwargs):
656 def __init__(self, work_dir=u'.', config=None, **kwargs):
657 super(WindowsHPCLauncher, self).__init__(
657 super(WindowsHPCLauncher, self).__init__(
658 work_dir=work_dir, config=config, **kwargs
658 work_dir=work_dir, config=config, **kwargs
659 )
659 )
660
660
661 @property
661 @property
662 def job_file(self):
662 def job_file(self):
663 return os.path.join(self.work_dir, self.job_file_name)
663 return os.path.join(self.work_dir, self.job_file_name)
664
664
665 def write_job_file(self, n):
665 def write_job_file(self, n):
666 raise NotImplementedError("Implement write_job_file in a subclass.")
666 raise NotImplementedError("Implement write_job_file in a subclass.")
667
667
668 def find_args(self):
668 def find_args(self):
669 return [u'job.exe']
669 return [u'job.exe']
670
670
671 def parse_job_id(self, output):
671 def parse_job_id(self, output):
672 """Take the output of the submit command and return the job id."""
672 """Take the output of the submit command and return the job id."""
673 m = re.search(self.job_id_regexp, output)
673 m = re.search(self.job_id_regexp, output)
674 if m is not None:
674 if m is not None:
675 job_id = m.group()
675 job_id = m.group()
676 else:
676 else:
677 raise LauncherError("Job id couldn't be determined: %s" % output)
677 raise LauncherError("Job id couldn't be determined: %s" % output)
678 self.job_id = job_id
678 self.job_id = job_id
679 self.log.info('Job started with job id: %r' % job_id)
679 self.log.info('Job started with job id: %r' % job_id)
680 return job_id
680 return job_id
681
681
682 def start(self, n):
682 def start(self, n):
683 """Start n copies of the process using the Win HPC job scheduler."""
683 """Start n copies of the process using the Win HPC job scheduler."""
684 self.write_job_file(n)
684 self.write_job_file(n)
685 args = [
685 args = [
686 'submit',
686 'submit',
687 '/jobfile:%s' % self.job_file,
687 '/jobfile:%s' % self.job_file,
688 '/scheduler:%s' % self.scheduler
688 '/scheduler:%s' % self.scheduler
689 ]
689 ]
690 self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
690 self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
691
691
692 output = check_output([self.job_cmd]+args,
692 output = check_output([self.job_cmd]+args,
693 env=os.environ,
693 env=os.environ,
694 cwd=self.work_dir,
694 cwd=self.work_dir,
695 stderr=STDOUT
695 stderr=STDOUT
696 )
696 )
697 job_id = self.parse_job_id(output)
697 job_id = self.parse_job_id(output)
698 self.notify_start(job_id)
698 self.notify_start(job_id)
699 return job_id
699 return job_id
700
700
701 def stop(self):
701 def stop(self):
702 args = [
702 args = [
703 'cancel',
703 'cancel',
704 self.job_id,
704 self.job_id,
705 '/scheduler:%s' % self.scheduler
705 '/scheduler:%s' % self.scheduler
706 ]
706 ]
707 self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
707 self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
708 try:
708 try:
709 output = check_output([self.job_cmd]+args,
709 output = check_output([self.job_cmd]+args,
710 env=os.environ,
710 env=os.environ,
711 cwd=self.work_dir,
711 cwd=self.work_dir,
712 stderr=STDOUT
712 stderr=STDOUT
713 )
713 )
714 except:
714 except:
715 output = 'The job already appears to be stoppped: %r' % self.job_id
715 output = 'The job already appears to be stoppped: %r' % self.job_id
716 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
716 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
717 return output
717 return output
718
718
719
719
720 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
720 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
721
721
722 job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
722 job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
723 help="WinHPC xml job file.")
723 help="WinHPC xml job file.")
724 extra_args = List([], config=False,
724 extra_args = List([], config=False,
725 help="extra args to pass to ipcontroller")
725 help="extra args to pass to ipcontroller")
726
726
727 def write_job_file(self, n):
727 def write_job_file(self, n):
728 job = IPControllerJob(config=self.config)
728 job = IPControllerJob(config=self.config)
729
729
730 t = IPControllerTask(config=self.config)
730 t = IPControllerTask(config=self.config)
731 # The tasks work directory is *not* the actual work directory of
731 # The tasks work directory is *not* the actual work directory of
732 # the controller. It is used as the base path for the stdout/stderr
732 # the controller. It is used as the base path for the stdout/stderr
733 # files that the scheduler redirects to.
733 # files that the scheduler redirects to.
734 t.work_directory = self.profile_dir
734 t.work_directory = self.profile_dir
735 # Add the profile_dir and from self.start().
735 # Add the profile_dir and from self.start().
736 t.controller_args.extend(self.extra_args)
736 t.controller_args.extend(self.extra_args)
737 job.add_task(t)
737 job.add_task(t)
738
738
739 self.log.info("Writing job description file: %s" % self.job_file)
739 self.log.info("Writing job description file: %s" % self.job_file)
740 job.write(self.job_file)
740 job.write(self.job_file)
741
741
742 @property
742 @property
743 def job_file(self):
743 def job_file(self):
744 return os.path.join(self.profile_dir, self.job_file_name)
744 return os.path.join(self.profile_dir, self.job_file_name)
745
745
746 def start(self, profile_dir):
746 def start(self, profile_dir):
747 """Start the controller by profile_dir."""
747 """Start the controller by profile_dir."""
748 self.extra_args = ['profile_dir=%s'%profile_dir]
748 self.extra_args = ['--profile_dir=%s'%profile_dir]
749 self.profile_dir = unicode(profile_dir)
749 self.profile_dir = unicode(profile_dir)
750 return super(WindowsHPCControllerLauncher, self).start(1)
750 return super(WindowsHPCControllerLauncher, self).start(1)
751
751
752
752
753 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
753 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
754
754
755 job_file_name = Unicode(u'ipengineset_job.xml', config=True,
755 job_file_name = Unicode(u'ipengineset_job.xml', config=True,
756 help="jobfile for ipengines job")
756 help="jobfile for ipengines job")
757 extra_args = List([], config=False,
757 extra_args = List([], config=False,
758 help="extra args to pas to ipengine")
758 help="extra args to pas to ipengine")
759
759
760 def write_job_file(self, n):
760 def write_job_file(self, n):
761 job = IPEngineSetJob(config=self.config)
761 job = IPEngineSetJob(config=self.config)
762
762
763 for i in range(n):
763 for i in range(n):
764 t = IPEngineTask(config=self.config)
764 t = IPEngineTask(config=self.config)
765 # The tasks work directory is *not* the actual work directory of
765 # The tasks work directory is *not* the actual work directory of
766 # the engine. It is used as the base path for the stdout/stderr
766 # the engine. It is used as the base path for the stdout/stderr
767 # files that the scheduler redirects to.
767 # files that the scheduler redirects to.
768 t.work_directory = self.profile_dir
768 t.work_directory = self.profile_dir
769 # Add the profile_dir and from self.start().
769 # Add the profile_dir and from self.start().
770 t.engine_args.extend(self.extra_args)
770 t.engine_args.extend(self.extra_args)
771 job.add_task(t)
771 job.add_task(t)
772
772
773 self.log.info("Writing job description file: %s" % self.job_file)
773 self.log.info("Writing job description file: %s" % self.job_file)
774 job.write(self.job_file)
774 job.write(self.job_file)
775
775
776 @property
776 @property
777 def job_file(self):
777 def job_file(self):
778 return os.path.join(self.profile_dir, self.job_file_name)
778 return os.path.join(self.profile_dir, self.job_file_name)
779
779
780 def start(self, n, profile_dir):
780 def start(self, n, profile_dir):
781 """Start the controller by profile_dir."""
781 """Start the controller by profile_dir."""
782 self.extra_args = ['profile_dir=%s'%profile_dir]
782 self.extra_args = ['--profile_dir=%s'%profile_dir]
783 self.profile_dir = unicode(profile_dir)
783 self.profile_dir = unicode(profile_dir)
784 return super(WindowsHPCEngineSetLauncher, self).start(n)
784 return super(WindowsHPCEngineSetLauncher, self).start(n)
785
785
786
786
787 #-----------------------------------------------------------------------------
787 #-----------------------------------------------------------------------------
788 # Batch (PBS) system launchers
788 # Batch (PBS) system launchers
789 #-----------------------------------------------------------------------------
789 #-----------------------------------------------------------------------------
790
790
791 class BatchSystemLauncher(BaseLauncher):
791 class BatchSystemLauncher(BaseLauncher):
792 """Launch an external process using a batch system.
792 """Launch an external process using a batch system.
793
793
794 This class is designed to work with UNIX batch systems like PBS, LSF,
794 This class is designed to work with UNIX batch systems like PBS, LSF,
795 GridEngine, etc. The overall model is that there are different commands
795 GridEngine, etc. The overall model is that there are different commands
796 like qsub, qdel, etc. that handle the starting and stopping of the process.
796 like qsub, qdel, etc. that handle the starting and stopping of the process.
797
797
798 This class also has the notion of a batch script. The ``batch_template``
798 This class also has the notion of a batch script. The ``batch_template``
799 attribute can be set to a string that is a template for the batch script.
799 attribute can be set to a string that is a template for the batch script.
800 This template is instantiated using string formatting. Thus the template can
800 This template is instantiated using string formatting. Thus the template can
801 use {n} fot the number of instances. Subclasses can add additional variables
801 use {n} fot the number of instances. Subclasses can add additional variables
802 to the template dict.
802 to the template dict.
803 """
803 """
804
804
805 # Subclasses must fill these in. See PBSEngineSet
805 # Subclasses must fill these in. See PBSEngineSet
806 submit_command = List([''], config=True,
806 submit_command = List([''], config=True,
807 help="The name of the command line program used to submit jobs.")
807 help="The name of the command line program used to submit jobs.")
808 delete_command = List([''], config=True,
808 delete_command = List([''], config=True,
809 help="The name of the command line program used to delete jobs.")
809 help="The name of the command line program used to delete jobs.")
810 job_id_regexp = Unicode('', config=True,
810 job_id_regexp = Unicode('', config=True,
811 help="""A regular expression used to get the job id from the output of the
811 help="""A regular expression used to get the job id from the output of the
812 submit_command.""")
812 submit_command.""")
813 batch_template = Unicode('', config=True,
813 batch_template = Unicode('', config=True,
814 help="The string that is the batch script template itself.")
814 help="The string that is the batch script template itself.")
815 batch_template_file = Unicode(u'', config=True,
815 batch_template_file = Unicode(u'', config=True,
816 help="The file that contains the batch template.")
816 help="The file that contains the batch template.")
817 batch_file_name = Unicode(u'batch_script', config=True,
817 batch_file_name = Unicode(u'batch_script', config=True,
818 help="The filename of the instantiated batch script.")
818 help="The filename of the instantiated batch script.")
819 queue = Unicode(u'', config=True,
819 queue = Unicode(u'', config=True,
820 help="The PBS Queue.")
820 help="The PBS Queue.")
821
821
822 # not configurable, override in subclasses
822 # not configurable, override in subclasses
823 # PBS Job Array regex
823 # PBS Job Array regex
824 job_array_regexp = Unicode('')
824 job_array_regexp = Unicode('')
825 job_array_template = Unicode('')
825 job_array_template = Unicode('')
826 # PBS Queue regex
826 # PBS Queue regex
827 queue_regexp = Unicode('')
827 queue_regexp = Unicode('')
828 queue_template = Unicode('')
828 queue_template = Unicode('')
829 # The default batch template, override in subclasses
829 # The default batch template, override in subclasses
830 default_template = Unicode('')
830 default_template = Unicode('')
831 # The full path to the instantiated batch script.
831 # The full path to the instantiated batch script.
832 batch_file = Unicode(u'')
832 batch_file = Unicode(u'')
833 # the format dict used with batch_template:
833 # the format dict used with batch_template:
834 context = Dict()
834 context = Dict()
835 # the Formatter instance for rendering the templates:
835 # the Formatter instance for rendering the templates:
836 formatter = Instance(EvalFormatter, (), {})
836 formatter = Instance(EvalFormatter, (), {})
837
837
838
838
839 def find_args(self):
839 def find_args(self):
840 return self.submit_command + [self.batch_file]
840 return self.submit_command + [self.batch_file]
841
841
842 def __init__(self, work_dir=u'.', config=None, **kwargs):
842 def __init__(self, work_dir=u'.', config=None, **kwargs):
843 super(BatchSystemLauncher, self).__init__(
843 super(BatchSystemLauncher, self).__init__(
844 work_dir=work_dir, config=config, **kwargs
844 work_dir=work_dir, config=config, **kwargs
845 )
845 )
846 self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
846 self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
847
847
848 def parse_job_id(self, output):
848 def parse_job_id(self, output):
849 """Take the output of the submit command and return the job id."""
849 """Take the output of the submit command and return the job id."""
850 m = re.search(self.job_id_regexp, output)
850 m = re.search(self.job_id_regexp, output)
851 if m is not None:
851 if m is not None:
852 job_id = m.group()
852 job_id = m.group()
853 else:
853 else:
854 raise LauncherError("Job id couldn't be determined: %s" % output)
854 raise LauncherError("Job id couldn't be determined: %s" % output)
855 self.job_id = job_id
855 self.job_id = job_id
856 self.log.info('Job submitted with job id: %r' % job_id)
856 self.log.info('Job submitted with job id: %r' % job_id)
857 return job_id
857 return job_id
858
858
859 def write_batch_script(self, n):
859 def write_batch_script(self, n):
860 """Instantiate and write the batch script to the work_dir."""
860 """Instantiate and write the batch script to the work_dir."""
861 self.context['n'] = n
861 self.context['n'] = n
862 self.context['queue'] = self.queue
862 self.context['queue'] = self.queue
863 # first priority is batch_template if set
863 # first priority is batch_template if set
864 if self.batch_template_file and not self.batch_template:
864 if self.batch_template_file and not self.batch_template:
865 # second priority is batch_template_file
865 # second priority is batch_template_file
866 with open(self.batch_template_file) as f:
866 with open(self.batch_template_file) as f:
867 self.batch_template = f.read()
867 self.batch_template = f.read()
868 if not self.batch_template:
868 if not self.batch_template:
869 # third (last) priority is default_template
869 # third (last) priority is default_template
870 self.batch_template = self.default_template
870 self.batch_template = self.default_template
871
871
872 # add jobarray or queue lines to user-specified template
872 # add jobarray or queue lines to user-specified template
873 # note that this is *only* when user did not specify a template.
873 # note that this is *only* when user did not specify a template.
874 regex = re.compile(self.job_array_regexp)
874 regex = re.compile(self.job_array_regexp)
875 # print regex.search(self.batch_template)
875 # print regex.search(self.batch_template)
876 if not regex.search(self.batch_template):
876 if not regex.search(self.batch_template):
877 self.log.info("adding job array settings to batch script")
877 self.log.info("adding job array settings to batch script")
878 firstline, rest = self.batch_template.split('\n',1)
878 firstline, rest = self.batch_template.split('\n',1)
879 self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
879 self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
880
880
881 regex = re.compile(self.queue_regexp)
881 regex = re.compile(self.queue_regexp)
882 # print regex.search(self.batch_template)
882 # print regex.search(self.batch_template)
883 if self.queue and not regex.search(self.batch_template):
883 if self.queue and not regex.search(self.batch_template):
884 self.log.info("adding PBS queue settings to batch script")
884 self.log.info("adding PBS queue settings to batch script")
885 firstline, rest = self.batch_template.split('\n',1)
885 firstline, rest = self.batch_template.split('\n',1)
886 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
886 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
887
887
888 script_as_string = self.formatter.format(self.batch_template, **self.context)
888 script_as_string = self.formatter.format(self.batch_template, **self.context)
889 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
889 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
890
890
891 with open(self.batch_file, 'w') as f:
891 with open(self.batch_file, 'w') as f:
892 f.write(script_as_string)
892 f.write(script_as_string)
893 os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
893 os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
894
894
895 def start(self, n, profile_dir):
895 def start(self, n, profile_dir):
896 """Start n copies of the process using a batch system."""
896 """Start n copies of the process using a batch system."""
897 # Here we save profile_dir in the context so they
897 # Here we save profile_dir in the context so they
898 # can be used in the batch script template as {profile_dir}
898 # can be used in the batch script template as {profile_dir}
899 self.context['profile_dir'] = profile_dir
899 self.context['profile_dir'] = profile_dir
900 self.profile_dir = unicode(profile_dir)
900 self.profile_dir = unicode(profile_dir)
901 self.write_batch_script(n)
901 self.write_batch_script(n)
902 output = check_output(self.args, env=os.environ)
902 output = check_output(self.args, env=os.environ)
903
903
904 job_id = self.parse_job_id(output)
904 job_id = self.parse_job_id(output)
905 self.notify_start(job_id)
905 self.notify_start(job_id)
906 return job_id
906 return job_id
907
907
908 def stop(self):
908 def stop(self):
909 output = check_output(self.delete_command+[self.job_id], env=os.environ)
909 output = check_output(self.delete_command+[self.job_id], env=os.environ)
910 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
910 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
911 return output
911 return output
912
912
913
913
914 class PBSLauncher(BatchSystemLauncher):
914 class PBSLauncher(BatchSystemLauncher):
915 """A BatchSystemLauncher subclass for PBS."""
915 """A BatchSystemLauncher subclass for PBS."""
916
916
917 submit_command = List(['qsub'], config=True,
917 submit_command = List(['qsub'], config=True,
918 help="The PBS submit command ['qsub']")
918 help="The PBS submit command ['qsub']")
919 delete_command = List(['qdel'], config=True,
919 delete_command = List(['qdel'], config=True,
920 help="The PBS delete command ['qsub']")
920 help="The PBS delete command ['qsub']")
921 job_id_regexp = Unicode(r'\d+', config=True,
921 job_id_regexp = Unicode(r'\d+', config=True,
922 help="Regular expresion for identifying the job ID [r'\d+']")
922 help="Regular expresion for identifying the job ID [r'\d+']")
923
923
924 batch_file = Unicode(u'')
924 batch_file = Unicode(u'')
925 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
925 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
926 job_array_template = Unicode('#PBS -t 1-{n}')
926 job_array_template = Unicode('#PBS -t 1-{n}')
927 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
927 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
928 queue_template = Unicode('#PBS -q {queue}')
928 queue_template = Unicode('#PBS -q {queue}')
929
929
930
930
931 class PBSControllerLauncher(PBSLauncher):
931 class PBSControllerLauncher(PBSLauncher):
932 """Launch a controller using PBS."""
932 """Launch a controller using PBS."""
933
933
934 batch_file_name = Unicode(u'pbs_controller', config=True,
934 batch_file_name = Unicode(u'pbs_controller', config=True,
935 help="batch file name for the controller job.")
935 help="batch file name for the controller job.")
936 default_template= Unicode("""#!/bin/sh
936 default_template= Unicode("""#!/bin/sh
937 #PBS -V
937 #PBS -V
938 #PBS -N ipcontroller
938 #PBS -N ipcontroller
939 %s --log-to-file profile_dir={profile_dir}
939 %s --log-to-file --profile_dir={profile_dir}
940 """%(' '.join(ipcontroller_cmd_argv)))
940 """%(' '.join(ipcontroller_cmd_argv)))
941
941
942 def start(self, profile_dir):
942 def start(self, profile_dir):
943 """Start the controller by profile or profile_dir."""
943 """Start the controller by profile or profile_dir."""
944 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
944 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
945 return super(PBSControllerLauncher, self).start(1, profile_dir)
945 return super(PBSControllerLauncher, self).start(1, profile_dir)
946
946
947
947
948 class PBSEngineSetLauncher(PBSLauncher):
948 class PBSEngineSetLauncher(PBSLauncher):
949 """Launch Engines using PBS"""
949 """Launch Engines using PBS"""
950 batch_file_name = Unicode(u'pbs_engines', config=True,
950 batch_file_name = Unicode(u'pbs_engines', config=True,
951 help="batch file name for the engine(s) job.")
951 help="batch file name for the engine(s) job.")
952 default_template= Unicode(u"""#!/bin/sh
952 default_template= Unicode(u"""#!/bin/sh
953 #PBS -V
953 #PBS -V
954 #PBS -N ipengine
954 #PBS -N ipengine
955 %s profile_dir={profile_dir}
955 %s --profile_dir={profile_dir}
956 """%(' '.join(ipengine_cmd_argv)))
956 """%(' '.join(ipengine_cmd_argv)))
957
957
958 def start(self, n, profile_dir):
958 def start(self, n, profile_dir):
959 """Start n engines by profile or profile_dir."""
959 """Start n engines by profile or profile_dir."""
960 self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
960 self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
961 return super(PBSEngineSetLauncher, self).start(n, profile_dir)
961 return super(PBSEngineSetLauncher, self).start(n, profile_dir)
962
962
963 #SGE is very similar to PBS
963 #SGE is very similar to PBS
964
964
965 class SGELauncher(PBSLauncher):
965 class SGELauncher(PBSLauncher):
966 """Sun GridEngine is a PBS clone with slightly different syntax"""
966 """Sun GridEngine is a PBS clone with slightly different syntax"""
967 job_array_regexp = Unicode('#\$\W+\-t')
967 job_array_regexp = Unicode('#\$\W+\-t')
968 job_array_template = Unicode('#$ -t 1-{n}')
968 job_array_template = Unicode('#$ -t 1-{n}')
969 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
969 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
970 queue_template = Unicode('#$ -q {queue}')
970 queue_template = Unicode('#$ -q {queue}')
971
971
972 class SGEControllerLauncher(SGELauncher):
972 class SGEControllerLauncher(SGELauncher):
973 """Launch a controller using SGE."""
973 """Launch a controller using SGE."""
974
974
975 batch_file_name = Unicode(u'sge_controller', config=True,
975 batch_file_name = Unicode(u'sge_controller', config=True,
976 help="batch file name for the ipontroller job.")
976 help="batch file name for the ipontroller job.")
977 default_template= Unicode(u"""#$ -V
977 default_template= Unicode(u"""#$ -V
978 #$ -S /bin/sh
978 #$ -S /bin/sh
979 #$ -N ipcontroller
979 #$ -N ipcontroller
980 %s --log-to-file profile_dir={profile_dir}
980 %s --log-to-file --profile_dir={profile_dir}
981 """%(' '.join(ipcontroller_cmd_argv)))
981 """%(' '.join(ipcontroller_cmd_argv)))
982
982
983 def start(self, profile_dir):
983 def start(self, profile_dir):
984 """Start the controller by profile or profile_dir."""
984 """Start the controller by profile or profile_dir."""
985 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
985 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
986 return super(SGEControllerLauncher, self).start(1, profile_dir)
986 return super(SGEControllerLauncher, self).start(1, profile_dir)
987
987
988 class SGEEngineSetLauncher(SGELauncher):
988 class SGEEngineSetLauncher(SGELauncher):
989 """Launch Engines with SGE"""
989 """Launch Engines with SGE"""
990 batch_file_name = Unicode(u'sge_engines', config=True,
990 batch_file_name = Unicode(u'sge_engines', config=True,
991 help="batch file name for the engine(s) job.")
991 help="batch file name for the engine(s) job.")
992 default_template = Unicode("""#$ -V
992 default_template = Unicode("""#$ -V
993 #$ -S /bin/sh
993 #$ -S /bin/sh
994 #$ -N ipengine
994 #$ -N ipengine
995 %s profile_dir={profile_dir}
995 %s --profile_dir={profile_dir}
996 """%(' '.join(ipengine_cmd_argv)))
996 """%(' '.join(ipengine_cmd_argv)))
997
997
998 def start(self, n, profile_dir):
998 def start(self, n, profile_dir):
999 """Start n engines by profile or profile_dir."""
999 """Start n engines by profile or profile_dir."""
1000 self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
1000 self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
1001 return super(SGEEngineSetLauncher, self).start(n, profile_dir)
1001 return super(SGEEngineSetLauncher, self).start(n, profile_dir)
1002
1002
1003
1003
1004 #-----------------------------------------------------------------------------
1004 #-----------------------------------------------------------------------------
1005 # A launcher for ipcluster itself!
1005 # A launcher for ipcluster itself!
1006 #-----------------------------------------------------------------------------
1006 #-----------------------------------------------------------------------------
1007
1007
1008
1008
1009 class IPClusterLauncher(LocalProcessLauncher):
1009 class IPClusterLauncher(LocalProcessLauncher):
1010 """Launch the ipcluster program in an external process."""
1010 """Launch the ipcluster program in an external process."""
1011
1011
1012 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1012 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1013 help="Popen command for ipcluster")
1013 help="Popen command for ipcluster")
1014 ipcluster_args = List(
1014 ipcluster_args = List(
1015 ['--clean-logs', '--log-to-file', 'log_level=%i'%logging.INFO], config=True,
1015 ['--clean-logs', '--log-to-file', '--log_level=%i'%logging.INFO], config=True,
1016 help="Command line arguments to pass to ipcluster.")
1016 help="Command line arguments to pass to ipcluster.")
1017 ipcluster_subcommand = Unicode('start')
1017 ipcluster_subcommand = Unicode('start')
1018 ipcluster_n = Int(2)
1018 ipcluster_n = Int(2)
1019
1019
1020 def find_args(self):
1020 def find_args(self):
1021 return self.ipcluster_cmd + ['--'+self.ipcluster_subcommand] + \
1021 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1022 ['n=%i'%self.ipcluster_n] + self.ipcluster_args
1022 ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
1023
1023
1024 def start(self):
1024 def start(self):
1025 self.log.info("Starting ipcluster: %r" % self.args)
1025 self.log.info("Starting ipcluster: %r" % self.args)
1026 return super(IPClusterLauncher, self).start()
1026 return super(IPClusterLauncher, self).start()
1027
1027
1028 #-----------------------------------------------------------------------------
1028 #-----------------------------------------------------------------------------
1029 # Collections of launchers
1029 # Collections of launchers
1030 #-----------------------------------------------------------------------------
1030 #-----------------------------------------------------------------------------
1031
1031
1032 local_launchers = [
1032 local_launchers = [
1033 LocalControllerLauncher,
1033 LocalControllerLauncher,
1034 LocalEngineLauncher,
1034 LocalEngineLauncher,
1035 LocalEngineSetLauncher,
1035 LocalEngineSetLauncher,
1036 ]
1036 ]
1037 mpi_launchers = [
1037 mpi_launchers = [
1038 MPIExecLauncher,
1038 MPIExecLauncher,
1039 MPIExecControllerLauncher,
1039 MPIExecControllerLauncher,
1040 MPIExecEngineSetLauncher,
1040 MPIExecEngineSetLauncher,
1041 ]
1041 ]
1042 ssh_launchers = [
1042 ssh_launchers = [
1043 SSHLauncher,
1043 SSHLauncher,
1044 SSHControllerLauncher,
1044 SSHControllerLauncher,
1045 SSHEngineLauncher,
1045 SSHEngineLauncher,
1046 SSHEngineSetLauncher,
1046 SSHEngineSetLauncher,
1047 ]
1047 ]
1048 winhpc_launchers = [
1048 winhpc_launchers = [
1049 WindowsHPCLauncher,
1049 WindowsHPCLauncher,
1050 WindowsHPCControllerLauncher,
1050 WindowsHPCControllerLauncher,
1051 WindowsHPCEngineSetLauncher,
1051 WindowsHPCEngineSetLauncher,
1052 ]
1052 ]
1053 pbs_launchers = [
1053 pbs_launchers = [
1054 PBSLauncher,
1054 PBSLauncher,
1055 PBSControllerLauncher,
1055 PBSControllerLauncher,
1056 PBSEngineSetLauncher,
1056 PBSEngineSetLauncher,
1057 ]
1057 ]
1058 sge_launchers = [
1058 sge_launchers = [
1059 SGELauncher,
1059 SGELauncher,
1060 SGEControllerLauncher,
1060 SGEControllerLauncher,
1061 SGEEngineSetLauncher,
1061 SGEEngineSetLauncher,
1062 ]
1062 ]
1063 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1063 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1064 + pbs_launchers + sge_launchers
1064 + pbs_launchers + sge_launchers
1065
1065
@@ -1,111 +1,111 b''
1 """toplevel setup/teardown for parallel tests."""
1 """toplevel setup/teardown for parallel tests."""
2
2
3 #-------------------------------------------------------------------------------
3 #-------------------------------------------------------------------------------
4 # Copyright (C) 2011 The IPython Development Team
4 # Copyright (C) 2011 The IPython Development Team
5 #
5 #
6 # Distributed under the terms of the BSD License. The full license is in
6 # Distributed under the terms of the BSD License. The full license is in
7 # the file COPYING, distributed as part of this software.
7 # the file COPYING, distributed as part of this software.
8 #-------------------------------------------------------------------------------
8 #-------------------------------------------------------------------------------
9
9
10 #-------------------------------------------------------------------------------
10 #-------------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-------------------------------------------------------------------------------
12 #-------------------------------------------------------------------------------
13
13
14 import os
14 import os
15 import tempfile
15 import tempfile
16 import time
16 import time
17 from subprocess import Popen
17 from subprocess import Popen
18
18
19 from IPython.utils.path import get_ipython_dir
19 from IPython.utils.path import get_ipython_dir
20 from IPython.parallel import Client
20 from IPython.parallel import Client
21 from IPython.parallel.apps.launcher import (LocalProcessLauncher,
21 from IPython.parallel.apps.launcher import (LocalProcessLauncher,
22 ipengine_cmd_argv,
22 ipengine_cmd_argv,
23 ipcontroller_cmd_argv,
23 ipcontroller_cmd_argv,
24 SIGKILL)
24 SIGKILL)
25
25
26 # globals
26 # globals
27 launchers = []
27 launchers = []
28 blackhole = open(os.devnull, 'w')
28 blackhole = open(os.devnull, 'w')
29
29
30 # Launcher class
30 # Launcher class
31 class TestProcessLauncher(LocalProcessLauncher):
31 class TestProcessLauncher(LocalProcessLauncher):
32 """subclass LocalProcessLauncher, to prevent extra sockets and threads being created on Windows"""
32 """subclass LocalProcessLauncher, to prevent extra sockets and threads being created on Windows"""
33 def start(self):
33 def start(self):
34 if self.state == 'before':
34 if self.state == 'before':
35 self.process = Popen(self.args,
35 self.process = Popen(self.args,
36 stdout=blackhole, stderr=blackhole,
36 stdout=blackhole, stderr=blackhole,
37 env=os.environ,
37 env=os.environ,
38 cwd=self.work_dir
38 cwd=self.work_dir
39 )
39 )
40 self.notify_start(self.process.pid)
40 self.notify_start(self.process.pid)
41 self.poll = self.process.poll
41 self.poll = self.process.poll
42 else:
42 else:
43 s = 'The process was already started and has state: %r' % self.state
43 s = 'The process was already started and has state: %r' % self.state
44 raise ProcessStateError(s)
44 raise ProcessStateError(s)
45
45
46 # nose setup/teardown
46 # nose setup/teardown
47
47
48 def setup():
48 def setup():
49 cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest')
49 cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest')
50 engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json')
50 engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json')
51 client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json')
51 client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json')
52 for json in (engine_json, client_json):
52 for json in (engine_json, client_json):
53 if os.path.exists(json):
53 if os.path.exists(json):
54 os.remove(json)
54 os.remove(json)
55
55
56 cp = TestProcessLauncher()
56 cp = TestProcessLauncher()
57 cp.cmd_and_args = ipcontroller_cmd_argv + \
57 cp.cmd_and_args = ipcontroller_cmd_argv + \
58 ['profile=iptest', 'log_level=50']
58 ['--profile=iptest', '--log_level=50']
59 cp.start()
59 cp.start()
60 launchers.append(cp)
60 launchers.append(cp)
61 tic = time.time()
61 tic = time.time()
62 while not os.path.exists(engine_json) or not os.path.exists(client_json):
62 while not os.path.exists(engine_json) or not os.path.exists(client_json):
63 if cp.poll() is not None:
63 if cp.poll() is not None:
64 print cp.poll()
64 print cp.poll()
65 raise RuntimeError("The test controller failed to start.")
65 raise RuntimeError("The test controller failed to start.")
66 elif time.time()-tic > 10:
66 elif time.time()-tic > 10:
67 raise RuntimeError("Timeout waiting for the test controller to start.")
67 raise RuntimeError("Timeout waiting for the test controller to start.")
68 time.sleep(0.1)
68 time.sleep(0.1)
69 add_engines(1)
69 add_engines(1)
70
70
71 def add_engines(n=1, profile='iptest'):
71 def add_engines(n=1, profile='iptest'):
72 rc = Client(profile=profile)
72 rc = Client(profile=profile)
73 base = len(rc)
73 base = len(rc)
74 eps = []
74 eps = []
75 for i in range(n):
75 for i in range(n):
76 ep = TestProcessLauncher()
76 ep = TestProcessLauncher()
77 ep.cmd_and_args = ipengine_cmd_argv + ['profile=%s'%profile, 'log_level=50']
77 ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log_level=50']
78 ep.start()
78 ep.start()
79 launchers.append(ep)
79 launchers.append(ep)
80 eps.append(ep)
80 eps.append(ep)
81 tic = time.time()
81 tic = time.time()
82 while len(rc) < base+n:
82 while len(rc) < base+n:
83 if any([ ep.poll() is not None for ep in eps ]):
83 if any([ ep.poll() is not None for ep in eps ]):
84 raise RuntimeError("A test engine failed to start.")
84 raise RuntimeError("A test engine failed to start.")
85 elif time.time()-tic > 10:
85 elif time.time()-tic > 10:
86 raise RuntimeError("Timeout waiting for engines to connect.")
86 raise RuntimeError("Timeout waiting for engines to connect.")
87 time.sleep(.1)
87 time.sleep(.1)
88 rc.spin()
88 rc.spin()
89 rc.close()
89 rc.close()
90 return eps
90 return eps
91
91
92 def teardown():
92 def teardown():
93 time.sleep(1)
93 time.sleep(1)
94 while launchers:
94 while launchers:
95 p = launchers.pop()
95 p = launchers.pop()
96 if p.poll() is None:
96 if p.poll() is None:
97 try:
97 try:
98 p.stop()
98 p.stop()
99 except Exception, e:
99 except Exception, e:
100 print e
100 print e
101 pass
101 pass
102 if p.poll() is None:
102 if p.poll() is None:
103 time.sleep(.25)
103 time.sleep(.25)
104 if p.poll() is None:
104 if p.poll() is None:
105 try:
105 try:
106 print 'cleaning up test process...'
106 print 'cleaning up test process...'
107 p.signal(SIGKILL)
107 p.signal(SIGKILL)
108 except:
108 except:
109 print "couldn't shutdown process: ", p
109 print "couldn't shutdown process: ", p
110 blackhole.close()
110 blackhole.close()
111
111
@@ -1,309 +1,309 b''
1 """Generic testing tools that do NOT depend on Twisted.
1 """Generic testing tools that do NOT depend on Twisted.
2
2
3 In particular, this module exposes a set of top-level assert* functions that
3 In particular, this module exposes a set of top-level assert* functions that
4 can be used in place of nose.tools.assert* in method generators (the ones in
4 can be used in place of nose.tools.assert* in method generators (the ones in
5 nose can not, at least as of nose 0.10.4).
5 nose can not, at least as of nose 0.10.4).
6
6
7 Note: our testing package contains testing.util, which does depend on Twisted
7 Note: our testing package contains testing.util, which does depend on Twisted
8 and provides utilities for tests that manage Deferreds. All testing support
8 and provides utilities for tests that manage Deferreds. All testing support
9 tools that only depend on nose, IPython or the standard library should go here
9 tools that only depend on nose, IPython or the standard library should go here
10 instead.
10 instead.
11
11
12
12
13 Authors
13 Authors
14 -------
14 -------
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 """
16 """
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Copyright (C) 2009 The IPython Development Team
21 # Copyright (C) 2009 The IPython Development Team
22 #
22 #
23 # Distributed under the terms of the BSD License. The full license is in
23 # Distributed under the terms of the BSD License. The full license is in
24 # the file COPYING, distributed as part of this software.
24 # the file COPYING, distributed as part of this software.
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Imports
28 # Imports
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30
30
31 import os
31 import os
32 import re
32 import re
33 import sys
33 import sys
34
34
35 try:
35 try:
36 # These tools are used by parts of the runtime, so we make the nose
36 # These tools are used by parts of the runtime, so we make the nose
37 # dependency optional at this point. Nose is a hard dependency to run the
37 # dependency optional at this point. Nose is a hard dependency to run the
38 # test suite, but NOT to use ipython itself.
38 # test suite, but NOT to use ipython itself.
39 import nose.tools as nt
39 import nose.tools as nt
40 has_nose = True
40 has_nose = True
41 except ImportError:
41 except ImportError:
42 has_nose = False
42 has_nose = False
43
43
44 from IPython.config.loader import Config
44 from IPython.config.loader import Config
45 from IPython.utils.process import find_cmd, getoutputerror
45 from IPython.utils.process import find_cmd, getoutputerror
46 from IPython.utils.text import list_strings
46 from IPython.utils.text import list_strings
47 from IPython.utils.io import temp_pyfile
47 from IPython.utils.io import temp_pyfile
48
48
49 from . import decorators as dec
49 from . import decorators as dec
50 from . import skipdoctest
50 from . import skipdoctest
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Globals
53 # Globals
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56 # Make a bunch of nose.tools assert wrappers that can be used in test
56 # Make a bunch of nose.tools assert wrappers that can be used in test
57 # generators. This will expose an assert* function for each one in nose.tools.
57 # generators. This will expose an assert* function for each one in nose.tools.
58
58
59 _tpl = """
59 _tpl = """
60 def %(name)s(*a,**kw):
60 def %(name)s(*a,**kw):
61 return nt.%(name)s(*a,**kw)
61 return nt.%(name)s(*a,**kw)
62 """
62 """
63
63
64 if has_nose:
64 if has_nose:
65 for _x in [a for a in dir(nt) if a.startswith('assert')]:
65 for _x in [a for a in dir(nt) if a.startswith('assert')]:
66 exec _tpl % dict(name=_x)
66 exec _tpl % dict(name=_x)
67
67
68 #-----------------------------------------------------------------------------
68 #-----------------------------------------------------------------------------
69 # Functions and classes
69 # Functions and classes
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71
71
72 # The docstring for full_path doctests differently on win32 (different path
72 # The docstring for full_path doctests differently on win32 (different path
73 # separator) so just skip the doctest there. The example remains informative.
73 # separator) so just skip the doctest there. The example remains informative.
74 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
74 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
75
75
76 @doctest_deco
76 @doctest_deco
77 def full_path(startPath,files):
77 def full_path(startPath,files):
78 """Make full paths for all the listed files, based on startPath.
78 """Make full paths for all the listed files, based on startPath.
79
79
80 Only the base part of startPath is kept, since this routine is typically
80 Only the base part of startPath is kept, since this routine is typically
81 used with a script's __file__ variable as startPath. The base of startPath
81 used with a script's __file__ variable as startPath. The base of startPath
82 is then prepended to all the listed files, forming the output list.
82 is then prepended to all the listed files, forming the output list.
83
83
84 Parameters
84 Parameters
85 ----------
85 ----------
86 startPath : string
86 startPath : string
87 Initial path to use as the base for the results. This path is split
87 Initial path to use as the base for the results. This path is split
88 using os.path.split() and only its first component is kept.
88 using os.path.split() and only its first component is kept.
89
89
90 files : string or list
90 files : string or list
91 One or more files.
91 One or more files.
92
92
93 Examples
93 Examples
94 --------
94 --------
95
95
96 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
96 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
97 ['/foo/a.txt', '/foo/b.txt']
97 ['/foo/a.txt', '/foo/b.txt']
98
98
99 >>> full_path('/foo',['a.txt','b.txt'])
99 >>> full_path('/foo',['a.txt','b.txt'])
100 ['/a.txt', '/b.txt']
100 ['/a.txt', '/b.txt']
101
101
102 If a single file is given, the output is still a list:
102 If a single file is given, the output is still a list:
103 >>> full_path('/foo','a.txt')
103 >>> full_path('/foo','a.txt')
104 ['/a.txt']
104 ['/a.txt']
105 """
105 """
106
106
107 files = list_strings(files)
107 files = list_strings(files)
108 base = os.path.split(startPath)[0]
108 base = os.path.split(startPath)[0]
109 return [ os.path.join(base,f) for f in files ]
109 return [ os.path.join(base,f) for f in files ]
110
110
111
111
112 def parse_test_output(txt):
112 def parse_test_output(txt):
113 """Parse the output of a test run and return errors, failures.
113 """Parse the output of a test run and return errors, failures.
114
114
115 Parameters
115 Parameters
116 ----------
116 ----------
117 txt : str
117 txt : str
118 Text output of a test run, assumed to contain a line of one of the
118 Text output of a test run, assumed to contain a line of one of the
119 following forms::
119 following forms::
120 'FAILED (errors=1)'
120 'FAILED (errors=1)'
121 'FAILED (failures=1)'
121 'FAILED (failures=1)'
122 'FAILED (errors=1, failures=1)'
122 'FAILED (errors=1, failures=1)'
123
123
124 Returns
124 Returns
125 -------
125 -------
126 nerr, nfail: number of errors and failures.
126 nerr, nfail: number of errors and failures.
127 """
127 """
128
128
129 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
129 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
130 if err_m:
130 if err_m:
131 nerr = int(err_m.group(1))
131 nerr = int(err_m.group(1))
132 nfail = 0
132 nfail = 0
133 return nerr, nfail
133 return nerr, nfail
134
134
135 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
135 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
136 if fail_m:
136 if fail_m:
137 nerr = 0
137 nerr = 0
138 nfail = int(fail_m.group(1))
138 nfail = int(fail_m.group(1))
139 return nerr, nfail
139 return nerr, nfail
140
140
141 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
141 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
142 re.MULTILINE)
142 re.MULTILINE)
143 if both_m:
143 if both_m:
144 nerr = int(both_m.group(1))
144 nerr = int(both_m.group(1))
145 nfail = int(both_m.group(2))
145 nfail = int(both_m.group(2))
146 return nerr, nfail
146 return nerr, nfail
147
147
148 # If the input didn't match any of these forms, assume no error/failures
148 # If the input didn't match any of these forms, assume no error/failures
149 return 0, 0
149 return 0, 0
150
150
151
151
152 # So nose doesn't think this is a test
152 # So nose doesn't think this is a test
153 parse_test_output.__test__ = False
153 parse_test_output.__test__ = False
154
154
155
155
156 def default_argv():
156 def default_argv():
157 """Return a valid default argv for creating testing instances of ipython"""
157 """Return a valid default argv for creating testing instances of ipython"""
158
158
159 return ['--quick', # so no config file is loaded
159 return ['--quick', # so no config file is loaded
160 # Other defaults to minimize side effects on stdout
160 # Other defaults to minimize side effects on stdout
161 'colors=NoColor', '--no-term-title','--no-banner',
161 '--colors=NoColor', '--no-term-title','--no-banner',
162 'autocall=0']
162 '--autocall=0']
163
163
164
164
165 def default_config():
165 def default_config():
166 """Return a config object with good defaults for testing."""
166 """Return a config object with good defaults for testing."""
167 config = Config()
167 config = Config()
168 config.TerminalInteractiveShell.colors = 'NoColor'
168 config.TerminalInteractiveShell.colors = 'NoColor'
169 config.TerminalTerminalInteractiveShell.term_title = False,
169 config.TerminalTerminalInteractiveShell.term_title = False,
170 config.TerminalInteractiveShell.autocall = 0
170 config.TerminalInteractiveShell.autocall = 0
171 config.HistoryManager.hist_file = u'test_hist.sqlite'
171 config.HistoryManager.hist_file = u'test_hist.sqlite'
172 config.HistoryManager.db_cache_size = 10000
172 config.HistoryManager.db_cache_size = 10000
173 return config
173 return config
174
174
175
175
176 def ipexec(fname, options=None):
176 def ipexec(fname, options=None):
177 """Utility to call 'ipython filename'.
177 """Utility to call 'ipython filename'.
178
178
179 Starts IPython witha minimal and safe configuration to make startup as fast
179 Starts IPython witha minimal and safe configuration to make startup as fast
180 as possible.
180 as possible.
181
181
182 Note that this starts IPython in a subprocess!
182 Note that this starts IPython in a subprocess!
183
183
184 Parameters
184 Parameters
185 ----------
185 ----------
186 fname : str
186 fname : str
187 Name of file to be executed (should have .py or .ipy extension).
187 Name of file to be executed (should have .py or .ipy extension).
188
188
189 options : optional, list
189 options : optional, list
190 Extra command-line flags to be passed to IPython.
190 Extra command-line flags to be passed to IPython.
191
191
192 Returns
192 Returns
193 -------
193 -------
194 (stdout, stderr) of ipython subprocess.
194 (stdout, stderr) of ipython subprocess.
195 """
195 """
196 if options is None: options = []
196 if options is None: options = []
197
197
198 # For these subprocess calls, eliminate all prompt printing so we only see
198 # For these subprocess calls, eliminate all prompt printing so we only see
199 # output from script execution
199 # output from script execution
200 prompt_opts = [ 'InteractiveShell.prompt_in1=""',
200 prompt_opts = [ '--InteractiveShell.prompt_in1=""',
201 'InteractiveShell.prompt_in2=""',
201 '--InteractiveShell.prompt_in2=""',
202 'InteractiveShell.prompt_out=""'
202 '--InteractiveShell.prompt_out=""'
203 ]
203 ]
204 cmdargs = ' '.join(default_argv() + prompt_opts + options)
204 cmdargs = ' '.join(default_argv() + prompt_opts + options)
205
205
206 _ip = get_ipython()
206 _ip = get_ipython()
207 test_dir = os.path.dirname(__file__)
207 test_dir = os.path.dirname(__file__)
208
208
209 ipython_cmd = find_cmd('ipython')
209 ipython_cmd = find_cmd('ipython')
210 # Absolute path for filename
210 # Absolute path for filename
211 full_fname = os.path.join(test_dir, fname)
211 full_fname = os.path.join(test_dir, fname)
212 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
212 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
213 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
213 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
214 return getoutputerror(full_cmd)
214 return getoutputerror(full_cmd)
215
215
216
216
217 def ipexec_validate(fname, expected_out, expected_err='',
217 def ipexec_validate(fname, expected_out, expected_err='',
218 options=None):
218 options=None):
219 """Utility to call 'ipython filename' and validate output/error.
219 """Utility to call 'ipython filename' and validate output/error.
220
220
221 This function raises an AssertionError if the validation fails.
221 This function raises an AssertionError if the validation fails.
222
222
223 Note that this starts IPython in a subprocess!
223 Note that this starts IPython in a subprocess!
224
224
225 Parameters
225 Parameters
226 ----------
226 ----------
227 fname : str
227 fname : str
228 Name of the file to be executed (should have .py or .ipy extension).
228 Name of the file to be executed (should have .py or .ipy extension).
229
229
230 expected_out : str
230 expected_out : str
231 Expected stdout of the process.
231 Expected stdout of the process.
232
232
233 expected_err : optional, str
233 expected_err : optional, str
234 Expected stderr of the process.
234 Expected stderr of the process.
235
235
236 options : optional, list
236 options : optional, list
237 Extra command-line flags to be passed to IPython.
237 Extra command-line flags to be passed to IPython.
238
238
239 Returns
239 Returns
240 -------
240 -------
241 None
241 None
242 """
242 """
243
243
244 import nose.tools as nt
244 import nose.tools as nt
245
245
246 out, err = ipexec(fname)
246 out, err = ipexec(fname)
247 #print 'OUT', out # dbg
247 #print 'OUT', out # dbg
248 #print 'ERR', err # dbg
248 #print 'ERR', err # dbg
249 # If there are any errors, we must check those befor stdout, as they may be
249 # If there are any errors, we must check those befor stdout, as they may be
250 # more informative than simply having an empty stdout.
250 # more informative than simply having an empty stdout.
251 if err:
251 if err:
252 if expected_err:
252 if expected_err:
253 nt.assert_equals(err.strip(), expected_err.strip())
253 nt.assert_equals(err.strip(), expected_err.strip())
254 else:
254 else:
255 raise ValueError('Running file %r produced error: %r' %
255 raise ValueError('Running file %r produced error: %r' %
256 (fname, err))
256 (fname, err))
257 # If no errors or output on stderr was expected, match stdout
257 # If no errors or output on stderr was expected, match stdout
258 nt.assert_equals(out.strip(), expected_out.strip())
258 nt.assert_equals(out.strip(), expected_out.strip())
259
259
260
260
261 class TempFileMixin(object):
261 class TempFileMixin(object):
262 """Utility class to create temporary Python/IPython files.
262 """Utility class to create temporary Python/IPython files.
263
263
264 Meant as a mixin class for test cases."""
264 Meant as a mixin class for test cases."""
265
265
266 def mktmp(self, src, ext='.py'):
266 def mktmp(self, src, ext='.py'):
267 """Make a valid python temp file."""
267 """Make a valid python temp file."""
268 fname, f = temp_pyfile(src, ext)
268 fname, f = temp_pyfile(src, ext)
269 self.tmpfile = f
269 self.tmpfile = f
270 self.fname = fname
270 self.fname = fname
271
271
272 def tearDown(self):
272 def tearDown(self):
273 if hasattr(self, 'tmpfile'):
273 if hasattr(self, 'tmpfile'):
274 # If the tmpfile wasn't made because of skipped tests, like in
274 # If the tmpfile wasn't made because of skipped tests, like in
275 # win32, there's nothing to cleanup.
275 # win32, there's nothing to cleanup.
276 self.tmpfile.close()
276 self.tmpfile.close()
277 try:
277 try:
278 os.unlink(self.fname)
278 os.unlink(self.fname)
279 except:
279 except:
280 # On Windows, even though we close the file, we still can't
280 # On Windows, even though we close the file, we still can't
281 # delete it. I have no clue why
281 # delete it. I have no clue why
282 pass
282 pass
283
283
284 pair_fail_msg = ("Testing function {0}\n\n"
284 pair_fail_msg = ("Testing function {0}\n\n"
285 "In:\n"
285 "In:\n"
286 " {1!r}\n"
286 " {1!r}\n"
287 "Expected:\n"
287 "Expected:\n"
288 " {2!r}\n"
288 " {2!r}\n"
289 "Got:\n"
289 "Got:\n"
290 " {3!r}\n")
290 " {3!r}\n")
291 def check_pairs(func, pairs):
291 def check_pairs(func, pairs):
292 """Utility function for the common case of checking a function with a
292 """Utility function for the common case of checking a function with a
293 sequence of input/output pairs.
293 sequence of input/output pairs.
294
294
295 Parameters
295 Parameters
296 ----------
296 ----------
297 func : callable
297 func : callable
298 The function to be tested. Should accept a single argument.
298 The function to be tested. Should accept a single argument.
299 pairs : iterable
299 pairs : iterable
300 A list of (input, expected_output) tuples.
300 A list of (input, expected_output) tuples.
301
301
302 Returns
302 Returns
303 -------
303 -------
304 None. Raises an AssertionError if any output does not match the expected
304 None. Raises an AssertionError if any output does not match the expected
305 value.
305 value.
306 """
306 """
307 for inp, expected in pairs:
307 for inp, expected in pairs:
308 out = func(inp)
308 out = func(inp)
309 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
309 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
@@ -1,159 +1,158 b''
1 """ Defines helper functions for creating kernel entry points and process
1 """ Defines helper functions for creating kernel entry points and process
2 launchers.
2 launchers.
3 """
3 """
4
4
5 # Standard library imports.
5 # Standard library imports.
6 import atexit
6 import atexit
7 import os
7 import os
8 import socket
8 import socket
9 from subprocess import Popen, PIPE
9 from subprocess import Popen, PIPE
10 import sys
10 import sys
11
11
12 # Local imports.
12 # Local imports.
13 from parentpoller import ParentPollerWindows
13 from parentpoller import ParentPollerWindows
14
14
15
15
16 def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
16 def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
17 ip=None, stdin=None, stdout=None, stderr=None,
17 ip=None, stdin=None, stdout=None, stderr=None,
18 executable=None, independent=False, extra_arguments=[]):
18 executable=None, independent=False, extra_arguments=[]):
19 """ Launches a localhost kernel, binding to the specified ports.
19 """ Launches a localhost kernel, binding to the specified ports.
20
20
21 Parameters
21 Parameters
22 ----------
22 ----------
23 code : str,
23 code : str,
24 A string of Python code that imports and executes a kernel entry point.
24 A string of Python code that imports and executes a kernel entry point.
25
25
26 shell_port : int, optional
26 shell_port : int, optional
27 The port to use for XREP channel.
27 The port to use for XREP channel.
28
28
29 iopub_port : int, optional
29 iopub_port : int, optional
30 The port to use for the SUB channel.
30 The port to use for the SUB channel.
31
31
32 stdin_port : int, optional
32 stdin_port : int, optional
33 The port to use for the REQ (raw input) channel.
33 The port to use for the REQ (raw input) channel.
34
34
35 hb_port : int, optional
35 hb_port : int, optional
36 The port to use for the hearbeat REP channel.
36 The port to use for the hearbeat REP channel.
37
37
38 ip : str, optional
38 ip : str, optional
39 The ip address the kernel will bind to.
39 The ip address the kernel will bind to.
40
40
41 stdin, stdout, stderr : optional (default None)
41 stdin, stdout, stderr : optional (default None)
42 Standards streams, as defined in subprocess.Popen.
42 Standards streams, as defined in subprocess.Popen.
43
43
44 executable : str, optional (default sys.executable)
44 executable : str, optional (default sys.executable)
45 The Python executable to use for the kernel process.
45 The Python executable to use for the kernel process.
46
46
47 independent : bool, optional (default False)
47 independent : bool, optional (default False)
48 If set, the kernel process is guaranteed to survive if this process
48 If set, the kernel process is guaranteed to survive if this process
49 dies. If not set, an effort is made to ensure that the kernel is killed
49 dies. If not set, an effort is made to ensure that the kernel is killed
50 when this process dies. Note that in this case it is still good practice
50 when this process dies. Note that in this case it is still good practice
51 to kill kernels manually before exiting.
51 to kill kernels manually before exiting.
52
52
53 extra_arguments = list, optional
53 extra_arguments = list, optional
54 A list of extra arguments to pass when executing the launch code.
54 A list of extra arguments to pass when executing the launch code.
55
55
56 Returns
56 Returns
57 -------
57 -------
58 A tuple of form:
58 A tuple of form:
59 (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
59 (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
60 where kernel_process is a Popen object and the ports are integers.
60 where kernel_process is a Popen object and the ports are integers.
61 """
61 """
62 # Find open ports as necessary.
62 # Find open ports as necessary.
63 ports = []
63 ports = []
64 ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
64 ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
65 int(stdin_port <= 0) + int(hb_port <= 0)
65 int(stdin_port <= 0) + int(hb_port <= 0)
66 for i in xrange(ports_needed):
66 for i in xrange(ports_needed):
67 sock = socket.socket()
67 sock = socket.socket()
68 sock.bind(('', 0))
68 sock.bind(('', 0))
69 ports.append(sock)
69 ports.append(sock)
70 for i, sock in enumerate(ports):
70 for i, sock in enumerate(ports):
71 port = sock.getsockname()[1]
71 port = sock.getsockname()[1]
72 sock.close()
72 sock.close()
73 ports[i] = port
73 ports[i] = port
74 if shell_port <= 0:
74 if shell_port <= 0:
75 shell_port = ports.pop(0)
75 shell_port = ports.pop(0)
76 if iopub_port <= 0:
76 if iopub_port <= 0:
77 iopub_port = ports.pop(0)
77 iopub_port = ports.pop(0)
78 if stdin_port <= 0:
78 if stdin_port <= 0:
79 stdin_port = ports.pop(0)
79 stdin_port = ports.pop(0)
80 if hb_port <= 0:
80 if hb_port <= 0:
81 hb_port = ports.pop(0)
81 hb_port = ports.pop(0)
82
82
83 # Build the kernel launch command.
83 # Build the kernel launch command.
84 if executable is None:
84 if executable is None:
85 executable = sys.executable
85 executable = sys.executable
86 arguments = [ executable, '-c', code, 'shell=%i'%shell_port,
86 arguments = [ executable, '-c', code, '--shell=%i'%shell_port,
87 'iopub=%i'%iopub_port, 'stdin=%i'%stdin_port,
87 '--iopub=%i'%iopub_port, '--stdin=%i'%stdin_port,
88 'hb=%i'%hb_port
88 '--hb=%i'%hb_port
89 ]
89 ]
90 if ip is not None:
90 if ip is not None:
91 arguments.append('ip=%s'%ip)
91 arguments.append('--ip=%s'%ip)
92 arguments.extend(extra_arguments)
92 arguments.extend(extra_arguments)
93
93
94 # Spawn a kernel.
94 # Spawn a kernel.
95 if sys.platform == 'win32':
95 if sys.platform == 'win32':
96 # Create a Win32 event for interrupting the kernel.
96 # Create a Win32 event for interrupting the kernel.
97 interrupt_event = ParentPollerWindows.create_interrupt_event()
97 interrupt_event = ParentPollerWindows.create_interrupt_event()
98 arguments += [ 'interrupt=%i'%interrupt_event ]
98 arguments += [ '--interrupt=%i'%interrupt_event ]
99
99
100 # If this process in running on pythonw, stdin, stdout, and stderr are
100 # If this process in running on pythonw, stdin, stdout, and stderr are
101 # invalid. Popen will fail unless they are suitably redirected. We don't
101 # invalid. Popen will fail unless they are suitably redirected. We don't
102 # read from the pipes, but they must exist.
102 # read from the pipes, but they must exist.
103 if sys.executable.endswith('pythonw.exe'):
103 if sys.executable.endswith('pythonw.exe'):
104 redirect = True
104 redirect = True
105 _stdin = PIPE if stdin is None else stdin
105 _stdin = PIPE if stdin is None else stdin
106 _stdout = PIPE if stdout is None else stdout
106 _stdout = PIPE if stdout is None else stdout
107 _stderr = PIPE if stderr is None else stderr
107 _stderr = PIPE if stderr is None else stderr
108 else:
108 else:
109 redirect = False
109 redirect = False
110 _stdin, _stdout, _stderr = stdin, stdout, stderr
110 _stdin, _stdout, _stderr = stdin, stdout, stderr
111
111
112 # If the kernel is running on pythonw and stdout/stderr are not been
112 # If the kernel is running on pythonw and stdout/stderr are not been
113 # re-directed, it will crash when more than 4KB of data is written to
113 # re-directed, it will crash when more than 4KB of data is written to
114 # stdout or stderr. This is a bug that has been with Python for a very
114 # stdout or stderr. This is a bug that has been with Python for a very
115 # long time; see http://bugs.python.org/issue706263.
115 # long time; see http://bugs.python.org/issue706263.
116 # A cleaner solution to this problem would be to pass os.devnull to
116 # A cleaner solution to this problem would be to pass os.devnull to
117 # Popen directly. Unfortunately, that does not work.
117 # Popen directly. Unfortunately, that does not work.
118 if executable.endswith('pythonw.exe'):
118 if executable.endswith('pythonw.exe'):
119 if stdout is None:
119 if stdout is None:
120 arguments.append('--no-stdout')
120 arguments.append('--no-stdout')
121 if stderr is None:
121 if stderr is None:
122 arguments.append('--no-stderr')
122 arguments.append('--no-stderr')
123
123
124 # Launch the kernel process.
124 # Launch the kernel process.
125 if independent:
125 if independent:
126 proc = Popen(arguments,
126 proc = Popen(arguments,
127 creationflags=512, # CREATE_NEW_PROCESS_GROUP
127 creationflags=512, # CREATE_NEW_PROCESS_GROUP
128 stdin=_stdin, stdout=_stdout, stderr=_stderr)
128 stdin=_stdin, stdout=_stdout, stderr=_stderr)
129 else:
129 else:
130 from _subprocess import DuplicateHandle, GetCurrentProcess, \
130 from _subprocess import DuplicateHandle, GetCurrentProcess, \
131 DUPLICATE_SAME_ACCESS
131 DUPLICATE_SAME_ACCESS
132 pid = GetCurrentProcess()
132 pid = GetCurrentProcess()
133 handle = DuplicateHandle(pid, pid, pid, 0,
133 handle = DuplicateHandle(pid, pid, pid, 0,
134 True, # Inheritable by new processes.
134 True, # Inheritable by new processes.
135 DUPLICATE_SAME_ACCESS)
135 DUPLICATE_SAME_ACCESS)
136 proc = Popen(arguments + ['parent=%i'%int(handle)],
136 proc = Popen(arguments + ['--parent=%i'%int(handle)],
137 stdin=_stdin, stdout=_stdout, stderr=_stderr)
137 stdin=_stdin, stdout=_stdout, stderr=_stderr)
138
138
139 # Attach the interrupt event to the Popen objet so it can be used later.
139 # Attach the interrupt event to the Popen objet so it can be used later.
140 proc.win32_interrupt_event = interrupt_event
140 proc.win32_interrupt_event = interrupt_event
141
141
142 # Clean up pipes created to work around Popen bug.
142 # Clean up pipes created to work around Popen bug.
143 if redirect:
143 if redirect:
144 if stdin is None:
144 if stdin is None:
145 proc.stdin.close()
145 proc.stdin.close()
146 if stdout is None:
146 if stdout is None:
147 proc.stdout.close()
147 proc.stdout.close()
148 if stderr is None:
148 if stderr is None:
149 proc.stderr.close()
149 proc.stderr.close()
150
150
151 else:
151 else:
152 if independent:
152 if independent:
153 proc = Popen(arguments, preexec_fn=lambda: os.setsid(),
153 proc = Popen(arguments, preexec_fn=lambda: os.setsid(),
154 stdin=stdin, stdout=stdout, stderr=stderr)
154 stdin=stdin, stdout=stdout, stderr=stderr)
155 else:
155 else:
156 proc = Popen(arguments + ['parent=1'],
156 proc = Popen(arguments + ['--parent=1'],
157 stdin=stdin, stdout=stdout, stderr=stderr)
157 stdin=stdin, stdout=stdout, stderr=stderr)
158
159 return proc, shell_port, iopub_port, stdin_port, hb_port
158 return proc, shell_port, iopub_port, stdin_port, hb_port
@@ -1,227 +1,227 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """An Application for launching a kernel
2 """An Application for launching a kernel
3
3
4 Authors
4 Authors
5 -------
5 -------
6 * MinRK
6 * MinRK
7 """
7 """
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2011 The IPython Development Team
9 # Copyright (C) 2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING.txt, distributed as part of this software.
12 # the file COPYING.txt, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 # Standard library imports.
19 # Standard library imports.
20 import os
20 import os
21 import sys
21 import sys
22
22
23 # System library imports.
23 # System library imports.
24 import zmq
24 import zmq
25
25
26 # IPython imports.
26 # IPython imports.
27 from IPython.core.ultratb import FormattedTB
27 from IPython.core.ultratb import FormattedTB
28 from IPython.core.application import (
28 from IPython.core.application import (
29 BaseIPythonApplication, base_flags, base_aliases
29 BaseIPythonApplication, base_flags, base_aliases
30 )
30 )
31 from IPython.utils import io
31 from IPython.utils import io
32 from IPython.utils.localinterfaces import LOCALHOST
32 from IPython.utils.localinterfaces import LOCALHOST
33 from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
33 from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
34 DottedObjectName)
34 DottedObjectName)
35 from IPython.utils.importstring import import_item
35 from IPython.utils.importstring import import_item
36 # local imports
36 # local imports
37 from IPython.zmq.heartbeat import Heartbeat
37 from IPython.zmq.heartbeat import Heartbeat
38 from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
38 from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
39 from IPython.zmq.session import Session
39 from IPython.zmq.session import Session
40
40
41
41
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43 # Flags and Aliases
43 # Flags and Aliases
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45
45
46 kernel_aliases = dict(base_aliases)
46 kernel_aliases = dict(base_aliases)
47 kernel_aliases.update({
47 kernel_aliases.update({
48 'ip' : 'KernelApp.ip',
48 'ip' : 'KernelApp.ip',
49 'hb' : 'KernelApp.hb_port',
49 'hb' : 'KernelApp.hb_port',
50 'shell' : 'KernelApp.shell_port',
50 'shell' : 'KernelApp.shell_port',
51 'iopub' : 'KernelApp.iopub_port',
51 'iopub' : 'KernelApp.iopub_port',
52 'stdin' : 'KernelApp.stdin_port',
52 'stdin' : 'KernelApp.stdin_port',
53 'parent': 'KernelApp.parent',
53 'parent': 'KernelApp.parent',
54 })
54 })
55 if sys.platform.startswith('win'):
55 if sys.platform.startswith('win'):
56 kernel_aliases['interrupt'] = 'KernelApp.interrupt'
56 kernel_aliases['interrupt'] = 'KernelApp.interrupt'
57
57
58 kernel_flags = dict(base_flags)
58 kernel_flags = dict(base_flags)
59 kernel_flags.update({
59 kernel_flags.update({
60 'no-stdout' : (
60 'no-stdout' : (
61 {'KernelApp' : {'no_stdout' : True}},
61 {'KernelApp' : {'no_stdout' : True}},
62 "redirect stdout to the null device"),
62 "redirect stdout to the null device"),
63 'no-stderr' : (
63 'no-stderr' : (
64 {'KernelApp' : {'no_stderr' : True}},
64 {'KernelApp' : {'no_stderr' : True}},
65 "redirect stderr to the null device"),
65 "redirect stderr to the null device"),
66 })
66 })
67
67
68
68
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70 # Application class for starting a Kernel
70 # Application class for starting a Kernel
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72
72
73 class KernelApp(BaseIPythonApplication):
73 class KernelApp(BaseIPythonApplication):
74 name='pykernel'
74 name='pykernel'
75 aliases = Dict(kernel_aliases)
75 aliases = Dict(kernel_aliases)
76 flags = Dict(kernel_flags)
76 flags = Dict(kernel_flags)
77 classes = [Session]
77 classes = [Session]
78 # the kernel class, as an importstring
78 # the kernel class, as an importstring
79 kernel_class = DottedObjectName('IPython.zmq.pykernel.Kernel')
79 kernel_class = DottedObjectName('IPython.zmq.pykernel.Kernel')
80 kernel = Any()
80 kernel = Any()
81 poller = Any() # don't restrict this even though current pollers are all Threads
81 poller = Any() # don't restrict this even though current pollers are all Threads
82 heartbeat = Instance(Heartbeat)
82 heartbeat = Instance(Heartbeat)
83 session = Instance('IPython.zmq.session.Session')
83 session = Instance('IPython.zmq.session.Session')
84 ports = Dict()
84 ports = Dict()
85
85
86 # inherit config file name from parent:
86 # inherit config file name from parent:
87 parent_appname = Unicode(config=True)
87 parent_appname = Unicode(config=True)
88 def _parent_appname_changed(self, name, old, new):
88 def _parent_appname_changed(self, name, old, new):
89 if self.config_file_specified:
89 if self.config_file_specified:
90 # it was manually specified, ignore
90 # it was manually specified, ignore
91 return
91 return
92 self.config_file_name = new.replace('-','_') + u'_config.py'
92 self.config_file_name = new.replace('-','_') + u'_config.py'
93 # don't let this count as specifying the config file
93 # don't let this count as specifying the config file
94 self.config_file_specified = False
94 self.config_file_specified = False
95
95
96 # connection info:
96 # connection info:
97 ip = Unicode(LOCALHOST, config=True,
97 ip = Unicode(LOCALHOST, config=True,
98 help="Set the IP or interface on which the kernel will listen.")
98 help="Set the IP or interface on which the kernel will listen.")
99 hb_port = Int(0, config=True, help="set the heartbeat port [default: random]")
99 hb_port = Int(0, config=True, help="set the heartbeat port [default: random]")
100 shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
100 shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
101 iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
101 iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
102 stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
102 stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
103
103
104 # streams, etc.
104 # streams, etc.
105 no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
105 no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
106 no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
106 no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
107 outstream_class = DottedObjectName('IPython.zmq.iostream.OutStream',
107 outstream_class = DottedObjectName('IPython.zmq.iostream.OutStream',
108 config=True, help="The importstring for the OutStream factory")
108 config=True, help="The importstring for the OutStream factory")
109 displayhook_class = DottedObjectName('IPython.zmq.displayhook.ZMQDisplayHook',
109 displayhook_class = DottedObjectName('IPython.zmq.displayhook.ZMQDisplayHook',
110 config=True, help="The importstring for the DisplayHook factory")
110 config=True, help="The importstring for the DisplayHook factory")
111
111
112 # polling
112 # polling
113 parent = Int(0, config=True,
113 parent = Int(0, config=True,
114 help="""kill this process if its parent dies. On Windows, the argument
114 help="""kill this process if its parent dies. On Windows, the argument
115 specifies the HANDLE of the parent process, otherwise it is simply boolean.
115 specifies the HANDLE of the parent process, otherwise it is simply boolean.
116 """)
116 """)
117 interrupt = Int(0, config=True,
117 interrupt = Int(0, config=True,
118 help="""ONLY USED ON WINDOWS
118 help="""ONLY USED ON WINDOWS
119 Interrupt this process when the parent is signalled.
119 Interrupt this process when the parent is signalled.
120 """)
120 """)
121
121
122 def init_crash_handler(self):
122 def init_crash_handler(self):
123 # Install minimal exception handling
123 # Install minimal exception handling
124 sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
124 sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
125 ostream=sys.__stdout__)
125 ostream=sys.__stdout__)
126
126
127 def init_poller(self):
127 def init_poller(self):
128 if sys.platform == 'win32':
128 if sys.platform == 'win32':
129 if self.interrupt or self.parent:
129 if self.interrupt or self.parent:
130 self.poller = ParentPollerWindows(self.interrupt, self.parent)
130 self.poller = ParentPollerWindows(self.interrupt, self.parent)
131 elif self.parent:
131 elif self.parent:
132 self.poller = ParentPollerUnix()
132 self.poller = ParentPollerUnix()
133
133
134 def _bind_socket(self, s, port):
134 def _bind_socket(self, s, port):
135 iface = 'tcp://%s' % self.ip
135 iface = 'tcp://%s' % self.ip
136 if port <= 0:
136 if port <= 0:
137 port = s.bind_to_random_port(iface)
137 port = s.bind_to_random_port(iface)
138 else:
138 else:
139 s.bind(iface + ':%i'%port)
139 s.bind(iface + ':%i'%port)
140 return port
140 return port
141
141
142 def init_sockets(self):
142 def init_sockets(self):
143 # Create a context, a session, and the kernel sockets.
143 # Create a context, a session, and the kernel sockets.
144 self.log.info("Starting the kernel at pid:", os.getpid())
144 self.log.info("Starting the kernel at pid:", os.getpid())
145 context = zmq.Context.instance()
145 context = zmq.Context.instance()
146 # Uncomment this to try closing the context.
146 # Uncomment this to try closing the context.
147 # atexit.register(context.term)
147 # atexit.register(context.term)
148
148
149 self.shell_socket = context.socket(zmq.XREP)
149 self.shell_socket = context.socket(zmq.XREP)
150 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
150 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
151 self.log.debug("shell XREP Channel on port: %i"%self.shell_port)
151 self.log.debug("shell XREP Channel on port: %i"%self.shell_port)
152
152
153 self.iopub_socket = context.socket(zmq.PUB)
153 self.iopub_socket = context.socket(zmq.PUB)
154 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
154 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
155 self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
155 self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
156
156
157 self.stdin_socket = context.socket(zmq.XREQ)
157 self.stdin_socket = context.socket(zmq.XREQ)
158 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
158 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
159 self.log.debug("stdin XREQ Channel on port: %i"%self.stdin_port)
159 self.log.debug("stdin XREQ Channel on port: %i"%self.stdin_port)
160
160
161 self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
161 self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
162 self.hb_port = self.heartbeat.port
162 self.hb_port = self.heartbeat.port
163 self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
163 self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
164
164
165 # Helper to make it easier to connect to an existing kernel, until we have
165 # Helper to make it easier to connect to an existing kernel, until we have
166 # single-port connection negotiation fully implemented.
166 # single-port connection negotiation fully implemented.
167 # set log-level to critical, to make sure it is output
167 # set log-level to critical, to make sure it is output
168 self.log.critical("To connect another client to this kernel, use:")
168 self.log.critical("To connect another client to this kernel, use:")
169 self.log.critical("--existing shell={0} iopub={1} stdin={2} hb={3}".format(
169 self.log.critical("--existing --shell={0} --iopub={1} --stdin={2} --hb={3}".format(
170 self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
170 self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
171
171
172
172
173 self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
173 self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
174 stdin=self.stdin_port, hb=self.hb_port)
174 stdin=self.stdin_port, hb=self.hb_port)
175
175
176 def init_session(self):
176 def init_session(self):
177 """create our session object"""
177 """create our session object"""
178 self.session = Session(config=self.config, username=u'kernel')
178 self.session = Session(config=self.config, username=u'kernel')
179
179
180 def init_blackhole(self):
180 def init_blackhole(self):
181 """redirects stdout/stderr to devnull if necessary"""
181 """redirects stdout/stderr to devnull if necessary"""
182 if self.no_stdout or self.no_stderr:
182 if self.no_stdout or self.no_stderr:
183 blackhole = file(os.devnull, 'w')
183 blackhole = file(os.devnull, 'w')
184 if self.no_stdout:
184 if self.no_stdout:
185 sys.stdout = sys.__stdout__ = blackhole
185 sys.stdout = sys.__stdout__ = blackhole
186 if self.no_stderr:
186 if self.no_stderr:
187 sys.stderr = sys.__stderr__ = blackhole
187 sys.stderr = sys.__stderr__ = blackhole
188
188
189 def init_io(self):
189 def init_io(self):
190 """Redirect input streams and set a display hook."""
190 """Redirect input streams and set a display hook."""
191 if self.outstream_class:
191 if self.outstream_class:
192 outstream_factory = import_item(str(self.outstream_class))
192 outstream_factory = import_item(str(self.outstream_class))
193 sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
193 sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
194 sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
194 sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
195 if self.displayhook_class:
195 if self.displayhook_class:
196 displayhook_factory = import_item(str(self.displayhook_class))
196 displayhook_factory = import_item(str(self.displayhook_class))
197 sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
197 sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
198
198
199 def init_kernel(self):
199 def init_kernel(self):
200 """Create the Kernel object itself"""
200 """Create the Kernel object itself"""
201 kernel_factory = import_item(str(self.kernel_class))
201 kernel_factory = import_item(str(self.kernel_class))
202 self.kernel = kernel_factory(config=self.config, session=self.session,
202 self.kernel = kernel_factory(config=self.config, session=self.session,
203 shell_socket=self.shell_socket,
203 shell_socket=self.shell_socket,
204 iopub_socket=self.iopub_socket,
204 iopub_socket=self.iopub_socket,
205 stdin_socket=self.stdin_socket,
205 stdin_socket=self.stdin_socket,
206 log=self.log
206 log=self.log
207 )
207 )
208 self.kernel.record_ports(self.ports)
208 self.kernel.record_ports(self.ports)
209
209
210 def initialize(self, argv=None):
210 def initialize(self, argv=None):
211 super(KernelApp, self).initialize(argv)
211 super(KernelApp, self).initialize(argv)
212 self.init_blackhole()
212 self.init_blackhole()
213 self.init_session()
213 self.init_session()
214 self.init_poller()
214 self.init_poller()
215 self.init_sockets()
215 self.init_sockets()
216 self.init_io()
216 self.init_io()
217 self.init_kernel()
217 self.init_kernel()
218
218
219 def start(self):
219 def start(self):
220 self.heartbeat.start()
220 self.heartbeat.start()
221 if self.poller is not None:
221 if self.poller is not None:
222 self.poller.start()
222 self.poller.start()
223 try:
223 try:
224 self.kernel.start()
224 self.kernel.start()
225 except KeyboardInterrupt:
225 except KeyboardInterrupt:
226 pass
226 pass
227
227
General Comments 0
You need to be logged in to leave comments. Login now