From 84ec5449b15dcebeae089dd5de304d181ba5d01f 2010-01-30 21:57:52 From: Brian Granger Date: 2010-01-30 21:57:52 Subject: [PATCH] Refactored the command line config system and other aspects of config. * Removed ``NoConfigDefault`` from our config logic entirely. It turns out that argparse has a nice ``argparse.SUPPRESS`` object that handles this nicely (r1283, r1284). * Completely cleared the previous config when :meth:`load_config` is called to allow it to be called more than once. * Added some additional tests and docs for the config system. * Removed the ``override_config`` and ``constructor_config`` config stages from :class:`~IPython.core.application.Applications` (r1293). * Completely refactored the command line config parser classes. We are now using a more object oriented design that allows for fine grained inheritance and good encapsulation. --- diff --git a/IPython/config/loader.py b/IPython/config/loader.py index 98e13e7..df79679 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -299,7 +299,7 @@ class CommandLineConfigLoader(ConfigLoader): class ArgParseConfigLoader(CommandLineConfigLoader): - def __init__(self, argv=None, arguments=(), *parser_args, **parser_kw): + def __init__(self, argv=None, *parser_args, **parser_kw): """Create a config loader for use with argparse. Parameters @@ -309,11 +309,6 @@ class ArgParseConfigLoader(CommandLineConfigLoader): If given, used to read command-line arguments from, otherwise sys.argv[1:] is used. - arguments : optional, tuple - A tuple of two element tuples each having the form (args, kwargs). - Each such pair is passed to parser.add_argument(*args, **kwargs) - in sequence to configure the parser. - parser_args : tuple A tuple of positional arguments that will be passed to the constructor of :class:`argparse.ArgumentParser`. @@ -326,7 +321,6 @@ class ArgParseConfigLoader(CommandLineConfigLoader): if argv == None: argv = sys.argv[1:] self.argv = argv - self.arguments = arguments self.parser_args = parser_args kwargs = dict(argument_default=argparse.SUPPRESS) kwargs.update(parser_kw) @@ -359,19 +353,9 @@ class ArgParseConfigLoader(CommandLineConfigLoader): def _create_parser(self): self.parser = ArgumentParser(*self.parser_args, **self.parser_kw) self._add_arguments() - self._add_other_arguments() def _add_arguments(self): - for argument in self.arguments: - # Remove any defaults in case people add them. We can't have - # command line default because all default are determined by - # traited class attributes. - argument[1].pop('default', None) - self.parser.add_argument(*argument[0], **argument[1]) - - def _add_other_arguments(self): - """Meant for subclasses to add their own arguments.""" - pass + raise NotImplementedError("subclasses must implement _add_arguments") def _parse_args(self, args): """self.parser->self.parsed_data""" diff --git a/IPython/config/tests/test_loader.py b/IPython/config/tests/test_loader.py index 1276eb5..390f1b4 100755 --- a/IPython/config/tests/test_loader.py +++ b/IPython/config/tests/test_loader.py @@ -61,18 +61,26 @@ class TestPyFileCL(TestCase): self.assertEquals(config.Foo.Bam.value, range(10)) self.assertEquals(config.D.C.value, 'hi there') - -arguments = ( - (('-f','--foo'), dict(dest='Global.foo', type=str)), - (('-b',), dict(dest='MyClass.bar', type=int)), - (('-n',), dict(dest='n', action='store_true')), - (('Global.bam',), dict(type=str)) - ) +class MyLoader1(ArgParseConfigLoader): + def _add_arguments(self): + p = self.parser + p.add_argument('-f', '--foo', dest='Global.foo', type=str) + p.add_argument('-b', dest='MyClass.bar', type=int) + p.add_argument('-n', dest='n', action='store_true') + p.add_argument('Global.bam', type=str) + +class MyLoader2(ArgParseConfigLoader): + def _add_arguments(self): + subparsers = self.parser.add_subparsers(dest='subparser_name') + subparser1 = subparsers.add_parser('1') + subparser1.add_argument('-x',dest='Global.x') + subparser2 = subparsers.add_parser('2') + subparser2.add_argument('y') class TestArgParseCL(TestCase): def test_basic(self): - cl = ArgParseConfigLoader(arguments=arguments) + cl = MyLoader1() config = cl.load_config('-f hi -b 10 -n wow'.split()) self.assertEquals(config.Global.foo, 'hi') self.assertEquals(config.MyClass.bar, 10) @@ -84,16 +92,7 @@ class TestArgParseCL(TestCase): self.assertEquals(config.Global.bam, 'wow') def test_add_arguments(self): - - class MyLoader(ArgParseConfigLoader): - def _add_arguments(self): - subparsers = self.parser.add_subparsers(dest='subparser_name') - subparser1 = subparsers.add_parser('1') - subparser1.add_argument('-x',dest='Global.x') - subparser2 = subparsers.add_parser('2') - subparser2.add_argument('y') - - cl = MyLoader() + cl = MyLoader2() config = cl.load_config('2 frobble'.split()) self.assertEquals(config.subparser_name, '2') self.assertEquals(config.y, 'frobble') @@ -102,10 +101,7 @@ class TestArgParseCL(TestCase): self.assertEquals(config.Global.x, 'frobble') def test_argv(self): - cl = ArgParseConfigLoader( - argv='-f hi -b 10 -n wow'.split(), - arguments=arguments - ) + cl = MyLoader1(argv='-f hi -b 10 -n wow'.split()) config = cl.load_config() self.assertEquals(config.Global.foo, 'hi') self.assertEquals(config.MyClass.bar, 10) diff --git a/IPython/core/application.py b/IPython/core/application.py index cc0c08c..1e4940f 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -48,94 +48,75 @@ class ApplicationError(Exception): pass -app_cl_args = ( - (('--ipython-dir', ), dict( - dest='Global.ipython_dir',type=unicode, - help= - """Set to override default location of the IPython directory - IPYTHON_DIR, stored as Global.ipython_dir. This can also be specified - through the environment variable IPYTHON_DIR.""", - metavar='Global.ipython_dir') ), - (('-p', '--profile',), dict( - dest='Global.profile',type=unicode, - help= - """The string name of the ipython profile to be used. Assume that your - config file is ipython_config-.py (looks in current dir first, - then in IPYTHON_DIR). This is a quick way to keep and load multiple - config files for different tasks, especially if include your basic one - in your more specialized ones. You can keep a basic - IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which - include this one and load extra things for particular tasks.""", - metavar='Global.profile') ), - (('--log-level',), dict( - dest="Global.log_level",type=int, - help='Set the log level (0,10,20,30,40,50). Default is 30.', - metavar='Global.log_level')), - (('--config-file',), dict( - dest='Global.config_file',type=unicode, - help= - """Set the config file name to override default. Normally IPython - loads ipython_config.py (from current directory) or - IPYTHON_DIR/ipython_config.py. If the loading of your config file - fails, IPython starts with a bare bones configuration (no modules - loaded at all).""", - metavar='Global.config_file')), - ) +class BaseAppConfigLoader(ArgParseConfigLoader): + """Default command line options for IPython based applications.""" + + def _add_ipython_dir(self, parser): + """Add the --ipython-dir option to the parser.""" + paa = parser.add_argument + paa('--ipython-dir', + dest='Global.ipython_dir',type=unicode, + help= + """Set to override default location of the IPython directory + IPYTHON_DIR, stored as Global.ipython_dir. This can also be + specified through the environment variable IPYTHON_DIR.""", + metavar='Global.ipython_dir') + + def _add_log_level(self, parser): + """Add the --log-level option to the parser.""" + paa = parser.add_argument + paa('--log-level', + dest="Global.log_level",type=int, + help='Set the log level (0,10,20,30,40,50). Default is 30.', + metavar='Global.log_level') + + def _add_arguments(self): + self._add_ipython_dir(self.parser) + self._add_log_level(self.parser) + class Application(object): """Load a config, construct components and set them running. - The configuration of an application can be done via four different Config - objects, which are loaded and ultimately merged into a single one used from - that point on by the app. These are: + The configuration of an application can be done via three different Config + objects, which are loaded and ultimately merged into a single one used + from that point on by the app. These are: 1. default_config: internal defaults, implemented in code. 2. file_config: read from the filesystem. 3. command_line_config: read from the system's command line flags. - 4. constructor_config: passed parametrically to the constructor. During initialization, 3 is actually read before 2, since at the command-line one may override the location of the file to be read. But the above is the order in which the merge is made. - - There is a final config object can be created and passed to the - constructor: override_config. If it exists, this completely overrides the - configs 2-4 above (the default is still used to ensure that all needed - fields at least are created). This makes it easier to create - parametrically (e.g. in testing or sphinx plugins) objects with a known - configuration, that are unaffected by whatever arguments may be present in - sys.argv or files in the user's various directories. """ name = u'ipython' description = 'IPython: an enhanced interactive Python shell.' - #: usage message printed by argparse. If None, auto-generate + #: Usage message printed by argparse. If None, auto-generate usage = None + #: The command line config loader. Subclass of ArgParseConfigLoader. + command_line_loader = BaseAppConfigLoader + #: The name of the config file to load. config_file_name = u'ipython_config.py' - #: Track the default and actual separately because some messages are - #: only printed if we aren't using the default. + #: The name of the default config file. Track separately from the actual + #: name because some logic happens only if we aren't using the default. default_config_file_name = config_file_name default_log_level = logging.WARN #: Set by --profile option profile_name = None #: User's ipython directory, typically ~/.ipython/ ipython_dir = None - #: internal defaults, implemented in code. + #: Internal defaults, implemented in code. default_config = None - #: read from the filesystem + #: Read from the filesystem. file_config = None - #: read from the system's command line flags + #: Read from the system's command line flags. command_line_config = None - #: passed parametrically to the constructor. - constructor_config = None - #: final override, if given supercedes file/command/constructor configs - override_config = None + #: The final config that will be passed to the component. + master_config = None #: A reference to the argv to be used (typically ends up being sys.argv[1:]) argv = None - #: Default command line arguments. Subclasses should create a new tuple - #: that *includes* these. - cl_arguments = app_cl_args - #: extra arguments computed by the command-line loader extra_args = None @@ -146,10 +127,8 @@ class Application(object): # Class choices for things that will be instantiated at runtime. _CrashHandler = crashhandler.CrashHandler - def __init__(self, argv=None, constructor_config=None, override_config=None): + def __init__(self, argv=None): self.argv = sys.argv[1:] if argv is None else argv - self.constructor_config = constructor_config - self.override_config = override_config self.init_logger() def init_logger(self): @@ -194,13 +173,12 @@ class Application(object): self.log_default_config() self.set_default_config_log_level() - if self.override_config is None: - # Command-line config - self.pre_load_command_line_config() - self.load_command_line_config() - self.set_command_line_config_log_level() - self.post_load_command_line_config() - self.log_command_line_config() + # Command-line config + self.pre_load_command_line_config() + self.load_command_line_config() + self.set_command_line_config_log_level() + self.post_load_command_line_config() + self.log_command_line_config() # Find resources needed for filesystem access, using information from # the above two @@ -209,13 +187,12 @@ class Application(object): self.find_config_file_name() self.find_config_file_paths() - if self.override_config is None: - # File-based config - self.pre_load_file_config() - self.load_file_config() - self.set_file_config_log_level() - self.post_load_file_config() - self.log_file_config() + # File-based config + self.pre_load_file_config() + self.load_file_config() + self.set_file_config_log_level() + self.post_load_file_config() + self.log_file_config() # Merge all config objects into a single one the app can then use self.merge_configs() @@ -270,11 +247,12 @@ class Application(object): def create_command_line_config(self): """Create and return a command line config loader.""" - return ArgParseConfigLoader(self.argv, self.cl_arguments, - description=self.description, - version=release.version, - usage=self.usage, - ) + return self.command_line_loader( + self.argv, + description=self.description, + version=release.version, + usage=self.usage + ) def pre_load_command_line_config(self): """Do actions just before loading the command line config.""" @@ -418,13 +396,9 @@ class Application(object): """Merge the default, command line and file config objects.""" config = Config() config._merge(self.default_config) - if self.override_config is None: - config._merge(self.file_config) - config._merge(self.command_line_config) - if self.constructor_config is not None: - config._merge(self.constructor_config) - else: - config._merge(self.override_config) + config._merge(self.file_config) + config._merge(self.command_line_config) + # XXX fperez - propose to Brian we rename master_config to simply # config, I think this is going to be heavily used in examples and # application code and the name is shorter/easier to find/remember. diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index df15f10..d0fc98f 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -221,3 +221,4 @@ class IPythonCrashHandler(CrashHandler): pass return ''.join(report) + diff --git a/IPython/core/ipapp.py b/IPython/core/ipapp.py index 01cd58f..c30df9b 100755 --- a/IPython/core/ipapp.py +++ b/IPython/core/ipapp.py @@ -28,7 +28,7 @@ import os import sys from IPython.core import crashhandler -from IPython.core.application import Application +from IPython.core.application import Application, BaseAppConfigLoader from IPython.core.iplib import InteractiveShell from IPython.config.loader import ( Config, @@ -42,305 +42,279 @@ from . import usage # Globals, utilities and helpers #----------------------------------------------------------------------------- +#: The default config file name for this application. default_config_file_name = u'ipython_config.py' -cl_args = ( - (('--autocall',), dict( - type=int, dest='InteractiveShell.autocall', - help= - """Make IPython automatically call any callable object even if you - didn't type explicit parentheses. For example, 'str 43' becomes - 'str(43)' automatically. The value can be '0' to disable the feature, - '1' for 'smart' autocall, where it is not applied if there are no more - arguments on the line, and '2' for 'full' autocall, where all callable - objects are automatically called (even if no arguments are present). - The default is '1'.""", - metavar='InteractiveShell.autocall') - ), - (('--autoindent',), dict( - action='store_true', dest='InteractiveShell.autoindent', - help='Turn on autoindenting.') - ), - (('--no-autoindent',), dict( - action='store_false', dest='InteractiveShell.autoindent', - help='Turn off autoindenting.') - ), - (('--automagic',), dict( - action='store_true', dest='InteractiveShell.automagic', - help='Turn on the auto calling of magic commands.' - 'Type %%magic at the IPython prompt for more information.') - ), - (('--no-automagic',), dict( - action='store_false', dest='InteractiveShell.automagic', - help='Turn off the auto calling of magic commands.') - ), - (('--autoedit-syntax',), dict( - action='store_true', dest='InteractiveShell.autoedit_syntax', - help='Turn on auto editing of files with syntax errors.') - ), - (('--no-autoedit-syntax',), dict( - action='store_false', dest='InteractiveShell.autoedit_syntax', - help='Turn off auto editing of files with syntax errors.') - ), - (('--banner',), dict( - action='store_true', dest='Global.display_banner', - help='Display a banner upon starting IPython.') - ), - (('--no-banner',), dict( - action='store_false', dest='Global.display_banner', - help="Don't display a banner upon starting IPython.") - ), - (('--cache-size',), dict( - type=int, dest='InteractiveShell.cache_size', - help= - """Set the size of the output cache. The default is 1000, you can - change it permanently in your config file. Setting it to 0 completely - disables the caching system, and the minimum value accepted is 20 (if - you provide a value less than 20, it is reset to 0 and a warning is - issued). This limit is defined because otherwise you'll spend more - time re-flushing a too small cache than working. - """, - metavar='InteractiveShell.cache_size') - ), - (('--classic',), dict( - action='store_true', dest='Global.classic', - help="Gives IPython a similar feel to the classic Python prompt.") - ), - (('--colors',), dict( - type=str, dest='InteractiveShell.colors', - help="Set the color scheme (NoColor, Linux, and LightBG).", - metavar='InteractiveShell.colors') - ), - (('--color-info',), dict( - action='store_true', dest='InteractiveShell.color_info', - help= - """IPython can display information about objects via a set of func- - tions, and optionally can use colors for this, syntax highlighting - source code and various other elements. However, because this - information is passed through a pager (like 'less') and many pagers get - confused with color codes, this option is off by default. You can test - it and turn it on permanently in your ipython_config.py file if it - works for you. Test it and turn it on permanently if it works with - your system. The magic function %%color_info allows you to toggle this - inter- actively for testing.""" - ) - ), - (('--no-color-info',), dict( - action='store_false', dest='InteractiveShell.color_info', - help="Disable using colors for info related things.") - ), - (('--confirm-exit',), dict( - action='store_true', dest='InteractiveShell.confirm_exit', - help= - """Set to confirm when you try to exit IPython with an EOF (Control-D - in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or - '%%Exit', you can force a direct exit without any confirmation. - """ - ) - ), - (('--no-confirm-exit',), dict( - action='store_false', dest='InteractiveShell.confirm_exit', - help="Don't prompt the user when exiting.") - ), - (('--deep-reload',), dict( - action='store_true', dest='InteractiveShell.deep_reload', - help= - """Enable deep (recursive) reloading by default. IPython can use the - deep_reload module which reloads changes in modules recursively (it - replaces the reload() function, so you don't need to change anything to - use it). deep_reload() forces a full reload of modules whose code may - have changed, which the default reload() function does not. When - deep_reload is off, IPython will use the normal reload(), but - deep_reload will still be available as dreload(). This fea- ture is off - by default [which means that you have both normal reload() and - dreload()].""") - ), - (('--no-deep-reload',), dict( - action='store_false', dest='InteractiveShell.deep_reload', - help="Disable deep (recursive) reloading by default.") - ), - (('--editor',), dict( - type=str, dest='InteractiveShell.editor', - help="Set the editor used by IPython (default to $EDITOR/vi/notepad).", - metavar='InteractiveShell.editor') - ), - (('--log','-l'), dict( - action='store_true', dest='InteractiveShell.logstart', - help="Start logging to the default log file (./ipython_log.py).") - ), - (('--logfile','-lf'), dict( - type=unicode, dest='InteractiveShell.logfile', - help="Start logging to logfile with this name.", - metavar='InteractiveShell.logfile') - ), - (('--log-append','-la'), dict( - type=unicode, dest='InteractiveShell.logappend', - help="Start logging to the given file in append mode.", - metavar='InteractiveShell.logfile') - ), - (('--pdb',), dict( - action='store_true', dest='InteractiveShell.pdb', - help="Enable auto calling the pdb debugger after every exception.") - ), - (('--no-pdb',), dict( - action='store_false', dest='InteractiveShell.pdb', - help="Disable auto calling the pdb debugger after every exception.") - ), - (('--pprint',), dict( - action='store_true', dest='InteractiveShell.pprint', - help="Enable auto pretty printing of results.") - ), - (('--no-pprint',), dict( - action='store_false', dest='InteractiveShell.pprint', - help="Disable auto auto pretty printing of results.") - ), - (('--prompt-in1','-pi1'), dict( - type=str, dest='InteractiveShell.prompt_in1', - help= - """Set the main input prompt ('In [\#]: '). Note that if you are using - numbered prompts, the number is represented with a '\#' in the string. - Don't forget to quote strings with spaces embedded in them. Most - bash-like escapes can be used to customize IPython's prompts, as well - as a few additional ones which are IPython-spe- cific. All valid - prompt escapes are described in detail in the Customization section of - the IPython manual.""", - metavar='InteractiveShell.prompt_in1') - ), - (('--prompt-in2','-pi2'), dict( - type=str, dest='InteractiveShell.prompt_in2', - help= - """Set the secondary input prompt (' .\D.: '). Similar to the previous - option, but used for the continuation prompts. The special sequence - '\D' is similar to '\#', but with all digits replaced by dots (so you - can have your continuation prompt aligned with your input prompt). - Default: ' .\D.: ' (note three spaces at the start for alignment with - 'In [\#]')""", - metavar='InteractiveShell.prompt_in2') - ), - (('--prompt-out','-po'), dict( - type=str, dest='InteractiveShell.prompt_out', - help="Set the output prompt ('Out[\#]:')", - metavar='InteractiveShell.prompt_out') - ), - (('--quick',), dict( - action='store_true', dest='Global.quick', - help="Enable quick startup with no config files.") - ), - (('--readline',), dict( - action='store_true', dest='InteractiveShell.readline_use', - help="Enable readline for command line usage.") - ), - (('--no-readline',), dict( - action='store_false', dest='InteractiveShell.readline_use', - help="Disable readline for command line usage.") - ), - (('--screen-length','-sl'), dict( - type=int, dest='InteractiveShell.screen_length', - help= - """Number of lines of your screen, used to control printing of very - long strings. Strings longer than this number of lines will be sent - through a pager instead of directly printed. The default value for - this is 0, which means IPython will auto-detect your screen size every - time it needs to print certain potentially long strings (this doesn't - change the behavior of the 'print' keyword, it's only triggered - internally). If for some reason this isn't working well (it needs - curses support), specify it yourself. Otherwise don't change the - default.""", - metavar='InteractiveShell.screen_length') - ), - (('--separate-in','-si'), dict( - type=str, dest='InteractiveShell.separate_in', - help="Separator before input prompts. Default '\\n'.", - metavar='InteractiveShell.separate_in') - ), - (('--separate-out','-so'), dict( - type=str, dest='InteractiveShell.separate_out', - help="Separator before output prompts. Default 0 (nothing).", - metavar='InteractiveShell.separate_out') - ), - (('--separate-out2','-so2'), dict( - type=str, dest='InteractiveShell.separate_out2', - help="Separator after output prompts. Default 0 (nonight).", - metavar='InteractiveShell.separate_out2') - ), - (('-no-sep',), dict( - action='store_true', dest='Global.nosep', - help="Eliminate all spacing between prompts.") - ), - (('--term-title',), dict( - action='store_true', dest='InteractiveShell.term_title', - help="Enable auto setting the terminal title.") - ), - (('--no-term-title',), dict( - action='store_false', dest='InteractiveShell.term_title', - help="Disable auto setting the terminal title.") - ), - (('--xmode',), dict( - type=str, dest='InteractiveShell.xmode', - help= - """Exception reporting mode ('Plain','Context','Verbose'). Plain: - similar to python's normal traceback printing. Context: prints 5 lines - of context source code around each line in the traceback. Verbose: - similar to Context, but additionally prints the variables currently - visible where the exception happened (shortening their strings if too - long). This can potentially be very slow, if you happen to have a huge - data structure whose string representation is complex to compute. - Your computer may appear to freeze for a while with cpu usage at 100%%. - If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting - it more than once). - """, - metavar='InteractiveShell.xmode') - ), - (('--ext',), dict( - type=str, dest='Global.extra_extension', - help="The dotted module name of an IPython extension to load.", - metavar='Global.extra_extension') - ), - (('-c',), dict( - type=str, dest='Global.code_to_run', - help="Execute the given command string.", - metavar='Global.code_to_run') - ), - (('-i',), dict( - action='store_true', dest='Global.force_interact', - help= - "If running code from the command line, become interactive afterwards." - ) - ), - - # Options to start with GUI control enabled from the beginning - (('--gui',), dict( - type=str, dest='Global.gui', - help="Enable GUI event loop integration ('qt', 'wx', 'gtk').", - metavar='gui-mode') - ), - - (('--pylab','-pylab'), dict( - type=str, dest='Global.pylab', - nargs='?', const='auto', metavar='gui-mode', - help="Pre-load matplotlib and numpy for interactive use. "+ - "If no value is given, the gui backend is matplotlib's, else use "+ - "one of: ['tk', 'qt', 'wx', 'gtk'].") - ), - - # Legacy GUI options. Leave them in for backwards compatibility, but the - # 'thread' names are really a misnomer now. - (('--wthread','-wthread'), dict( - action='store_true', dest='Global.wthread', - help="Enable wxPython event loop integration "+ - "(DEPRECATED, use --gui wx)") - ), - (('--q4thread','--qthread','-q4thread','-qthread'), dict( - action='store_true', dest='Global.q4thread', - help="Enable Qt4 event loop integration. Qt3 is no longer supported. "+ - "(DEPRECATED, use --gui qt)") - ), - (('--gthread','-gthread'), dict( - action='store_true', dest='Global.gthread', - help="Enable GTK event loop integration. "+ - "(DEPRECATED, use --gui gtk)") - ), -) + +class IPAppConfigLoader(BaseAppConfigLoader): + + def _add_arguments(self): + super(IPAppConfigLoader, self)._add_arguments() + paa = self.parser.add_argument + paa('-p', + '--profile', dest='Global.profile', type=unicode, + help= + """The string name of the ipython profile to be used. Assume that your + config file is ipython_config-.py (looks in current dir first, + then in IPYTHON_DIR). This is a quick way to keep and load multiple + config files for different tasks, especially if include your basic one + in your more specialized ones. You can keep a basic + IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which + include this one and load extra things for particular tasks.""", + metavar='Global.profile') + paa('--config-file', + dest='Global.config_file', type=unicode, + help= + """Set the config file name to override default. Normally IPython + loads ipython_config.py (from current directory) or + IPYTHON_DIR/ipython_config.py. If the loading of your config file + fails, IPython starts with a bare bones configuration (no modules + loaded at all).""", + metavar='Global.config_file') + paa('--autocall', + dest='InteractiveShell.autocall', type=int, + help= + """Make IPython automatically call any callable object even if you + didn't type explicit parentheses. For example, 'str 43' becomes + 'str(43)' automatically. The value can be '0' to disable the feature, + '1' for 'smart' autocall, where it is not applied if there are no more + arguments on the line, and '2' for 'full' autocall, where all callable + objects are automatically called (even if no arguments are present). + The default is '1'.""", + metavar='InteractiveShell.autocall') + paa('--autoindent', + action='store_true', dest='InteractiveShell.autoindent', + help='Turn on autoindenting.') + paa('--no-autoindent', + action='store_false', dest='InteractiveShell.autoindent', + help='Turn off autoindenting.') + paa('--automagic', + action='store_true', dest='InteractiveShell.automagic', + help= + """Turn on the auto calling of magic commands. Type %%magic at the + IPython prompt for more information.""") + paa('--no-automagic', + action='store_false', dest='InteractiveShell.automagic', + help='Turn off the auto calling of magic commands.') + paa('--autoedit-syntax', + action='store_true', dest='InteractiveShell.autoedit_syntax', + help='Turn on auto editing of files with syntax errors.') + paa('--no-autoedit-syntax', + action='store_false', dest='InteractiveShell.autoedit_syntax', + help='Turn off auto editing of files with syntax errors.') + paa('--banner', + action='store_true', dest='Global.display_banner', + help='Display a banner upon starting IPython.') + paa('--no-banner', + action='store_false', dest='Global.display_banner', + help="Don't display a banner upon starting IPython.") + paa('--cache-size', + type=int, dest='InteractiveShell.cache_size', + help= + """Set the size of the output cache. The default is 1000, you can + change it permanently in your config file. Setting it to 0 completely + disables the caching system, and the minimum value accepted is 20 (if + you provide a value less than 20, it is reset to 0 and a warning is + issued). This limit is defined because otherwise you'll spend more + time re-flushing a too small cache than working""", + metavar='InteractiveShell.cache_size') + paa('--classic', + action='store_true', dest='Global.classic', + help="Gives IPython a similar feel to the classic Python prompt.") + paa('--colors', + type=str, dest='InteractiveShell.colors', + help="Set the color scheme (NoColor, Linux, and LightBG).", + metavar='InteractiveShell.colors') + paa('--color-info', + action='store_true', dest='InteractiveShell.color_info', + help= + """IPython can display information about objects via a set of func- + tions, and optionally can use colors for this, syntax highlighting + source code and various other elements. However, because this + information is passed through a pager (like 'less') and many pagers get + confused with color codes, this option is off by default. You can test + it and turn it on permanently in your ipython_config.py file if it + works for you. Test it and turn it on permanently if it works with + your system. The magic function %%color_info allows you to toggle this + inter- actively for testing.""") + paa('--no-color-info', + action='store_false', dest='InteractiveShell.color_info', + help="Disable using colors for info related things.") + paa('--confirm-exit', + action='store_true', dest='InteractiveShell.confirm_exit', + help= + """Set to confirm when you try to exit IPython with an EOF (Control-D + in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or + '%%Exit', you can force a direct exit without any confirmation.""") + paa('--no-confirm-exit', + action='store_false', dest='InteractiveShell.confirm_exit', + help="Don't prompt the user when exiting.") + paa('--deep-reload', + action='store_true', dest='InteractiveShell.deep_reload', + help= + """Enable deep (recursive) reloading by default. IPython can use the + deep_reload module which reloads changes in modules recursively (it + replaces the reload() function, so you don't need to change anything to + use it). deep_reload() forces a full reload of modules whose code may + have changed, which the default reload() function does not. When + deep_reload is off, IPython will use the normal reload(), but + deep_reload will still be available as dreload(). This fea- ture is off + by default [which means that you have both normal reload() and + dreload()].""") + paa('--no-deep-reload', + action='store_false', dest='InteractiveShell.deep_reload', + help="Disable deep (recursive) reloading by default.") + paa('--editor', + type=str, dest='InteractiveShell.editor', + help="Set the editor used by IPython (default to $EDITOR/vi/notepad).", + metavar='InteractiveShell.editor') + paa('--log','-l', + action='store_true', dest='InteractiveShell.logstart', + help="Start logging to the default log file (./ipython_log.py).") + paa('--logfile','-lf', + type=unicode, dest='InteractiveShell.logfile', + help="Start logging to logfile with this name.", + metavar='InteractiveShell.logfile') + paa('--log-append','-la', + type=unicode, dest='InteractiveShell.logappend', + help="Start logging to the given file in append mode.", + metavar='InteractiveShell.logfile') + paa('--pdb', + action='store_true', dest='InteractiveShell.pdb', + help="Enable auto calling the pdb debugger after every exception.") + paa('--no-pdb', + action='store_false', dest='InteractiveShell.pdb', + help="Disable auto calling the pdb debugger after every exception.") + paa('--pprint', + action='store_true', dest='InteractiveShell.pprint', + help="Enable auto pretty printing of results.") + paa('--no-pprint', + action='store_false', dest='InteractiveShell.pprint', + help="Disable auto auto pretty printing of results.") + paa('--prompt-in1','-pi1', + type=str, dest='InteractiveShell.prompt_in1', + help= + """Set the main input prompt ('In [\#]: '). Note that if you are using + numbered prompts, the number is represented with a '\#' in the string. + Don't forget to quote strings with spaces embedded in them. Most + bash-like escapes can be used to customize IPython's prompts, as well + as a few additional ones which are IPython-spe- cific. All valid + prompt escapes are described in detail in the Customization section of + the IPython manual.""", + metavar='InteractiveShell.prompt_in1') + paa('--prompt-in2','-pi2', + type=str, dest='InteractiveShell.prompt_in2', + help= + """Set the secondary input prompt (' .\D.: '). Similar to the previous + option, but used for the continuation prompts. The special sequence + '\D' is similar to '\#', but with all digits replaced by dots (so you + can have your continuation prompt aligned with your input prompt). + Default: ' .\D.: ' (note three spaces at the start for alignment with + 'In [\#]')""", + metavar='InteractiveShell.prompt_in2') + paa('--prompt-out','-po', + type=str, dest='InteractiveShell.prompt_out', + help="Set the output prompt ('Out[\#]:')", + metavar='InteractiveShell.prompt_out') + paa('--quick', + action='store_true', dest='Global.quick', + help="Enable quick startup with no config files.") + paa('--readline', + action='store_true', dest='InteractiveShell.readline_use', + help="Enable readline for command line usage.") + paa('--no-readline', + action='store_false', dest='InteractiveShell.readline_use', + help="Disable readline for command line usage.") + paa('--screen-length','-sl', + type=int, dest='InteractiveShell.screen_length', + help= + """Number of lines of your screen, used to control printing of very + long strings. Strings longer than this number of lines will be sent + through a pager instead of directly printed. The default value for + this is 0, which means IPython will auto-detect your screen size every + time it needs to print certain potentially long strings (this doesn't + change the behavior of the 'print' keyword, it's only triggered + internally). If for some reason this isn't working well (it needs + curses support), specify it yourself. Otherwise don't change the + default.""", + metavar='InteractiveShell.screen_length') + paa('--separate-in','-si', + type=str, dest='InteractiveShell.separate_in', + help="Separator before input prompts. Default '\\n'.", + metavar='InteractiveShell.separate_in') + paa('--separate-out','-so', + type=str, dest='InteractiveShell.separate_out', + help="Separator before output prompts. Default 0 (nothing).", + metavar='InteractiveShell.separate_out') + paa('--separate-out2','-so2', + type=str, dest='InteractiveShell.separate_out2', + help="Separator after output prompts. Default 0 (nonight).", + metavar='InteractiveShell.separate_out2') + paa('-no-sep', + action='store_true', dest='Global.nosep', + help="Eliminate all spacing between prompts.") + paa('--term-title', + action='store_true', dest='InteractiveShell.term_title', + help="Enable auto setting the terminal title.") + paa('--no-term-title', + action='store_false', dest='InteractiveShell.term_title', + help="Disable auto setting the terminal title.") + paa('--xmode', + type=str, dest='InteractiveShell.xmode', + help= + """Exception reporting mode ('Plain','Context','Verbose'). Plain: + similar to python's normal traceback printing. Context: prints 5 lines + of context source code around each line in the traceback. Verbose: + similar to Context, but additionally prints the variables currently + visible where the exception happened (shortening their strings if too + long). This can potentially be very slow, if you happen to have a huge + data structure whose string representation is complex to compute. + Your computer may appear to freeze for a while with cpu usage at 100%%. + If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting + it more than once). + """, + metavar='InteractiveShell.xmode') + paa('--ext', + type=str, dest='Global.extra_extension', + help="The dotted module name of an IPython extension to load.", + metavar='Global.extra_extension') + paa('-c', + type=str, dest='Global.code_to_run', + help="Execute the given command string.", + metavar='Global.code_to_run') + paa('-i', + action='store_true', dest='Global.force_interact', + help= + "If running code from the command line, become interactive afterwards.") + + # Options to start with GUI control enabled from the beginning + paa('--gui', + type=str, dest='Global.gui', + help="Enable GUI event loop integration ('qt', 'wx', 'gtk').", + metavar='gui-mode') + paa('--pylab','-pylab', + type=str, dest='Global.pylab', + nargs='?', const='auto', metavar='gui-mode', + help="Pre-load matplotlib and numpy for interactive use. "+ + "If no value is given, the gui backend is matplotlib's, else use "+ + "one of: ['tk', 'qt', 'wx', 'gtk'].") + + # Legacy GUI options. Leave them in for backwards compatibility, but the + # 'thread' names are really a misnomer now. + paa('--wthread', '-wthread', + action='store_true', dest='Global.wthread', + help= + """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""") + paa('--q4thread', '--qthread', '-q4thread', '-qthread', + action='store_true', dest='Global.q4thread', + help= + """Enable Qt4 event loop integration. Qt3 is no longer supported. + (DEPRECATED, use --gui qt)""") + paa('--gthread', '-gthread', + action='store_true', dest='Global.gthread', + help= + """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""") + #----------------------------------------------------------------------------- # Main classes and functions @@ -350,44 +324,12 @@ class IPythonApp(Application): name = u'ipython' #: argparse formats better the 'usage' than the 'description' field description = None - #: usage message printed by argparse. If None, auto-generate usage = usage.cl_usage - + command_line_loader = IPAppConfigLoader config_file_name = default_config_file_name - cl_arguments = Application.cl_arguments + cl_args - # Private and configuration attributes _CrashHandler = crashhandler.IPythonCrashHandler - - def __init__(self, argv=None, - constructor_config=None, override_config=None, - **shell_params): - """Create a new IPythonApp. - - See the parent class for details on how configuration is handled. - - Parameters - ---------- - argv : optional, list - If given, used as the command-line argv environment to read arguments - from. - - constructor_config : optional, Config - If given, additional config that is merged last, after internal - defaults, command-line and file-based configs. - - override_config : optional, Config - If given, config that overrides all others unconditionally (except - for internal defaults, which ensure that all parameters exist). - - shell_params : optional, dict - All other keywords are passed to the :class:`iplib.InteractiveShell` - constructor. - """ - super(IPythonApp, self).__init__(argv, constructor_config, - override_config) - self.shell_params = shell_params def create_default_config(self): super(IPythonApp, self).create_default_config() @@ -473,8 +415,7 @@ class IPythonApp(Application): sys.path.insert(0, '') # Create an InteractiveShell instance - self.shell = InteractiveShell(None, self.master_config, - **self.shell_params ) + self.shell = InteractiveShell(None, self.master_config) def post_construct(self): """Do actions after construct, but before starting the app.""" @@ -646,3 +587,4 @@ def launch_new_instance(): """Create and run a full blown IPython instance""" app = IPythonApp() app.start() + diff --git a/IPython/kernel/clusterdir.py b/IPython/kernel/clusterdir.py index 7ed28ae..e43caaa 100755 --- a/IPython/kernel/clusterdir.py +++ b/IPython/kernel/clusterdir.py @@ -25,7 +25,7 @@ import warnings from twisted.python import log from IPython.config.loader import PyFileConfigLoader -from IPython.core.application import Application +from IPython.core.application import Application, BaseAppConfigLoader from IPython.core.component import Component from IPython.utils.path import ( get_ipython_package_dir, @@ -46,7 +46,7 @@ warnings.filterwarnings('ignore', 'the sha module is deprecated', DeprecationWarning) #----------------------------------------------------------------------------- -# Classes and functions +# Module errors #----------------------------------------------------------------------------- class ClusterDirError(Exception): @@ -57,6 +57,10 @@ class PIDFileError(Exception): pass +#----------------------------------------------------------------------------- +# Class for managing cluster directories +#----------------------------------------------------------------------------- + class ClusterDir(Component): """An object to manage the cluster directory and its resources. @@ -232,38 +236,63 @@ class ClusterDir(Component): return ClusterDir(cluster_dir) -# Default command line options for IPython cluster applications. -cl_args = ( - (('--ipython-dir',), dict( - dest='Global.ipython_dir',type=unicode, - help='Set to override default location of Global.ipython_dir.', - metavar='Global.ipython_dir') ), - (('-p', '--profile',), dict( - dest='Global.profile',type=unicode, - help= - """The string name of the profile to be used. This determines the name - of the cluster dir as: cluster_. The default profile is named - 'default'. The cluster directory is resolve this way if the - --cluster-dir option is not used.""", - metavar='Global.profile') ), - (('--cluster-dir',), dict( - dest='Global.cluster_dir',type=unicode, - help="""Set the cluster dir. This overrides the logic used by the - --profile option.""", - metavar='Global.cluster_dir') ), - (('--work-dir',), dict( - dest='Global.work_dir',type=unicode, - help='Set the working dir for the process.', - metavar='Global.work_dir') ), - (('--clean-logs',), dict( - dest='Global.clean_logs', action='store_true', - help='Delete old log flies before starting.') ), - (('--no-clean-logs',), dict( - dest='Global.clean_logs', action='store_false', - help="Don't Delete old log flies before starting.") ), - ) +#----------------------------------------------------------------------------- +# Command line options +#----------------------------------------------------------------------------- + +class ClusterDirConfigLoader(BaseAppConfigLoader): + + def _add_cluster_profile(self, parser): + paa = parser.add_argument + paa('-p', '--profile', + dest='Global.profile',type=unicode, + help= + """The string name of the profile to be used. This determines the name + of the cluster dir as: cluster_. The default profile is named + 'default'. The cluster directory is resolve this way if the + --cluster-dir option is not used.""", + metavar='Global.profile') + + def _add_cluster_dir(self, parser): + paa = parser.add_argument + paa('--cluster-dir', + dest='Global.cluster_dir',type=unicode, + help="""Set the cluster dir. This overrides the logic used by the + --profile option.""", + metavar='Global.cluster_dir') + + def _add_work_dir(self, parser): + paa = parser.add_argument + paa('--work-dir', + dest='Global.work_dir',type=unicode, + help='Set the working dir for the process.', + metavar='Global.work_dir') + + def _add_clean_logs(self, parser): + paa = parser.add_argument + paa('--clean-logs', + dest='Global.clean_logs', action='store_true', + help='Delete old log flies before starting.') + + def _add_no_clean_logs(self, parser): + paa = parser.add_argument + paa('--no-clean-logs', + dest='Global.clean_logs', action='store_false', + help="Don't Delete old log flies before starting.") + + def _add_arguments(self): + super(ClusterDirConfigLoader, self)._add_arguments() + self._add_cluster_profile(self.parser) + self._add_cluster_dir(self.parser) + self._add_work_dir(self.parser) + self._add_clean_logs(self.parser) + self._add_no_clean_logs(self.parser) +#----------------------------------------------------------------------------- +# Main application +#----------------------------------------------------------------------------- + class ApplicationWithClusterDir(Application): """An application that puts everything into a cluster directory. @@ -283,10 +312,9 @@ class ApplicationWithClusterDir(Application): dir and named the value of the ``config_file_name`` class attribute. """ + command_line_loader = ClusterDirConfigLoader auto_create_cluster_dir = True - cl_arguments = Application.cl_arguments + cl_args - def create_default_config(self): super(ApplicationWithClusterDir, self).create_default_config() self.default_config.Global.profile = u'default' @@ -462,3 +490,4 @@ class ApplicationWithClusterDir(Application): return pid else: raise PIDFileError('pid file not found: %s' % pid_file) + diff --git a/IPython/kernel/ipclusterapp.py b/IPython/kernel/ipclusterapp.py index c19bcc0..346be61 100755 --- a/IPython/kernel/ipclusterapp.py +++ b/IPython/kernel/ipclusterapp.py @@ -22,159 +22,162 @@ import signal if os.name=='posix': from twisted.scripts._twistd_unix import daemonize -from IPython.core import release +from twisted.internet import reactor, defer +from twisted.python import log, failure + + from IPython.external.argparse import ArgumentParser, SUPPRESS -from IPython.config.loader import ArgParseConfigLoader from IPython.utils.importstring import import_item - from IPython.kernel.clusterdir import ( - ApplicationWithClusterDir, ClusterDirError, PIDFileError + ApplicationWithClusterDir, ClusterDirConfigLoader, + ClusterDirError, PIDFileError ) -from twisted.internet import reactor, defer -from twisted.python import log, failure - #----------------------------------------------------------------------------- -# The ipcluster application +# Module level variables #----------------------------------------------------------------------------- +default_config_file_name = u'ipcluster_config.py' + + +_description = """\ +Start an IPython cluster for parallel computing.\n\n + +An IPython cluster consists of 1 controller and 1 or more engines. +This command automates the startup of these processes using a wide +range of startup methods (SSH, local processes, PBS, mpiexec, +Windows HPC Server 2008). To start a cluster with 4 engines on your +local host simply do "ipcluster start -n 4". For more complex usage +you will typically do "ipcluster create -p mycluster", then edit +configuration files, followed by "ipcluster start -p mycluster -n 4". +""" + + # Exit codes for ipcluster # This will be the exit code if the ipcluster appears to be running because # a .pid file exists ALREADY_STARTED = 10 + # This will be the exit code if ipcluster stop is run, but there is not .pid # file to be found. ALREADY_STOPPED = 11 -class IPClusterCLLoader(ArgParseConfigLoader): +#----------------------------------------------------------------------------- +# Command line options +#----------------------------------------------------------------------------- + - def _add_other_arguments(self): +class IPClusterAppConfigLoader(ClusterDirConfigLoader): + + def _add_arguments(self): + # Don't call ClusterDirConfigLoader._add_arguments as we don't want + # its defaults on self.parser. Instead, we will put those on + # default options on our subparsers. + # This has all the common options that all subcommands use - parent_parser1 = ArgumentParser(add_help=False, - argument_default=SUPPRESS) - parent_parser1.add_argument('--ipython-dir', - dest='Global.ipython_dir',type=unicode, - help='Set to override default location of Global.ipython_dir.', - metavar='Global.ipython_dir') - parent_parser1.add_argument('--log-level', - dest="Global.log_level",type=int, - help='Set the log level (0,10,20,30,40,50). Default is 30.', - metavar='Global.log_level') + parent_parser1 = ArgumentParser( + add_help=False, + argument_default=SUPPRESS + ) + self._add_ipython_dir(parent_parser1) + self._add_log_level(parent_parser1) # This has all the common options that other subcommands use - parent_parser2 = ArgumentParser(add_help=False, - argument_default=SUPPRESS) - parent_parser2.add_argument('-p','--profile', - dest='Global.profile',type=unicode, - help='The string name of the profile to be used. This determines ' - 'the name of the cluster dir as: cluster_. The default profile ' - 'is named "default". The cluster directory is resolve this way ' - 'if the --cluster-dir option is not used.', - metavar='Global.profile') - parent_parser2.add_argument('--cluster-dir', - dest='Global.cluster_dir',type=unicode, - help='Set the cluster dir. This overrides the logic used by the ' - '--profile option.', - metavar='Global.cluster_dir'), - parent_parser2.add_argument('--work-dir', - dest='Global.work_dir',type=unicode, - help='Set the working dir for the process.', - metavar='Global.work_dir') - parent_parser2.add_argument('--log-to-file', - action='store_true', dest='Global.log_to_file', - help='Log to a file in the log directory (default is stdout)' + parent_parser2 = ArgumentParser( + add_help=False, + argument_default=SUPPRESS ) + self._add_cluster_profile(parent_parser2) + self._add_cluster_dir(parent_parser2) + self._add_work_dir(parent_parser2) + paa = parent_parser2.add_argument + paa('--log-to-file', + action='store_true', dest='Global.log_to_file', + help='Log to a file in the log directory (default is stdout)') + # Create the object used to create the subparsers. subparsers = self.parser.add_subparsers( dest='Global.subcommand', title='ipcluster subcommands', - description='ipcluster has a variety of subcommands. ' - 'The general way of running ipcluster is "ipcluster ' - ' [options]""', - help='For more help, type "ipcluster -h"') + description= + """ipcluster has a variety of subcommands. The general way of + running ipcluster is 'ipcluster [options]'""", + help="For more help, type 'ipcluster -h'" + ) + # The "list" subcommand parser parser_list = subparsers.add_parser( 'list', help='List all clusters in cwd and ipython_dir.', parents=[parent_parser1] ) + # The "create" subcommand parser parser_create = subparsers.add_parser( 'create', help='Create a new cluster directory.', parents=[parent_parser1, parent_parser2] ) - parser_create.add_argument( - '--reset-config', + paa = parser_create.add_argument + paa('--reset-config', dest='Global.reset_config', action='store_true', - help='Recopy the default config files to the cluster directory. ' - 'You will loose any modifications you have made to these files.' - ) + help= + """Recopy the default config files to the cluster directory. + You will loose any modifications you have made to these files.""") + # The "start" subcommand parser parser_start = subparsers.add_parser( 'start', help='Start a cluster.', parents=[parent_parser1, parent_parser2] ) - parser_start.add_argument( - '-n', '--number', + paa = parser_start.add_argument + paa('-n', '--number', type=int, dest='Global.n', help='The number of engines to start.', - metavar='Global.n' - ) - parser_start.add_argument('--clean-logs', + metavar='Global.n') + paa('--clean-logs', dest='Global.clean_logs', action='store_true', - help='Delete old log flies before starting.', - ) - parser_start.add_argument('--no-clean-logs', + help='Delete old log flies before starting.') + paa('--no-clean-logs', dest='Global.clean_logs', action='store_false', - help="Don't delete old log flies before starting.", - ) - parser_start.add_argument('--daemon', + help="Don't delete old log flies before starting.") + paa('--daemon', dest='Global.daemonize', action='store_true', - help='Daemonize the ipcluster program. This implies --log-to-file', - ) - parser_start.add_argument('--no-daemon', + help='Daemonize the ipcluster program. This implies --log-to-file') + paa('--no-daemon', dest='Global.daemonize', action='store_false', - help="Dont't daemonize the ipcluster program.", - ) + help="Dont't daemonize the ipcluster program.") - parser_start = subparsers.add_parser( + # The "stop" subcommand parser + parser_stop = subparsers.add_parser( 'stop', help='Stop a cluster.', parents=[parent_parser1, parent_parser2] ) - parser_start.add_argument('--signal', + paa = parser_stop.add_argument + paa('--signal', dest='Global.signal', type=int, help="The signal number to use in stopping the cluster (default=2).", - metavar="Global.signal", - ) - - -default_config_file_name = u'ipcluster_config.py' + metavar="Global.signal") -_description = """Start an IPython cluster for parallel computing.\n\n - -An IPython cluster consists of 1 controller and 1 or more engines. -This command automates the startup of these processes using a wide -range of startup methods (SSH, local processes, PBS, mpiexec, -Windows HPC Server 2008). To start a cluster with 4 engines on your -local host simply do "ipcluster start -n 4". For more complex usage -you will typically do "ipcluster create -p mycluster", then edit -configuration files, followed by "ipcluster start -p mycluster -n 4". -""" +#----------------------------------------------------------------------------- +# Main application +#----------------------------------------------------------------------------- class IPClusterApp(ApplicationWithClusterDir): name = u'ipcluster' description = _description + usage = None + command_line_loader = IPClusterAppConfigLoader config_file_name = default_config_file_name default_log_level = logging.INFO auto_create_cluster_dir = False @@ -191,13 +194,6 @@ class IPClusterApp(ApplicationWithClusterDir): self.default_config.Global.signal = 2 self.default_config.Global.daemonize = False - def create_command_line_config(self): - """Create and return a command line config loader.""" - return IPClusterCLLoader( - description=self.description, - version=release.version - ) - def find_resources(self): subcommand = self.command_line_config.Global.subcommand if subcommand=='list': @@ -451,6 +447,7 @@ class IPClusterApp(ApplicationWithClusterDir): # old .pid files. self.remove_pid_file() + def launch_new_instance(): """Create and run the IPython cluster.""" app = IPClusterApp() diff --git a/IPython/kernel/ipcontrollerapp.py b/IPython/kernel/ipcontrollerapp.py index 7cffb54..2468710 100755 --- a/IPython/kernel/ipcontrollerapp.py +++ b/IPython/kernel/ipcontrollerapp.py @@ -25,12 +25,34 @@ from twisted.internet import reactor from twisted.python import log from IPython.config.loader import Config -from IPython.core.application import Application from IPython.kernel import controllerservice -from IPython.kernel.clusterdir import ApplicationWithClusterDir +from IPython.kernel.clusterdir import ( + ApplicationWithClusterDir, + ClusterDirConfigLoader +) from IPython.kernel.fcutil import FCServiceFactory from IPython.utils.traitlets import Instance, Unicode + +#----------------------------------------------------------------------------- +# Module level variables +#----------------------------------------------------------------------------- + + +#: The default config file name for this application +default_config_file_name = u'ipcontroller_config.py' + + +_description = """Start the IPython controller for parallel computing. + +The IPython controller provides a gateway between the IPython engines and +clients. The controller needs to be started before the engines and can be +configured using command line options or using a cluster directory. Cluster +directories contain config, log and security files and are usually located in +your .ipython directory and named as "cluster_". See the --profile +and --cluster-dir options for details. +""" + #----------------------------------------------------------------------------- # Default interfaces #----------------------------------------------------------------------------- @@ -90,97 +112,82 @@ class FCEngineServiceFactory(FCServiceFactory): #----------------------------------------------------------------------------- -# The main application +# Command line options #----------------------------------------------------------------------------- -cl_args = ( - # Client config - (('--client-ip',), dict( - type=str, dest='FCClientServiceFactory.ip', - help='The IP address or hostname the controller will listen on for ' - 'client connections.', - metavar='FCClientServiceFactory.ip') - ), - (('--client-port',), dict( - type=int, dest='FCClientServiceFactory.port', - help='The port the controller will listen on for client connections. ' - 'The default is to use 0, which will autoselect an open port.', - metavar='FCClientServiceFactory.port') - ), - (('--client-location',), dict( - type=str, dest='FCClientServiceFactory.location', - help='The hostname or IP that clients should connect to. This does ' - 'not control which interface the controller listens on. Instead, this ' - 'determines the hostname/IP that is listed in the FURL, which is how ' - 'clients know where to connect. Useful if the controller is listening ' - 'on multiple interfaces.', - metavar='FCClientServiceFactory.location') - ), - # Engine config - (('--engine-ip',), dict( - type=str, dest='FCEngineServiceFactory.ip', - help='The IP address or hostname the controller will listen on for ' - 'engine connections.', - metavar='FCEngineServiceFactory.ip') - ), - (('--engine-port',), dict( - type=int, dest='FCEngineServiceFactory.port', - help='The port the controller will listen on for engine connections. ' - 'The default is to use 0, which will autoselect an open port.', - metavar='FCEngineServiceFactory.port') - ), - (('--engine-location',), dict( - type=str, dest='FCEngineServiceFactory.location', - help='The hostname or IP that engines should connect to. This does ' - 'not control which interface the controller listens on. Instead, this ' - 'determines the hostname/IP that is listed in the FURL, which is how ' - 'engines know where to connect. Useful if the controller is listening ' - 'on multiple interfaces.', - metavar='FCEngineServiceFactory.location') - ), - # Global config - (('--log-to-file',), dict( - action='store_true', dest='Global.log_to_file', - help='Log to a file in the log directory (default is stdout)') - ), - (('-r','--reuse-furls'), dict( - action='store_true', dest='Global.reuse_furls', - help='Try to reuse all FURL files. If this is not set all FURL files ' - 'are deleted before the controller starts. This must be set if ' - 'specific ports are specified by --engine-port or --client-port.') - ), - (('--no-secure',), dict( - action='store_false', dest='Global.secure', - help='Turn off SSL encryption for all connections.') - ), - (('--secure',), dict( - action='store_true', dest='Global.secure', - help='Turn off SSL encryption for all connections.') - ) -) +class IPControllerAppConfigLoader(ClusterDirConfigLoader): + + def _add_arguments(self): + super(IPControllerAppConfigLoader, self)._add_arguments() + paa = self.parser.add_argument + # Client config + paa('--client-ip', + type=str, dest='FCClientServiceFactory.ip', + help='The IP address or hostname the controller will listen on for ' + 'client connections.', + metavar='FCClientServiceFactory.ip') + paa('--client-port', + type=int, dest='FCClientServiceFactory.port', + help='The port the controller will listen on for client connections. ' + 'The default is to use 0, which will autoselect an open port.', + metavar='FCClientServiceFactory.port') + paa('--client-location',), dict( + type=str, dest='FCClientServiceFactory.location', + help='The hostname or IP that clients should connect to. This does ' + 'not control which interface the controller listens on. Instead, this ' + 'determines the hostname/IP that is listed in the FURL, which is how ' + 'clients know where to connect. Useful if the controller is listening ' + 'on multiple interfaces.', + metavar='FCClientServiceFactory.location') + # Engine config + paa('--engine-ip', + type=str, dest='FCEngineServiceFactory.ip', + help='The IP address or hostname the controller will listen on for ' + 'engine connections.', + metavar='FCEngineServiceFactory.ip') + paa('--engine-port', + type=int, dest='FCEngineServiceFactory.port', + help='The port the controller will listen on for engine connections. ' + 'The default is to use 0, which will autoselect an open port.', + metavar='FCEngineServiceFactory.port') + paa('--engine-location', + type=str, dest='FCEngineServiceFactory.location', + help='The hostname or IP that engines should connect to. This does ' + 'not control which interface the controller listens on. Instead, this ' + 'determines the hostname/IP that is listed in the FURL, which is how ' + 'engines know where to connect. Useful if the controller is listening ' + 'on multiple interfaces.', + metavar='FCEngineServiceFactory.location') + # Global config + paa('--log-to-file', + action='store_true', dest='Global.log_to_file', + help='Log to a file in the log directory (default is stdout)') + paa('-r','--reuse-furls', + action='store_true', dest='Global.reuse_furls', + help='Try to reuse all FURL files. If this is not set all FURL files ' + 'are deleted before the controller starts. This must be set if ' + 'specific ports are specified by --engine-port or --client-port.') + paa('--no-secure', + action='store_false', dest='Global.secure', + help='Turn off SSL encryption for all connections.') + paa('--secure', + action='store_true', dest='Global.secure', + help='Turn off SSL encryption for all connections.') -_description = """Start the IPython controller for parallel computing. - -The IPython controller provides a gateway between the IPython engines and -clients. The controller needs to be started before the engines and can be -configured using command line options or using a cluster directory. Cluster -directories contain config, log and security files and are usually located in -your .ipython directory and named as "cluster_". See the --profile -and --cluster-dir options for details. -""" - -default_config_file_name = u'ipcontroller_config.py' +#----------------------------------------------------------------------------- +# The main application +#----------------------------------------------------------------------------- class IPControllerApp(ApplicationWithClusterDir): name = u'ipcontroller' description = _description + command_line_loader = IPControllerAppConfigLoader config_file_name = default_config_file_name auto_create_cluster_dir = True - cl_arguments = Application.cl_arguments + cl_args def create_default_config(self): super(IPControllerApp, self).create_default_config() diff --git a/IPython/kernel/ipengineapp.py b/IPython/kernel/ipengineapp.py index dbd3e35..129af6b 100755 --- a/IPython/kernel/ipengineapp.py +++ b/IPython/kernel/ipengineapp.py @@ -22,39 +22,21 @@ from twisted.application import service from twisted.internet import reactor from twisted.python import log -from IPython.core.application import Application -from IPython.kernel.clusterdir import ApplicationWithClusterDir +from IPython.kernel.clusterdir import ( + ApplicationWithClusterDir, + ClusterDirConfigLoader +) from IPython.kernel.engineconnector import EngineConnector from IPython.kernel.engineservice import EngineService from IPython.kernel.fcutil import Tub from IPython.utils.importstring import import_item #----------------------------------------------------------------------------- -# The main application +# Module level variables #----------------------------------------------------------------------------- -cl_args = ( - # Controller config - (('--furl-file',), dict( - type=unicode, dest='Global.furl_file', - help='The full location of the file containing the FURL of the ' - 'controller. If this is not given, the FURL file must be in the ' - 'security directory of the cluster directory. This location is ' - 'resolved using the --profile and --app-dir options.', - metavar='Global.furl_file') - ), - # MPI - (('--mpi',), dict( - type=str, dest='MPI.use', - help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', - metavar='MPI.use') - ), - # Global config - (('--log-to-file',), dict( - action='store_true', dest='Global.log_to_file', - help='Log to a file in the log directory (default is stdout)') - ) -) +#: The default config file name for this application +default_config_file_name = u'ipengine_config.py' mpi4py_init = """from mpi4py import MPI as mpi @@ -62,6 +44,7 @@ mpi.size = mpi.COMM_WORLD.Get_size() mpi.rank = mpi.COMM_WORLD.Get_rank() """ + pytrilinos_init = """from PyTrilinos import Epetra class SimpleStruct: pass @@ -71,9 +54,6 @@ mpi.size = 0 """ -default_config_file_name = u'ipengine_config.py' - - _description = """Start an IPython engine for parallel computing.\n\n IPython engines run in parallel and perform computations on behalf of a client @@ -84,14 +64,47 @@ usually located in your .ipython directory and named as "cluster_". See the --profile and --cluster-dir options for details. """ +#----------------------------------------------------------------------------- +# Command line options +#----------------------------------------------------------------------------- + + +class IPEngineAppConfigLoader(ClusterDirConfigLoader): + + def _add_arguments(self): + super(IPEngineAppConfigLoader, self)._add_arguments() + paa = self.parser.add_argument + # Controller config + paa('--furl-file', + type=unicode, dest='Global.furl_file', + help='The full location of the file containing the FURL of the ' + 'controller. If this is not given, the FURL file must be in the ' + 'security directory of the cluster directory. This location is ' + 'resolved using the --profile and --app-dir options.', + metavar='Global.furl_file') + # MPI + paa('--mpi', + type=str, dest='MPI.use', + help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', + metavar='MPI.use') + # Global config + paa('--log-to-file', + action='store_true', dest='Global.log_to_file', + help='Log to a file in the log directory (default is stdout)') + + +#----------------------------------------------------------------------------- +# Main application +#----------------------------------------------------------------------------- + class IPEngineApp(ApplicationWithClusterDir): name = u'ipengine' description = _description + command_line_loader = IPEngineAppConfigLoader config_file_name = default_config_file_name auto_create_cluster_dir = True - cl_arguments = Application.cl_arguments + cl_args def create_default_config(self): super(IPEngineApp, self).create_default_config()