From 832789bdf908d377a0115543f485ac38d30ed66b 2011-08-18 00:30:06 From: Fernando Perez Date: 2011-08-18 00:30:06 Subject: [PATCH] Merge pull request #674 from minrk/argparse use argparse to parse aliases & flags. This now allows calling "ipython --pylab qt" without having to use "--pylab=qt" always. --- diff --git a/IPython/config/application.py b/IPython/config/application.py index eb7623d..ec71818 100644 --- a/IPython/config/application.py +++ b/IPython/config/application.py @@ -27,7 +27,7 @@ from copy import deepcopy from IPython.config.configurable import SingletonConfigurable from IPython.config.loader import ( - KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError + KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError ) from IPython.utils.traitlets import ( @@ -46,8 +46,6 @@ from IPython.utils.text import indent, wrap_paragraphs, dedent # merge flags&aliases into options option_description = """ -IPython command-line arguments are passed as '--', or '--='. - Arguments that take values are actually convenience aliases to full Configurables, whose aliases are listed on the help line. For more information on full configurables, see '--help-all'. @@ -210,6 +208,8 @@ class Application(SingletonConfigurable): help = cls.class_get_trait_help(trait).splitlines() # reformat first line help[0] = help[0].replace(longname, alias) + ' (%s)'%longname + if len(alias) == 1: + help[0] = help[0].replace('--%s='%alias, '-%s '%alias) lines.extend(help) # lines.append('') print os.linesep.join(lines) @@ -221,7 +221,8 @@ class Application(SingletonConfigurable): lines = [] for m, (cfg,help) in self.flags.iteritems(): - lines.append('--'+m) + prefix = '--' if len(m) > 1 else '-' + lines.append(prefix+m) lines.append(indent(dedent(help.strip()))) # lines.append('') print os.linesep.join(lines) @@ -350,7 +351,7 @@ class Application(SingletonConfigurable): self.print_version() self.exit(0) - loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases, + loader = KVArgParseConfigLoader(argv=argv, aliases=self.aliases, flags=self.flags) try: config = loader.load_config() diff --git a/IPython/config/loader.py b/IPython/config/loader.py index bc904af..1e1620e 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -326,6 +326,31 @@ class CommandLineConfigLoader(ConfigLoader): here. """ + def _exec_config_str(self, lhs, rhs): + exec_str = 'self.config.' + lhs + '=' + rhs + try: + # Try to see if regular Python syntax will work. This + # won't handle strings as the quote marks are removed + # by the system shell. + exec exec_str in locals(), globals() + except (NameError, SyntaxError): + # This case happens if the rhs is a string but without + # the quote marks. Use repr, to get quote marks, and + # 'u' prefix and see if + # it succeeds. If it still fails, we let it raise. + exec_str = u'self.config.' + lhs + '=' + repr(rhs) + exec exec_str in locals(), globals() + + def _load_flag(self, cfg): + """update self.config from a flag, which can be a dict or Config""" + if isinstance(cfg, (dict, Config)): + # don't clobber whole config sections, update + # each section from config: + for sec,c in cfg.iteritems(): + self.config[sec].update(c) + else: + raise ValueError("Invalid flag: '%s'"%raw) + # raw --identifier=value pattern # but *also* accept '-' as wordsep, for aliases # accepts: --foo=a @@ -463,29 +488,12 @@ class KeyValueConfigLoader(CommandLineConfigLoader): if '.' not in lhs: # probably a mistyped alias, but not technically illegal warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs) - exec_str = 'self.config.' + lhs + '=' + rhs - try: - # Try to see if regular Python syntax will work. This - # won't handle strings as the quote marks are removed - # by the system shell. - exec exec_str in locals(), globals() - except (NameError, SyntaxError): - # This case happens if the rhs is a string but without - # the quote marks. Use repr, to get quote marks, and - # 'u' prefix and see if - # it succeeds. If it still fails, we let it raise. - exec_str = u'self.config.' + lhs + '=' + repr(rhs) - exec exec_str in locals(), globals() + self._exec_config_str(lhs, rhs) + elif flag_pattern.match(raw): if item in flags: cfg,help = flags[item] - if isinstance(cfg, (dict, Config)): - # don't clobber whole config sections, update - # each section from config: - for sec,c in cfg.iteritems(): - self.config[sec].update(c) - else: - raise ValueError("Invalid flag: '%s'"%raw) + self._load_flag(cfg) else: raise ArgumentError("Unrecognized flag: '%s'"%raw) elif raw.startswith('-'): @@ -503,7 +511,7 @@ class KeyValueConfigLoader(CommandLineConfigLoader): class ArgParseConfigLoader(CommandLineConfigLoader): """A loader that uses the argparse module to load from the command line.""" - def __init__(self, argv=None, *parser_args, **parser_kw): + def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw): """Create a config loader for use with argparse. Parameters @@ -527,16 +535,20 @@ class ArgParseConfigLoader(CommandLineConfigLoader): The resulting Config object. """ super(CommandLineConfigLoader, self).__init__() - if argv == None: + self.clear() + if argv is None: argv = sys.argv[1:] self.argv = argv + self.aliases = aliases or {} + self.flags = flags or {} + self.parser_args = parser_args self.version = parser_kw.pop("version", None) kwargs = dict(argument_default=argparse.SUPPRESS) kwargs.update(parser_kw) self.parser_kw = kwargs - def load_config(self, argv=None): + def load_config(self, argv=None, aliases=None, flags=None): """Parse command line arguments and return as a Config object. Parameters @@ -549,7 +561,11 @@ class ArgParseConfigLoader(CommandLineConfigLoader): self.clear() if argv is None: argv = self.argv - self._create_parser() + if aliases is None: + aliases = self.aliases + if flags is None: + flags = self.flags + self._create_parser(aliases, flags) self._parse_args(argv) self._convert_to_config() return self.config @@ -560,11 +576,11 @@ class ArgParseConfigLoader(CommandLineConfigLoader): else: return [] - def _create_parser(self): + def _create_parser(self, aliases=None, flags=None): self.parser = ArgumentParser(*self.parser_args, **self.parser_kw) - self._add_arguments() + self._add_arguments(aliases, flags) - def _add_arguments(self): + def _add_arguments(self, aliases=None, flags=None): raise NotImplementedError("subclasses must implement _add_arguments") def _parse_args(self, args): @@ -582,7 +598,69 @@ class ArgParseConfigLoader(CommandLineConfigLoader): def _convert_to_config(self): """self.parsed_data->self.config""" for k, v in vars(self.parsed_data).iteritems(): - exec_str = 'self.config.' + k + '= v' - exec exec_str in locals(), globals() + exec "self.config.%s = v"%k in locals(), globals() +class KVArgParseConfigLoader(ArgParseConfigLoader): + """A config loader that loads aliases and flags with argparse, + but will use KVLoader for the rest. This allows better parsing + of common args, such as `ipython -c 'print 5'`, but still gets + arbitrary config with `ipython --InteractiveShell.use_readline=False`""" + + def _convert_to_config(self): + """self.parsed_data->self.config""" + for k, v in vars(self.parsed_data).iteritems(): + self._exec_config_str(k, v) + def _add_arguments(self, aliases=None, flags=None): + self.alias_flags = {} + # print aliases, flags + if aliases is None: + aliases = self.aliases + if flags is None: + flags = self.flags + paa = self.parser.add_argument + for key,value in aliases.iteritems(): + if key in flags: + # flags + nargs = '?' + else: + nargs = None + if len(key) is 1: + paa('-'+key, '--'+key, type=str, dest=value, nargs=nargs) + else: + paa('--'+key, type=str, dest=value, nargs=nargs) + for key, (value, help) in flags.iteritems(): + if key in self.aliases: + # + self.alias_flags[self.aliases[key]] = value + continue + if len(key) is 1: + paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value) + else: + paa('--'+key, action='append_const', dest='_flags', const=value) + + def _convert_to_config(self): + """self.parsed_data->self.config, parse unrecognized extra args via KVLoader.""" + # remove subconfigs list from namespace before transforming the Namespace + if '_flags' in self.parsed_data: + subcs = self.parsed_data._flags + del self.parsed_data._flags + else: + subcs = [] + + for k, v in vars(self.parsed_data).iteritems(): + if v is None: + # it was a flag that shares the name of an alias + subcs.append(self.alias_flags[k]) + else: + # eval the KV assignment + self._exec_config_str(k, v) + + for subc in subcs: + self._load_flag(subc) + + if self.extra_args: + sub_parser = KeyValueConfigLoader() + sub_parser.load_config(self.extra_args) + self.config._merge(sub_parser.config) + self.extra_args = sub_parser.extra_args diff --git a/IPython/config/tests/test_loader.py b/IPython/config/tests/test_loader.py index da97f03..6373d37 100755 --- a/IPython/config/tests/test_loader.py +++ b/IPython/config/tests/test_loader.py @@ -69,7 +69,7 @@ class TestPyFileCL(TestCase): self.assertEquals(config.D.C.value, 'hi there') class MyLoader1(ArgParseConfigLoader): - def _add_arguments(self): + def _add_arguments(self, aliases=None, flags=None): p = self.parser p.add_argument('-f', '--foo', dest='Global.foo', type=str) p.add_argument('-b', dest='MyClass.bar', type=int) @@ -77,7 +77,7 @@ class MyLoader1(ArgParseConfigLoader): p.add_argument('Global.bam', type=str) class MyLoader2(ArgParseConfigLoader): - def _add_arguments(self): + def _add_arguments(self, aliases=None, flags=None): subparsers = self.parser.add_subparsers(dest='subparser_name') subparser1 = subparsers.add_parser('1') subparser1.add_argument('-x',dest='Global.x') diff --git a/docs/source/config/overview.txt b/docs/source/config/overview.txt index 1c75b95..3717de3 100644 --- a/docs/source/config/overview.txt +++ b/docs/source/config/overview.txt @@ -386,23 +386,39 @@ Is the same as adding: to your config file. Key/Value arguments *always* take a value, separated by '=' and no spaces. +Common Arguments +**************** + +Since the strictness and verbosity of the KVLoader above are not ideal for everyday +use, common arguments can be specified as flags_ or aliases_. + +Flags and Aliases are handled by :mod:`argparse` instead, allowing for more flexible +parsing. In general, flags and aliases are prefixed by ``--``, except for those +that are single characters, in which case they can be specified with a single ``-``, e.g.: + +.. code-block:: bash + + $> ipython -i -c "import numpy; x=numpy.linspace(0,1)" --profile testing --colors=lightbg + Aliases ------- -For convenience, applications have a mapping of commonly -used traits, so you don't have to specify the whole class name. For these **aliases**, the class need not be specified: +For convenience, applications have a mapping of commonly used traits, so you don't have +to specify the whole class name: .. code-block:: bash + $> ipython --profile myprofile + # and $> ipython --profile='myprofile' - # is equivalent to + # are equivalent to $> ipython --BaseIPythonApplication.profile='myprofile' Flags ----- Applications can also be passed **flags**. Flags are options that take no -arguments, and are always prefixed with ``--``. They are simply wrappers for +arguments. They are simply wrappers for setting one or more configurables with predefined values, often True/False. For instance: @@ -412,13 +428,17 @@ For instance: $> ipcontroller --debug # is equivalent to $> ipcontroller --Application.log_level=DEBUG - # and + # and $> ipython --pylab # is equivalent to $> ipython --pylab=auto + # or + $> ipython --no-banner + # is equivalent to + $> ipython --TerminalIPythonApp.display_banner=False Subcommands ------------ +*********** Some IPython applications have **subcommands**. Subcommands are modeled after diff --git a/docs/source/interactive/reference.txt b/docs/source/interactive/reference.txt index 1ba2f28..550b1ab 100644 --- a/docs/source/interactive/reference.txt +++ b/docs/source/interactive/reference.txt @@ -83,7 +83,7 @@ All options with a [no] prepended can be specified in negated form ``--[no-]banner`` Print the initial information banner (default on). - ``--c=`` + ``-c `` execute the given command string. This is similar to the -c option in the normal Python interpreter. @@ -158,7 +158,7 @@ All options with a [no] prepended can be specified in negated form ipython_log.py in your current directory (which prevents logs from multiple IPython sessions from trampling each other). You can use this to later restore a session by loading your - logfile with ``ipython --i ipython_log.py`` + logfile with ``ipython -i ipython_log.py`` ``--logplay=`` diff --git a/docs/source/parallel/parallel_intro.txt b/docs/source/parallel/parallel_intro.txt index a3d3eeb..842001d 100644 --- a/docs/source/parallel/parallel_intro.txt +++ b/docs/source/parallel/parallel_intro.txt @@ -205,7 +205,7 @@ simply start a controller and engines on a single host using the :command:`ipcluster` command. To start a controller and 4 engines on your localhost, just do:: - $ ipcluster start --n=4 + $ ipcluster start -n 4 More details about starting the IPython controller and engines can be found :ref:`here ` diff --git a/docs/source/parallel/parallel_mpi.txt b/docs/source/parallel/parallel_mpi.txt index 3f6733c..b7d12d0 100644 --- a/docs/source/parallel/parallel_mpi.txt +++ b/docs/source/parallel/parallel_mpi.txt @@ -52,7 +52,7 @@ The easiest approach is to use the `MPIExec` Launchers in :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`:: - $ ipcluster start --n=4 --elauncher=MPIExecEngineSetLauncher + $ ipcluster start -n 4 --elauncher=MPIExecEngineSetLauncher This approach is best as interrupting :command:`ipcluster` will automatically stop and clean up the controller and engines. @@ -105,7 +105,7 @@ distributed array. Save the following text in a file called :file:`psum.py`: Now, start an IPython cluster:: - $ ipcluster start --profile=mpi --n=4 + $ ipcluster start --profile=mpi -n 4 .. note:: diff --git a/docs/source/parallel/parallel_multiengine.txt b/docs/source/parallel/parallel_multiengine.txt index 0d288e2..f8fc5f5 100644 --- a/docs/source/parallel/parallel_multiengine.txt +++ b/docs/source/parallel/parallel_multiengine.txt @@ -19,7 +19,7 @@ To follow along with this tutorial, you will need to start the IPython controller and four IPython engines. The simplest way of doing this is to use the :command:`ipcluster` command:: - $ ipcluster start --n=4 + $ ipcluster start -n 4 For more detailed information about starting the controller and engines, see our :ref:`introduction ` to using IPython for parallel computing. diff --git a/docs/source/parallel/parallel_process.txt b/docs/source/parallel/parallel_process.txt index 851e619..93ab220 100644 --- a/docs/source/parallel/parallel_process.txt +++ b/docs/source/parallel/parallel_process.txt @@ -109,7 +109,7 @@ The simplest way to use ipcluster requires no configuration, and will launch a controller and a number of engines on the local machine. For instance, to start one controller and 4 engines on localhost, just do:: - $ ipcluster start --n=4 + $ ipcluster start -n 4 To see other command line options, do:: @@ -174,7 +174,7 @@ There, instruct ipcluster to use the MPIExec launchers by adding the lines: If the default MPI configuration is correct, then you can now start your cluster, with:: - $ ipcluster start --n=4 --profile=mpi + $ ipcluster start -n 4 --profile=mpi This does the following: @@ -324,7 +324,7 @@ connections on all its interfaces, by adding in :file:`ipcontroller_config`: You can now run the cluster with:: - $ ipcluster start --profile=pbs --n=128 + $ ipcluster start --profile=pbs -n 128 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`. diff --git a/docs/source/parallel/parallel_task.txt b/docs/source/parallel/parallel_task.txt index aa32e34..28d34d6 100644 --- a/docs/source/parallel/parallel_task.txt +++ b/docs/source/parallel/parallel_task.txt @@ -24,7 +24,7 @@ To follow along with this tutorial, you will need to start the IPython controller and four IPython engines. The simplest way of doing this is to use the :command:`ipcluster` command:: - $ ipcluster start --n=4 + $ ipcluster start -n 4 For more detailed information about starting the controller and engines, see our :ref:`introduction ` to using IPython for parallel computing. diff --git a/docs/source/parallel/parallel_winhpc.txt b/docs/source/parallel/parallel_winhpc.txt index 01e5cd1..83dedf7 100644 --- a/docs/source/parallel/parallel_winhpc.txt +++ b/docs/source/parallel/parallel_winhpc.txt @@ -257,7 +257,7 @@ Starting the cluster profile Once a cluster profile has been configured, starting an IPython cluster using the profile is simple:: - ipcluster start --profile=mycluster --n=32 + ipcluster start --profile=mycluster -n 32 The ``-n`` option tells :command:`ipcluster` how many engines to start (in this case 32). Stopping the cluster is as simple as typing Control-C.