diff --git a/IPython/frontend/consoleapp.py b/IPython/frontend/consoleapp.py new file mode 100644 index 0000000..2776eef --- /dev/null +++ b/IPython/frontend/consoleapp.py @@ -0,0 +1,352 @@ +""" A minimal application base mixin for all ZMQ based IPython frontends. + +This is not a complete console app, as subprocess will not be able to receive +input, there is no real readline support, among other limitations. This is a +refactoring of what used to be the IPython/frontend/qt/console/qtconsoleapp.py + +Authors: + +* Evan Patterson +* Min RK +* Erik Tollerud +* Fernando Perez +* Bussonnier Matthias +* Thomas Kluyver +* Paul Ivanov + +""" + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# stdlib imports +import atexit +import json +import os +import signal +import sys +import uuid + + +# Local imports +from IPython.config.application import boolean_flag +from IPython.config.configurable import Configurable +from IPython.core.profiledir import ProfileDir +from IPython.lib.kernel import tunnel_to_kernel, find_connection_file, swallow_argv +from IPython.zmq.blockingkernelmanager import BlockingKernelManager +from IPython.utils.path import filefind +from IPython.utils.py3compat import str_to_bytes +from IPython.utils.traitlets import ( + Dict, List, Unicode, CUnicode, Int, CBool, Any +) +from IPython.zmq.ipkernel import ( + flags as ipkernel_flags, + aliases as ipkernel_aliases, + IPKernelApp +) +from IPython.zmq.session import Session, default_secure +from IPython.zmq.zmqshell import ZMQInteractiveShell + +#----------------------------------------------------------------------------- +# Network Constants +#----------------------------------------------------------------------------- + +from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS + +#----------------------------------------------------------------------------- +# Globals +#----------------------------------------------------------------------------- + + +#----------------------------------------------------------------------------- +# Aliases and Flags +#----------------------------------------------------------------------------- + +flags = dict(ipkernel_flags) + +# the flags that are specific to the frontend +# these must be scrubbed before being passed to the kernel, +# or it will raise an error on unrecognized flags +app_flags = { + 'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}}, + "Connect to an existing kernel. If no argument specified, guess most recent"), +} +app_flags.update(boolean_flag( + 'confirm-exit', 'IPythonConsoleApp.confirm_exit', + """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', + to force a direct exit without any confirmation. + """, + """Don't prompt the user when exiting. This will terminate the kernel + if it is owned by the frontend, and leave it alive if it is external. + """ +)) +flags.update(app_flags) + +aliases = dict(ipkernel_aliases) + +# also scrub aliases from the frontend +app_aliases = dict( + hb = 'IPythonConsoleApp.hb_port', + shell = 'IPythonConsoleApp.shell_port', + iopub = 'IPythonConsoleApp.iopub_port', + stdin = 'IPythonConsoleApp.stdin_port', + ip = 'IPythonConsoleApp.ip', + existing = 'IPythonConsoleApp.existing', + f = 'IPythonConsoleApp.connection_file', + + + ssh = 'IPythonConsoleApp.sshserver', +) +aliases.update(app_aliases) + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# IPythonConsole +#----------------------------------------------------------------------------- + + +class IPythonConsoleApp(Configurable): + name = 'ipython-console-mixin' + default_config_file_name='ipython_config.py' + + description = """ + The IPython Mixin Console. + + This class contains the common portions of console client (QtConsole, + ZMQ-based terminal console, etc). It is not a full console, in that + launched terminal subprocesses will not be able to accept input. + + The Console using this mixing supports various extra features beyond + the single-process Terminal IPython shell, such as connecting to + existing kernel, via: + + ipython --existing + + as well as tunnel via SSH + + """ + + classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session] + flags = Dict(flags) + aliases = Dict(aliases) + kernel_manager_class = BlockingKernelManager + + kernel_argv = List(Unicode) + # frontend flags&aliases to be stripped when building kernel_argv + frontend_flags = Any(app_flags) + frontend_aliases = Any(app_aliases) + + pure = CBool(False, config=True, + help="Use a pure Python kernel instead of an IPython kernel.") + # create requested profiles by default, if they don't exist: + auto_create = CBool(True) + # connection info: + ip = Unicode(LOCALHOST, config=True, + help="""Set the kernel\'s IP address [default localhost]. + If the IP address is something other than localhost, then + Consoles on other machines will be able to connect + to the Kernel, so be careful!""" + ) + + sshserver = Unicode('', config=True, + help="""The SSH server to use to connect to the kernel.""") + sshkey = Unicode('', config=True, + help="""Path to the ssh key to use for logging in to the ssh server.""") + + hb_port = Int(0, config=True, + help="set the heartbeat port [default: random]") + shell_port = Int(0, config=True, + help="set the shell (XREP) port [default: random]") + iopub_port = Int(0, config=True, + help="set the iopub (PUB) port [default: random]") + stdin_port = Int(0, config=True, + help="set the stdin (XREQ) port [default: random]") + connection_file = Unicode('', config=True, + help="""JSON file in which to store connection info [default: kernel-.json] + + This file will contain the IP, ports, and authentication key needed to connect + clients to this kernel. By default, this file will be created in the security-dir + of the current profile, but can be specified by absolute path. + """) + def _connection_file_default(self): + return 'kernel-%i.json' % os.getpid() + + existing = CUnicode('', config=True, + help="""Connect to an already running kernel""") + + confirm_exit = CBool(True, config=True, + help=""" + Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', + to force a direct exit without any confirmation.""", + ) + + + def build_kernel_argv(self, argv=None): + """build argv to be passed to kernel subprocess""" + if argv is None: + argv = sys.argv[1:] + self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags) + # kernel should inherit default config file from frontend + self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name) + + def init_connection_file(self): + """find the connection file, and load the info if found. + + The current working directory and the current profile's security + directory will be searched for the file if it is not given by + absolute path. + + When attempting to connect to an existing kernel and the `--existing` + argument does not match an existing file, it will be interpreted as a + fileglob, and the matching file in the current profile's security dir + with the latest access time will be used. + + After this method is called, self.connection_file contains the *full path* + to the connection file, never just its name. + """ + if self.existing: + try: + cf = find_connection_file(self.existing) + except Exception: + self.log.critical("Could not find existing kernel connection file %s", self.existing) + self.exit(1) + self.log.info("Connecting to existing kernel: %s" % cf) + self.connection_file = cf + else: + # not existing, check if we are going to write the file + # and ensure that self.connection_file is a full path, not just the shortname + try: + cf = find_connection_file(self.connection_file) + except Exception: + # file might not exist + if self.connection_file == os.path.basename(self.connection_file): + # just shortname, put it in security dir + cf = os.path.join(self.profile_dir.security_dir, self.connection_file) + else: + cf = self.connection_file + self.connection_file = cf + + # should load_connection_file only be used for existing? + # as it is now, this allows reusing ports if an existing + # file is requested + try: + self.load_connection_file() + except Exception: + self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) + self.exit(1) + + def load_connection_file(self): + """load ip/port/hmac config from JSON connection file""" + # this is identical to KernelApp.load_connection_file + # perhaps it can be centralized somewhere? + try: + fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) + except IOError: + self.log.debug("Connection File not found: %s", self.connection_file) + return + self.log.debug(u"Loading connection file %s", fname) + with open(fname) as f: + s = f.read() + cfg = json.loads(s) + if self.ip == LOCALHOST and 'ip' in cfg: + # not overridden by config or cl_args + self.ip = cfg['ip'] + for channel in ('hb', 'shell', 'iopub', 'stdin'): + name = channel + '_port' + if getattr(self, name) == 0 and name in cfg: + # not overridden by config or cl_args + setattr(self, name, cfg[name]) + if 'key' in cfg: + self.config.Session.key = str_to_bytes(cfg['key']) + + def init_ssh(self): + """set up ssh tunnels, if needed.""" + if not self.sshserver and not self.sshkey: + return + + if self.sshkey and not self.sshserver: + # specifying just the key implies that we are connecting directly + self.sshserver = self.ip + self.ip = LOCALHOST + + # build connection dict for tunnels: + info = dict(ip=self.ip, + shell_port=self.shell_port, + iopub_port=self.iopub_port, + stdin_port=self.stdin_port, + hb_port=self.hb_port + ) + + self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver)) + + # tunnels return a new set of ports, which will be on localhost: + self.ip = LOCALHOST + try: + newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) + except: + # even catch KeyboardInterrupt + self.log.error("Could not setup tunnels", exc_info=True) + self.exit(1) + + self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports + + cf = self.connection_file + base,ext = os.path.splitext(cf) + base = os.path.basename(base) + self.connection_file = os.path.basename(base)+'-ssh'+ext + self.log.critical("To connect another client via this tunnel, use:") + self.log.critical("--existing %s" % self.connection_file) + + def _new_connection_file(self): + cf = '' + while not cf: + # we don't need a 128b id to distinguish kernels, use more readable + # 48b node segment (12 hex chars). Users running more than 32k simultaneous + # kernels can subclass. + ident = str(uuid.uuid4()).split('-')[-1] + cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident) + # only keep if it's actually new. Protect against unlikely collision + # in 48b random search space + cf = cf if not os.path.exists(cf) else '' + return cf + + def init_kernel_manager(self): + # Don't let Qt or ZMQ swallow KeyboardInterupts. + signal.signal(signal.SIGINT, signal.SIG_DFL) + + # Create a KernelManager and start a kernel. + self.kernel_manager = self.kernel_manager_class( + ip=self.ip, + shell_port=self.shell_port, + iopub_port=self.iopub_port, + stdin_port=self.stdin_port, + hb_port=self.hb_port, + connection_file=self.connection_file, + config=self.config, + ) + # start the kernel + if not self.existing: + kwargs = dict(ipython=not self.pure) + kwargs['extra_arguments'] = self.kernel_argv + self.kernel_manager.start_kernel(**kwargs) + elif self.sshserver: + # ssh, write new connection file + self.kernel_manager.write_connection_file() + atexit.register(self.kernel_manager.cleanup_connection_file) + self.kernel_manager.start_channels() + + + def initialize(self, argv=None): + """ + Classes which mix this class in should call: + IPythonConsoleApp.initialize(self,argv) + """ + self.init_connection_file() + default_secure(self.config) + self.init_ssh() + self.init_kernel_manager() + diff --git a/IPython/frontend/html/notebook/notebookapp.py b/IPython/frontend/html/notebook/notebookapp.py index d2bdcf6..e7a00c0 100644 --- a/IPython/frontend/html/notebook/notebookapp.py +++ b/IPython/frontend/html/notebook/notebookapp.py @@ -55,6 +55,7 @@ from .notebookmanager import NotebookManager from IPython.config.application import catch_config_error from IPython.core.application import BaseIPythonApplication from IPython.core.profiledir import ProfileDir +from IPython.lib.kernel import swallow_argv from IPython.zmq.session import Session, default_secure from IPython.zmq.zmqshell import ZMQInteractiveShell from IPython.zmq.ipkernel import ( @@ -283,27 +284,10 @@ class NotebookApp(BaseIPythonApplication): if argv is None: argv = sys.argv[1:] - self.kernel_argv = list(argv) # copy + # Scrub frontend-specific flags + self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags) # Kernel should inherit default config file from frontend self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name) - # Scrub frontend-specific flags - for a in argv: - if a.startswith('-') and a.lstrip('-') in notebook_flags: - self.kernel_argv.remove(a) - swallow_next = False - for a in argv: - if swallow_next: - self.kernel_argv.remove(a) - swallow_next = False - continue - if a.startswith('-'): - split = a.lstrip('-').split('=') - alias = split[0] - if alias in notebook_aliases: - self.kernel_argv.remove(a) - if len(split) == 1: - # alias passed with arg via space - swallow_next = True def init_configurables(self): # Don't let Qt or ZMQ swallow KeyboardInterupts. diff --git a/IPython/frontend/html/notebook/tests/test_kernelsession.py b/IPython/frontend/html/notebook/tests/test_kernelsession.py index 10b37fc..8b457f5 100644 --- a/IPython/frontend/html/notebook/tests/test_kernelsession.py +++ b/IPython/frontend/html/notebook/tests/test_kernelsession.py @@ -22,5 +22,6 @@ class TestKernelManager(TestCase): self.assert_('shell_port' in port_dict) self.assert_('hb_port' in port_dict) km.get_kernel(kid) + km.kill_kernel(kid) diff --git a/IPython/frontend/qt/console/qtconsoleapp.py b/IPython/frontend/qt/console/qtconsoleapp.py index bd4cec2..7b3e608 100644 --- a/IPython/frontend/qt/console/qtconsoleapp.py +++ b/IPython/frontend/qt/console/qtconsoleapp.py @@ -11,6 +11,7 @@ Authors: * Fernando Perez * Bussonnier Matthias * Thomas Kluyver +* Paul Ivanov """ @@ -26,7 +27,7 @@ import sys import uuid # System library imports -from IPython.external.qt import QtGui +from IPython.external.qt import QtCore, QtGui # Local imports from IPython.config.application import boolean_flag, catch_config_error @@ -44,14 +45,14 @@ from IPython.utils.py3compat import str_to_bytes from IPython.utils.traitlets import ( Dict, List, Unicode, Integer, CaselessStrEnum, CBool, Any ) -from IPython.zmq.ipkernel import ( - flags as ipkernel_flags, - aliases as ipkernel_aliases, - IPKernelApp -) +from IPython.zmq.ipkernel import IPKernelApp from IPython.zmq.session import Session, default_secure from IPython.zmq.zmqshell import ZMQInteractiveShell +from IPython.frontend.consoleapp import ( + IPythonConsoleApp, app_aliases, app_flags, flags, aliases + ) + #----------------------------------------------------------------------------- # Network Constants #----------------------------------------------------------------------------- @@ -71,10 +72,9 @@ ipython qtconsole --pylab=inline # start with pylab in inline plotting mode # Aliases and Flags #----------------------------------------------------------------------------- -flags = dict(ipkernel_flags) +# start with copy of flags +flags = dict(flags) qt_flags = { - 'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}}, - "Connect to an existing kernel. If no argument specified, guess most recent"), 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}}, "Use a pure Python kernel instead of an IPython kernel."), 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}}, @@ -85,27 +85,14 @@ qt_flags.update(boolean_flag( "use a GUI widget for tab completion", "use plaintext output for completion" )) -qt_flags.update(boolean_flag( - 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit', - """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', - to force a direct exit without any confirmation. - """, - """Don't prompt the user when exiting. This will terminate the kernel - if it is owned by the frontend, and leave it alive if it is external. - """ -)) +# and app_flags from the Console Mixin +qt_flags.update(app_flags) +# add frontend flags to the full set flags.update(qt_flags) -aliases = dict(ipkernel_aliases) - +# start with copy of front&backend aliases list +aliases = dict(aliases) qt_aliases = dict( - hb = 'IPythonQtConsoleApp.hb_port', - shell = 'IPythonQtConsoleApp.shell_port', - iopub = 'IPythonQtConsoleApp.iopub_port', - stdin = 'IPythonQtConsoleApp.stdin_port', - ip = 'IPythonQtConsoleApp.ip', - existing = 'IPythonQtConsoleApp.existing', - f = 'IPythonQtConsoleApp.connection_file', style = 'IPythonWidget.syntax_style', stylesheet = 'IPythonQtConsoleApp.stylesheet', @@ -113,10 +100,18 @@ qt_aliases = dict( editor = 'IPythonWidget.editor', paging = 'ConsoleWidget.paging', - ssh = 'IPythonQtConsoleApp.sshserver', ) +# and app_aliases from the Console Mixin +qt_aliases.update(app_aliases) +# add frontend aliases to the full set aliases.update(qt_aliases) +# get flags&aliases into sets, and remove a couple that +# shouldn't be scrubbed from backend flags: +qt_aliases = set(qt_aliases.keys()) +qt_aliases.remove('colors') +qt_flags = set(qt_flags.keys()) + #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- @@ -126,9 +121,8 @@ aliases.update(qt_aliases) #----------------------------------------------------------------------------- -class IPythonQtConsoleApp(BaseIPythonApplication): +class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp): name = 'ipython-qtconsole' - default_config_file_name='ipython_config.py' description = """ The IPython QtConsole. @@ -150,50 +144,13 @@ class IPythonQtConsoleApp(BaseIPythonApplication): classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session] flags = Dict(flags) aliases = Dict(aliases) - - kernel_argv = List(Unicode) - - # create requested profiles by default, if they don't exist: - auto_create = CBool(True) - # connection info: - ip = Unicode(LOCALHOST, config=True, - help="""Set the kernel\'s IP address [default localhost]. - If the IP address is something other than localhost, then - Consoles on other machines will be able to connect - to the Kernel, so be careful!""" - ) - - sshserver = Unicode('', config=True, - help="""The SSH server to use to connect to the kernel.""") - sshkey = Unicode('', config=True, - help="""Path to the ssh key to use for logging in to the ssh server.""") - - hb_port = Integer(0, config=True, - help="set the heartbeat port [default: random]") - shell_port = Integer(0, config=True, - help="set the shell (XREP) port [default: random]") - iopub_port = Integer(0, config=True, - help="set the iopub (PUB) port [default: random]") - stdin_port = Integer(0, config=True, - help="set the stdin (XREQ) port [default: random]") - connection_file = Unicode('', config=True, - help="""JSON file in which to store connection info [default: kernel-.json] - - This file will contain the IP, ports, and authentication key needed to connect - clients to this kernel. By default, this file will be created in the security-dir - of the current profile, but can be specified by absolute path. - """) - def _connection_file_default(self): - return 'kernel-%i.json' % os.getpid() - - existing = Unicode('', config=True, - help="""Connect to an already running kernel""") + frontend_flags = Any(qt_flags) + frontend_aliases = Any(qt_aliases) + kernel_manager_class = QtKernelManager stylesheet = Unicode('', config=True, help="path to a custom CSS stylesheet") - pure = CBool(False, config=True, - help="Use a pure Python kernel instead of an IPython kernel.") plain = CBool(False, config=True, help="Use a plaintext widget instead of rich text (plain can't print/save).") @@ -209,178 +166,13 @@ class IPythonQtConsoleApp(BaseIPythonApplication): _plain_changed = _pure_changed - confirm_exit = CBool(True, config=True, - help=""" - Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', - to force a direct exit without any confirmation.""", - ) - # the factory for creating a widget widget_factory = Any(RichIPythonWidget) def parse_command_line(self, argv=None): super(IPythonQtConsoleApp, self).parse_command_line(argv) - if argv is None: - argv = sys.argv[1:] - self.kernel_argv = list(argv) # copy - # kernel should inherit default config file from frontend - self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name) - # Scrub frontend-specific flags - swallow_next = False - was_flag = False - # copy again, in case some aliases have the same name as a flag - # argv = list(self.kernel_argv) - for a in argv: - if swallow_next: - swallow_next = False - # last arg was an alias, remove the next one - # *unless* the last alias has a no-arg flag version, in which - # case, don't swallow the next arg if it's also a flag: - if not (was_flag and a.startswith('-')): - self.kernel_argv.remove(a) - continue - if a.startswith('-'): - split = a.lstrip('-').split('=') - alias = split[0] - if alias in qt_aliases: - self.kernel_argv.remove(a) - if len(split) == 1: - # alias passed with arg via space - swallow_next = True - # could have been a flag that matches an alias, e.g. `existing` - # in which case, we might not swallow the next arg - was_flag = alias in qt_flags - elif alias in qt_flags: - # strip flag, but don't swallow next, as flags don't take args - self.kernel_argv.remove(a) - - def init_connection_file(self): - """find the connection file, and load the info if found. - - The current working directory and the current profile's security - directory will be searched for the file if it is not given by - absolute path. - - When attempting to connect to an existing kernel and the `--existing` - argument does not match an existing file, it will be interpreted as a - fileglob, and the matching file in the current profile's security dir - with the latest access time will be used. - """ - if self.existing: - try: - cf = find_connection_file(self.existing) - except Exception: - self.log.critical("Could not find existing kernel connection file %s", self.existing) - self.exit(1) - self.log.info("Connecting to existing kernel: %s" % cf) - self.connection_file = cf - # should load_connection_file only be used for existing? - # as it is now, this allows reusing ports if an existing - # file is requested - try: - self.load_connection_file() - except Exception: - self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) - self.exit(1) - - def load_connection_file(self): - """load ip/port/hmac config from JSON connection file""" - # this is identical to KernelApp.load_connection_file - # perhaps it can be centralized somewhere? - try: - fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) - except IOError: - self.log.debug("Connection File not found: %s", self.connection_file) - return - self.log.debug(u"Loading connection file %s", fname) - with open(fname) as f: - s = f.read() - cfg = json.loads(s) - if self.ip == LOCALHOST and 'ip' in cfg: - # not overridden by config or cl_args - self.ip = cfg['ip'] - for channel in ('hb', 'shell', 'iopub', 'stdin'): - name = channel + '_port' - if getattr(self, name) == 0 and name in cfg: - # not overridden by config or cl_args - setattr(self, name, cfg[name]) - if 'key' in cfg: - self.config.Session.key = str_to_bytes(cfg['key']) - - def init_ssh(self): - """set up ssh tunnels, if needed.""" - if not self.sshserver and not self.sshkey: - return - - if self.sshkey and not self.sshserver: - # specifying just the key implies that we are connecting directly - self.sshserver = self.ip - self.ip = LOCALHOST - - # build connection dict for tunnels: - info = dict(ip=self.ip, - shell_port=self.shell_port, - iopub_port=self.iopub_port, - stdin_port=self.stdin_port, - hb_port=self.hb_port - ) - - self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver)) - - # tunnels return a new set of ports, which will be on localhost: - self.ip = LOCALHOST - try: - newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) - except: - # even catch KeyboardInterrupt - self.log.error("Could not setup tunnels", exc_info=True) - self.exit(1) - - self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports - - cf = self.connection_file - base,ext = os.path.splitext(cf) - base = os.path.basename(base) - self.connection_file = os.path.basename(base)+'-ssh'+ext - self.log.critical("To connect another client via this tunnel, use:") - self.log.critical("--existing %s" % self.connection_file) - - def _new_connection_file(self): - return os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % uuid.uuid4()) - - def init_kernel_manager(self): - # Don't let Qt or ZMQ swallow KeyboardInterupts. - signal.signal(signal.SIGINT, signal.SIG_DFL) - sec = self.profile_dir.security_dir - try: - cf = filefind(self.connection_file, ['.', sec]) - except IOError: - # file might not exist - if self.connection_file == os.path.basename(self.connection_file): - # just shortname, put it in security dir - cf = os.path.join(sec, self.connection_file) - else: - cf = self.connection_file - - # Create a KernelManager and start a kernel. - self.kernel_manager = QtKernelManager( - ip=self.ip, - shell_port=self.shell_port, - iopub_port=self.iopub_port, - stdin_port=self.stdin_port, - hb_port=self.hb_port, - connection_file=cf, - config=self.config, - ) - # start the kernel - if not self.existing: - kwargs = dict(ipython=not self.pure) - kwargs['extra_arguments'] = self.kernel_argv - self.kernel_manager.start_kernel(**kwargs) - elif self.sshserver: - # ssh, write new connection file - self.kernel_manager.write_connection_file() - self.kernel_manager.start_channels() + self.build_kernel_argv(argv) + def new_frontend_master(self): """ Create and return new frontend attached to new kernel, launched on localhost. @@ -517,15 +309,26 @@ class IPythonQtConsoleApp(BaseIPythonApplication): else: raise IOError("Stylesheet %r not found."%self.stylesheet) + def init_signal(self): + """allow clean shutdown on sigint""" + signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-2)) + # need a timer, so that QApplication doesn't block until a real + # Qt event fires (can require mouse movement) + # timer trick from http://stackoverflow.com/q/4938723/938949 + timer = QtCore.QTimer() + # Let the interpreter run each 200 ms: + timer.timeout.connect(lambda: None) + timer.start(200) + # hold onto ref, so the timer doesn't get cleaned up + self._sigint_timer = timer + @catch_config_error def initialize(self, argv=None): super(IPythonQtConsoleApp, self).initialize(argv) - self.init_connection_file() - default_secure(self.config) - self.init_ssh() - self.init_kernel_manager() + IPythonConsoleApp.initialize(self,argv) self.init_qt_elements() self.init_colors() + self.init_signal() def start(self): diff --git a/IPython/frontend/terminal/console/__init__.py b/IPython/frontend/terminal/console/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/IPython/frontend/terminal/console/__init__.py diff --git a/IPython/frontend/terminal/console/app.py b/IPython/frontend/terminal/console/app.py new file mode 100644 index 0000000..b836abe --- /dev/null +++ b/IPython/frontend/terminal/console/app.py @@ -0,0 +1,152 @@ +""" A minimal application using the ZMQ-based terminal IPython frontend. + +This is not a complete console app, as subprocess will not be able to receive +input, there is no real readline support, among other limitations. + +Authors: + +* Min RK +* Paul Ivanov + +""" + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +import signal +import sys +import time + +from IPython.frontend.terminal.ipapp import TerminalIPythonApp, frontend_flags as term_flags + +from IPython.utils.traitlets import ( + Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any +) +from IPython.utils.warn import warn,error + +from IPython.zmq.ipkernel import IPKernelApp +from IPython.zmq.session import Session, default_secure +from IPython.zmq.zmqshell import ZMQInteractiveShell +from IPython.frontend.consoleapp import ( + IPythonConsoleApp, app_aliases, app_flags, aliases, app_aliases, flags + ) + +from IPython.frontend.terminal.console.interactiveshell import ZMQTerminalInteractiveShell + +#----------------------------------------------------------------------------- +# Globals +#----------------------------------------------------------------------------- + +_examples = """ +ipython console # start the ZMQ-based console +ipython console --existing # connect to an existing ipython session +""" + +#----------------------------------------------------------------------------- +# Flags and Aliases +#----------------------------------------------------------------------------- + +# copy flags from mixin: +flags = dict(flags) +# start with mixin frontend flags: +frontend_flags = dict(app_flags) +# add TerminalIPApp flags: +frontend_flags.update(term_flags) +# pylab is not frontend-specific in two-process IPython +frontend_flags.pop('pylab') +# disable quick startup, as it won't propagate to the kernel anyway +frontend_flags.pop('quick') +# update full dict with frontend flags: +flags.update(frontend_flags) + +# copy flags from mixin +aliases = dict(aliases) +# start with mixin frontend flags +frontend_aliases = dict(app_aliases) +# load updated frontend flags into full dict +aliases.update(frontend_aliases) + +# get flags&aliases into sets, and remove a couple that +# shouldn't be scrubbed from backend flags: +frontend_aliases = set(frontend_aliases.keys()) +frontend_flags = set(frontend_flags.keys()) + + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- + + +class ZMQTerminalIPythonApp(TerminalIPythonApp, IPythonConsoleApp): + name = "ipython-console" + """Start a terminal frontend to the IPython zmq kernel.""" + + description = """ + The IPython terminal-based Console. + + This launches a Console application inside a terminal. + + The Console supports various extra features beyond the traditional + single-process Terminal IPython shell, such as connecting to an + existing ipython session, via: + + ipython console --existing + + where the previous session could have been created by another ipython + console, an ipython qtconsole, or by opening an ipython notebook. + + """ + examples = _examples + + classes = List([IPKernelApp, ZMQTerminalInteractiveShell, Session]) + flags = Dict(flags) + aliases = Dict(aliases) + frontend_aliases = Any(frontend_aliases) + frontend_flags = Any(frontend_flags) + + subcommands = Dict() + + def parse_command_line(self, argv=None): + super(ZMQTerminalIPythonApp, self).parse_command_line(argv) + self.build_kernel_argv(argv) + + def init_shell(self): + IPythonConsoleApp.initialize(self) + # relay sigint to kernel + signal.signal(signal.SIGINT, self.handle_sigint) + self.shell = ZMQTerminalInteractiveShell.instance(config=self.config, + display_banner=False, profile_dir=self.profile_dir, + ipython_dir=self.ipython_dir, kernel_manager=self.kernel_manager) + + def handle_sigint(self, *args): + if self.shell._executing: + if self.kernel_manager.has_kernel: + # interrupt already gets passed to subprocess by signal handler. + # Only if we prevent that should we need to explicitly call + # interrupt_kernel, until which time, this would result in a + # double-interrupt: + # self.kernel_manager.interrupt_kernel() + pass + else: + self.shell.write_err('\n') + error("Cannot interrupt kernels we didn't start.\n") + else: + # raise the KeyboardInterrupt if we aren't waiting for execution, + # so that the interact loop advances, and prompt is redrawn, etc. + raise KeyboardInterrupt + + + def init_code(self): + # no-op in the frontend, code gets run in the backend + pass + +def launch_new_instance(): + """Create and run a full blown IPython instance""" + app = ZMQTerminalIPythonApp.instance() + app.initialize() + app.start() + + +if __name__ == '__main__': + launch_new_instance() + diff --git a/IPython/frontend/terminal/console/completer.py b/IPython/frontend/terminal/console/completer.py new file mode 100644 index 0000000..817ee18 --- /dev/null +++ b/IPython/frontend/terminal/console/completer.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +import readline +from Queue import Empty + +class ZMQCompleter(object): + """Client-side completion machinery. + + How it works: self.complete will be called multiple times, with + state=0,1,2,... When state=0 it should compute ALL the completion matches, + and then return them for each value of state.""" + + def __init__(self, shell, km): + self.shell = shell + self.km = km + self.matches = [] + + def complete_request(self,text): + line = readline.get_line_buffer() + cursor_pos = readline.get_endidx() + + # send completion request to kernel + # Give the kernel up to 0.5s to respond + msg_id = self.km.shell_channel.complete(text=text, line=line, + cursor_pos=cursor_pos) + + msg = self.km.shell_channel.get_msg(timeout=0.5) + if msg['parent_header']['msg_id'] == msg_id: + return msg["content"]["matches"] + return [] + + def rlcomplete(self, text, state): + if state == 0: + try: + self.matches = self.complete_request(text) + except Empty: + print('WARNING: Kernel timeout on tab completion.') + + try: + return self.matches[state] + except IndexError: + return None + + def complete(self, text, line, cursor_pos=None): + return self.rlcomplete(text, 0) diff --git a/IPython/frontend/terminal/console/interactiveshell.py b/IPython/frontend/terminal/console/interactiveshell.py new file mode 100644 index 0000000..1c9075a --- /dev/null +++ b/IPython/frontend/terminal/console/interactiveshell.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- +"""Frontend of ipython working with python-zmq + +Ipython's frontend, is a ipython interface that send request to kernel and proccess the kernel's outputs. + +For more details, see the ipython-zmq design +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +from __future__ import print_function + +import bdb +import signal +import sys +import time + +from Queue import Empty + +from IPython.core.alias import AliasManager, AliasError +from IPython.core import page +from IPython.utils.warn import warn, error, fatal +from IPython.utils import io + +from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell +from IPython.frontend.terminal.console.completer import ZMQCompleter + + +class ZMQTerminalInteractiveShell(TerminalInteractiveShell): + """A subclass of TerminalInteractiveShell that uses the 0MQ kernel""" + _executing = False + + def __init__(self, *args, **kwargs): + self.km = kwargs.pop('kernel_manager') + self.session_id = self.km.session.session + super(ZMQTerminalInteractiveShell, self).__init__(*args, **kwargs) + + def init_completer(self): + """Initialize the completion machinery. + + This creates completion machinery that can be used by client code, + either interactively in-process (typically triggered by the readline + library), programatically (such as in test suites) or out-of-prcess + (typically over the network by remote frontends). + """ + from IPython.core.completerlib import (module_completer, + magic_run_completer, cd_completer) + + self.Completer = ZMQCompleter(self, self.km) + + + self.set_hook('complete_command', module_completer, str_key = 'import') + self.set_hook('complete_command', module_completer, str_key = 'from') + self.set_hook('complete_command', magic_run_completer, str_key = '%run') + self.set_hook('complete_command', cd_completer, str_key = '%cd') + + # Only configure readline if we truly are using readline. IPython can + # do tab-completion over the network, in GUIs, etc, where readline + # itself may be absent + if self.has_readline: + self.set_readline_completer() + + def run_cell(self, cell, store_history=True): + """Run a complete IPython cell. + + Parameters + ---------- + cell : str + The code (including IPython code such as %magic functions) to run. + store_history : bool + If True, the raw and translated cell will be stored in IPython's + history. For user code calling back into IPython's machinery, this + should be set to False. + """ + if (not cell) or cell.isspace(): + return + + if cell.strip() == 'exit': + # explicitly handle 'exit' command + return self.ask_exit() + + self._executing = True + # flush stale replies, which could have been ignored, due to missed heartbeats + while self.km.shell_channel.msg_ready(): + self.km.shell_channel.get_msg() + # shell_channel.execute takes 'hidden', which is the inverse of store_hist + msg_id = self.km.shell_channel.execute(cell, not store_history) + while not self.km.shell_channel.msg_ready() and self.km.is_alive: + try: + self.handle_stdin_request(timeout=0.05) + except Empty: + # display intermediate print statements, etc. + self.handle_iopub() + pass + if self.km.shell_channel.msg_ready(): + self.handle_execute_reply(msg_id) + self._executing = False + + #----------------- + # message handlers + #----------------- + + def handle_execute_reply(self, msg_id): + msg = self.km.shell_channel.get_msg() + if msg["parent_header"].get("msg_id", None) == msg_id: + + self.handle_iopub() + + content = msg["content"] + status = content['status'] + + if status == 'aborted': + self.write('Aborted\n') + return + elif status == 'ok': + # print execution payloads as well: + for item in content["payload"]: + text = item.get('text', None) + if text: + page.page(text) + + elif status == 'error': + for frame in content["traceback"]: + print(frame, file=io.stderr) + + self.execution_count = int(content["execution_count"] + 1) + + + def handle_iopub(self): + """ Method to procces subscribe channel's messages + + This method reads a message and processes the content in different + outputs like stdout, stderr, pyout and status + + Arguments: + sub_msg: message receive from kernel in the sub socket channel + capture by kernel manager. + """ + while self.km.sub_channel.msg_ready(): + sub_msg = self.km.sub_channel.get_msg() + msg_type = sub_msg['header']['msg_type'] + parent = sub_msg["parent_header"] + if (not parent) or self.session_id == parent['session']: + if msg_type == 'status' : + if sub_msg["content"]["execution_state"] == "busy" : + pass + + elif msg_type == 'stream' : + if sub_msg["content"]["name"] == "stdout": + print(sub_msg["content"]["data"], file=io.stdout, end="") + io.stdout.flush() + elif sub_msg["content"]["name"] == "stderr" : + print(sub_msg["content"]["data"], file=io.stderr, end="") + io.stderr.flush() + + elif msg_type == 'pyout': + self.execution_count = int(sub_msg["content"]["execution_count"]) + format_dict = sub_msg["content"]["data"] + # taken from DisplayHook.__call__: + hook = self.displayhook + hook.start_displayhook() + hook.write_output_prompt() + hook.write_format_data(format_dict) + hook.log_output(format_dict) + hook.finish_displayhook() + + def handle_stdin_request(self, timeout=0.1): + """ Method to capture raw_input + """ + msg_rep = self.km.stdin_channel.get_msg(timeout=timeout) + # in case any iopub came while we were waiting: + self.handle_iopub() + if self.session_id == msg_rep["parent_header"].get("session"): + # wrap SIGINT handler + real_handler = signal.getsignal(signal.SIGINT) + def double_int(sig,frame): + # call real handler (forwards sigint to kernel), + # then raise local interrupt, stopping local raw_input + real_handler(sig,frame) + raise KeyboardInterrupt + signal.signal(signal.SIGINT, double_int) + + try: + raw_data = raw_input(msg_rep["content"]["prompt"]) + except EOFError: + # turn EOFError into EOF character + raw_data = '\x04' + except KeyboardInterrupt: + sys.stdout.write('\n') + return + finally: + # restore SIGINT handler + signal.signal(signal.SIGINT, real_handler) + + # only send stdin reply if there *was not* another request + # or execution finished while we were reading. + if not (self.km.stdin_channel.msg_ready() or self.km.shell_channel.msg_ready()): + self.km.stdin_channel.input(raw_data) + + def mainloop(self, display_banner=False): + while True: + try: + self.interact(display_banner=display_banner) + #self.interact_with_readline() + # XXX for testing of a readline-decoupled repl loop, call + # interact_with_readline above + break + except KeyboardInterrupt: + # this should not be necessary, but KeyboardInterrupt + # handling seems rather unpredictable... + self.write("\nKeyboardInterrupt in interact()\n") + + def wait_for_kernel(self, timeout=None): + """method to wait for a kernel to be ready""" + tic = time.time() + self.km.hb_channel.unpause() + while True: + self.run_cell('1', False) + if self.km.hb_channel.is_beating(): + # heart failure was not the reason this returned + break + else: + # heart failed + if timeout is not None and (time.time() - tic) > timeout: + return False + return True + + def interact(self, display_banner=None): + """Closely emulate the interactive Python console.""" + + # batch run -> do not interact + if self.exit_now: + return + + if display_banner is None: + display_banner = self.display_banner + + if isinstance(display_banner, basestring): + self.show_banner(display_banner) + elif display_banner: + self.show_banner() + + more = False + + # run a non-empty no-op, so that we don't get a prompt until + # we know the kernel is ready. This keeps the connection + # message above the first prompt. + if not self.wait_for_kernel(3): + error("Kernel did not respond\n") + return + + if self.has_readline: + self.readline_startup_hook(self.pre_readline) + hlen_b4_cell = self.readline.get_current_history_length() + else: + hlen_b4_cell = 0 + # exit_now is set by a call to %Exit or %Quit, through the + # ask_exit callback. + + while not self.exit_now: + if not self.km.is_alive: + # kernel died, prompt for action or exit + action = "restart" if self.km.has_kernel else "wait for restart" + ans = self.ask_yes_no("kernel died, %s ([y]/n)?" % action, default='y') + if ans: + if self.km.has_kernel: + self.km.restart_kernel(True) + self.wait_for_kernel(3) + else: + self.exit_now = True + continue + try: + # protect prompt block from KeyboardInterrupt + # when sitting on ctrl-C + self.hooks.pre_prompt_hook() + if more: + try: + prompt = self.prompt_manager.render('in2') + except Exception: + self.showtraceback() + if self.autoindent: + self.rl_do_indent = True + + else: + try: + prompt = self.separate_in + self.prompt_manager.render('in') + except Exception: + self.showtraceback() + + line = self.raw_input(prompt) + if self.exit_now: + # quick exit on sys.std[in|out] close + break + if self.autoindent: + self.rl_do_indent = False + + except KeyboardInterrupt: + #double-guard against keyboardinterrupts during kbdint handling + try: + self.write('\nKeyboardInterrupt\n') + source_raw = self.input_splitter.source_raw_reset()[1] + hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell) + more = False + except KeyboardInterrupt: + pass + except EOFError: + if self.autoindent: + self.rl_do_indent = False + if self.has_readline: + self.readline_startup_hook(None) + self.write('\n') + self.exit() + except bdb.BdbQuit: + warn('The Python debugger has exited with a BdbQuit exception.\n' + 'Because of how pdb handles the stack, it is impossible\n' + 'for IPython to properly format this particular exception.\n' + 'IPython will resume normal operation.') + except: + # exceptions here are VERY RARE, but they can be triggered + # asynchronously by signal handlers, for example. + self.showtraceback() + else: + self.input_splitter.push(line) + more = self.input_splitter.push_accepts_more() + if (self.SyntaxTB.last_syntax_error and + self.autoedit_syntax): + self.edit_syntax_error() + if not more: + source_raw = self.input_splitter.source_reset() + hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell) + self.run_cell(source_raw) + + + # Turn off the exit flag, so the mainloop can be restarted if desired + self.exit_now = False diff --git a/IPython/frontend/terminal/console/tests/__init__.py b/IPython/frontend/terminal/console/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/IPython/frontend/terminal/console/tests/__init__.py diff --git a/IPython/frontend/terminal/console/tests/test_console.py b/IPython/frontend/terminal/console/tests/test_console.py new file mode 100644 index 0000000..5dd3f33 --- /dev/null +++ b/IPython/frontend/terminal/console/tests/test_console.py @@ -0,0 +1,59 @@ +"""Tests for two-process terminal frontend + +Currenlty only has the most simple test possible, starting a console and running +a single command. + +Authors: + +* Min RK +""" + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import time + +import nose.tools as nt +from nose import SkipTest + +from IPython.testing import decorators as dec +from IPython.testing import tools as tt +from IPython.utils import py3compat +from IPython.utils.process import find_cmd + +#----------------------------------------------------------------------------- +# Test functions begin +#----------------------------------------------------------------------------- + +@dec.skip_win32 +def test_console_starts(): + """test that `ipython console` starts a terminal""" + from IPython.external import pexpect + + # weird IOErrors prevent this from firing sometimes: + ipython_cmd = None + for i in range(5): + try: + ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython') + except IOError: + time.sleep(0.1) + else: + break + if ipython_cmd is None: + raise SkipTest("Could not determine ipython command") + + p = pexpect.spawn(ipython_cmd, args=['console', '--colors=NoColor']) + idx = p.expect([r'In \[\d+\]', pexpect.EOF], timeout=4) + nt.assert_equals(idx, 0, "expected in prompt") + p.sendline('5') + idx = p.expect([r'Out\[\d+\]: 5', pexpect.EOF], timeout=1) + nt.assert_equals(idx, 0, "expected out prompt") + idx = p.expect([r'In \[\d+\]', pexpect.EOF], timeout=1) + nt.assert_equals(idx, 0, "expected second in prompt") + # send ctrl-D;ctrl-D to exit + p.sendeof() + p.sendeof() + p.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=1) + if p.isalive(): + p.terminate() diff --git a/IPython/frontend/terminal/ipapp.py b/IPython/frontend/terminal/ipapp.py index 1ab9435..abd4730 100755 --- a/IPython/frontend/terminal/ipapp.py +++ b/IPython/frontend/terminal/ipapp.py @@ -69,6 +69,9 @@ ipython --profile=foo # start with profile foo ipython qtconsole # start the qtconsole GUI application ipython qtconsole -h # show the help string for the qtconsole subcmd +ipython console # start the terminal-based console application +ipython console -h # show the help string for the console subcmd + ipython profile create foo # create profile foo w/ default config files ipython profile -h # show the help string for the profile subcmd """ @@ -112,7 +115,8 @@ class IPAppCrashHandler(CrashHandler): #----------------------------------------------------------------------------- flags = dict(base_flags) flags.update(shell_flags) -addflag = lambda *args: flags.update(boolean_flag(*args)) +frontend_flags = {} +addflag = lambda *args: frontend_flags.update(boolean_flag(*args)) addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax', 'Turn on auto editing of files with syntax errors.', 'Turn off auto editing of files with syntax errors.' @@ -143,7 +147,7 @@ classic_config.InteractiveShell.separate_out2 = '' classic_config.InteractiveShell.colors = 'NoColor' classic_config.InteractiveShell.xmode = 'Plain' -flags['classic']=( +frontend_flags['classic']=( classic_config, "Gives IPython a similar feel to the classic Python prompt." ) @@ -153,21 +157,22 @@ flags['classic']=( # help="Start logging to the default log file (./ipython_log.py).") # # # quick is harder to implement -flags['quick']=( +frontend_flags['quick']=( {'TerminalIPythonApp' : {'quick' : True}}, "Enable quick startup with no config files." ) -flags['i'] = ( +frontend_flags['i'] = ( {'TerminalIPythonApp' : {'force_interact' : True}}, """If running code from the command line, become interactive afterwards. Note: can also be given simply as '-i.'""" ) -flags['pylab'] = ( +frontend_flags['pylab'] = ( {'TerminalIPythonApp' : {'pylab' : 'auto'}}, """Pre-load matplotlib and numpy for interactive use with the default matplotlib backend.""" ) +flags.update(frontend_flags) aliases = dict(base_aliases) aliases.update(shell_aliases) @@ -209,7 +214,7 @@ class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): """Launch the IPython Qt Console.""" ), notebook=('IPython.frontend.html.notebook.notebookapp.NotebookApp', - """Launch the IPython HTML Notebook Server""" + """Launch the IPython HTML Notebook Server.""" ), profile = ("IPython.core.profileapp.ProfileApp", "Create and manage IPython profiles." @@ -217,6 +222,9 @@ class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): kernel = ("IPython.zmq.ipkernel.IPKernelApp", "Start a kernel without an attached frontend." ), + console=('IPython.frontend.terminal.console.app.ZMQTerminalIPythonApp', + """Launch the IPython terminal-based Console.""" + ), )) # *do* autocreate requested profile, but don't create the config file. diff --git a/IPython/lib/kernel.py b/IPython/lib/kernel.py index df9ec6b..345ecee 100644 --- a/IPython/lib/kernel.py +++ b/IPython/lib/kernel.py @@ -253,3 +253,63 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): return tuple(lports) +def swallow_argv(argv, aliases=None, flags=None): + """strip frontend-specific aliases and flags from an argument list + + For use primarily in frontend apps that want to pass a subset of command-line + arguments through to a subprocess, where frontend-specific flags and aliases + should be removed from the list. + + Parameters + ---------- + + argv : list(str) + The starting argv, to be filtered + aliases : container of aliases (dict, list, set, etc.) + The frontend-specific aliases to be removed + flags : container of flags (dict, list, set, etc.) + The frontend-specific flags to be removed + + Returns + ------- + + argv : list(str) + The argv list, excluding flags and aliases that have been stripped + """ + + if aliases is None: + aliases = set() + if flags is None: + flags = set() + + stripped = list(argv) # copy + + swallow_next = False + was_flag = False + for a in argv: + if swallow_next: + swallow_next = False + # last arg was an alias, remove the next one + # *unless* the last alias has a no-arg flag version, in which + # case, don't swallow the next arg if it's also a flag: + if not (was_flag and a.startswith('-')): + stripped.remove(a) + continue + if a.startswith('-'): + split = a.lstrip('-').split('=') + alias = split[0] + if alias in aliases: + stripped.remove(a) + if len(split) == 1: + # alias passed with arg via space + swallow_next = True + # could have been a flag that matches an alias, e.g. `existing` + # in which case, we might not swallow the next arg + was_flag = alias in flags + elif alias in flags and len(split) == 1: + # strip flag, but don't swallow next, as flags don't take args + stripped.remove(a) + + # return shortened list + return stripped + diff --git a/IPython/lib/tests/test_kernel.py b/IPython/lib/tests/test_kernel.py new file mode 100644 index 0000000..87a2a1b --- /dev/null +++ b/IPython/lib/tests/test_kernel.py @@ -0,0 +1,63 @@ +"""Tests for kernel utility functions + +Authors +------- +* MinRK +""" +#----------------------------------------------------------------------------- +# Copyright (c) 2011, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib imports +from unittest import TestCase + +# Third-party imports +import nose.tools as nt + +# Our own imports +from IPython.testing import decorators as dec +from IPython.lib import kernel + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +@dec.parametric +def test_swallow_argv(): + tests = [ + # expected , argv , aliases, flags + (['-a', '5'], ['-a', '5'], None, None), + (['5'], ['-a', '5'], None, ['a']), + ([], ['-a', '5'], ['a'], None), + ([], ['-a', '5'], ['a'], ['a']), + ([], ['--foo'], None, ['foo']), + (['--foo'], ['--foo'], ['foobar'], []), + ([], ['--foo', '5'], ['foo'], []), + ([], ['--foo=5'], ['foo'], []), + (['--foo=5'], ['--foo=5'], [], ['foo']), + (['5'], ['--foo', '5'], [], ['foo']), + (['bar'], ['--foo', '5', 'bar'], ['foo'], ['foo']), + (['bar'], ['--foo=5', 'bar'], ['foo'], ['foo']), + (['5','bar'], ['--foo', '5', 'bar'], None, ['foo']), + (['bar'], ['--foo', '5', 'bar'], ['foo'], None), + (['bar'], ['--foo=5', 'bar'], ['foo'], None), + ] + for expected, argv, aliases, flags in tests: + stripped = kernel.swallow_argv(argv, aliases=aliases, flags=flags) + message = '\n'.join(['', + "argv: %r" % argv, + "aliases: %r" % aliases, + "flags : %r" % flags, + "expected : %r" % expected, + "returned : %r" % stripped, + ]) + yield nt.assert_equal(expected, stripped, message) + diff --git a/IPython/testing/_paramtestpy2.py b/IPython/testing/_paramtestpy2.py index 368ab09..f4bfa26 100644 --- a/IPython/testing/_paramtestpy2.py +++ b/IPython/testing/_paramtestpy2.py @@ -12,6 +12,7 @@ # Imports #----------------------------------------------------------------------------- +import sys import unittest from compiler.consts import CO_GENERATOR @@ -45,7 +46,7 @@ class ParametricTestCase(unittest.TestCase): except KeyboardInterrupt: raise except: - result.addError(self, self._exc_info()) + result.addError(self, sys.exc_info()) return # Test execution ok = False @@ -56,18 +57,18 @@ class ParametricTestCase(unittest.TestCase): # We stop the loop break except self.failureException: - result.addFailure(self, self._exc_info()) + result.addFailure(self, sys.exc_info()) except KeyboardInterrupt: raise except: - result.addError(self, self._exc_info()) + result.addError(self, sys.exc_info()) # TearDown try: self.tearDown() except KeyboardInterrupt: raise except: - result.addError(self, self._exc_info()) + result.addError(self, sys.exc_info()) ok = False if ok: result.addSuccess(self) diff --git a/IPython/zmq/blockingkernelmanager.py b/IPython/zmq/blockingkernelmanager.py index 9dbe1c3..581a844 100644 --- a/IPython/zmq/blockingkernelmanager.py +++ b/IPython/zmq/blockingkernelmanager.py @@ -16,6 +16,7 @@ from __future__ import print_function # Stdlib from Queue import Queue, Empty +from threading import Event # Our own from IPython.utils import io @@ -104,18 +105,42 @@ class BlockingShellSocketChannel(ShellSocketChannel): class BlockingStdInSocketChannel(StdInSocketChannel): + def __init__(self, context, session, address=None): + super(BlockingStdInSocketChannel, self).__init__(context, session, address) + self._in_queue = Queue() + def call_handlers(self, msg): #io.rprint('[[Rep]]', msg) # dbg - pass + self._in_queue.put(msg) + + def get_msg(self, block=True, timeout=None): + "Gets a message if there is one that is ready." + return self._in_queue.get(block, timeout) + + def get_msgs(self): + """Get all messages that are currently ready.""" + msgs = [] + while True: + try: + msgs.append(self.get_msg(block=False)) + except Empty: + break + return msgs + + def msg_ready(self): + "Is there a message that has been received?" + return not self._in_queue.empty() class BlockingHBSocketChannel(HBSocketChannel): - # This kernel needs rapid monitoring capabilities - time_to_dead = 0.2 + # This kernel needs quicker monitoring, shorten to 1 sec. + # less than 0.5s is unreliable, and will get occasional + # false reports of missed beats. + time_to_dead = 1. def call_handlers(self, since_last_heartbeat): - #io.rprint('[[Heart]]', since_last_heartbeat) # dbg + """pause beating on missed heartbeat""" pass diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 9b8bdc3..edc5c9a 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -22,7 +22,9 @@ import sys import time import traceback import logging - +from signal import ( + signal, default_int_handler, SIGINT, SIG_IGN +) # System library imports. import zmq @@ -168,6 +170,9 @@ class Kernel(Configurable): def start(self): """ Start the kernel main loop. """ + # a KeyboardInterrupt (SIGINT) can occur on any python statement, so + # let's ignore (SIG_IGN) them until we're in a place to handle them properly + signal(SIGINT,SIG_IGN) poller = zmq.Poller() poller.register(self.shell_socket, zmq.POLLIN) # loop while self.eventloop has not been overridden @@ -182,12 +187,20 @@ class Kernel(Configurable): # due to pyzmq Issue #130 try: poller.poll(10*1000*self._poll_interval) + # restore raising of KeyboardInterrupt + signal(SIGINT, default_int_handler) self.do_one_iteration() except: raise + finally: + # prevent raising of KeyboardInterrupt + signal(SIGINT,SIG_IGN) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel io.raw_print("KeyboardInterrupt caught in kernel") + # stop ignoring sigint, now that we are out of our own loop, + # we don't want to prevent future code from handling it + signal(SIGINT, default_int_handler) if self.eventloop is not None: try: self.eventloop(self) @@ -456,6 +469,9 @@ class Kernel(Configurable): self.log.error("Got bad raw_input reply: ") self.log.error(str(Message(parent))) value = '' + if value == '\x04': + # EOF + raise EOFError return value def _complete(self, msg): diff --git a/IPython/zmq/kernelmanager.py b/IPython/zmq/kernelmanager.py index f7f856d..4c3b8c1 100644 --- a/IPython/zmq/kernelmanager.py +++ b/IPython/zmq/kernelmanager.py @@ -471,83 +471,89 @@ class HBSocketChannel(ZMQSocketChannel): poller = None _running = None _pause = None + _beating = None def __init__(self, context, session, address): super(HBSocketChannel, self).__init__(context, session, address) self._running = False - self._pause = True + self._pause =True + self.poller = zmq.Poller() def _create_socket(self): + if self.socket is not None: + # close previous socket, before opening a new one + self.poller.unregister(self.socket) + self.socket.close() self.socket = self.context.socket(zmq.REQ) - self.socket.setsockopt(zmq.IDENTITY, self.session.bsession) + self.socket.setsockopt(zmq.LINGER, 0) self.socket.connect('tcp://%s:%i' % self.address) - self.poller = zmq.Poller() + self.poller.register(self.socket, zmq.POLLIN) + + def _poll(self, start_time): + """poll for heartbeat replies until we reach self.time_to_dead + + Ignores interrupts, and returns the result of poll(), which + will be an empty list if no messages arrived before the timeout, + or the event tuple if there is a message to receive. + """ + + until_dead = self.time_to_dead - (time.time() - start_time) + # ensure poll at least once + until_dead = max(until_dead, 1e-3) + events = [] + while True: + try: + events = self.poller.poll(1000 * until_dead) + except zmq.ZMQError as e: + if e.errno == errno.EINTR: + # ignore interrupts during heartbeat + # this may never actually happen + until_dead = self.time_to_dead - (time.time() - start_time) + until_dead = max(until_dead, 1e-3) + pass + else: + raise + else: + break + return events def run(self): """The thread's main activity. Call start() instead.""" self._create_socket() self._running = True + self._beating = True + while self._running: if self._pause: + # just sleep, and skip the rest of the loop time.sleep(self.time_to_dead) + continue + + since_last_heartbeat = 0.0 + # io.rprint('Ping from HB channel') # dbg + # no need to catch EFSM here, because the previous event was + # either a recv or connect, which cannot be followed by EFSM + self.socket.send(b'ping') + request_time = time.time() + ready = self._poll(request_time) + if ready: + self._beating = True + # the poll above guarantees we have something to recv + self.socket.recv() + # sleep the remainder of the cycle + remainder = self.time_to_dead - (time.time() - request_time) + if remainder > 0: + time.sleep(remainder) + continue else: - since_last_heartbeat = 0.0 - request_time = time.time() - try: - #io.rprint('Ping from HB channel') # dbg - self.socket.send(b'ping') - except zmq.ZMQError, e: - #io.rprint('*** HB Error:', e) # dbg - if e.errno == zmq.EFSM: - #io.rprint('sleep...', self.time_to_dead) # dbg - time.sleep(self.time_to_dead) - self._create_socket() - else: - raise - else: - while True: - try: - self.socket.recv(zmq.NOBLOCK) - except zmq.ZMQError, e: - #io.rprint('*** HB Error 2:', e) # dbg - if e.errno == zmq.EAGAIN: - before_poll = time.time() - until_dead = self.time_to_dead - (before_poll - - request_time) - - # When the return value of poll() is an empty - # list, that is when things have gone wrong - # (zeromq bug). As long as it is not an empty - # list, poll is working correctly even if it - # returns quickly. Note: poll timeout is in - # milliseconds. - if until_dead > 0.0: - while True: - try: - self.poller.poll(1000 * until_dead) - except zmq.ZMQError as e: - if e.errno == errno.EINTR: - continue - else: - raise - else: - break - - since_last_heartbeat = time.time()-request_time - if since_last_heartbeat > self.time_to_dead: - self.call_handlers(since_last_heartbeat) - break - else: - # FIXME: We should probably log this instead. - raise - else: - until_dead = self.time_to_dead - (time.time() - - request_time) - if until_dead > 0.0: - #io.rprint('sleep...', self.time_to_dead) # dbg - time.sleep(until_dead) - break + # nothing was received within the time limit, signal heart failure + self._beating = False + since_last_heartbeat = time.time() - request_time + self.call_handlers(since_last_heartbeat) + # and close/reopen the socket, because the REQ/REP cycle has been broken + self._create_socket() + continue def pause(self): """Pause the heartbeat.""" @@ -558,8 +564,8 @@ class HBSocketChannel(ZMQSocketChannel): self._pause = False def is_beating(self): - """Is the heartbeat running and not paused.""" - if self.is_alive() and not self._pause: + """Is the heartbeat running and responsive (and not paused).""" + if self.is_alive() and not self._pause and self._beating: return True else: return False @@ -573,7 +579,7 @@ class HBSocketChannel(ZMQSocketChannel): Subclasses should override this method to handle incoming messages. It is important to remember that this method is called in the thread - so that some logic must be done to ensure that the application leve + so that some logic must be done to ensure that the application level handlers are called in the application thread. """ raise NotImplementedError('call_handlers must be defined in a subclass.') @@ -639,14 +645,8 @@ class KernelManager(HasTraits): self.session = Session(config=self.config) def __del__(self): - if self._connection_file_written: - # cleanup connection files on full shutdown of kernel we started - self._connection_file_written = False - try: - os.remove(self.connection_file) - except IOError: - pass - + self.cleanup_connection_file() + #-------------------------------------------------------------------------- # Channel management methods: @@ -694,6 +694,19 @@ class KernelManager(HasTraits): # Kernel process management methods: #-------------------------------------------------------------------------- + def cleanup_connection_file(self): + """cleanup connection file *if we wrote it* + + Will not raise if the connection file was already removed somehow. + """ + if self._connection_file_written: + # cleanup connection files on full shutdown of kernel we started + self._connection_file_written = False + try: + os.remove(self.connection_file) + except OSError: + pass + def load_connection_file(self): """load connection info from JSON dict in self.connection_file""" with open(self.connection_file) as f: @@ -893,16 +906,18 @@ class KernelManager(HasTraits): @property def is_alive(self): """Is the kernel process still running?""" - # FIXME: not using a heartbeat means this method is broken for any - # remote kernel, it's only capable of handling local kernels. if self.has_kernel: if self.kernel.poll() is None: return True else: return False + elif self._hb_channel is not None: + # We didn't start the kernel with this KernelManager so we + # use the heartbeat. + return self._hb_channel.is_beating() else: - # We didn't start the kernel with this KernelManager so we don't - # know if it is running. We should use a heartbeat for this case. + # no heartbeat and not local, we can't tell if it's running, + # so naively return True return True #-------------------------------------------------------------------------- diff --git a/IPython/zmq/zmqshell.py b/IPython/zmq/zmqshell.py index f603ebe..60ed391 100644 --- a/IPython/zmq/zmqshell.py +++ b/IPython/zmq/zmqshell.py @@ -44,12 +44,6 @@ from IPython.zmq.displayhook import ZMQShellDisplayHook, _encode_binary from IPython.zmq.session import extract_header from session import Session -#----------------------------------------------------------------------------- -# Globals and side-effects -#----------------------------------------------------------------------------- - -# Install the payload version of page. -install_payload_page() #----------------------------------------------------------------------------- # Functions and classes @@ -126,6 +120,9 @@ class ZMQInteractiveShell(InteractiveShell): # subprocesses as much as possible. env['PAGER'] = 'cat' env['GIT_PAGER'] = 'cat' + + # And install the payload version of page. + install_payload_page() def auto_rewrite_input(self, cmd): """Called to show the auto-rewritten input for autocall and friends.