From 5d0418a89b3c6e91e885b286de285c54f3e7fbc2 2013-09-18 23:12:00 From: MinRK Date: 2013-09-18 23:12:00 Subject: [PATCH] avoid executing code in utils.localinterfaces at import time moves away from global constants to utility functions. The code is still only executed once, but instead of at import time, it is executed at first request. --- diff --git a/IPython/consoleapp.py b/IPython/consoleapp.py index 26554c4..dddcb8a 100644 --- a/IPython/consoleapp.py +++ b/IPython/consoleapp.py @@ -53,7 +53,7 @@ from IPython.kernel.connect import ConnectionFileMixin # Network Constants #----------------------------------------------------------------------------- -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost #----------------------------------------------------------------------------- # Globals @@ -254,7 +254,7 @@ class IPythonConsoleApp(ConnectionFileMixin): with open(fname) as f: cfg = json.load(f) self.transport = cfg.get('transport', 'tcp') - self.ip = cfg.get('ip', LOCALHOST) + self.ip = cfg.get('ip', localhost()) for channel in ('hb', 'shell', 'iopub', 'stdin', 'control'): name = channel + '_port' @@ -282,7 +282,7 @@ class IPythonConsoleApp(ConnectionFileMixin): if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = ip - ip = LOCALHOST + ip = localhost() # build connection dict for tunnels: info = dict(ip=ip, @@ -295,7 +295,7 @@ class IPythonConsoleApp(ConnectionFileMixin): self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: - self.ip = LOCALHOST + self.ip = localhost() try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except: diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py index efe5313..955f6df 100644 --- a/IPython/html/notebookapp.py +++ b/IPython/html/notebookapp.py @@ -78,7 +78,7 @@ from IPython.kernel.zmq.kernelapp import ( kernel_aliases, ) from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils import submodule from IPython.utils.traitlets import ( Dict, Unicode, Integer, List, Bool, Bytes, @@ -293,9 +293,11 @@ class NotebookApp(BaseIPythonApplication): # Network related information. - ip = Unicode(LOCALHOST, config=True, + ip = Unicode(config=True, help="The IP address the notebook server will listen on." ) + def _ip_default(self): + return localhost() def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' @@ -694,7 +696,7 @@ class NotebookApp(BaseIPythonApplication): info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).") if self.open_browser or self.file_to_run: - ip = self.ip or LOCALHOST + ip = self.ip or localhost() try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: diff --git a/IPython/kernel/connect.py b/IPython/kernel/connect.py index 83167ad..8ce939d 100644 --- a/IPython/kernel/connect.py +++ b/IPython/kernel/connect.py @@ -36,7 +36,7 @@ from IPython.external.ssh import tunnel # IPython imports from IPython.config import Configurable from IPython.core.profiledir import ProfileDir -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.path import filefind, get_ipython_dir from IPython.utils.py3compat import str_to_bytes, bytes_to_str from IPython.utils.traitlets import ( @@ -49,7 +49,7 @@ from IPython.utils.traitlets import ( #----------------------------------------------------------------------------- def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, - control_port=0, ip=LOCALHOST, key=b'', transport='tcp', + control_port=0, ip='', key=b'', transport='tcp', signature_scheme='hmac-sha256', ): """Generates a JSON config file, including the selection of random ports. @@ -90,6 +90,8 @@ def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, and 'sha256' is the default hash function. """ + if not ip: + ip = localhost() # default to temporary connector file if not fname: fname = tempfile.mktemp('.json') @@ -391,7 +393,7 @@ class ConnectionFileMixin(Configurable): transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) - ip = Unicode(LOCALHOST, config=True, + ip = Unicode(config=True, help="""Set the kernel\'s IP address [default localhost]. If the IP address is something other than localhost, then Consoles on other machines will be able to connect @@ -405,7 +407,7 @@ class ConnectionFileMixin(Configurable): else: return 'kernel-ipc' else: - return LOCALHOST + return localhost() def _ip_changed(self, name, old, new): if new == '*': diff --git a/IPython/kernel/manager.py b/IPython/kernel/manager.py index cf5efe1..a26423e 100644 --- a/IPython/kernel/manager.py +++ b/IPython/kernel/manager.py @@ -1,5 +1,4 @@ -"""Base class to manage a running kernel -""" +"""Base class to manage a running kernel""" #----------------------------------------------------------------------------- # Copyright (C) 2013 The IPython Development Team @@ -24,7 +23,7 @@ import zmq # Local imports from IPython.config.configurable import LoggingConfigurable from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCAL_IPS +from IPython.utils.localinterfaces import is_local_ip, local_ips from IPython.utils.traitlets import ( Any, Instance, Unicode, List, Bool, Type, DottedObjectName ) @@ -185,11 +184,11 @@ class KernelManager(LoggingConfigurable, ConnectionFileMixin): keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - if self.transport == 'tcp' and self.ip not in LOCAL_IPS: + if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " - "Currently valid addresses are: %s"%LOCAL_IPS + "Currently valid addresses are: %s" % local_ips() ) # write connection file / get default ports diff --git a/IPython/kernel/tests/test_multikernelmanager.py b/IPython/kernel/tests/test_multikernelmanager.py index 1136af2..3838b72 100644 --- a/IPython/kernel/tests/test_multikernelmanager.py +++ b/IPython/kernel/tests/test_multikernelmanager.py @@ -7,7 +7,7 @@ from unittest import TestCase from IPython.testing import decorators as dec from IPython.config.loader import Config -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.kernel import KernelManager from IPython.kernel.multikernelmanager import MultiKernelManager @@ -64,7 +64,7 @@ class TestKernelManager(TestCase): def test_tcp_cinfo(self): km = self._get_tcp_km() - self._run_cinfo(km, 'tcp', LOCALHOST) + self._run_cinfo(km, 'tcp', localhost()) @dec.skip_win32 def test_ipc_lifecycle(self): diff --git a/IPython/kernel/zmq/heartbeat.py b/IPython/kernel/zmq/heartbeat.py index a018abb..ccfd5dc 100644 --- a/IPython/kernel/zmq/heartbeat.py +++ b/IPython/kernel/zmq/heartbeat.py @@ -19,7 +19,7 @@ from threading import Thread import zmq -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost #----------------------------------------------------------------------------- # Code @@ -29,7 +29,9 @@ from IPython.utils.localinterfaces import LOCALHOST class Heartbeat(Thread): "A simple ping-pong style heartbeat that runs in a thread." - def __init__(self, context, addr=('tcp', LOCALHOST, 0)): + def __init__(self, context, addr=None): + if addr is None: + addr = ('tcp', localhost(), 0) Thread.__init__(self) self.context = context self.transport, self.ip, self.port = addr diff --git a/IPython/kernel/zmq/kernelapp.py b/IPython/kernel/zmq/kernelapp.py index 3b100fd..7461bbf 100644 --- a/IPython/kernel/zmq/kernelapp.py +++ b/IPython/kernel/zmq/kernelapp.py @@ -39,7 +39,7 @@ from IPython.core.shellapp import ( InteractiveShellApp, shell_flags, shell_aliases ) from IPython.utils import io -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.path import filefind from IPython.utils.py3compat import str_to_bytes from IPython.utils.traitlets import ( @@ -156,7 +156,8 @@ class IPKernelApp(BaseIPythonApplication, InteractiveShellApp): else: return 'kernel-ipc' else: - return LOCALHOST + return localhost() + hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]") shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]") diff --git a/IPython/parallel/apps/ipcontrollerapp.py b/IPython/parallel/apps/ipcontrollerapp.py index 28c651c..a4b879d 100755 --- a/IPython/parallel/apps/ipcontrollerapp.py +++ b/IPython/parallel/apps/ipcontrollerapp.py @@ -11,7 +11,7 @@ Authors: """ #----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team +# Copyright (C) 2008 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. @@ -44,7 +44,7 @@ from IPython.parallel.apps.baseapp import ( catch_config_error, ) from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS +from IPython.utils.localinterfaces import localhost, public_ips from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError from IPython.kernel.zmq.session import ( @@ -224,13 +224,13 @@ class IPControllerApp(BaseParallelApplication): location = cdict['location'] if not location: - if PUBLIC_IPS: - location = PUBLIC_IPS[-1] + if public_ips(): + location = public_ips()[-1] else: self.log.warn("Could not identify this machine's IP, assuming %s." " You may need to specify '--location=' to help" - " IPython decide when to connect via loopback." % LOCALHOST) - location = LOCALHOST + " IPython decide when to connect via loopback." % localhost() ) + location = localhost() cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) self.log.info("writing connection info to %s", fname) diff --git a/IPython/parallel/apps/logwatcher.py b/IPython/parallel/apps/logwatcher.py index 80be410..c6ed6a3 100644 --- a/IPython/parallel/apps/logwatcher.py +++ b/IPython/parallel/apps/logwatcher.py @@ -26,7 +26,7 @@ import zmq from zmq.eventloop import ioloop, zmqstream from IPython.config.configurable import LoggingConfigurable -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import Int, Unicode, Instance, List #----------------------------------------------------------------------------- @@ -44,8 +44,10 @@ class LogWatcher(LoggingConfigurable): # configurables topics = List([''], config=True, help="The ZMQ topics to subscribe to. Default is to subscribe to all messages") - url = Unicode('tcp://%s:20202' % LOCALHOST, config=True, + url = Unicode(config=True, help="ZMQ url on which to listen for log messages") + def _url_default(self): + return 'tcp://%s:20202' % localhost() # internals stream = Instance('zmq.eventloop.zmqstream.ZMQStream') diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index e6dc974..0b0c722 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -37,7 +37,7 @@ from IPython.core.profiledir import ProfileDir, ProfileDirError from IPython.utils.capture import RichOutput from IPython.utils.coloransi import TermColors from IPython.utils.jsonutil import rekey -from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS +from IPython.utils.localinterfaces import localhost, is_local_ip from IPython.utils.path import get_ipython_dir from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode, @@ -433,13 +433,13 @@ class Client(HasTraits): url = cfg['registration'] - if location is not None and addr == LOCALHOST: + if location is not None and addr == localhost(): # location specified, and connection is expected to be local - if location not in LOCAL_IPS and not sshserver: + if not is_local_ip(location) and not sshserver: # load ssh from JSON *only* if the controller is not on # this machine sshserver=cfg['ssh'] - if location not in LOCAL_IPS and not sshserver: + if not is_local_ip(location) and not sshserver: # warn if no ssh specified, but SSH is probably needed # This is only a warning, because the most likely cause # is a local Controller on a laptop whose IP is dynamic diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index ae0b3a8..e049fc9 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -30,7 +30,7 @@ from zmq.eventloop.zmqstream import ZMQStream # internal: from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import ( HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName @@ -177,20 +177,25 @@ class HubFactory(RegistrationFactory): def _notifier_port_default(self): return util.select_random_ports(1)[0] - engine_ip = Unicode(LOCALHOST, config=True, + engine_ip = Unicode(config=True, help="IP on which to listen for engine connections. [default: loopback]") + def _engine_ip_default(self): + return localhost() engine_transport = Unicode('tcp', config=True, help="0MQ transport for engine connections. [default: tcp]") - client_ip = Unicode(LOCALHOST, config=True, + client_ip = Unicode(config=True, help="IP on which to listen for client connections. [default: loopback]") client_transport = Unicode('tcp', config=True, help="0MQ transport for client connections. [default : tcp]") - monitor_ip = Unicode(LOCALHOST, config=True, + monitor_ip = Unicode(config=True, help="IP on which to listen for monitor messages. [default: loopback]") monitor_transport = Unicode('tcp', config=True, help="0MQ transport for monitor messages. [default : tcp]") + + _client_ip_default = _monitor_ip_default = _engine_ip_default + monitor_url = Unicode('') diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index 208212f..e60f2bf 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -24,7 +24,7 @@ from zmq.eventloop import ioloop, zmqstream from IPython.external.ssh import tunnel # internal -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import ( Instance, Dict, Integer, Type, Float, Integer, Unicode, CBytes, Bool ) @@ -184,13 +184,13 @@ class EngineFactory(RegistrationFactory): if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) - mport = mon.bind_to_random_port('tcp://%s' % LOCALHOST) + mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) - hb_monitor = "tcp://%s:%i" % (LOCALHOST, mport) + hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity) heart.start() diff --git a/IPython/parallel/factory.py b/IPython/parallel/factory.py index 3b2a770..4636136 100644 --- a/IPython/parallel/factory.py +++ b/IPython/parallel/factory.py @@ -24,7 +24,7 @@ import zmq from zmq.eventloop.ioloop import IOLoop from IPython.config.configurable import Configurable -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import Integer, Instance, Unicode from IPython.parallel.util import select_random_ports @@ -40,16 +40,18 @@ class RegistrationFactory(SessionFactory): url = Unicode('', config=True, help="""The 0MQ url used for registration. This sets transport, ip, and port - in one variable. For example: url='tcp://%s:12345' or + in one variable. For example: url='tcp://127.0.0.1:12345' or url='epgm://*:90210'""" - % LOCALHOST) # url takes precedence over ip,regport,transport + ) # url takes precedence over ip,regport,transport transport = Unicode('tcp', config=True, help="""The 0MQ transport for communications. This will likely be the default of 'tcp', but other values include 'ipc', 'epgm', 'inproc'.""") - ip = Unicode(LOCALHOST, config=True, + ip = Unicode(config=True, help="""The IP address for registration. This is generally either '127.0.0.1' for loopback only or '*' for all interfaces. - [default: '%s']""" % LOCALHOST) + """) + def _ip_default(self): + return localhost() regport = Integer(config=True, help="""The port on which the Hub listens for registration.""") def _regport_default(self): diff --git a/IPython/parallel/util.py b/IPython/parallel/util.py index a725d5a..cf0824a 100644 --- a/IPython/parallel/util.py +++ b/IPython/parallel/util.py @@ -43,7 +43,7 @@ from IPython.external.decorator import decorator # IPython imports from IPython.config.application import Application -from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS +from IPython.utils.localinterfaces import localhost, is_public_ip, public_ips from IPython.kernel.zmq.log import EnginePUBHandler from IPython.kernel.zmq.serialize import ( unserialize_object, serialize_object, pack_apply_message, unpack_apply_message @@ -187,9 +187,9 @@ def disambiguate_ip_address(ip, location=None): """turn multi-ip interfaces '0.0.0.0' and '*' into connectable ones, based on the location (default interpretation of location is localhost).""" if ip in ('0.0.0.0', '*'): - if location is None or location in PUBLIC_IPS or not PUBLIC_IPS: + if location is None or is_public_ip(location) or not public_ips(): # If location is unspecified or cannot be determined, assume local - ip = LOCALHOST + ip = localhost() elif location: return location return ip diff --git a/IPython/qt/console/qtconsoleapp.py b/IPython/qt/console/qtconsoleapp.py index e8ddbf4..3ef7fa1 100644 --- a/IPython/qt/console/qtconsoleapp.py +++ b/IPython/qt/console/qtconsoleapp.py @@ -74,7 +74,7 @@ from IPython.consoleapp import ( # Network Constants #----------------------------------------------------------------------------- -from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS +from IPython.utils.localinterfaces import is_local_ip #----------------------------------------------------------------------------- # Globals @@ -250,7 +250,7 @@ class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp): QtGui.QApplication.setWindowIcon(self.app.icon) ip = self.ip - local_kernel = (not self.existing) or ip in LOCAL_IPS + local_kernel = (not self.existing) or is_local_ip(ip) self.widget = self.widget_factory(config=self.config, local_kernel=local_kernel) self.init_colors(self.widget) diff --git a/IPython/utils/localinterfaces.py b/IPython/utils/localinterfaces.py index 6418459..e67a35f 100644 --- a/IPython/utils/localinterfaces.py +++ b/IPython/utils/localinterfaces.py @@ -29,27 +29,80 @@ from .data import uniq_stable #----------------------------------------------------------------------------- LOCAL_IPS = [] -try: - LOCAL_IPS = socket.gethostbyname_ex('localhost')[2] -except socket.error: - pass - PUBLIC_IPS = [] -try: - hostname = socket.gethostname() - PUBLIC_IPS = socket.gethostbyname_ex(hostname)[2] - # try hostname.local, in case hostname has been short-circuited to loopback - if not hostname.endswith('.local') and all(ip.startswith('127') for ip in PUBLIC_IPS): - PUBLIC_IPS = socket.gethostbyname_ex(socket.gethostname() + '.local')[2] -except socket.error: - pass -finally: - PUBLIC_IPS = uniq_stable(PUBLIC_IPS) - LOCAL_IPS.extend(PUBLIC_IPS) - -# include all-interface aliases: 0.0.0.0 and '' -LOCAL_IPS.extend(['0.0.0.0', '']) - -LOCAL_IPS = uniq_stable(LOCAL_IPS) - -LOCALHOST = LOCAL_IPS[0] + +LOCALHOST = '127.0.0.1' + +def _only_once(f): + """decorator to only run a function once""" + f.called = False + def wrapped(): + if f.called: + return + ret = f() + f.called = True + return ret + return wrapped + +def _requires_ips(f): + """decorator to ensure load_ips has been run before f""" + def ips_loaded(*args, **kwargs): + _load_ips() + return f(*args, **kwargs) + return ips_loaded + +@_only_once +def _load_ips(): + """load the IPs that point to this machine + + This function will only ever be called once. + """ + global LOCALHOST + try: + LOCAL_IPS[:] = socket.gethostbyname_ex('localhost')[2] + except socket.error: + pass + + try: + hostname = socket.gethostname() + PUBLIC_IPS[:] = socket.gethostbyname_ex(hostname)[2] + # try hostname.local, in case hostname has been short-circuited to loopback + if not hostname.endswith('.local') and all(ip.startswith('127') for ip in PUBLIC_IPS): + PUBLIC_IPS[:] = socket.gethostbyname_ex(socket.gethostname() + '.local')[2] + except socket.error: + pass + finally: + PUBLIC_IPS[:] = uniq_stable(PUBLIC_IPS) + LOCAL_IPS.extend(PUBLIC_IPS) + + # include all-interface aliases: 0.0.0.0 and '' + LOCAL_IPS.extend(['0.0.0.0', '']) + + LOCAL_IPS[:] = uniq_stable(LOCAL_IPS) + + LOCALHOST = LOCAL_IPS[0] + +@_requires_ips +def local_ips(): + """return the IP addresses that point to this machine""" + return LOCAL_IPS + +@_requires_ips +def public_ips(): + """return the IP addresses for this machine that are visible to other machines""" + return PUBLIC_IPS + +@_requires_ips +def localhost(): + """return ip for localhost (almost always 127.0.0.1)""" + return LOCALHOST + +@_requires_ips +def is_local_ip(ip): + """does `ip` point to this machine?""" + return ip in LOCAL_IPS + +@_requires_ips +def is_public_ip(ip): + """is `ip` a publicly visible address?""" + return ip in PUBLIC_IPS