Merge pull request #1262 from minrk/hbgil

Heartbeat no longer shares the app's Context

Fixes in both the single and parallel kernels, preventing the heartbeat thread from sharing the zmq Context with the rest of the process.  Non-copying sends require grabbing the GIL from the zmq io-thread in order to free memory, which could let Python get in the way of the heartbeat.

Test script and notebook added to examples directory.

closes  #1260

docs/examples/tests/heartbeat/gilsleep.ipynb

@@ -0,0 +1,56 b''
+{
+ "metadata": {
+  "name": "gilsleep"
+ }, 
+ "nbformat": 2, 
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown", 
+     "source": [
+      "Holding the GIL for too long could disrupt the heartbeat due to non-copying sends.", 
+      "", 
+      "The following cell repeatedly calls a function that holds the GIL for five seconds.", 
+      "", 
+      "The heartbeat will fail after a few iterations prior to fixing Issue [#1260](https://github.com/ipython/ipython/issues/1260)."
+     ]
+    }, 
+    {
+     "cell_type": "code", 
+     "collapsed": false, 
+     "input": [
+      "import sys", 
+      "import time", 
+      "", 
+      "from cython import inline", 
+      "", 
+      "def gilsleep(t):", 
+      "    \"\"\"gil-holding sleep with cython.inline\"\"\"", 
+      "    code = '\\n'.join([", 
+      "        'from posix cimport unistd',", 
+      "        'unistd.sleep(t)',", 
+      "    ])", 
+      "    while True:", 
+      "        inline(code, quiet=True, t=t)", 
+      "        print time.time()", 
+      "        sys.stdout.flush() # this is important", 
+      "", 
+      "gilsleep(5)"
+     ], 
+     "language": "python", 
+     "outputs": [], 
+     "prompt_number": 1
+    }, 
+    {
+     "cell_type": "code", 
+     "collapsed": true, 
+     "input": [], 
+     "language": "python", 
+     "outputs": [], 
+     "prompt_number": " "
+    }
+   ]
+  }
+ ]
+}
  No newline at end of file

docs/examples/tests/heartbeat/hb_gil.py

@@ -0,0 +1,31 b''
+"""
+Run this script in the qtconsole with one of:
+
+    %loadpy hb_gil.py
+
+or
+    %run hb_gil.py
+
+Holding the GIL for too long could disrupt the heartbeat.
+
+See Issue #1260: https://github.com/ipython/ipython/issues/1260
+
+"""
+
+import sys
+import time
+
+from cython import inline
+
+def gilsleep(t):
+    """gil-holding sleep with cython.inline"""
+    code = '\n'.join([
+        'from posix cimport unistd',
+        'unistd.sleep(t)',
+    ])
+    while True:
+        inline(code, quiet=True, t=t)
+        print time.time()
+        sys.stdout.flush() # this is important
+
+gilsleep(5)

IPython/parallel/controller/heartmonitor.py

@@ -40,6 +40,10 b' class Heart(object):'
     id=None
     def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.DEALER, heart_id=None):
         self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
+        # do not allow the device to share global Context.instance,
+        # which is the default behavior in pyzmq > 2.1.10
+        self.device.context_factory = zmq.Context
+        
         self.device.daemon=True
         self.device.connect_in(in_addr)
         self.device.connect_out(out_addr)

IPython/zmq/kernelapp.py

@@ -216,8 +216,11 b' class KernelApp(BaseIPythonApplication):'
         self.stdin_socket = context.socket(zmq.ROUTER)
         self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
         self.log.debug("stdin ROUTER Channel on port: %i"%self.stdin_port)
-
-        self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
+        
+        # heartbeat doesn't share context, because it mustn't be blocked
+        # by the GIL, which is accessed by libzmq when freeing zero-copy messages
+        hb_ctx = zmq.Context()
+        self.heartbeat = Heartbeat(hb_ctx, (self.ip, self.hb_port))
         self.hb_port = self.heartbeat.port
         self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)