Backport PR #5166: remove mktemp usage

mktemp is insecure, by the time the generated filename is used someone
else might have modified it.

IPython/core/page.py

@@ -1,348 +1,351 b''
 # encoding: utf-8
 """
 Paging capabilities for IPython.core
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 
 For now this uses ipapi, so it can't be in IPython.utils.  If we can get
 rid of that dependency, we could move it there.
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import os
 import re
 import sys
 import tempfile
 
 from io import UnsupportedOperation
 
 from IPython import get_ipython
 from IPython.core.error import TryNext
 from IPython.utils.data import chop
 from IPython.utils import io
 from IPython.utils.process import system
 from IPython.utils.terminal import get_terminal_size
 from IPython.utils import py3compat
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng, start=0, screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print(os.linesep.join(screens[0]), file=io.stdout)
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print(last_escape + hunk, file=io.stdout)
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print(last_escape + os.linesep.join(screens[-1]), file=io.stdout)
 
 def _detect_screen_size(screen_lines_def):
     """Attempt to work out the number of lines on the screen.
 
     This is called by page(). It can raise an error (e.g. when run in the
     test suite), so it's separated out so it can easily be called in a try block.
     """
     TERM = os.environ.get('TERM',None)
     if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
         # curses causes problems on many terminals other than xterm, and
         # some termios calls lock up on Sun OS5.
         return screen_lines_def
     
     try:
         import termios
         import curses
     except ImportError:
         return screen_lines_def
     
     # There is a bug in curses, where *sometimes* it fails to properly
     # initialize, and then after the endwin() call is made, the
     # terminal is left in an unusable state.  Rather than trying to
     # check everytime for this (by requesting and comparing termios
     # flags each time), we just save the initial terminal state and
     # unconditionally reset it every time.  It's cheaper than making
     # the checks.
     term_flags = termios.tcgetattr(sys.stdout)
 
     # Curses modifies the stdout buffer size by default, which messes
     # up Python's normal stdout buffering.  This would manifest itself
     # to IPython users as delayed printing on stdout after having used
     # the pager.
     #
     # We can prevent this by manually setting the NCURSES_NO_SETBUF
     # environment variable.  For more details, see:
     # http://bugs.python.org/issue10144
     NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
     os.environ['NCURSES_NO_SETBUF'] = ''
 
     # Proceed with curses initialization
     try:
         scr = curses.initscr()
     except AttributeError:
         # Curses on Solaris may not be complete, so we can't use it there
         return screen_lines_def
     
     screen_lines_real,screen_cols = scr.getmaxyx()
     curses.endwin()
 
     # Restore environment
     if NCURSES_NO_SETBUF is None:
         del os.environ['NCURSES_NO_SETBUF']
     else:
         os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF
 
     # Restore terminal state in case endwin() didn't.
     termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
     # Now we have what we needed: the screen size in rows/columns
     return screen_lines_real
     #print '***Screen size:',screen_lines_real,'lines x',\
     #screen_cols,'columns.' # dbg
 
 def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
 
     # first, try the hook
     ip = get_ipython()
     if ip:
         try:
             ip.hooks.show_in_pager(strng)
             return
         except TryNext:
             pass
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print(strng)
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.splitlines()[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         try:
             screen_lines += _detect_screen_size(screen_lines_def)
         except (TypeError, UnsupportedOperation):
             print(str_toprint, file=io.stdout)
             return
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print(str_toprint, file=io.stdout)
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
-                tmpname = tempfile.mktemp('.txt')
-                tmpfile = open(tmpname,'wt')
-                tmpfile.write(strng)
-                tmpfile.close()
-                cmd = "%s < %s" % (pager_cmd,tmpname)
-                if os.system(cmd):
-                  retval = 1
-                else:
-                  retval = None
-                os.remove(tmpname)
+                fd, tmpname = tempfile.mkstemp('.txt')
+                try:
+                    os.close(fd)
+                    with open(tmpname, 'wt') as tmpfile:
+                        tmpfile.write(strng)
+                        cmd = "%s < %s" % (pager_cmd, tmpname)
+                    # tmpfile needs to be closed for windows
+                    if os.system(cmd):
+                        retval = 1
+                    else:
+                        retval = None
+                finally:
+                    os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd, 'w')
                 try:
                     pager_encoding = pager.encoding or sys.stdout.encoding
                     pager.write(py3compat.cast_bytes_py2(
                         strng, encoding=pager_encoding))
                 finally:
                     retval = pager.close()
             except IOError as msg:  # broken pipe when user quits
                 if msg.args == (32, 'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 
 def page_file(fname, start=0, pager_cmd=None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         system(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print('Unable to show file',repr(fname))
 
 
 def get_pager_cmd(pager_cmd=None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one.
     """
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 
 def get_pager_start(pager, start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 
 # (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         io.stdout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         io.stdout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print(header, end=' ')
     if len(str) < width:
         print(str)
         snip = 0
     else:
         whalf = int((width -5)/2)
         print(str[:whalf] + ' <...> ' + str[-whalf:])
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip

IPython/kernel/connect.py

@@ -1,556 +1,557 b''
 """Utilities for connecting to kernels
 
 Authors:
 
 * Min Ragan-Kelley
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
 import glob
 import json
 import os
 import socket
 import sys
 from getpass import getpass
 from subprocess import Popen, PIPE
 import tempfile
 
 import zmq
 
 # external imports
 from IPython.external.ssh import tunnel
 
 # IPython imports
 from IPython.config import Configurable
 from IPython.core.profiledir import ProfileDir
 from IPython.utils.localinterfaces import LOCALHOST
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils.py3compat import str_to_bytes, bytes_to_str, cast_bytes_py2
 from IPython.utils.traitlets import (
     Bool, Integer, Unicode, CaselessStrEnum,
 )
 
 
 #-----------------------------------------------------------------------------
 # Working with Connection Files
 #-----------------------------------------------------------------------------
 
 def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
                          control_port=0, ip=LOCALHOST, key=b'', transport='tcp',
                          signature_scheme='hmac-sha256',
                          ):
     """Generates a JSON config file, including the selection of random ports.
     
     Parameters
     ----------
 
     fname : unicode
         The path to the file to write
 
     shell_port : int, optional
         The port to use for ROUTER (shell) channel.
 
     iopub_port : int, optional
         The port to use for the SUB channel.
 
     stdin_port : int, optional
         The port to use for the ROUTER (raw input) channel.
 
     control_port : int, optional
         The port to use for the ROUTER (control) channel.
 
     hb_port : int, optional
         The port to use for the heartbeat REP channel.
 
     ip  : str, optional
         The ip address the kernel will bind to.
 
     key : str, optional
         The Session key used for message authentication.
     
     signature_scheme : str, optional
         The scheme used for message authentication.
         This has the form 'digest-hash', where 'digest'
         is the scheme used for digests, and 'hash' is the name of the hash function
         used by the digest scheme.
         Currently, 'hmac' is the only supported digest scheme,
         and 'sha256' is the default hash function.
 
     """
     # default to temporary connector file
     if not fname:
-        fname = tempfile.mktemp('.json')
+        fd, fname = tempfile.mkstemp('.json')
+        os.close(fd)
     
     # Find open ports as necessary.
     
     ports = []
     ports_needed = int(shell_port <= 0) + \
                    int(iopub_port <= 0) + \
                    int(stdin_port <= 0) + \
                    int(control_port <= 0) + \
                    int(hb_port <= 0)
     if transport == 'tcp':
         for i in range(ports_needed):
             sock = socket.socket()
             sock.bind(('', 0))
             ports.append(sock)
         for i, sock in enumerate(ports):
             port = sock.getsockname()[1]
             sock.close()
             ports[i] = port
     else:
         N = 1
         for i in range(ports_needed):
             while os.path.exists("%s-%s" % (ip, str(N))):
                 N += 1
             ports.append(N)
             N += 1
     if shell_port <= 0:
         shell_port = ports.pop(0)
     if iopub_port <= 0:
         iopub_port = ports.pop(0)
     if stdin_port <= 0:
         stdin_port = ports.pop(0)
     if control_port <= 0:
         control_port = ports.pop(0)
     if hb_port <= 0:
         hb_port = ports.pop(0)
     
     cfg = dict( shell_port=shell_port,
                 iopub_port=iopub_port,
                 stdin_port=stdin_port,
                 control_port=control_port,
                 hb_port=hb_port,
               )
     cfg['ip'] = ip
     cfg['key'] = bytes_to_str(key)
     cfg['transport'] = transport
     cfg['signature_scheme'] = signature_scheme
     
     with open(fname, 'w') as f:
         f.write(json.dumps(cfg, indent=2))
     
     return fname, cfg
 
 
 def get_connection_file(app=None):
     """Return the path to the connection file of an app
     
     Parameters
     ----------
     app : IPKernelApp instance [optional]
         If unspecified, the currently running app will be used
     """
     if app is None:
         from IPython.kernel.zmq.kernelapp import IPKernelApp
         if not IPKernelApp.initialized():
             raise RuntimeError("app not specified, and not in a running Kernel")
 
         app = IPKernelApp.instance()
     return filefind(app.connection_file, ['.', app.profile_dir.security_dir])
 
 
 def find_connection_file(filename, profile=None):
     """find a connection file, and return its absolute path.
     
     The current working directory and the profile's security
     directory will be searched for the file if it is not given by
     absolute path.
     
     If profile is unspecified, then the current running application's
     profile will be used, or 'default', if not run from IPython.
     
     If the argument does not match an existing file, it will be interpreted as a
     fileglob, and the matching file in the profile's security dir with
     the latest access time will be used.
     
     Parameters
     ----------
     filename : str
         The connection file or fileglob to search for.
     profile : str [optional]
         The name of the profile to use when searching for the connection file,
         if different from the current IPython session or 'default'.
     
     Returns
     -------
     str : The absolute path of the connection file.
     """
     from IPython.core.application import BaseIPythonApplication as IPApp
     try:
         # quick check for absolute path, before going through logic
         return filefind(filename)
     except IOError:
         pass
     
     if profile is None:
         # profile unspecified, check if running from an IPython app
         if IPApp.initialized():
             app = IPApp.instance()
             profile_dir = app.profile_dir
         else:
             # not running in IPython, use default profile
             profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')
     else:
         # find profiledir by profile name:
         profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
     security_dir = profile_dir.security_dir
     
     try:
         # first, try explicit name
         return filefind(filename, ['.', security_dir])
     except IOError:
         pass
     
     # not found by full name
     
     if '*' in filename:
         # given as a glob already
         pat = filename
     else:
         # accept any substring match
         pat = '*%s*' % filename
     matches = glob.glob( os.path.join(security_dir, pat) )
     if not matches:
         raise IOError("Could not find %r in %r" % (filename, security_dir))
     elif len(matches) == 1:
         return matches[0]
     else:
         # get most recent match, by access time:
         return sorted(matches, key=lambda f: os.stat(f).st_atime)[-1]
 
 
 def get_connection_info(connection_file=None, unpack=False, profile=None):
     """Return the connection information for the current Kernel.
     
     Parameters
     ----------
     connection_file : str [optional]
         The connection file to be used. Can be given by absolute path, or
         IPython will search in the security directory of a given profile.
         If run from IPython, 
         
         If unspecified, the connection file for the currently running
         IPython Kernel will be used, which is only allowed from inside a kernel.
     unpack : bool [default: False]
         if True, return the unpacked dict, otherwise just the string contents
         of the file.
     profile : str [optional]
         The name of the profile to use when searching for the connection file,
         if different from the current IPython session or 'default'.
         
     
     Returns
     -------
     The connection dictionary of the current kernel, as string or dict,
     depending on `unpack`.
     """
     if connection_file is None:
         # get connection file from current kernel
         cf = get_connection_file()
     else:
         # connection file specified, allow shortnames:
         cf = find_connection_file(connection_file, profile=profile)
     
     with open(cf) as f:
         info = f.read()
     
     if unpack:
         info = json.loads(info)
         # ensure key is bytes:
         info['key'] = str_to_bytes(info.get('key', ''))
     return info
 
 
 def connect_qtconsole(connection_file=None, argv=None, profile=None):
     """Connect a qtconsole to the current kernel.
     
     This is useful for connecting a second qtconsole to a kernel, or to a
     local notebook.
     
     Parameters
     ----------
     connection_file : str [optional]
         The connection file to be used. Can be given by absolute path, or
         IPython will search in the security directory of a given profile.
         If run from IPython, 
         
         If unspecified, the connection file for the currently running
         IPython Kernel will be used, which is only allowed from inside a kernel.
     argv : list [optional]
         Any extra args to be passed to the console.
     profile : str [optional]
         The name of the profile to use when searching for the connection file,
         if different from the current IPython session or 'default'.
     
     
     Returns
     -------
     subprocess.Popen instance running the qtconsole frontend
     """
     argv = [] if argv is None else argv
     
     if connection_file is None:
         # get connection file from current kernel
         cf = get_connection_file()
     else:
         cf = find_connection_file(connection_file, profile=profile)
     
     cmd = ';'.join([
         "from IPython.qt.console import qtconsoleapp",
         "qtconsoleapp.main()"
     ])
     
     return Popen([sys.executable, '-c', cmd, '--existing', cf] + argv,
         stdout=PIPE, stderr=PIPE, close_fds=(sys.platform != 'win32'),
     )
 
 
 def tunnel_to_kernel(connection_info, sshserver, sshkey=None):
     """tunnel connections to a kernel via ssh
     
     This will open four SSH tunnels from localhost on this machine to the
     ports associated with the kernel.  They can be either direct
     localhost-localhost tunnels, or if an intermediate server is necessary,
     the kernel must be listening on a public IP.
     
     Parameters
     ----------
     connection_info : dict or str (path)
         Either a connection dict, or the path to a JSON connection file
     sshserver : str
         The ssh sever to use to tunnel to the kernel. Can be a full
         `user@server:port` string. ssh config aliases are respected.
     sshkey : str [optional]
         Path to file containing ssh key to use for authentication.
         Only necessary if your ssh config does not already associate
         a keyfile with the host.
     
     Returns
     -------
     
     (shell, iopub, stdin, hb) : ints
         The four ports on localhost that have been forwarded to the kernel.
     """
     if isinstance(connection_info, basestring):
         # it's a path, unpack it
         with open(connection_info) as f:
             connection_info = json.loads(f.read())
     
     cf = connection_info
     
     lports = tunnel.select_random_ports(4)
     rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf['hb_port']
     
     remote_ip = cf['ip']
     
     if tunnel.try_passwordless_ssh(sshserver, sshkey):
         password=False
     else:
         password = getpass("SSH Password for %s: " % cast_bytes_py2(sshserver))
     
     for lp,rp in zip(lports, rports):
         tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password)
     
     return tuple(lports)
 
 
 #-----------------------------------------------------------------------------
 # Mixin for classes that work with connection files
 #-----------------------------------------------------------------------------
 
 channel_socket_types = {
     'hb' : zmq.REQ,
     'shell' : zmq.DEALER,
     'iopub' : zmq.SUB,
     'stdin' : zmq.DEALER,
     'control': zmq.DEALER,
 }
 
 port_names = [ "%s_port" % channel for channel in ('shell', 'stdin', 'iopub', 'hb', 'control')]
 
 class ConnectionFileMixin(Configurable):
     """Mixin for configurable classes that work with connection files"""
 
     # The addresses for the communication channels
     connection_file = Unicode('')
     _connection_file_written = Bool(False)
 
     transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True)
 
     ip = Unicode(LOCALHOST, config=True,
         help="""Set the kernel\'s IP address [default localhost].
         If the IP address is something other than localhost, then
         Consoles on other machines will be able to connect
         to the Kernel, so be careful!"""
     )
 
     def _ip_default(self):
         if self.transport == 'ipc':
             if self.connection_file:
                 return os.path.splitext(self.connection_file)[0] + '-ipc'
             else:
                 return 'kernel-ipc'
         else:
             return LOCALHOST
 
     def _ip_changed(self, name, old, new):
         if new == '*':
             self.ip = '0.0.0.0'
 
     # protected traits
 
     shell_port = Integer(0)
     iopub_port = Integer(0)
     stdin_port = Integer(0)
     control_port = Integer(0)
     hb_port = Integer(0)
 
     @property
     def ports(self):
         return [ getattr(self, name) for name in port_names ]
 
     #--------------------------------------------------------------------------
     # Connection and ipc file management
     #--------------------------------------------------------------------------
 
     def get_connection_info(self):
         """return the connection info as a dict"""
         return dict(
             transport=self.transport,
             ip=self.ip,
             shell_port=self.shell_port,
             iopub_port=self.iopub_port,
             stdin_port=self.stdin_port,
             hb_port=self.hb_port,
             control_port=self.control_port,
             signature_scheme=self.session.signature_scheme,
             key=self.session.key,
         )
 
     def cleanup_connection_file(self):
         """Cleanup connection file *if we wrote it*
 
         Will not raise if the connection file was already removed somehow.
         """
         if self._connection_file_written:
             # cleanup connection files on full shutdown of kernel we started
             self._connection_file_written = False
             try:
                 os.remove(self.connection_file)
             except (IOError, OSError, AttributeError):
                 pass
 
     def cleanup_ipc_files(self):
         """Cleanup ipc files if we wrote them."""
         if self.transport != 'ipc':
             return
         for port in self.ports:
             ipcfile = "%s-%i" % (self.ip, port)
             try:
                 os.remove(ipcfile)
             except (IOError, OSError):
                 pass
 
     def write_connection_file(self):
         """Write connection info to JSON dict in self.connection_file."""
         if self._connection_file_written:
             return
 
         self.connection_file, cfg = write_connection_file(self.connection_file,
             transport=self.transport, ip=self.ip, key=self.session.key,
             stdin_port=self.stdin_port, iopub_port=self.iopub_port,
             shell_port=self.shell_port, hb_port=self.hb_port,
             control_port=self.control_port,
             signature_scheme=self.session.signature_scheme,
         )
         # write_connection_file also sets default ports:
         for name in port_names:
             setattr(self, name, cfg[name])
 
         self._connection_file_written = True
 
     def load_connection_file(self):
         """Load connection info from JSON dict in self.connection_file."""
         with open(self.connection_file) as f:
             cfg = json.loads(f.read())
 
         self.transport = cfg.get('transport', 'tcp')
         self.ip = cfg['ip']
         for name in port_names:
             setattr(self, name, cfg[name])
         if 'key' in cfg:
             self.session.key = str_to_bytes(cfg['key'])
         if cfg.get('signature_scheme'):
             self.session.signature_scheme = cfg['signature_scheme']
 
     #--------------------------------------------------------------------------
     # Creating connected sockets
     #--------------------------------------------------------------------------
 
     def _make_url(self, channel):
         """Make a ZeroMQ URL for a given channel."""
         transport = self.transport
         ip = self.ip
         port = getattr(self, '%s_port' % channel)
 
         if transport == 'tcp':
             return "tcp://%s:%i" % (ip, port)
         else:
             return "%s://%s-%s" % (transport, ip, port)
 
     def _create_connected_socket(self, channel, identity=None):
         """Create a zmq Socket and connect it to the kernel."""
         url = self._make_url(channel)
         socket_type = channel_socket_types[channel]
         self.log.info("Connecting to: %s" % url)
         sock = self.context.socket(socket_type)
         if identity:
             sock.identity = identity
         sock.connect(url)
         return sock
 
     def connect_iopub(self, identity=None):
         """return zmq Socket connected to the IOPub channel"""
         sock = self._create_connected_socket('iopub', identity=identity)
         sock.setsockopt(zmq.SUBSCRIBE, b'')
         return sock
 
     def connect_shell(self, identity=None):
         """return zmq Socket connected to the Shell channel"""
         return self._create_connected_socket('shell', identity=identity)
 
     def connect_stdin(self, identity=None):
         """return zmq Socket connected to the StdIn channel"""
         return self._create_connected_socket('stdin', identity=identity)
 
     def connect_hb(self, identity=None):
         """return zmq Socket connected to the Heartbeat channel"""
         return self._create_connected_socket('hb', identity=identity)
 
     def connect_control(self, identity=None):
         """return zmq Socket connected to the Heartbeat channel"""
         return self._create_connected_socket('control', identity=identity)
 
 
 __all__ = [
     'write_connection_file',
     'get_connection_file',
     'find_connection_file',
     'get_connection_info',
     'connect_qtconsole',
     'tunnel_to_kernel',
 ]

IPython/parallel/tests/test_client.py

@@ -1,542 +1,541 b''
 """Tests for parallel client.py
 
 Authors:
 
 * Min RK
 """
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from __future__ import division
 
 import time
 from datetime import datetime
-from tempfile import mktemp
 
 import zmq
 
 from IPython import parallel
 from IPython.parallel.client import client as clientmod
 from IPython.parallel import error
 from IPython.parallel import AsyncResult, AsyncHubResult
 from IPython.parallel import LoadBalancedView, DirectView
 
 from clienttest import ClusterTestCase, segfault, wait, add_engines
 
 def setup():
     add_engines(4, total=True)
 
 class TestClient(ClusterTestCase):
     
     def test_ids(self):
         n = len(self.client.ids)
         self.add_engines(2)
         self.assertEqual(len(self.client.ids), n+2)
     
     def test_view_indexing(self):
         """test index access for views"""
         self.minimum_engines(4)
         targets = self.client._build_targets('all')[-1]
         v = self.client[:]
         self.assertEqual(v.targets, targets)
         t = self.client.ids[2]
         v = self.client[t]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, t)
         t = self.client.ids[2:4]
         v = self.client[t]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, t)
         v = self.client[::2]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[::2])
         v = self.client[1::3]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[1::3])
         v = self.client[:-3]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[:-3])
         v = self.client[-1]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[-1])
         self.assertRaises(TypeError, lambda : self.client[None])
     
     def test_lbview_targets(self):
         """test load_balanced_view targets"""
         v = self.client.load_balanced_view()
         self.assertEqual(v.targets, None)
         v = self.client.load_balanced_view(-1)
         self.assertEqual(v.targets, [self.client.ids[-1]])
         v = self.client.load_balanced_view('all')
         self.assertEqual(v.targets, None)
     
     def test_dview_targets(self):
         """test direct_view targets"""
         v = self.client.direct_view()
         self.assertEqual(v.targets, 'all')
         v = self.client.direct_view('all')
         self.assertEqual(v.targets, 'all')
         v = self.client.direct_view(-1)
         self.assertEqual(v.targets, self.client.ids[-1])
     
     def test_lazy_all_targets(self):
         """test lazy evaluation of rc.direct_view('all')"""
         v = self.client.direct_view()
         self.assertEqual(v.targets, 'all')
         
         def double(x):
             return x*2
         seq = range(100)
         ref = [ double(x) for x in seq ]
         
         # add some engines, which should be used
         self.add_engines(1)
         n1 = len(self.client.ids)
         
         # simple apply
         r = v.apply_sync(lambda : 1)
         self.assertEqual(r, [1] * n1)
         
         # map goes through remotefunction
         r = v.map_sync(double, seq)
         self.assertEqual(r, ref)
 
         # add a couple more engines, and try again
         self.add_engines(2)
         n2 = len(self.client.ids)
         self.assertNotEqual(n2, n1)
         
         # apply
         r = v.apply_sync(lambda : 1)
         self.assertEqual(r, [1] * n2)
         
         # map
         r = v.map_sync(double, seq)
         self.assertEqual(r, ref)
     
     def test_targets(self):
         """test various valid targets arguments"""
         build = self.client._build_targets
         ids = self.client.ids
         idents,targets = build(None)
         self.assertEqual(ids, targets)
     
     def test_clear(self):
         """test clear behavior"""
         self.minimum_engines(2)
         v = self.client[:]
         v.block=True
         v.push(dict(a=5))
         v.pull('a')
         id0 = self.client.ids[-1]
         self.client.clear(targets=id0, block=True)
         a = self.client[:-1].get('a')
         self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
         self.client.clear(block=True)
         for i in self.client.ids:
             self.assertRaisesRemote(NameError, self.client[i].get, 'a')
     
     def test_get_result(self):
         """test getting results from the Hub."""
         c = clientmod.Client(profile='iptest')
         t = c.ids[-1]
         ar = c[t].apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = self.client.get_result(ar.msg_ids[0])
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEqual(ahr.get(), ar.get())
         ar2 = self.client.get_result(ar.msg_ids[0])
         self.assertFalse(isinstance(ar2, AsyncHubResult))
         c.close()
     
     def test_get_execute_result(self):
         """test getting execute results from the Hub."""
         c = clientmod.Client(profile='iptest')
         t = c.ids[-1]
         cell = '\n'.join([
             'import time',
             'time.sleep(0.25)',
             '5'
         ])
         ar = c[t].execute("import time; time.sleep(1)", silent=False)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = self.client.get_result(ar.msg_ids[0])
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEqual(ahr.get().pyout, ar.get().pyout)
         ar2 = self.client.get_result(ar.msg_ids[0])
         self.assertFalse(isinstance(ar2, AsyncHubResult))
         c.close()
     
     def test_ids_list(self):
         """test client.ids"""
         ids = self.client.ids
         self.assertEqual(ids, self.client._ids)
         self.assertFalse(ids is self.client._ids)
         ids.remove(ids[-1])
         self.assertNotEqual(ids, self.client._ids)
     
     def test_queue_status(self):
         ids = self.client.ids
         id0 = ids[0]
         qs = self.client.queue_status(targets=id0)
         self.assertTrue(isinstance(qs, dict))
         self.assertEqual(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
         allqs = self.client.queue_status()
         self.assertTrue(isinstance(allqs, dict))
         intkeys = list(allqs.keys())
         intkeys.remove('unassigned')
         self.assertEqual(sorted(intkeys), sorted(self.client.ids))
         unassigned = allqs.pop('unassigned')
         for eid,qs in allqs.items():
             self.assertTrue(isinstance(qs, dict))
             self.assertEqual(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
 
     def test_shutdown(self):
         ids = self.client.ids
         id0 = ids[0]
         self.client.shutdown(id0, block=True)
         while id0 in self.client.ids:
             time.sleep(0.1)
             self.client.spin()
         
         self.assertRaises(IndexError, lambda : self.client[id0])
         
     def test_result_status(self):
         pass
         # to be written
     
     def test_db_query_dt(self):
         """test db query by date"""
         hist = self.client.hub_history()
         middle = self.client.db_query({'msg_id' : hist[len(hist)//2]})[0]
         tic = middle['submitted']
         before = self.client.db_query({'submitted' : {'$lt' : tic}})
         after = self.client.db_query({'submitted' : {'$gte' : tic}})
         self.assertEqual(len(before)+len(after),len(hist))
         for b in before:
             self.assertTrue(b['submitted'] < tic)
         for a in after:
             self.assertTrue(a['submitted'] >= tic)
         same = self.client.db_query({'submitted' : tic})
         for s in same:
             self.assertTrue(s['submitted'] == tic)
     
     def test_db_query_keys(self):
         """test extracting subset of record keys"""
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
         for rec in found:
             self.assertEqual(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))
     
     def test_db_query_default_keys(self):
         """default db_query excludes buffers"""
         found = self.client.db_query({'msg_id': {'$ne' : ''}})
         for rec in found:
             keys = set(rec.keys())
             self.assertFalse('buffers' in keys, "'buffers' should not be in: %s" % keys)
             self.assertFalse('result_buffers' in keys, "'result_buffers' should not be in: %s" % keys)
     
     def test_db_query_msg_id(self):
         """ensure msg_id is always in db queries"""
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
         for rec in found:
             self.assertTrue('msg_id' in rec.keys())
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted'])
         for rec in found:
             self.assertTrue('msg_id' in rec.keys())
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['msg_id'])
         for rec in found:
             self.assertTrue('msg_id' in rec.keys())
     
     def test_db_query_get_result(self):
         """pop in db_query shouldn't pop from result itself"""
         self.client[:].apply_sync(lambda : 1)
         found = self.client.db_query({'msg_id': {'$ne' : ''}})
         rc2 = clientmod.Client(profile='iptest')
         # If this bug is not fixed, this call will hang:
         ar = rc2.get_result(self.client.history[-1])
         ar.wait(2)
         self.assertTrue(ar.ready())
         ar.get()
         rc2.close()
     
     def test_db_query_in(self):
         """test db query with '$in','$nin' operators"""
         hist = self.client.hub_history()
         even = hist[::2]
         odd = hist[1::2]
         recs = self.client.db_query({ 'msg_id' : {'$in' : even}})
         found = [ r['msg_id'] for r in recs ]
         self.assertEqual(set(even), set(found))
         recs = self.client.db_query({ 'msg_id' : {'$nin' : even}})
         found = [ r['msg_id'] for r in recs ]
         self.assertEqual(set(odd), set(found))
     
     def test_hub_history(self):
         hist = self.client.hub_history()
         recs = self.client.db_query({ 'msg_id' : {"$ne":''}})
         recdict = {}
         for rec in recs:
             recdict[rec['msg_id']] = rec
         
         latest = datetime(1984,1,1)
         for msg_id in hist:
             rec = recdict[msg_id]
             newt = rec['submitted']
             self.assertTrue(newt >= latest)
             latest = newt
         ar = self.client[-1].apply_async(lambda : 1)
         ar.get()
         time.sleep(0.25)
         self.assertEqual(self.client.hub_history()[-1:],ar.msg_ids)
     
     def _wait_for_idle(self):
         """wait for an engine to become idle, according to the Hub"""
         rc = self.client
         
         # step 1. wait for all requests to be noticed
         # timeout 5s, polling every 100ms
         msg_ids = set(rc.history)
         hub_hist = rc.hub_history()
         for i in range(50):
             if msg_ids.difference(hub_hist):
                 time.sleep(0.1)
                 hub_hist = rc.hub_history()
             else:
                 break
         
         self.assertEqual(len(msg_ids.difference(hub_hist)), 0)
         
         # step 2. wait for all requests to be done
         # timeout 5s, polling every 100ms
         qs = rc.queue_status()
         for i in range(50):
             if qs['unassigned'] or any(qs[eid]['tasks'] for eid in rc.ids):
                 time.sleep(0.1)
                 qs = rc.queue_status()
             else:
                 break
         
         # ensure Hub up to date:
         self.assertEqual(qs['unassigned'], 0)
         for eid in rc.ids:
             self.assertEqual(qs[eid]['tasks'], 0)
     
     
     def test_resubmit(self):
         def f():
             import random
             return random.random()
         v = self.client.load_balanced_view()
         ar = v.apply_async(f)
         r1 = ar.get(1)
         # give the Hub a chance to notice:
         self._wait_for_idle()
         ahr = self.client.resubmit(ar.msg_ids)
         r2 = ahr.get(1)
         self.assertFalse(r1 == r2)
 
     def test_resubmit_chain(self):
         """resubmit resubmitted tasks"""
         v = self.client.load_balanced_view()
         ar = v.apply_async(lambda x: x, 'x'*1024)
         ar.get()
         self._wait_for_idle()
         ars = [ar]
         
         for i in range(10):
             ar = ars[-1]
             ar2 = self.client.resubmit(ar.msg_ids)
         
         [ ar.get() for ar in ars ]
 
     def test_resubmit_header(self):
         """resubmit shouldn't clobber the whole header"""
         def f():
             import random
             return random.random()
         v = self.client.load_balanced_view()
         v.retries = 1
         ar = v.apply_async(f)
         r1 = ar.get(1)
         # give the Hub a chance to notice:
         self._wait_for_idle()
         ahr = self.client.resubmit(ar.msg_ids)
         ahr.get(1)
         time.sleep(0.5)
         records = self.client.db_query({'msg_id': {'$in': ar.msg_ids + ahr.msg_ids}}, keys='header')
         h1,h2 = [ r['header'] for r in records ]
         for key in set(h1.keys()).union(set(h2.keys())):
             if key in ('msg_id', 'date'):
                 self.assertNotEqual(h1[key], h2[key])
             else:
                 self.assertEqual(h1[key], h2[key])
 
     def test_resubmit_aborted(self):
         def f():
             import random
             return random.random()
         v = self.client.load_balanced_view()
         # restrict to one engine, so we can put a sleep
         # ahead of the task, so it will get aborted
         eid = self.client.ids[-1]
         v.targets = [eid]
         sleep = v.apply_async(time.sleep, 0.5)
         ar = v.apply_async(f)
         ar.abort()
         self.assertRaises(error.TaskAborted, ar.get)
         # Give the Hub a chance to get up to date:
         self._wait_for_idle()
         ahr = self.client.resubmit(ar.msg_ids)
         r2 = ahr.get(1)
 
     def test_resubmit_inflight(self):
         """resubmit of inflight task"""
         v = self.client.load_balanced_view()
         ar = v.apply_async(time.sleep,1)
         # give the message a chance to arrive
         time.sleep(0.2)
         ahr = self.client.resubmit(ar.msg_ids)
         ar.get(2)
         ahr.get(2)
 
     def test_resubmit_badkey(self):
         """ensure KeyError on resubmit of nonexistant task"""
         self.assertRaisesRemote(KeyError, self.client.resubmit, ['invalid'])
 
     def test_purge_hub_results(self):
         # ensure there are some tasks
         for i in range(5):
             self.client[:].apply_sync(lambda : 1)
         # Wait for the Hub to realise the result is done:
         # This prevents a race condition, where we
         # might purge a result the Hub still thinks is pending.
         self._wait_for_idle()
         rc2 = clientmod.Client(profile='iptest')
         hist = self.client.hub_history()
         ahr = rc2.get_result([hist[-1]])
         ahr.wait(10)
         self.client.purge_hub_results(hist[-1])
         newhist = self.client.hub_history()
         self.assertEqual(len(newhist)+1,len(hist))
         rc2.spin()
         rc2.close()
 
     def test_purge_local_results(self):
         # ensure there are some tasks
         res = []
         for i in range(5):
             res.append(self.client[:].apply_async(lambda : 1))
         self._wait_for_idle()
         self.client.wait(10) # wait for the results to come back
         before = len(self.client.results)
         self.assertEqual(len(self.client.metadata),before)
         self.client.purge_local_results(res[-1])
         self.assertEqual(len(self.client.results),before-len(res[-1]), msg="Not removed from results")
         self.assertEqual(len(self.client.metadata),before-len(res[-1]), msg="Not removed from metadata")
 
     def test_purge_local_results_outstanding(self):
         v = self.client[-1]
         ar = v.apply_async(lambda : 1)
         msg_id = ar.msg_ids[0]
         ar.get()
         self._wait_for_idle()
         ar2 = v.apply_async(time.sleep, 1)
         self.assertIn(msg_id, self.client.results)
         self.assertIn(msg_id, self.client.metadata)
         self.client.purge_local_results(ar)
         self.assertNotIn(msg_id, self.client.results)
         self.assertNotIn(msg_id, self.client.metadata)
         with self.assertRaises(RuntimeError):
             self.client.purge_local_results(ar2)
         ar2.get()
         self.client.purge_local_results(ar2)
 
     def test_purge_all_local_results_outstanding(self):
         v = self.client[-1]
         ar = v.apply_async(time.sleep, 1)
         with self.assertRaises(RuntimeError):
             self.client.purge_local_results('all')
         ar.get()
         self.client.purge_local_results('all')
 
     def test_purge_all_hub_results(self):
         self.client.purge_hub_results('all')
         hist = self.client.hub_history()
         self.assertEqual(len(hist), 0)
 
     def test_purge_all_local_results(self):
         self.client.purge_local_results('all')
         self.assertEqual(len(self.client.results), 0, msg="Results not empty")
         self.assertEqual(len(self.client.metadata), 0, msg="metadata not empty")
 
     def test_purge_all_results(self):
         # ensure there are some tasks
         for i in range(5):
             self.client[:].apply_sync(lambda : 1)
         self.client.wait(10)
         self._wait_for_idle()
         self.client.purge_results('all')
         self.assertEqual(len(self.client.results), 0, msg="Results not empty")
         self.assertEqual(len(self.client.metadata), 0, msg="metadata not empty")
         hist = self.client.hub_history()
         self.assertEqual(len(hist), 0, msg="hub history not empty")
         
     def test_purge_everything(self):
         # ensure there are some tasks
         for i in range(5):
             self.client[:].apply_sync(lambda : 1)
         self.client.wait(10)
         self._wait_for_idle()
         self.client.purge_everything()
         # The client results
         self.assertEqual(len(self.client.results), 0, msg="Results not empty")
         self.assertEqual(len(self.client.metadata), 0, msg="metadata not empty")
         # The client "bookkeeping"
         self.assertEqual(len(self.client.session.digest_history), 0, msg="session digest not empty")
         self.assertEqual(len(self.client.history), 0, msg="client history not empty")
         # the hub results
         hist = self.client.hub_history()
         self.assertEqual(len(hist), 0, msg="hub history not empty")
         
     
     def test_spin_thread(self):
         self.client.spin_thread(0.01)
         ar = self.client[-1].apply_async(lambda : 1)
         time.sleep(0.1)
         self.assertTrue(ar.wall_time < 0.1,
             "spin should have kept wall_time < 0.1, but got %f" % ar.wall_time
         )
     
     def test_stop_spin_thread(self):
         self.client.spin_thread(0.01)
         self.client.stop_spin_thread()
         ar = self.client[-1].apply_async(lambda : 1)
         time.sleep(0.15)
         self.assertTrue(ar.wall_time > 0.1,
             "Shouldn't be spinning, but got wall_time=%f" % ar.wall_time
         )
     
     def test_activate(self):
         ip = get_ipython()
         magics = ip.magics_manager.magics
         self.assertTrue('px' in magics['line'])
         self.assertTrue('px' in magics['cell'])
         v0 = self.client.activate(-1, '0')
         self.assertTrue('px0' in magics['line'])
         self.assertTrue('px0' in magics['cell'])
         self.assertEqual(v0.targets, self.client.ids[-1])
         v0 = self.client.activate('all', 'all')
         self.assertTrue('pxall' in magics['line'])
         self.assertTrue('pxall' in magics['cell'])
         self.assertEqual(v0.targets, 'all')

IPython/parallel/tests/test_view.py

@@ -1,801 +1,800 b''
 # -*- coding: utf-8 -*-
 """test View objects
 
 Authors:
 
 * Min RK
 """
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import sys
 import platform
 import time
 from collections import namedtuple
-from tempfile import mktemp
+from tempfile import NamedTemporaryFile
 from StringIO import StringIO
 
 import zmq
 from nose import SkipTest
 from nose.plugins.attrib import attr
 
 from IPython.testing import decorators as dec
 from IPython.testing.ipunittest import ParametricTestCase
 from IPython.utils.io import capture_output
 
 from IPython import parallel  as pmod
 from IPython.parallel import error
 from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult
 from IPython.parallel import DirectView
 from IPython.parallel.util import interactive
 
 from IPython.parallel.tests import add_engines
 
 from .clienttest import ClusterTestCase, crash, wait, skip_without
 
 def setup():
     add_engines(3, total=True)
 
 point = namedtuple("point", "x y")
 
 class TestView(ClusterTestCase, ParametricTestCase):
     
     def setUp(self):
         # On Win XP, wait for resource cleanup, else parallel test group fails
         if platform.system() == "Windows" and platform.win32_ver()[0] == "XP":
             # 1 sec fails. 1.5 sec seems ok. Using 2 sec for margin of safety
             time.sleep(2)
         super(TestView, self).setUp()
 
     @attr('crash')
     def test_z_crash_mux(self):
         """test graceful handling of engine death (direct)"""
         # self.add_engines(1)
         eid = self.client.ids[-1]
         ar = self.client[eid].apply_async(crash)
         self.assertRaisesRemote(error.EngineError, ar.get, 10)
         eid = ar.engine_id
         tic = time.time()
         while eid in self.client.ids and time.time()-tic < 5:
             time.sleep(.01)
             self.client.spin()
         self.assertFalse(eid in self.client.ids, "Engine should have died")
     
     def test_push_pull(self):
         """test pushing and pulling"""
         data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
         t = self.client.ids[-1]
         v = self.client[t]
         push = v.push
         pull = v.pull
         v.block=True
         nengines = len(self.client)
         push({'data':data})
         d = pull('data')
         self.assertEqual(d, data)
         self.client[:].push({'data':data})
         d = self.client[:].pull('data', block=True)
         self.assertEqual(d, nengines*[data])
         ar = push({'data':data}, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         ar = self.client[:].pull('data', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         self.assertEqual(r, nengines*[data])
         self.client[:].push(dict(a=10,b=20))
         r = self.client[:].pull(('a','b'), block=True)
         self.assertEqual(r, nengines*[[10,20]])
     
     def test_push_pull_function(self):
         "test pushing and pulling functions"
         def testf(x):
             return 2.0*x
         
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=True
         push = v.push
         pull = v.pull
         execute = v.execute
         push({'testf':testf})
         r = pull('testf')
         self.assertEqual(r(1.0), testf(1.0))
         execute('r = testf(10)')
         r = pull('r')
         self.assertEqual(r, testf(10))
         ar = self.client[:].push({'testf':testf}, block=False)
         ar.get()
         ar = self.client[:].pull('testf', block=False)
         rlist = ar.get()
         for r in rlist:
             self.assertEqual(r(1.0), testf(1.0))
         execute("def g(x): return x*x")
         r = pull(('testf','g'))
         self.assertEqual((r[0](10),r[1](10)), (testf(10), 100))
     
     def test_push_function_globals(self):
         """test that pushed functions have access to globals"""
         @interactive
         def geta():
             return a
         # self.add_engines(1)
         v = self.client[-1]
         v.block=True
         v['f'] = geta
         self.assertRaisesRemote(NameError, v.execute, 'b=f()')
         v.execute('a=5')
         v.execute('b=f()')
         self.assertEqual(v['b'], 5)
     
     def test_push_function_defaults(self):
         """test that pushed functions preserve default args"""
         def echo(a=10):
             return a
         v = self.client[-1]
         v.block=True
         v['f'] = echo
         v.execute('b=f()')
         self.assertEqual(v['b'], 10)
 
     def test_get_result(self):
         """test getting results from the Hub."""
         c = pmod.Client(profile='iptest')
         # self.add_engines(1)
         t = c.ids[-1]
         v = c[t]
         v2 = self.client[t]
         ar = v.apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = v2.get_result(ar.msg_ids[0])
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEqual(ahr.get(), ar.get())
         ar2 = v2.get_result(ar.msg_ids[0])
         self.assertFalse(isinstance(ar2, AsyncHubResult))
         c.spin()
         c.close()
     
     def test_run_newline(self):
         """test that run appends newline to files"""
-        tmpfile = mktemp()
-        with open(tmpfile, 'w') as f:
+        with NamedTemporaryFile('w', delete=False) as f:
             f.write("""def g():
                 return 5
                 """)
         v = self.client[-1]
-        v.run(tmpfile, block=True)
+        v.run(f.name, block=True)
         self.assertEqual(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
 
     def test_apply_tracked(self):
         """test tracking for apply"""
         # self.add_engines(1)
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=False
         def echo(n=1024*1024, **kwargs):
             with v.temp_flags(**kwargs):
                 return v.apply(lambda x: x, 'x'*n)
         ar = echo(1, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         ar = echo(track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEqual(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         
     def test_push_tracked(self):
         t = self.client.ids[-1]
         ns = dict(x='x'*1024*1024)
         v = self.client[t]
         ar = v.push(ns, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = v.push(ns, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         ar._tracker.wait()
         self.assertEqual(ar.sent, ar._tracker.done)
         self.assertTrue(ar.sent)
         ar.get()
         
     def test_scatter_tracked(self):
         t = self.client.ids
         x='x'*1024*1024
         ar = self.client[t].scatter('x', x, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = self.client[t].scatter('x', x, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEqual(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         ar.get()
     
     def test_remote_reference(self):
         v = self.client[-1]
         v['a'] = 123
         ra = pmod.Reference('a')
         b = v.apply_sync(lambda x: x, ra)
         self.assertEqual(b, 123)
 
 
     def test_scatter_gather(self):
         view = self.client[:]
         seq1 = range(16)
         view.scatter('a', seq1)
         seq2 = view.gather('a', block=True)
         self.assertEqual(seq2, seq1)
         self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy(self):
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         view = self.client[:]
         a = numpy.arange(64)
         view.scatter('a', a, block=True)
         b = view.gather('a', block=True)
         assert_array_equal(b, a)
     
     def test_scatter_gather_lazy(self):
         """scatter/gather with targets='all'"""
         view = self.client.direct_view(targets='all')
         x = range(64)
         view.scatter('x', x)
         gathered = view.gather('x', block=True)
         self.assertEqual(gathered, x)
         
 
     @dec.known_failure_py3
     @skip_without('numpy')
     def test_push_numpy_nocopy(self):
         import numpy
         view = self.client[:]
         a = numpy.arange(64)
         view['A'] = a
         @interactive
         def check_writeable(x):
             return x.flags.writeable
         
         for flag in view.apply_sync(check_writeable, pmod.Reference('A')):
             self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
         
         view.push(dict(B=a))
         for flag in view.apply_sync(check_writeable, pmod.Reference('B')):
             self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
     
     @skip_without('numpy')
     def test_apply_numpy(self):
         """view.apply(f, ndarray)"""
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         
         A = numpy.random.random((100,100))
         view = self.client[-1]
         for dt in [ 'int32', 'uint8', 'float32', 'float64' ]:
             B = A.astype(dt)
             C = view.apply_sync(lambda x:x, B)
             assert_array_equal(B,C)
     
     @skip_without('numpy')
     def test_push_pull_recarray(self):
         """push/pull recarrays"""
         import numpy
         from numpy.testing.utils import assert_array_equal
         
         view = self.client[-1]
         
         R = numpy.array([
             (1, 'hi', 0.),
             (2**30, 'there', 2.5),
             (-99999, 'world', -12345.6789),
         ], [('n', int), ('s', '|S10'), ('f', float)])
         
         view['RR'] = R
         R2 = view['RR']
         
         r_dtype, r_shape = view.apply_sync(interactive(lambda : (RR.dtype, RR.shape)))
         self.assertEqual(r_dtype, R.dtype)
         self.assertEqual(r_shape, R.shape)
         self.assertEqual(R2.dtype, R.dtype)
         self.assertEqual(R2.shape, R.shape)
         assert_array_equal(R2, R)
 
     @skip_without('pandas')
     def test_push_pull_timeseries(self):
         """push/pull pandas.TimeSeries"""
         import pandas
         
         ts = pandas.TimeSeries(range(10))
         
         view = self.client[-1]
         
         view.push(dict(ts=ts), block=True)
         rts = view['ts']
         
         self.assertEqual(type(rts), type(ts))
         self.assertTrue((ts == rts).all())
     
     def test_map(self):
         view = self.client[:]
         def f(x):
             return x**2
         data = range(16)
         r = view.map_sync(f, data)
         self.assertEqual(r, map(f, data))
     
     def test_map_iterable(self):
         """test map on iterables (direct)"""
         view = self.client[:]
         # 101 is prime, so it won't be evenly distributed
         arr = range(101)
         # ensure it will be an iterator, even in Python 3
         it = iter(arr)
         r = view.map_sync(lambda x: x, it)
         self.assertEqual(r, list(arr))
 
     @skip_without('numpy')
     def test_map_numpy(self):
         """test map on numpy arrays (direct)"""
         import numpy
         from numpy.testing.utils import assert_array_equal
 
         view = self.client[:]
         # 101 is prime, so it won't be evenly distributed
         arr = numpy.arange(101)
         r = view.map_sync(lambda x: x, arr)
         assert_array_equal(r, arr)
     
     def test_scatter_gather_nonblocking(self):
         data = range(16)
         view = self.client[:]
         view.scatter('a', data, block=False)
         ar = view.gather('a', block=False)
         self.assertEqual(ar.get(), data)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy_nonblocking(self):
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         a = numpy.arange(64)
         view = self.client[:]
         ar = view.scatter('a', a, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         amr = view.gather('a', block=False)
         self.assertTrue(isinstance(amr, AsyncMapResult))
         assert_array_equal(amr.get(), a)
 
     def test_execute(self):
         view = self.client[:]
         # self.client.debug=True
         execute = view.execute
         ar = execute('c=30', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         ar = execute('d=[0,1,2]', block=False)
         self.client.wait(ar, 1)
         self.assertEqual(len(ar.get()), len(self.client))
         for c in view['c']:
             self.assertEqual(c, 30)
     
     def test_abort(self):
         view = self.client[-1]
         ar = view.execute('import time; time.sleep(1)', block=False)
         ar2 = view.apply_async(lambda : 2)
         ar3 = view.apply_async(lambda : 3)
         view.abort(ar2)
         view.abort(ar3.msg_ids)
         self.assertRaises(error.TaskAborted, ar2.get)
         self.assertRaises(error.TaskAborted, ar3.get)
     
     def test_abort_all(self):
         """view.abort() aborts all outstanding tasks"""
         view = self.client[-1]
         ars = [ view.apply_async(time.sleep, 0.25) for i in range(10) ]
         view.abort()
         view.wait(timeout=5)
         for ar in ars[5:]:
             self.assertRaises(error.TaskAborted, ar.get)
     
     def test_temp_flags(self):
         view = self.client[-1]
         view.block=True
         with view.temp_flags(block=False):
             self.assertFalse(view.block)
         self.assertTrue(view.block)
     
     @dec.known_failure_py3
     def test_importer(self):
         view = self.client[-1]
         view.clear(block=True)
         with view.importer:
             import re
         
         @interactive
         def findall(pat, s):
             # this globals() step isn't necessary in real code
             # only to prevent a closure in the test
             re = globals()['re']
             return re.findall(pat, s)
         
         self.assertEqual(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split())
     
     def test_unicode_execute(self):
         """test executing unicode strings"""
         v = self.client[-1]
         v.block=True
         if sys.version_info[0] >= 3:
             code="a='é'"
         else:
             code=u"a=u'é'"
         v.execute(code)
         self.assertEqual(v['a'], u'é')
         
     def test_unicode_apply_result(self):
         """test unicode apply results"""
         v = self.client[-1]
         r = v.apply_sync(lambda : u'é')
         self.assertEqual(r, u'é')
     
     def test_unicode_apply_arg(self):
         """test passing unicode arguments to apply"""
         v = self.client[-1]
         
         @interactive
         def check_unicode(a, check):
             assert isinstance(a, unicode), "%r is not unicode"%a
             assert isinstance(check, bytes), "%r is not bytes"%check
             assert a.encode('utf8') == check, "%s != %s"%(a,check)
         
         for s in [ u'é', u'ßø®∫',u'asdf' ]:
             try:
                 v.apply_sync(check_unicode, s, s.encode('utf8'))
             except error.RemoteError as e:
                 if e.ename == 'AssertionError':
                     self.fail(e.evalue)
                 else:
                     raise e
     
     def test_map_reference(self):
         """view.map(<Reference>, *seqs) should work"""
         v = self.client[:]
         v.scatter('n', self.client.ids, flatten=True)
         v.execute("f = lambda x,y: x*y")
         rf = pmod.Reference('f')
         nlist = list(range(10))
         mlist = nlist[::-1]
         expected = [ m*n for m,n in zip(mlist, nlist) ]
         result = v.map_sync(rf, mlist, nlist)
         self.assertEqual(result, expected)
 
     def test_apply_reference(self):
         """view.apply(<Reference>, *args) should work"""
         v = self.client[:]
         v.scatter('n', self.client.ids, flatten=True)
         v.execute("f = lambda x: n*x")
         rf = pmod.Reference('f')
         result = v.apply_sync(rf, 5)
         expected = [ 5*id for id in self.client.ids ]
         self.assertEqual(result, expected)
     
     def test_eval_reference(self):
         v = self.client[self.client.ids[0]]
         v['g'] = range(5)
         rg = pmod.Reference('g[0]')
         echo = lambda x:x
         self.assertEqual(v.apply_sync(echo, rg), 0)
     
     def test_reference_nameerror(self):
         v = self.client[self.client.ids[0]]
         r = pmod.Reference('elvis_has_left')
         echo = lambda x:x
         self.assertRaisesRemote(NameError, v.apply_sync, echo, r)
 
     def test_single_engine_map(self):
         e0 = self.client[self.client.ids[0]]
         r = range(5)
         check = [ -1*i for i in r ]
         result = e0.map_sync(lambda x: -1*x, r)
         self.assertEqual(result, check)
     
     def test_len(self):
         """len(view) makes sense"""
         e0 = self.client[self.client.ids[0]]
         yield self.assertEqual(len(e0), 1)
         v = self.client[:]
         yield self.assertEqual(len(v), len(self.client.ids))
         v = self.client.direct_view('all')
         yield self.assertEqual(len(v), len(self.client.ids))
         v = self.client[:2]
         yield self.assertEqual(len(v), 2)
         v = self.client[:1]
         yield self.assertEqual(len(v), 1)
         v = self.client.load_balanced_view()
         yield self.assertEqual(len(v), len(self.client.ids))
         # parametric tests seem to require manual closing?
         self.client.close()
 
     
     # begin execute tests
     
     def test_execute_reply(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         ar = e0.execute("5", silent=False)
         er = ar.get()
         self.assertEqual(str(er), "<ExecuteReply[%i]: 5>" % er.execution_count)
         self.assertEqual(er.pyout['data']['text/plain'], '5')
 
     def test_execute_reply_stdout(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         ar = e0.execute("print (5)", silent=False)
         er = ar.get()
         self.assertEqual(er.stdout.strip(), '5')
         
     def test_execute_pyout(self):
         """execute triggers pyout with silent=False"""
         view = self.client[:]
         ar = view.execute("5", silent=False, block=True)
         
         expected = [{'text/plain' : '5'}] * len(view)
         mimes = [ out['data'] for out in ar.pyout ]
         self.assertEqual(mimes, expected)
     
     def test_execute_silent(self):
         """execute does not trigger pyout with silent=True"""
         view = self.client[:]
         ar = view.execute("5", block=True)
         expected = [None] * len(view)
         self.assertEqual(ar.pyout, expected)
     
     def test_execute_magic(self):
         """execute accepts IPython commands"""
         view = self.client[:]
         view.execute("a = 5")
         ar = view.execute("%whos", block=True)
         # this will raise, if that failed
         ar.get(5)
         for stdout in ar.stdout:
             lines = stdout.splitlines()
             self.assertEqual(lines[0].split(), ['Variable', 'Type', 'Data/Info'])
             found = False
             for line in lines[2:]:
                 split = line.split()
                 if split == ['a', 'int', '5']:
                     found = True
                     break
             self.assertTrue(found, "whos output wrong: %s" % stdout)
     
     def test_execute_displaypub(self):
         """execute tracks display_pub output"""
         view = self.client[:]
         view.execute("from IPython.core.display import *")
         ar = view.execute("[ display(i) for i in range(5) ]", block=True)
         
         expected = [ {u'text/plain' : unicode(j)} for j in range(5) ]
         for outputs in ar.outputs:
             mimes = [ out['data'] for out in outputs ]
             self.assertEqual(mimes, expected)
     
     def test_apply_displaypub(self):
         """apply tracks display_pub output"""
         view = self.client[:]
         view.execute("from IPython.core.display import *")
         
         @interactive
         def publish():
             [ display(i) for i in range(5) ]
         
         ar = view.apply_async(publish)
         ar.get(5)
         expected = [ {u'text/plain' : unicode(j)} for j in range(5) ]
         for outputs in ar.outputs:
             mimes = [ out['data'] for out in outputs ]
             self.assertEqual(mimes, expected)
     
     def test_execute_raises(self):
         """exceptions in execute requests raise appropriately"""
         view = self.client[-1]
         ar = view.execute("1/0")
         self.assertRaisesRemote(ZeroDivisionError, ar.get, 2)
     
     def test_remoteerror_render_exception(self):
         """RemoteErrors get nice tracebacks"""
         view = self.client[-1]
         ar = view.execute("1/0")
         ip = get_ipython()
         ip.user_ns['ar'] = ar
         with capture_output() as io:
             ip.run_cell("ar.get(2)")
         
         self.assertTrue('ZeroDivisionError' in io.stdout, io.stdout)
     
     def test_compositeerror_render_exception(self):
         """CompositeErrors get nice tracebacks"""
         view = self.client[:]
         ar = view.execute("1/0")
         ip = get_ipython()
         ip.user_ns['ar'] = ar
 
         with capture_output() as io:
             ip.run_cell("ar.get(2)")
         
         count = min(error.CompositeError.tb_limit, len(view))
         
         self.assertEqual(io.stdout.count('ZeroDivisionError'), count * 2, io.stdout)
         self.assertEqual(io.stdout.count('by zero'), count, io.stdout)
         self.assertEqual(io.stdout.count(':execute'), count, io.stdout)
     
     def test_compositeerror_truncate(self):
         """Truncate CompositeErrors with many exceptions"""
         view = self.client[:]
         msg_ids = []
         for i in range(10):
             ar = view.execute("1/0")
             msg_ids.extend(ar.msg_ids)
         
         ar = self.client.get_result(msg_ids)
         try:
             ar.get()
         except error.CompositeError as _e:
             e = _e
         else:
             self.fail("Should have raised CompositeError")
         
         lines = e.render_traceback()
         with capture_output() as io:
             e.print_traceback()
         
         self.assertTrue("more exceptions" in lines[-1])
         count = e.tb_limit
         
         self.assertEqual(io.stdout.count('ZeroDivisionError'), 2 * count, io.stdout)
         self.assertEqual(io.stdout.count('by zero'), count, io.stdout)
         self.assertEqual(io.stdout.count(':execute'), count, io.stdout)
     
     @dec.skipif_not_matplotlib
     def test_magic_pylab(self):
         """%pylab works on engines"""
         view = self.client[-1]
         ar = view.execute("%pylab inline")
         # at least check if this raised:
         reply = ar.get(5)
         # include imports, in case user config
         ar = view.execute("plot(rand(100))", silent=False)
         reply = ar.get(5)
         self.assertEqual(len(reply.outputs), 1)
         output = reply.outputs[0]
         self.assertTrue("data" in output)
         data = output['data']
         self.assertTrue("image/png" in data)
     
     def test_func_default_func(self):
         """interactively defined function as apply func default"""
         def foo():
             return 'foo'
         
         def bar(f=foo):
             return f()
         
         view = self.client[-1]
         ar = view.apply_async(bar)
         r = ar.get(10)
         self.assertEqual(r, 'foo')
     def test_data_pub_single(self):
         view = self.client[-1]
         ar = view.execute('\n'.join([
             'from IPython.kernel.zmq.datapub import publish_data',
             'for i in range(5):',
             '  publish_data(dict(i=i))'
         ]), block=False)
         self.assertTrue(isinstance(ar.data, dict))
         ar.get(5)
         self.assertEqual(ar.data, dict(i=4))
 
     def test_data_pub(self):
         view = self.client[:]
         ar = view.execute('\n'.join([
             'from IPython.kernel.zmq.datapub import publish_data',
             'for i in range(5):',
             '  publish_data(dict(i=i))'
         ]), block=False)
         self.assertTrue(all(isinstance(d, dict) for d in ar.data))
         ar.get(5)
         self.assertEqual(ar.data, [dict(i=4)] * len(ar))
     
     def test_can_list_arg(self):
         """args in lists are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x: x, [rA])
         r = ar.get(5)
         self.assertEqual(r, [128])
 
     def test_can_dict_arg(self):
         """args in dicts are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x: x, dict(foo=rA))
         r = ar.get(5)
         self.assertEqual(r, dict(foo=128))
 
     def test_can_list_kwarg(self):
         """kwargs in lists are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x=5: x, x=[rA])
         r = ar.get(5)
         self.assertEqual(r, [128])
 
     def test_can_dict_kwarg(self):
         """kwargs in dicts are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x=5: x, dict(foo=rA))
         r = ar.get(5)
         self.assertEqual(r, dict(foo=128))
     
     def test_map_ref(self):
         """view.map works with references"""
         view = self.client[:]
         ranks = sorted(self.client.ids)
         view.scatter('rank', ranks, flatten=True)
         rrank = pmod.Reference('rank')
         
         amr = view.map_async(lambda x: x*2, [rrank] * len(view))
         drank = amr.get(5)
         self.assertEqual(drank, [ r*2 for r in ranks ])
         
     def test_nested_getitem_setitem(self):
         """get and set with view['a.b']"""
         view = self.client[-1]
         view.execute('\n'.join([
             'class A(object): pass',
             'a = A()',
             'a.b = 128',
             ]), block=True)
         ra = pmod.Reference('a')
 
         r = view.apply_sync(lambda x: x.b, ra)
         self.assertEqual(r, 128)
         self.assertEqual(view['a.b'], 128)
 
         view['a.b'] = 0
 
         r = view.apply_sync(lambda x: x.b, ra)
         self.assertEqual(r, 0)
         self.assertEqual(view['a.b'], 0)
     
     def test_return_namedtuple(self):
         def namedtuplify(x, y):
             from IPython.parallel.tests.test_view import point
             return point(x, y)
         
         view = self.client[-1]
         p = view.apply_sync(namedtuplify, 1, 2)
         self.assertEqual(p.x, 1)
         self.assertEqual(p.y, 2)
 
     def test_apply_namedtuple(self):
         def echoxy(p):
             return p.y, p.x
         
         view = self.client[-1]
         tup = view.apply_sync(echoxy, point(1, 2))
         self.assertEqual(tup, (2,1))
 

IPython/testing/tools.py

@@ -1,442 +1,444 b''
 """Generic testing tools.
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import tempfile
 
 from contextlib import contextmanager
 from io import StringIO
 from subprocess import Popen, PIPE
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
     # dependency optional at this point.  Nose is a hard dependency to run the
     # test suite, but NOT to use ipython itself.
     import nose.tools as nt
     has_nose = True
 except ImportError:
     has_nose = False
 
 from IPython.config.loader import Config
 from IPython.utils.process import get_output_error_code
 from IPython.utils.text import list_strings
 from IPython.utils.io import temp_pyfile, Tee
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 
 from . import decorators as dec
 from . import skipdoctest
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 # The docstring for full_path doctests differently on win32 (different path
 # separator) so just skip the doctest there.  The example remains informative.
 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
 @doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> full_path('/foo','a.txt')
     ['/a.txt']
     """
 
     files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
       
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail: number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
 
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
 
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
 
 
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
             '--colors=NoColor', '--no-term-title','--no-banner',
             '--autocall=0']
 
 
 def default_config():
     """Return a config object with good defaults for testing."""
     config = Config()
     config.TerminalInteractiveShell.colors = 'NoColor'
     config.TerminalTerminalInteractiveShell.term_title = False,
     config.TerminalInteractiveShell.autocall = 0
-    config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
+    f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
+    config.HistoryManager.hist_file = f.name
+    f.close()
     config.HistoryManager.db_cache_size = 10000
     return config
 
 
 def get_ipython_cmd(as_string=False):
     """
     Return appropriate IPython command line name. By default, this will return
     a list that can be used with subprocess.Popen, for example, but passing
     `as_string=True` allows for returning the IPython command as a string.
 
     Parameters
     ----------
     as_string: bool
         Flag to allow to return the command as a string.
     """
     # FIXME: remove workaround for 2.6 support
     if sys.version_info[:2] > (2,6):
         ipython_cmd = [sys.executable, "-m", "IPython"]
     else:
         ipython_cmd = ["ipython"]
 
     if as_string:
         ipython_cmd = " ".join(ipython_cmd)
 
     return ipython_cmd
 
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 
     Starts IPython with a minimal and safe configuration to make startup as fast
     as possible.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of file to be executed (should have .py or .ipy extension).
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     (stdout, stderr) of ipython subprocess.
     """
     if options is None: options = []
 
     # For these subprocess calls, eliminate all prompt printing so we only see
     # output from script execution
     prompt_opts = [ '--PromptManager.in_template=""',
                     '--PromptManager.in2_template=""',
                     '--PromptManager.out_template=""'
     ]
     cmdargs = default_argv() + prompt_opts + options
 
     test_dir = os.path.dirname(__file__)
 
     ipython_cmd = get_ipython_cmd()
     # Absolute path for filename
     full_fname = os.path.join(test_dir, fname)
     full_cmd = ipython_cmd + cmdargs + [full_fname]
     p = Popen(full_cmd, stdout=PIPE, stderr=PIPE)
     out, err = p.communicate()
     out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
     # `import readline` causes 'ESC[?1034h' to be output sometimes,
     # so strip that out before doing comparisons
     if out:
         out = re.sub(r'\x1b\[[^h]+h', '', out)
     return out, err
 
 
 def ipexec_validate(fname, expected_out, expected_err='',
                     options=None):
     """Utility to call 'ipython filename' and validate output/error.
 
     This function raises an AssertionError if the validation fails.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of the file to be executed (should have .py or .ipy extension).
 
     expected_out : str
       Expected stdout of the process.
 
     expected_err : optional, str
       Expected stderr of the process.
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     None
     """
 
     import nose.tools as nt
 
     out, err = ipexec(fname, options)
     #print 'OUT', out  # dbg
     #print 'ERR', err  # dbg
     # If there are any errors, we must check those befor stdout, as they may be
     # more informative than simply having an empty stdout.
     if err:
         if expected_err:
             nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
         else:
             raise ValueError('Running file %r produced error: %r' %
                              (fname, err))
     # If no errors or output on stderr was expected, match stdout
     nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
 
 
 class TempFileMixin(object):
     """Utility class to create temporary Python/IPython files.
 
     Meant as a mixin class for test cases."""
 
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
         self.tmpfile = f
         self.fname = fname
 
     def tearDown(self):
         if hasattr(self, 'tmpfile'):
             # If the tmpfile wasn't made because of skipped tests, like in
             # win32, there's nothing to cleanup.
             self.tmpfile.close()
             try:
                 os.unlink(self.fname)
             except:
                 # On Windows, even though we close the file, we still can't
                 # delete it.  I have no clue why
                 pass
 
 pair_fail_msg = ("Testing {0}\n\n"
                 "In:\n"
                 "  {1!r}\n"
                 "Expected:\n"
                 "  {2!r}\n"
                 "Got:\n"
                 "  {3!r}\n")
 def check_pairs(func, pairs):
     """Utility function for the common case of checking a function with a
     sequence of input/output pairs.
 
     Parameters
     ----------
     func : callable
       The function to be tested. Should accept a single argument.
     pairs : iterable
       A list of (input, expected_output) tuples.
 
     Returns
     -------
     None. Raises an AssertionError if any output does not match the expected
     value.
     """
     name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
     for inp, expected in pairs:
         out = func(inp)
         assert out == expected, pair_fail_msg.format(name, inp, expected, out)
 
 
 if py3compat.PY3:
     MyStringIO = StringIO
 else:
     # In Python 2, stdout/stderr can have either bytes or unicode written to them,
     # so we need a class that can handle both.
     class MyStringIO(StringIO):
         def write(self, s):
             s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING)
             super(MyStringIO, self).write(s)
 
 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
 -------
 {2!s}
 -------
 """
 
 class AssertPrints(object):
     """Context manager for testing that code prints certain text.
     
     Examples
     --------
     >>> with AssertPrints("abc", suppress=False):
     ...     print "abcd"
     ...     print "def"
     ... 
     abcd
     def
     """
     def __init__(self, s, channel='stdout', suppress=True):
         self.s = s
         if isinstance(self.s, py3compat.string_types):
             self.s = [self.s]
         self.channel = channel
         self.suppress = suppress
     
     def __enter__(self):
         self.orig_stream = getattr(sys, self.channel)
         self.buffer = MyStringIO()
         self.tee = Tee(self.buffer, channel=self.channel)
         setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
     
     def __exit__(self, etype, value, traceback):
         self.tee.flush()
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         for s in self.s:
             assert s in printed, notprinted_msg.format(s, self.channel, printed)
         return False
 
 printed_msg = """Found {0!r} in printed output (on {1}):
 -------
 {2!s}
 -------
 """
 
 class AssertNotPrints(AssertPrints):
     """Context manager for checking that certain output *isn't* produced.
     
     Counterpart of AssertPrints"""
     def __exit__(self, etype, value, traceback):
         self.tee.flush()
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         for s in self.s:
             assert s not in printed, printed_msg.format(s, self.channel, printed)
         return False
 
 @contextmanager
 def mute_warn():
     from IPython.utils import warn
     save_warn = warn.warn
     warn.warn = lambda *a, **kw: None
     try:
         yield
     finally:
         warn.warn = save_warn
 
 @contextmanager
 def make_tempfile(name):
     """ Create an empty, named, temporary file for the duration of the context.
     """
     f = open(name, 'w')
     f.close()
     try:
         yield
     finally:
         os.unlink(name)
 
 
 @contextmanager
 def monkeypatch(obj, name, attr):
     """
     Context manager to replace attribute named `name` in `obj` with `attr`.
     """
     orig = getattr(obj, name)
     setattr(obj, name, attr)
     yield
     setattr(obj, name, orig)
 
 
 def help_output_test(subcommand=''):
     """test that `ipython [subcommand] -h` works"""
     cmd = ' '.join(get_ipython_cmd() + [subcommand, '-h'])
     out, err, rc = get_output_error_code(cmd)
     nt.assert_equal(rc, 0, err)
     nt.assert_not_in("Traceback", err)
     nt.assert_in("Options", out)
     nt.assert_in("--help-all", out)
     return out, err
 
 
 def help_all_output_test(subcommand=''):
     """test that `ipython [subcommand] --help-all` works"""
     cmd = ' '.join(get_ipython_cmd() + [subcommand, '--help-all'])
     out, err, rc = get_output_error_code(cmd)
     nt.assert_equal(rc, 0, err)
     nt.assert_not_in("Traceback", err)
     nt.assert_in("Options", out)
     nt.assert_in("Class parameters", out)
     return out, err