Merge pull request #2882 from takluyver/utils-cleanup

Utils cleanup.

This removes various unused code that was sitting in IPython.utils.

I checked that we don't use these functions and modules at present, but we may want to leave some of them for future use or for extension authors. For the first pass, I've been deliberately quite aggressive about removing things.

IPython/core/magics/code.py

@@ -27,7 +27,6 b' from IPython.core.oinspect import find_file, find_source_lines'
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
 from IPython.utils.contexts import preserve_keys
-from IPython.utils.io import file_read
 from IPython.utils.path import get_py_filename, unquote_filename
 from IPython.utils.warn import warn
 
@@ -531,7 +530,8 b' class CodeMagics(Magics):'
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
         if args.strip() == 'pasted_block':
-            self.shell.user_ns['pasted_block'] = file_read(filename)
+            with open(filename, 'r') as f:
+                self.shell.user_ns['pasted_block'] = f.read()
 
         if 'x' in opts:  # -x prevents actual execution
             print
@@ -541,8 +541,9 b' class CodeMagics(Magics):'
                 if not is_temp:
                     self.shell.user_ns['__file__'] = filename
                 if 'r' in opts:    # Untranslated IPython code
-                    self.shell.run_cell(file_read(filename),
-                                        store_history=False)
+                    with open(filename, 'r') as f:
+                        source = f.read()
+                    self.shell.run_cell(source, store_history=False)
                 else:
                     self.shell.safe_execfile(filename, self.shell.user_ns,
                                              self.shell.user_ns)

IPython/core/magics/osm.py

@@ -31,7 +31,6 b' from IPython.core.magic import ('
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils.io import nlprint
 from IPython.utils.openpy import source_to_unicode
 from IPython.utils.path import unquote_filename
 from IPython.utils.process import abbrev_cwd
@@ -403,7 +402,7 b' class OSMagics(Magics):'
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
-        %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
+        %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
@@ -425,14 +424,15 b' class OSMagics(Magics):'
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
+                fin = min(fin, len(dh))
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
-        nlprint(dh,
-                header = 'Directory history (kept in _dh)',
-                start=ini,stop=fin)
+        print 'Directory history (kept in _dh)'
+        for i in range(ini, fin):
+            print "%d: %s" % (i, dh[i])
 
     @skip_doctest
     @line_magic

IPython/core/ultratb.py

@@ -104,9 +104,9 b' from IPython.core.display_trap import DisplayTrap'
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import openpy
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
-from IPython.utils import pyfile
 from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
 from IPython.utils.warn import info, error
@@ -837,7 +837,7 b' class VerboseTB(TBTools):'
                 continue
             elif file.endswith(('.pyc','.pyo')):
                 # Look up the corresponding source file.
-                file = pyfile.source_from_cache(file)
+                file = openpy.source_from_cache(file)
 
             def linereader(file=file, lnum=[lnum], getline=ulinecache.getline):
                 line = getline(file, lnum[0])

IPython/extensions/autoreload.py

@@ -119,7 +119,7 b' try:'
 except NameError:
     from imp import reload
 
-from IPython.utils import pyfile
+from IPython.utils import openpy
 from IPython.utils.py3compat import PY3
 
 #------------------------------------------------------------------------------
@@ -207,12 +207,12 b' class ModuleReloader(object):'
             path, ext = os.path.splitext(filename)
 
             if ext.lower() == '.py':
-                pyc_filename = pyfile.cache_from_source(filename)
+                pyc_filename = openpy.cache_from_source(filename)
                 py_filename = filename
             else:
                 pyc_filename = filename
                 try:
-                    py_filename = pyfile.source_from_cache(filename)
+                    py_filename = openpy.source_from_cache(filename)
                 except ValueError:
                     continue
 

IPython/lib/demo.py

@@ -184,7 +184,6 b' import shlex'
 import sys
 
 from IPython.utils import io
-from IPython.utils.io import file_read
 from IPython.utils.text import marquee
 from IPython.utils import openpy
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
@@ -380,7 +379,8 b' class Demo(object):'
 
         filename = self.shell.mktempfile(self.src_blocks[index])
         self.shell.hooks.editor(filename,1)
-        new_block = file_read(filename)
+        with open(filename, 'r') as f:
+            new_block = f.read()
         # update the source and colored block
         self.src_blocks[index] = new_block
         self.src_blocks_colored[index] = self.ip_colorize(new_block)

IPython/utils/_process_posix.py

@@ -22,7 +22,6 b' import sys'
 from IPython.external import pexpect
 
 # Our own
-from .autoattr import auto_attr
 from ._process_common import getoutput, arg_split
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
@@ -55,14 +54,16 b' class ProcessHandler(object):'
     logfile = None
 
     # Shell to call for subprocesses to execute
-    sh = None
+    _sh = None
 
-    @auto_attr
+    @property
     def sh(self):
-        sh = pexpect.which('sh')
-        if sh is None:
-            raise OSError('"sh" shell not found')
-        return sh
+        if self._sh is None:        
+            self._sh = pexpect.which('sh')
+            if self._sh is None:
+                raise OSError('"sh" shell not found')
+        
+        return self._sh
 
     def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
         """Arguments are used for pexpect calls."""

IPython/utils/data.py

@@ -9,16 +9,6 b''
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import types
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
@@ -32,65 +22,14 b' def uniq_stable(elems):'
     return [x for x in elems if x not in seen and not seen.add(x)]
 
 
-def sort_compare(lst1, lst2, inplace=1):
-    """Sort and compare two lists.
-
-    By default it does it in place, thus modifying the lists. Use inplace = 0
-    to avoid that (at the cost of temporary copy creation)."""
-    if not inplace:
-        lst1 = lst1[:]
-        lst2 = lst2[:]
-    lst1.sort(); lst2.sort()
-    return lst1 == lst2
-
-
-def list2dict(lst):
-    """Takes a list of (key,value) pairs and turns it into a dict."""
-
-    dic = {}
-    for k,v in lst: dic[k] = v
-    return dic
-
-
-def list2dict2(lst, default=''):
-    """Takes a list and turns it into a dict.
-    Much slower than list2dict, but more versatile. This version can take
-    lists with sublists of arbitrary length (including sclars)."""
-
-    dic = {}
-    for elem in lst:
-        if type(elem) in (types.ListType,types.TupleType):
-            size = len(elem)
-            if  size == 0:
-                pass
-            elif size == 1:
-                dic[elem] = default
-            else:
-                k,v = elem[0], elem[1:]
-                if len(v) == 1: v = v[0]
-                dic[k] = v
-        else:
-            dic[elem] = default
-    return dic
-
-
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
-
-
-def get_slice(seq, start=0, stop=None, step=1):
-    """Get a slice of a sequence with variable step. Specify start,stop,step."""
-    if stop == None:
-        stop = len(seq)
-    item = lambda i: seq[i]
-    return map(item,xrange(start,stop,step))
-
+    
 
 def chop(seq, size):
     """Chop a sequence into chunks of the given size."""
-    chunk = lambda i: seq[i:i+size]
-    return map(chunk,xrange(0,len(seq),size))
+    return [seq[i:i+size] for i in xrange(0,len(seq),size)]
 
 

IPython/utils/io.py

@@ -154,64 +154,6 b' class Tee(object):'
             self.close()
 
 
-def file_read(filename):
-    """Read a file and close it.  Returns the file source."""
-    fobj = open(filename,'r');
-    source = fobj.read();
-    fobj.close()
-    return source
-
-
-def file_readlines(filename):
-    """Read a file and close it.  Returns the file source using readlines()."""
-    fobj = open(filename,'r');
-    lines = fobj.readlines();
-    fobj.close()
-    return lines
-
-
-def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
-    """Take multiple lines of input.
-
-    A list with each line of input as a separate element is returned when a
-    termination string is entered (defaults to a single '.'). Input can also
-    terminate via EOF (^D in Unix, ^Z-RET in Windows).
-
-    Lines of input which end in \\ are joined into single entries (and a
-    secondary continuation prompt is issued as long as the user terminates
-    lines with \\). This allows entering very long strings which are still
-    meant to be treated as single entities.
-    """
-
-    try:
-        if header:
-            header += '\n'
-        lines = [raw_input(header + ps1)]
-    except EOFError:
-        return []
-    terminate = [terminate_str]
-    try:
-        while lines[-1:] != terminate:
-            new_line = raw_input(ps1)
-            while new_line.endswith('\\'):
-                new_line = new_line[:-1] + raw_input(ps2)
-            lines.append(new_line)
-
-        return lines[:-1]  # don't return the termination command
-    except EOFError:
-        print()
-        return lines
-
-
-def raw_input_ext(prompt='',  ps2='... '):
-    """Similar to raw_input(), but accepts extended lines if input ends with \\."""
-
-    line = raw_input(prompt)
-    while line.endswith('\\'):
-        line = line[:-1] + raw_input(ps2)
-    return line
-
-
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
@@ -242,44 +184,6 b' def ask_yes_no(prompt,default=None):'
     return answers[ans]
 
 
-class NLprinter:
-    """Print an arbitrarily nested list, indicating index numbers.
-
-    An instance of this class called nlprint is available and callable as a
-    function.
-
-    nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
-    and using 'sep' to separate the index from the value. """
-
-    def __init__(self):
-        self.depth = 0
-
-    def __call__(self,lst,pos='',**kw):
-        """Prints the nested list numbering levels."""
-        kw.setdefault('indent',' ')
-        kw.setdefault('sep',': ')
-        kw.setdefault('start',0)
-        kw.setdefault('stop',len(lst))
-        # we need to remove start and stop from kw so they don't propagate
-        # into a recursive call for a nested list.
-        start = kw['start']; del kw['start']
-        stop = kw['stop']; del kw['stop']
-        if self.depth == 0 and 'header' in kw.keys():
-            print(kw['header'])
-
-        for idx in range(start,stop):
-            elem = lst[idx]
-            newpos = pos + str(idx)
-            if type(elem)==type([]):
-                self.depth += 1
-                self.__call__(elem, newpos+",", **kw)
-                self.depth -= 1
-            else:
-                print(kw['indent']*self.depth + newpos + kw["sep"] + repr(elem))
-
-nlprint = NLprinter()
-
-
 def temp_pyfile(src, ext='.py'):
     """Make a temporary python file, return filename and filehandle.
 

IPython/utils/ipstruct.py

@@ -18,8 +18,6 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-from IPython.utils.data import list2dict2
-
 __all__ = ['Struct']
 
 #-----------------------------------------------------------------------------
@@ -370,7 +368,7 b' class Struct(dict):'
         add_s    = lambda old,new: old + ' ' + new
 
         # default policy is to keep current keys when there's a conflict
-        conflict_solve = list2dict2(self.keys(), default = preserve)
+        conflict_solve = dict.fromkeys(self, preserve)
 
         # the conflict_solve dictionary is given by the user 'inverted': we
         # need a name-function mapping, it comes as a function -> names

IPython/utils/openpy.py

@@ -8,6 +8,7 b' from __future__ import absolute_import'
 
 import io
 from io import TextIOWrapper, BytesIO
+import os.path
 import re
 
 cookie_re = re.compile(ur"coding[:=]\s*([-\w.]+)", re.UNICODE)
@@ -217,3 +218,22 b' def _list_readline(x):'
     def readline():
         return next(x)
     return readline
+
+# Code for going between .py files and cached .pyc files ----------------------
+
+try:    # Python 3.2, see PEP 3147
+    from imp import source_from_cache, cache_from_source
+except ImportError:
+    # Python <= 3.1: .pyc files go next to .py
+    def source_from_cache(path):
+        basename, ext = os.path.splitext(path)
+        if ext not in ('.pyc', '.pyo'):
+            raise ValueError('Not a cached Python file extension', ext)
+        # Should we look for .pyw files?
+        return basename + '.py'
+    
+    def cache_from_source(path, debug_override=None):
+        if debug_override is None:
+            debug_override = __debug__
+        basename, ext = os.path.splitext(path)
+        return basename + '.pyc' if debug_override else '.pyo'

IPython/utils/text.py

@@ -30,25 +30,12 b' from string import Formatter'
 from IPython.external.path import path
 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
 from IPython.utils import py3compat
-from IPython.utils.io import nlprint
 from IPython.utils.data import flatten
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
-def unquote_ends(istr):
-    """Remove a single pair of quotes from the endpoints of a string."""
-
-    if not istr:
-        return istr
-    if (istr[0]=="'" and istr[-1]=="'") or \
-       (istr[0]=='"' and istr[-1]=='"'):
-        return istr[1:-1]
-    else:
-        return istr
-
-
 class LSString(str):
     """String derivative with a special access attributes.
 
@@ -265,105 +252,11 b' class SList(list):'
 #         arg.hideonce = False
 #         return
 #
-#     nlprint(arg)
+#     nlprint(arg)   # This was a nested list printer, now removed.
 #
 # print_slist = result_display.when_type(SList)(print_slist)
 
 
-def esc_quotes(strng):
-    """Return the input string with single and double quotes escaped out"""
-
-    return strng.replace('"','\\"').replace("'","\\'")
-
-
-def qw(words,flat=0,sep=None,maxsplit=-1):
-    """Similar to Perl's qw() operator, but with some more options.
-
-    qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
-
-    words can also be a list itself, and with flat=1, the output will be
-    recursively flattened.
-
-    Examples:
-
-    >>> qw('1 2')
-    ['1', '2']
-
-    >>> qw(['a b','1 2',['m n','p q']])
-    [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
-
-    >>> qw(['a b','1 2',['m n','p q']],flat=1)
-    ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
-    """
-
-    if isinstance(words, basestring):
-        return [word.strip() for word in words.split(sep,maxsplit)
-                if word and not word.isspace() ]
-    if flat:
-        return flatten(map(qw,words,[1]*len(words)))
-    return map(qw,words)
-
-
-def qwflat(words,sep=None,maxsplit=-1):
-    """Calls qw(words) in flat mode. It's just a convenient shorthand."""
-    return qw(words,1,sep,maxsplit)
-
-
-def qw_lol(indata):
-    """qw_lol('a b') -> [['a','b']],
-    otherwise it's just a call to qw().
-
-    We need this to make sure the modules_some keys *always* end up as a
-    list of lists."""
-
-    if isinstance(indata, basestring):
-        return [qw(indata)]
-    else:
-        return qw(indata)
-
-
-def grep(pat,list,case=1):
-    """Simple minded grep-like function.
-    grep(pat,list) returns occurrences of pat in list, None on failure.
-
-    It only does simple string matching, with no support for regexps. Use the
-    option case=0 for case-insensitive matching."""
-
-    # This is pretty crude. At least it should implement copying only references
-    # to the original data in case it's big. Now it copies the data for output.
-    out=[]
-    if case:
-        for term in list:
-            if term.find(pat)>-1: out.append(term)
-    else:
-        lpat=pat.lower()
-        for term in list:
-            if term.lower().find(lpat)>-1: out.append(term)
-
-    if len(out): return out
-    else: return None
-
-
-def dgrep(pat,*opts):
-    """Return grep() on dir()+dir(__builtins__).
-
-    A very common use of grep() when working interactively."""
-
-    return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
-
-
-def idgrep(pat):
-    """Case-insensitive dgrep()"""
-
-    return dgrep(pat,0)
-
-
-def igrep(pat,list):
-    """Synonym for case-insensitive grep."""
-
-    return grep(pat,list,case=0)
-
-
 def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.