Merge pull request #1606 from Carreau/loadpycat

Share code for %pycat and %loadpy, make %pycat aware of URLs

IPython/core/interactiveshell.py

@@ -22,23 +22,17 b' import __future__'
 import abc
 import ast
 import atexit
-import codeop
-import inspect
 import os
 import re
 import runpy
 import sys
 import tempfile
 import types
-
-try:
-    from contextlib import nested
-except:
-    from IPython.utils.nested_context import nested
+import urllib
+from io import open as io_open
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.core import debugger, oinspect
-from IPython.core import history as ipcorehist
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import shadowns
@@ -50,7 +44,7 b' from IPython.core.compilerop import CachingCompiler'
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.displaypub import DisplayPublisher
-from IPython.core.error import TryNext, UsageError
+from IPython.core.error import UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.formatters import DisplayFormatter
@@ -68,19 +62,20 b' from IPython.core.prompts import PromptManager'
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import py3compat
+from IPython.utils import openpy
 from IPython.utils.doctestreload import doctest_reload
-from IPython.utils.io import ask_yes_no, rprint
+from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
-from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
+from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename
 from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
-from IPython.utils.text import (num_ini_spaces, format_screen, LSString, SList,
+from IPython.utils.text import (format_screen, LSString, SList,
                                 DollarFormatter)
 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
-from IPython.utils.warn import warn, error, fatal
+from IPython.utils.warn import warn, error
 import IPython.core.hooks
 
 #-----------------------------------------------------------------------------
@@ -2742,21 +2737,29 b' class InteractiveShell(SingletonConfigurable, Magic):'
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
-    def find_user_code(self, target, raw=True):
-        """Get a code string from history, file, or a string or macro.
+    def find_user_code(self, target, raw=True, py_only=False):
+        """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
 
         Parameters
         ----------
+
         target : str
+
           A string specifying code to retrieve. This will be tried respectively
-          as: ranges of input history (see %history for syntax), a filename, or
-          an expression evaluating to a string or Macro in the user namespace.
+          as: ranges of input history (see %history for syntax), url,
+          correspnding .py file, filename, or an expression evaluating to a
+          string or Macro in the user namespace.
+
         raw : bool
           If true (default), retrieve raw history. Has no effect on the other
           retrieval mechanisms.
 
+        py_only : bool (default False)
+          Only try to fetch python code, do not try alternative methods to decode file
+          if unicode fails.
+
         Returns
         -------
         A string of code.
@@ -2768,14 +2771,37 b' class InteractiveShell(SingletonConfigurable, Magic):'
         code = self.extract_input_lines(target, raw=raw)  # Grab history
         if code:
             return code
-        if os.path.isfile(target):                        # Read file
-            return open(target, "r").read()
+        utarget = unquote_filename(target)
+        try:
+            if utarget.startswith(('http://', 'https://')):
+                return openpy.read_py_url(utarget, skip_encoding_cookie=True)
+        except UnicodeDecodeError:
+            if not py_only :
+                response = urllib.urlopen(target)
+                return response.read().decode('latin1')
+            raise ValueError(("'%s' seem to be unreadable.") % utarget)
+
+        potential_target = [target]
+        try :
+            potential_target.insert(0,get_py_filename(target))
+        except IOError:
+            pass
+
+        for tgt in potential_target :
+            if os.path.isfile(tgt):                        # Read file
+                try :
+                    return openpy.read_py_file(tgt, skip_encoding_cookie=True)
+                except UnicodeDecodeError :
+                    if not py_only :
+                        with io_open(tgt,'r', encoding='latin1') as f :
+                            return f.read()
+                    raise ValueError(("'%s' seem to be unreadable.") % target)
 
         try:                                              # User namespace
             codeobj = eval(target, self.user_ns)
         except Exception:
-            raise ValueError(("'%s' was not found in history, as a file, nor in"
-                                " the user namespace.") % target)
+            raise ValueError(("'%s' was not found in history, as a file, url, "
+                                "nor in the user namespace.") % target)
         if isinstance(codeobj, basestring):
             return codeobj
         elif isinstance(codeobj, Macro):

IPython/core/magic.py

@@ -19,12 +19,10 b' import __builtin__ as builtin_mod'
 import __future__
 import bdb
 import inspect
-import imp
 import io
 import json
 import os
 import sys
-import shutil
 import re
 import time
 import gc
@@ -44,27 +42,23 b' except ImportError:'
     except ImportError:
         profile = pstats = None
 
-import IPython
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
 from IPython.core.error import StdinNotImplementedError
-from IPython.core.fakemodule import FakeModule
-from IPython.core.profiledir import ProfileDir
 from IPython.core.macro import Macro
 from IPython.core import magic_arguments, page
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.core.pylabtools import mpl_runner
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
-from IPython.utils import openpy
 from IPython.utils.encoding import DEFAULT_ENCODING
 from IPython.utils.io import file_read, nlprint
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, unquote_filename
 from IPython.utils.process import arg_split, abbrev_cwd
 from IPython.utils.terminal import set_term_title
-from IPython.utils.text import LSString, SList, format_screen
+from IPython.utils.text import format_screen
 from IPython.utils.timing import clock, clock2
 from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
@@ -2219,8 +2213,8 b' Currently the magic system has the following functions:\\n"""'
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
-            ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
-            if ans.lower() not in ['y','yes']:
+            overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
+            if not overwrite :
                 print 'Operation cancelled.'
                 return
         try:
@@ -2271,27 +2265,54 b' Currently the magic system has the following functions:\\n"""'
         return response_data['html_url']
 
     def magic_loadpy(self, arg_s):
-        """Load a .py python script into the GUI console.
+        """Alias of `%load`
 
-        This magic command can either take a local filename or a url::
+        `%loadpy` has gained some flexibility and droped the requirement of a `.py`
+        extension. So it has been renamed simply into %load. You can look at
+        `%load`'s docstring for more info.
+        """
+        self.magic_load(arg_s)
+
+    def magic_load(self, arg_s):
+        """Load code into the current frontend.
+
+        Usage:\\
+          %load [options] source
+
+          where source can be a filename, URL, input history range or macro
+
+        Options:
+        --------
+          -y : Don't ask confirmation for loading source above 200 000 characters.
+
+        This magic command can either take a local filename, a URL, an history
+        range (see %history) or a macro as argument, it will prompt for
+        confirmation before loading source with more than 200 000 characters, unless
+        -y flag is passed or if the frontend does not support raw_input::
 
-        %loadpy myscript.py
-        %loadpy http://www.example.com/myscript.py
+        %load myscript.py
+        %load 7-27
+        %load myMacro
+        %load http://www.example.com/myscript.py
         """
-        arg_s = unquote_filename(arg_s)
-        remote_url = arg_s.startswith(('http://', 'https://'))
-        local_url = not remote_url
-        if local_url and not arg_s.endswith('.py'):
-            # Local files must be .py; for remote URLs it's possible that the
-            # fetch URL doesn't have a .py in it (many servers have an opaque
-            # URL, such as scipy-central.org).
-            raise ValueError('%%loadpy only works with .py files: %s' % arg_s)
+        opts,args = self.parse_options(arg_s,'y')
         
-        # openpy takes care of finding the source encoding (per PEP 263)
-        if remote_url:
-            contents = openpy.read_py_url(arg_s, skip_encoding_cookie=True)
-        else:
-            contents = openpy.read_py_file(arg_s, skip_encoding_cookie=True)
+        contents = self.shell.find_user_code(args)
+        l = len(contents)
+
+        # 200 000 is ~ 2500 full 80 caracter lines
+        # so in average, more than 5000 lines
+        if l > 200000 and 'y' not in opts:
+            try:
+                ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
+                " (%d characters). Continue (y/[N]) ?" % l), default='n' )
+            except StdinNotImplementedError:
+                #asume yes if raw input not implemented
+                ans = True
+
+            if ans is False :
+                print 'Operation cancelled.'
+                return
         
         self.set_next_input(contents)
 
@@ -3323,22 +3344,26 b' Defaulting color scheme to \'NoColor\'"""'
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
+
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
-        to be Python source and will show it with syntax highlighting. """
+        to be Python source and will show it with syntax highlighting.
+
+        This magic command can either take a local filename, an url,
+        an history range (see %history) or a macro as argument ::
+
+        %pycat myscript.py
+        %pycat 7-27
+        %pycat myMacro
+        %pycat http://www.example.com/myscript.py
+        """
 
         try:
-            filename = get_py_filename(parameter_s)
-            cont = file_read(filename)
-        except IOError:
-            try:
-                cont = eval(parameter_s,self.user_ns)
-            except NameError:
-                cont = None
-        if cont is None:
-            print "Error: no such file or variable"
+            cont = self.shell.find_user_code(parameter_s)
+        except ValueError, IOError:
+            print "Error: no such file, variable, URL, history range or macro"
             return
 
         page.page(self.shell.pycolorize(cont))

IPython/frontend/html/notebook/notebookmanager.py

@@ -41,7 +41,7 b' class NotebookManager(LoggingConfigurable):'
     save_script = Bool(False, config=True,
         help="""Automatically create a Python script when saving the notebook.
         
-        For easier use of import, %run and %loadpy across notebooks, a
+        For easier use of import, %run and %load across notebooks, a
         <notebook-name>.py script will be created next to any
         <notebook-name>.ipynb on each save.  This can also be set with the
         short `--script` flag.

IPython/frontend/qt/console/mainwindow.py

@@ -597,7 +597,7 b' class MainWindow(QtGui.QMainWindow):'
 
         # list of protected magic that don't like to be called without argument
         # append '?' to the end to print the docstring when called from the menu
-        protected_magic = set(["more","less","load_ext","pycat","loadpy","save"])
+        protected_magic = set(["more","less","load_ext","pycat","loadpy","load","save"])
         magics=re.findall('\w+', listofmagic)
         for magic in magics:
             if magic in protected_magic:

IPython/utils/openpy.py

@@ -6,7 +6,6 b' Much of the code is taken from the tokenize module in Python 3.2.'
 """
 from __future__ import absolute_import
 
-import __builtin__
 import io
 from io import TextIOWrapper
 import re
@@ -190,3 +189,4 b" def read_py_url(url, errors='replace', skip_encoding_cookie=True):"
         return "".join(strip_encoding_cookie(text))
     else:
         return text.read()
+

docs/examples/notebooks/00_notebook_tour.ipynb

@@ -1014,7 +1014,7 b''
      "source": [
       "# Loading external codes",
       "* Drag and drop a ``.py`` in the dashboard",
-      "* Use ``%loadpy`` with any local or remote url: [the Matplotlib Gallery!](http://matplotlib.sourceforge.net/gallery.html)",
+      "* Use ``%load`` with any local or remote url: [the Matplotlib Gallery!](http://matplotlib.sourceforge.net/gallery.html)",
       "",
       "In this notebook we've kept the output saved so you can see the result, but you should run the next",
       "cell yourself (with an active internet connection)."
@@ -1024,7 +1024,7 b''
      "cell_type": "code",
      "collapsed": true,
      "input": [
-      "%loadpy http://matplotlib.sourceforge.net/mpl_examples/pylab_examples/integral_demo.py"
+      "%load http://matplotlib.sourceforge.net/mpl_examples/pylab_examples/integral_demo.py"
      ],
      "language": "python",
      "outputs": [],

docs/examples/tests/heartbeat/hb_gil.py

@@ -1,7 +1,7 b''
 """
 Run this script in the qtconsole with one of:
 
-    %loadpy hb_gil.py
+    %load hb_gil.py
 
 or
     %run hb_gil.py

docs/source/interactive/qtconsole.txt

@@ -33,18 +33,18 b' is not yet configurable.'
    point in a multiline block, you can force its execution (without having to
    go to the bottom) with :kbd:`Shift-Enter`.
 
-``%loadpy``
-===========
+``%load``
+=========
 
-The new ``%loadpy`` magic takes any python script (must end in '.py'), and
-pastes its contents as your next input, so you can edit it before
-executing. The script may be on your machine, but you can also specify a url,
-and it will download the script from the web. This is particularly useful for
-playing with examples from documentation, such as matplotlib.
+The new ``%load`` magic (previously ``%loadpy``) takes any script, and pastes
+its contents as your next input, so you can edit it before executing. The
+script may be on your machine, but you can also specify an history range, or a
+url, and it will download the script from the web. This is particularly useful
+for playing with examples from documentation, such as matplotlib.
 
 .. sourcecode:: ipython
 
-    In [6]: %loadpy http://matplotlib.sourceforge.net/plot_directive/mpl_examples/mplot3d/contour3d_demo.py
+    In [6]: %load http://matplotlib.sourceforge.net/plot_directive/mpl_examples/mplot3d/contour3d_demo.py
 
     In [7]: from mpl_toolkits.mplot3d import axes3d
        ...: import matplotlib.pyplot as plt