Use openpy module for %loadpy magic.

IPython/core/magic.py

@@ -55,6 +55,7 b' from IPython.core.prefilter import ESC_MAGIC'
 from IPython.core.pylabtools import mpl_runner
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
+from IPython.utils import openpy
 from IPython.utils.io import file_read, nlprint
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, unquote_filename
@@ -2261,28 +2262,15 b' Currently the magic system has the following functions:\\n"""'
             # Local files must be .py; for remote URLs it's possible that the
             # fetch URL doesn't have a .py in it (many servers have an opaque
             # URL, such as scipy-central.org).
-            raise ValueError('%%load only works with .py files: %s' % arg_s)
+            raise ValueError('%%loadpy only works with .py files: %s' % arg_s)
+        
+        # openpy takes care of finding the source encoding (per PEP 263)
         if remote_url:
-            import urllib2
-            fileobj = urllib2.urlopen(arg_s)
-            # While responses have a .info().getencoding() way of asking for
-            # their encoding, in *many* cases the return value is bogus.  In
-            # the wild, servers serving utf-8 but declaring latin-1 are
-            # extremely common, as the old HTTP standards specify latin-1 as
-            # the default but many modern filesystems use utf-8.  So we can NOT
-            # rely on the headers.  Short of building complex encoding-guessing
-            # logic, going with utf-8 is a simple solution likely to be right
-            # in most real-world cases.
-            linesource = fileobj.read().decode('utf-8', 'replace').splitlines()
-            fileobj.close()
+            contents = openpy.read_py_url(arg_s, skip_encoding_cookie=True)
         else:
-            with open(arg_s) as fileobj:
-                linesource = fileobj.read().splitlines()
-        
-        # Strip out encoding declarations
-        lines = [l for l in linesource if not _encoding_declaration_re.match(l)]
+            contents = openpy.read_py_file(arg_s, skip_encoding_cookie=True)
         
-        self.set_next_input(os.linesep.join(lines))
+        self.set_next_input(contents)
 
     def _find_edit_target(self, args, opts, last_call):
         """Utility method used by magic_edit to find what to edit."""

IPython/utils/openpy.py

@@ -4,6 +4,7 b' as per PEP 263.'
 
 Much of the code is taken from the tokenize module in Python 3.2.
 """
+from __future__ import absolute_import
 
 import __builtin__
 import io
@@ -107,7 +108,7 b' except ImportError:'
 try:
     # Available in Python 3.2 and above.
     from tokenize import open
-except:
+except ImportError:
     # Copied from Python 3.2 tokenize
     def open(filename):
         """Open a file in read only mode using the encoding detected by
@@ -118,19 +119,7 b' except:'
         buffer.seek(0)
         text = TextIOWrapper(buffer, encoding, line_buffering=True)
         text.mode = 'r'
-        return text
-
-def open_url(url, errors='replace'):
-    """Open a URL to a raw Python file, using the encoding detected by
-    detect_encoding().
-    """
-    response = urllib.urlopen(url)
-    buffer = io.BufferedRandom(response)
-    encoding, lines = detect_encoding(buffer.readline)
-    buffer.seek(0)
-    text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
-    text.mode = 'r'
-    return text
+        return text   
 
 def strip_encoding_cookie(filelike):
     """Generator to pull lines from a text-mode file, skipping the encoding
@@ -150,9 +139,24 b' def strip_encoding_cookie(filelike):'
     for line in it:
         yield line
 
-def read_py_file(filename, skip_encoding_cookie=True):
-    f = open(filename)   # the open function defined in this module.
+def read_py_file(filename, errors='replace', skip_encoding_cookie=True):
+    with open(filename) as f:   # the open function defined in this module.
+        if skip_encoding_cookie:
+            return "".join(strip_encoding_cookie(f))
+        else:
+            return f.read()
+
+def read_py_url(url, errors='replace', skip_encoding_cookie=True):
+    """Open a URL to a raw Python file, using the encoding detected by
+    detect_encoding().
+    """
+    response = urllib.urlopen(url)
+    buffer = io.BytesIO(response.read())
+    encoding, lines = detect_encoding(buffer.readline)
+    buffer.seek(0)
+    text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
+    text.mode = 'r'
     if skip_encoding_cookie:
-        return "".join(strip_encoding_cookie(f))
+        return "".join(strip_encoding_cookie(text))
     else:
-        return f.read()
+        return text.read()