From acdb8b0b08057ab41351806aef9bd8a9b6491d63 2013-07-16 23:12:44
From: MinRK <benjaminrk@gmail.com>
Date: 2013-07-16 23:12:44
Subject: [PATCH] add strip_encoding_cookie transformer

closes #3245
---

diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index 70a08e8..6028266 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -75,6 +75,7 @@ from IPython.utils.py3compat import cast_unicode
 from IPython.core.inputtransformer import (leading_indent,
                                            classic_prompt,
                                            ipy_prompt,
+                                           strip_encoding_cookie,
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
@@ -488,6 +489,7 @@ class IPythonInputSplitter(InputSplitter):
             self.physical_line_transforms = [leading_indent(),
                                              classic_prompt(),
                                              ipy_prompt(),
+                                             strip_encoding_cookie(),
                                             ]
         
         self.assemble_logical_lines = assemble_logical_lines()
diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py
index ddddde3..95abe72 100644
--- a/IPython/core/inputtransformer.py
+++ b/IPython/core/inputtransformer.py
@@ -5,6 +5,7 @@ from StringIO import StringIO
 
 from IPython.core.splitinput import LineInfo
 from IPython.utils import tokenize2
+from IPython.utils.openpy import cookie_comment_re
 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
 
 #-----------------------------------------------------------------------------
@@ -422,6 +423,30 @@ def leading_indent():
                 line = (yield line)
 
 
+@CoroutineInputTransformer.wrap
+def strip_encoding_cookie():
+    """Remove encoding comment if found in first two lines
+    
+    If the first or second line has the `# coding: utf-8` comment,
+    it will be removed.
+    """
+    line = ''
+    while True:
+        line = (yield line)
+        # check comment on first two lines
+        for i in range(2):
+            if line is None:
+                break
+            if cookie_comment_re.match(line):
+                line = (yield "")
+            else:
+                line = (yield line)
+        
+        # no-op on the rest of the cell
+        while line is not None:
+            line = (yield line)
+
+
 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                               r'\s*=\s*!\s*(?P<cmd>.*)')
 assign_system_template = '%s = get_ipython().getoutput(%r)'