Merge branch 'rising-intonation'

IPython/core/inputsplitter.py

@@ -70,6 +70,8 b' import ast'
 import codeop
 import re
 import sys
+import tokenize
+from StringIO import StringIO
 
 # IPython modules
 from IPython.utils.text import make_quoted_expr
@@ -155,6 +157,24 b' def remove_comments(src):'
     """
 
     return re.sub('#.*', '', src)
+    
+def has_comment(src):
+    """Indicate whether an input line has (i.e. ends in, or is) a comment.
+    
+    This uses tokenize, so it can distinguish comments from # inside strings.
+    
+    Parameters
+    ----------
+    src : string
+      A single line input string.
+    
+    Returns
+    -------
+    Boolean: True if source has a comment.
+    """
+    readline = StringIO(src).readline
+    toktypes = set(t[0] for t in tokenize.generate_tokens(readline))
+    return(tokenize.COMMENT in toktypes)
 
 
 def get_input_encoding():
@@ -173,20 +193,14 b' def get_input_encoding():'
 #-----------------------------------------------------------------------------
 
 class InputSplitter(object):
-    """An object that can split Python source input in executable blocks.
-
-    This object is designed to be used in one of two basic modes:
+    """An object that can accumulate lines of Python source before execution.
 
-    1. By feeding it python source line-by-line, using :meth:`push`.  In this
-       mode, it will return on each push whether the currently pushed code
-       could be executed already.  In addition, it provides a method called
+    This object is designed to be fed python source line-by-line, using
+       :meth:`push`. It will return on each push whether the currently pushed
+       code could be executed already. In addition, it provides a method called
        :meth:`push_accepts_more` that can be used to query whether more input
        can be pushed into a single interactive block.
 
-    2. By calling :meth:`split_blocks` with a single, multiline Python string,
-       that is then split into blocks each of which can be executed
-       interactively as a single statement.
-
     This is a simple example of how an interactive terminal-based client can use
     this tool::
 
@@ -347,9 +361,6 b' class InputSplitter(object):'
         *line-oriented* frontends, since it means that intermediate blank lines
         are not allowed in function definitions (or any other indented block).
 
-        Block-oriented frontends that have a separate keyboard event to
-        indicate execution should use the :meth:`split_blocks` method instead.
-
         If the current input produces a syntax error, this method immediately
         returns False but does *not* raise the syntax error exception, as
         typically clients will want to send invalid syntax to an execution
@@ -658,6 +669,42 b' def transform_ipy_prompt(line):'
         return line
 
 
+def _make_help_call(target, esc, lspace, next_input=None):
+    """Prepares a pinfo(2)/psearch call from a target name and the escape
+    (i.e. ? or ??)"""
+    method  = 'pinfo2' if esc == '??' \
+                else 'psearch' if '*' in target \
+                else 'pinfo'
+        
+    if next_input:
+        tpl = '%sget_ipython().magic(u"%s %s", next_input=%s)'
+        return tpl % (lspace, method, target, make_quoted_expr(next_input))
+    else:
+        return '%sget_ipython().magic(u"%s %s")' % (lspace, method, target)
+
+_initial_space_re = re.compile(r'\s*')
+_help_end_re = re.compile(r"""(%?
+                              [a-zA-Z_*][a-zA-Z0-9_*]*       # Variable name
+                              (\.[a-zA-Z_*][a-zA-Z0-9_*]*)*   # .etc.etc
+                              )
+                              (\?\??)$                       # ? or ??""",
+                              re.VERBOSE)
+def transform_help_end(line):
+    """Translate lines with ?/?? at the end"""
+    m = _help_end_re.search(line)
+    if m is None or has_comment(line):
+        return line
+    target = m.group(1)
+    esc = m.group(3)
+    lspace = _initial_space_re.match(line).group(0)
+    newline = _make_help_call(target, esc, lspace)
+    
+    # If we're mid-command, put it back on the next prompt for the user.
+    next_input = line.rstrip('?') if line.strip() != m.group(0) else None
+        
+    return _make_help_call(target, esc, lspace, next_input)
+
+
 class EscapedTransformer(object):
     """Class to transform lines that are explicitly escaped out."""
 
@@ -694,28 +741,8 b' class EscapedTransformer(object):'
         # A naked help line should just fire the intro help screen
         if not line_info.line[1:]:
             return 'get_ipython().show_usage()'
-
-        # There may be one or two '?' at the end, move them to the front so that
-        # the rest of the logic can assume escapes are at the start
-        l_ori = line_info
-        line = line_info.line
-        if line.endswith('?'):
-            line = line[-1] + line[:-1]
-        if line.endswith('?'):
-            line = line[-1] + line[:-1]
-        line_info = LineInfo(line)
-
-        # From here on, simply choose which level of detail to get, and
-        # special-case the psearch syntax
-        pinfo = 'pinfo' # default
-        if '*' in line_info.line:
-            pinfo = 'psearch'
-        elif line_info.esc == '??':
-            pinfo = 'pinfo2'
-
-        tpl = '%sget_ipython().magic(u"%s %s")'
-        return tpl % (line_info.lspace, pinfo,
-                      ' '.join([line_info.fpart, line_info.rest]).strip())
+        
+        return _make_help_call(line_info.fpart, line_info.esc, line_info.lspace)
 
     @staticmethod
     def _tr_magic(line_info):
@@ -756,14 +783,9 b' class EscapedTransformer(object):'
         # Get line endpoints, where the escapes can be
         line_info = LineInfo(line)
 
-        # If the escape is not at the start, only '?' needs to be special-cased.
-        # All other escapes are only valid at the start
         if not line_info.esc in self.tr:
-            if line.endswith(ESC_HELP):
-                return self._tr_help(line_info)
-            else:
-                # If we don't recognize the escape, don't modify the line
-                return line
+            # If we don't recognize the escape, don't modify the line
+            return line
 
         return self.tr[line_info.esc](line_info)
 
@@ -815,9 +837,9 b' class IPythonInputSplitter(InputSplitter):'
 
         lines_list = lines.splitlines()
 
-        transforms = [transform_escaped, transform_assign_system,
-                      transform_assign_magic, transform_ipy_prompt,
-                      transform_classic_prompt]
+        transforms = [transform_ipy_prompt, transform_classic_prompt,
+                      transform_escaped, transform_help_end,
+                      transform_assign_system, transform_assign_magic]
 
         # Transform logic
         #

IPython/core/interactiveshell.py

@@ -1703,7 +1703,8 b' class InteractiveShell(SingletonConfigurable, Magic):'
         [D:\ipython]|1> _ip.set_next_input("Hello Word")
         [D:\ipython]|2> Hello Word_  # cursor is here        
         """
-
+        if isinstance(s, unicode):
+            s = s.encode(self.stdin_encoding, 'replace')
         self.rl_next_input = s
 
     # Maybe move this to the terminal subclass?
@@ -1841,7 +1842,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
         from . import history
         history.init_ipython(self)
 
-    def magic(self,arg_s):
+    def magic(self, arg_s, next_input=None):
         """Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
@@ -1858,6 +1859,11 b' class InteractiveShell(SingletonConfigurable, Magic):'
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
+        # Allow setting the next input - this is used if the user does `a=abs?`.
+        # We do this first so that magic functions can override it.
+        if next_input:
+            self.set_next_input(next_input)
+            
         args = arg_s.split(' ',1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)

IPython/core/tests/test_inputsplitter.py

@@ -25,6 +25,7 b' import nose.tools as nt'
 
 # Our own
 from IPython.core import inputsplitter as isp
+from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Semi-complete examples (also used as tests)
@@ -92,9 +93,7 b' def test_spaces():'
              ('\tx', 1),
              ('\t x', 2),
              ]
-    
-    for s, nsp in tests:
-        nt.assert_equal(isp.num_ini_spaces(s), nsp)
+    tt.check_pairs(isp.num_ini_spaces, tests)
 
 
 def test_remove_comments():
@@ -106,9 +105,19 b' def test_remove_comments():'
              ('line # c \nline#c2  \nline\nline #c\n\n',
               'line \nline\nline\nline \n\n'),
              ]
-
-    for inp, out in tests:
-        nt.assert_equal(isp.remove_comments(inp), out)
+    tt.check_pairs(isp.remove_comments, tests)
+        
+def test_has_comment():
+    tests = [('text', False),
+             ('text #comment', True),
+             ('text #comment\n', True),
+             ('#comment', True),
+             ('#comment\n', True),
+             ('a = "#string"', False),
+             ('a = "#string" # comment', True),
+             ('a #comment not "string"', True),
+             ]
+    tt.check_pairs(isp.has_comment, tests)
 
 
 def test_get_input_encoding():
@@ -434,12 +443,22 b' syntax = \\'
        [ ('?', 'get_ipython().show_usage()'),
          ('?x1', 'get_ipython().magic(u"pinfo x1")'),
          ('??x2', 'get_ipython().magic(u"pinfo2 x2")'),
-         ('x3?', 'get_ipython().magic(u"pinfo x3")'),
-         ('x4??', 'get_ipython().magic(u"pinfo2 x4")'),
-         ('%hist?', 'get_ipython().magic(u"pinfo %hist")'),
-         ('f*?', 'get_ipython().magic(u"psearch f*")'),
-         ('ax.*aspe*?', 'get_ipython().magic(u"psearch ax.*aspe*")'),
+         ('?a.*s', 'get_ipython().magic(u"psearch a.*s")'),
+         ('?%hist', 'get_ipython().magic(u"pinfo %hist")'),
+         ('?abc = qwe', 'get_ipython().magic(u"pinfo abc")'),
          ],
+         
+      end_help =
+      [ ('x3?', 'get_ipython().magic(u"pinfo x3")'),
+        ('x4??', 'get_ipython().magic(u"pinfo2 x4")'),
+        ('%hist?', 'get_ipython().magic(u"pinfo %hist")'),
+        ('f*?', 'get_ipython().magic(u"psearch f*")'),
+        ('ax.*aspe*?', 'get_ipython().magic(u"psearch ax.*aspe*")'),
+        ('a = abc?', 'get_ipython().magic(u"pinfo abc", next_input=u"a = abc")'),
+        ('a = abc.qe??', 'get_ipython().magic(u"pinfo2 abc.qe", next_input=u"a = abc.qe")'),
+        ('a = *.items?', 'get_ipython().magic(u"psearch *.items", next_input=u"a = *.items")'),
+        ('a*2 #comment?', 'a*2 #comment?'),
+        ],
 
        # Explicit magic calls
        escaped_magic =
@@ -471,7 +490,15 b' syntax = \\'
          ('  /f y', '  f(y)'),
          ('/f a b', 'f(a, b)'),
          ],
-
+         
+       # Check that we transform prompts before other transforms
+       mixed =
+       [ ('In [1]: %lsmagic', 'get_ipython().magic(u"lsmagic")'),
+         ('>>> %lsmagic', 'get_ipython().magic(u"lsmagic")'),
+         ('In [2]: !ls', 'get_ipython().system(u"ls")'),
+         ('In [3]: abs?', 'get_ipython().magic(u"pinfo abs")'),
+         ('In [4]: b = %who', 'b = get_ipython().magic(u"who")'),
+         ],
        )
 
 # multiline syntax examples.  Each of these should be a list of lists, with
@@ -496,11 +523,11 b' syntax_ml = \\'
 
 
 def test_assign_system():
-    transform_checker(syntax['assign_system'], isp.transform_assign_system)
+    tt.check_pairs(isp.transform_assign_system, syntax['assign_system'])
 
     
 def test_assign_magic():
-    transform_checker(syntax['assign_magic'], isp.transform_assign_magic)
+    tt.check_pairs(isp.transform_assign_magic, syntax['assign_magic'])
 
 
 def test_classic_prompt():
@@ -513,34 +540,36 b' def test_ipy_prompt():'
     transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, isp.transform_ipy_prompt)
-    
+
+def test_end_help():
+    tt.check_pairs(isp.transform_help_end, syntax['end_help'])
 
 def test_escaped_noesc():
-    transform_checker(syntax['escaped_noesc'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_noesc'])
 
 
 def test_escaped_shell():
-    transform_checker(syntax['escaped_shell'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_shell'])
 
 
 def test_escaped_help():
-    transform_checker(syntax['escaped_help'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_help'])
 
 
 def test_escaped_magic():
-    transform_checker(syntax['escaped_magic'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_magic'])
 
 
 def test_escaped_quote():
-    transform_checker(syntax['escaped_quote'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_quote'])
 
 
 def test_escaped_quote2():
-    transform_checker(syntax['escaped_quote2'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_quote2'])
 
 
 def test_escaped_paren():
-    transform_checker(syntax['escaped_paren'], isp.transform_escaped)
+    tt.check_pairs(isp.transform_escaped, syntax['escaped_paren'])
 
 
 class IPythonInputTestCase(InputSplitterTestCase):

IPython/testing/tools.py

@@ -281,3 +281,29 b' class TempFileMixin(object):'
                 # delete it.  I have no clue why
                 pass
 
+pair_fail_msg = ("Testing function {0}\n\n"
+                "In:\n"
+                "  {1!r}\n"
+                "Expected:\n"
+                "  {2!r}\n"
+                "Got:\n"
+                "  {3!r}\n")
+def check_pairs(func, pairs):
+    """Utility function for the common case of checking a function with a 
+    sequence of input/output pairs.
+    
+    Parameters
+    ----------
+    func : callable
+      The function to be tested. Should accept a single argument.
+    pairs : iterable
+      A list of (input, expected_output) tuples.
+    
+    Returns
+    -------
+    None. Raises an AssertionError if any output does not match the expected
+    value.
+    """
+    for inp, expected in pairs:
+        out = func(inp)
+        assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)

docs/source/whatsnew/development.txt

@@ -33,7 +33,7 b' ZMQ architecture'
 There is a new GUI framework for IPython, based on a client-server model in
 which multiple clients can communicate with one IPython kernel, using the
 ZeroMQ messaging framework. There is already a Qt console client, which can
-be started by calling ``ipython-qtconsole``. The protocol is :ref:`documented
+be started by calling ``ipython qtconsole``. The protocol is :ref:`documented
 <messaging>`.
 
 The parallel computing framework has also been rewritten using ZMQ. The
@@ -43,6 +43,10 b' new :mod:`IPython.parallel` module.'
 New features
 ------------
 
+* You can now get help on an object halfway through typing a command. For
+  instance, typing ``a = zip?`` shows the details of :func:`zip`. It also
+  leaves the command at the next prompt so you can carry on with it.
+
 * The input history is now written to an SQLite database. The API for
   retrieving items from the history has also been redesigned.
 
@@ -63,7 +67,7 b' New features'
 
 * The methods of :class:`~IPython.core.iplib.InteractiveShell` have
   been organized into sections to make it easier to turn more sections
-  of functionality into componenets.
+  of functionality into components.
 
 * The embedded shell has been refactored into a truly standalone subclass of
   :class:`InteractiveShell` called :class:`InteractiveShellEmbed`.  All