Backport PR #5816: Add output to IPython directive when in verbatim mode.

Addresses  #5776.

IPython/nbconvert/utils/lexers.py

@@ -491,7 +491,9 b' class IPyLexer(Lexer):'
         self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
 
     def get_tokens_unprocessed(self, text):
-        if re.match(r'(In \[[0-9]+\]:)', text.strip()):
+        # Search for the input prompt anywhere...this allows code blocks to
+        # begin with comments as well.
+        if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
             lex = self.IPythonConsoleLexer
         else:
             lex = self.IPythonLexer

IPython/sphinxext/ipython_directive.py

@@ -86,11 +86,14 b' documentation.'
 Pseudo-Decorators
 =================
 
+Note: Only one decorator is supported per input. If more than one decorator
+is specified, then only the last one is used.
+
 In addition to the Pseudo-Decorators/options described at the above link,
-several enhancements have been made.
-The directive will emit a message to the console at build-time if
-code-execution resulted in an exception or warning. You can suppress these on
-a per-block basis by specifying the :okexcept: or :okwarning: options:
+several enhancements have been made. The directive will emit a message to the
+console at build-time if code-execution resulted in an exception or warning.
+You can suppress these on a per-block basis by specifying the :okexcept:
+or :okwarning: options:
 
 .. code-block:: rst
 
@@ -203,8 +206,8 b' def block_parser(part, rgxin, rgxout, fmtin, fmtout):'
             continue
 
         if line_stripped.startswith('@'):
-            # we're assuming at most one decorator -- may need to
-            # rethink
+            # Here is where we assume there is, at most, one decorator.
+            # Might need to rethink this.
             decorator = line_stripped
             continue
 
@@ -442,16 +445,58 b' class EmbeddedSphinxShell(object):'
                     ret.append(formatted_line)
 
         if not is_suppress and len(rest.strip()) and is_verbatim:
-            # the "rest" is the standard output of the
-            # input, which needs to be added in
-            # verbatim mode
+            # The "rest" is the standard output of the input. This needs to be
+            # added when in verbatim mode. If there is no "rest", then we don't
+            # add it, as the new line will be added by the processed output.
             ret.append(rest)
 
+        # Fetch the processed output. (This is not the submitted output.)
         self.cout.seek(0)
-        output = self.cout.read()
+        processed_output = self.cout.read()
         if not is_suppress and not is_semicolon:
-            ret.append(output)
-        elif is_semicolon: # get spacing right
+            #
+            # In IPythonDirective.run, the elements of `ret` are eventually
+            # combined such that '' entries correspond to newlines. So if
+            # `processed_output` is equal to '', then the adding it to `ret`
+            # ensures that there is a blank line between consecutive inputs
+            # that have no outputs, as in:
+            #
+            #    In [1]: x = 4
+            #
+            #    In [2]: x = 5
+            #
+            # When there is processed output, it has a '\n' at the tail end. So
+            # adding the output to `ret` will provide the necessary spacing
+            # between consecutive input/output blocks, as in:
+            #
+            #   In [1]: x
+            #   Out[1]: 5
+            #
+            #   In [2]: x
+            #   Out[2]: 5
+            #
+            # When there is stdout from the input, it also has a '\n' at the
+            # tail end, and so this ensures proper spacing as well. E.g.:
+            #
+            #   In [1]: print x
+            #   5
+            #
+            #   In [2]: x = 5
+            #
+            # When in verbatim mode, `processed_output` is empty (because
+            # nothing was passed to IP. Sometimes the submitted code block has
+            # an Out[] portion and sometimes it does not. When it does not, we
+            # need to ensure proper spacing, so we have to add '' to `ret`.
+            # However, if there is an Out[] in the submitted code, then we do
+            # not want to add a newline as `process_output` has stuff to add.
+            # The difficulty is that `process_input` doesn't know if
+            # `process_output` will be called---so it doesn't know if there is
+            # Out[] in the code block. The requires that we include a hack in
+            # `process_block`. See the comments there.
+            #
+            ret.append(processed_output)
+        elif is_semicolon:
+            # Make sure there is a newline after the semicolon.
             ret.append('')
 
         # context information
@@ -463,12 +508,12 b' class EmbeddedSphinxShell(object):'
 
         # output any exceptions raised during execution to stdout
         # unless :okexcept: has been specified.
-        if not is_okexcept and "Traceback" in output:
+        if not is_okexcept and "Traceback" in processed_output:
             s =  "\nException in %s at block ending on line %s\n" % (filename, lineno)
             s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
             sys.stdout.write('\n\n>>>' + ('-' * 73))
             sys.stdout.write(s)
-            sys.stdout.write(output)
+            sys.stdout.write(processed_output)
             sys.stdout.write('<<<' + ('-' * 73) + '\n\n')
 
         # output any warning raised during execution to stdout
@@ -486,21 +531,25 b' class EmbeddedSphinxShell(object):'
                 sys.stdout.write('<<<' + ('-' * 73) + '\n')
 
         self.cout.truncate(0)
-        return (ret, input_lines, output, is_doctest, decorator, image_file,
-                    image_directive)
+
+        return (ret, input_lines, processed_output,
+                is_doctest, decorator, image_file, image_directive)
 
 
-    def process_output(self, data, output_prompt,
-                       input_lines, output, is_doctest, decorator, image_file):
+    def process_output(self, data, output_prompt, input_lines, output,
+                       is_doctest, decorator, image_file):
         """
         Process data block for OUTPUT token.
 
         """
+        # Recall: `data` is the submitted output, and `output` is the processed
+        # output from `input_lines`.
+
         TAB = ' ' * 4
 
         if is_doctest and output is not None:
 
-            found = output
+            found = output # This is the processed output
             found = found.strip()
             submitted = data.strip()
 
@@ -542,6 +591,31 b' class EmbeddedSphinxShell(object):'
             else:
                 self.custom_doctest(decorator, input_lines, found, submitted)
 
+        # When in verbatim mode, this holds additional submitted output
+        # to be written in the final Sphinx output.
+        # https://github.com/ipython/ipython/issues/5776
+        out_data = []
+
+        is_verbatim = decorator=='@verbatim' or self.is_verbatim
+        if is_verbatim and data.strip():
+            # Note that `ret` in `process_block` has '' as its last element if
+            # the code block was in verbatim mode. So if there is no submitted
+            # output, then we will have proper spacing only if we do not add
+            # an additional '' to `out_data`. This is why we condition on
+            # `and data.strip()`.
+
+            # The submitted output has no output prompt. If we want the
+            # prompt and the code to appear, we need to join them now
+            # instead of adding them separately---as this would create an
+            # undesired newline. How we do this ultimately depends on the
+            # format of the output regex. I'll do what works for the default
+            # prompt for now, and we might have to adjust if it doesn't work
+            # in other cases. Finally, the submitted output does not have
+            # a trailing newline, so we must add it manually.
+            out_data.append("{0} {1}\n".format(output_prompt, data))
+
+        return out_data
+
     def process_comment(self, data):
         """Process data fPblock for COMMENT token."""
         if not self.is_suppress:
@@ -579,14 +653,22 b' class EmbeddedSphinxShell(object):'
             if token == COMMENT:
                 out_data = self.process_comment(data)
             elif token == INPUT:
-                (out_data, input_lines, output, is_doctest, decorator,
-                    image_file, image_directive) = \
+                (out_data, input_lines, output, is_doctest,
+                 decorator, image_file, image_directive) = \
                           self.process_input(data, input_prompt, lineno)
             elif token == OUTPUT:
                 out_data = \
-                    self.process_output(data, output_prompt,
-                                        input_lines, output, is_doctest,
-                                        decorator, image_file)
+                    self.process_output(data, output_prompt, input_lines,
+                                        output, is_doctest, decorator,
+                                        image_file)
+                if out_data:
+                    # Then there was user submitted output in verbatim mode.
+                    # We need to remove the last element of `ret` that was
+                    # added in `process_input`, as it is '' and would introduce
+                    # an undesirable newline.
+                    assert(ret[-1] == '')
+                    del ret[-1]
+
             if out_data:
                 ret.extend(out_data)
 
@@ -843,7 +925,8 b' class IPythonDirective(Directive):'
             if len(block):
                 rows, figure = self.shell.process_block(block)
                 for row in rows:
-                    lines.extend(['   %s'%line for line in row.split('\n')])
+                    lines.extend(['   {0}'.format(line)
+                                  for line in row.split('\n')])
 
                 if figure is not None:
                     figures.append(figure)
@@ -853,7 +936,7 b' class IPythonDirective(Directive):'
             lines.extend(figure.split('\n'))
             lines.append('')
 
-        if len(lines)>2:
+        if len(lines) > 2:
             if debug:
                 print('\n'.join(lines))
             else: