Backport PR #3974: nbconvert: Fix app tests on Window7 w/ Python 3.3

closes  #3945

Please see issue for more information.

IPython/nbconvert/nbconvertapp.py

@@ -1,321 +1,320 b''
 #!/usr/bin/env python
 """NBConvert is a utility for conversion of .ipynb files.
 
 Command-line interface for the NbConvert conversion utility.
 """
 #-----------------------------------------------------------------------------
 #Copyright (c) 2013, the IPython Development Team.
 #
 #Distributed under the terms of the Modified BSD License.
 #
 #The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 #Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib imports
 from __future__ import print_function
 
 import logging
 import sys
 import os
 import glob
 
 # From IPython
 from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
 from IPython.config import catch_config_error, Configurable
 from IPython.utils.traitlets import (
     Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.text import dedent
 
 from .exporters.export import get_export_names, exporter_map
 from IPython.nbconvert import exporters, transformers, writers, post_processors
 from .utils.base import NbConvertBase
 from .utils.exceptions import ConversionException
 
 #-----------------------------------------------------------------------------
 #Classes and functions
 #-----------------------------------------------------------------------------
 
 class DottedOrNone(DottedObjectName):
     """
     A string holding a valid dotted object name in Python, such as A.b3._c
     Also allows for None type."""
     
     default_value = u''
 
     def validate(self, obj, value):
         if value is not None and len(value) > 0:
             return super(DottedOrNone, self).validate(obj, value)
         else:
             return value
             
 nbconvert_aliases = {}
 nbconvert_aliases.update(base_aliases)
 nbconvert_aliases.update({
     'to' : 'NbConvertApp.export_format',
     'template' : 'Exporter.template_file',
-    'notebooks' : 'NbConvertApp.notebooks',
     'writer' : 'NbConvertApp.writer_class',
     'post': 'NbConvertApp.post_processor_class',
     'output': 'NbConvertApp.output_base'
 })
 
 nbconvert_flags = {}
 nbconvert_flags.update(base_flags)
 nbconvert_flags.update({
     'stdout' : (
         {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
         "Write notebook output to stdout instead of files."
         )
 })
 
 
 class NbConvertApp(BaseIPythonApplication):
     """Application used to convert to and from notebook file type (*.ipynb)"""
 
     name = 'ipython-nbconvert'
     aliases = nbconvert_aliases
     flags = nbconvert_flags
     
     def _log_level_default(self):
         return logging.INFO
     
     def _classes_default(self):
         classes = [NbConvertBase]
         for pkg in (exporters, transformers, writers):
             for name in dir(pkg):
                 cls = getattr(pkg, name)
                 if isinstance(cls, type) and issubclass(cls, Configurable):
                     classes.append(cls)
         return classes
 
     description = Unicode(
         u"""This application is used to convert notebook files (*.ipynb)
         to various other formats.
 
         WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""")
 
     output_base = Unicode('', config=True, help='''overwrite base name use for output files.
             can only  be use when converting one notebook at a time.
             ''')
 
     examples = Unicode(u"""
         The simplest way to use nbconvert is
         
         > ipython nbconvert mynotebook.ipynb
         
         which will convert mynotebook.ipynb to the default format (probably HTML).
         
         You can specify the export format with `--to`.
         Options include {0}
         
         > ipython nbconvert --to latex mynotebook.ipnynb
 
         Both HTML and LaTeX support multiple output templates. LaTeX includes
         'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You 
         can specify the flavor of the format used.
 
         > ipython nbconvert --to html --template basic mynotebook.ipynb
         
         You can also pipe the output to stdout, rather than a file
         
         > ipython nbconvert mynotebook.ipynb --stdout
 
         A post-processor can be used to compile a PDF
 
         > ipython nbconvert mynotebook.ipynb --to latex --post PDF
         
         You can get (and serve) a Reveal.js-powered slideshow
         
         > ipython nbconvert myslides.ipynb --to slides --post serve
         
         Multiple notebooks can be given at the command line in a couple of 
         different ways:
   
         > ipython nbconvert notebook*.ipynb
         > ipython nbconvert notebook1.ipynb notebook2.ipynb
         
         or you can specify the notebooks list in a config file, containing::
         
             c.NbConvertApp.notebooks = ["my_notebook.ipynb"]
         
         > ipython nbconvert --config mycfg.py
         """.format(get_export_names()))
 
     # Writer specific variables
     writer = Instance('IPython.nbconvert.writers.base.WriterBase',  
                       help="""Instance of the writer class used to write the 
                       results of the conversion.""")
     writer_class = DottedObjectName('FilesWriter', config=True, 
                                     help="""Writer class used to write the 
                                     results of the conversion""")
     writer_aliases = {'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter',
                       'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                       'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
     writer_factory = Type()
 
     def _writer_class_changed(self, name, old, new):
         if new.lower() in self.writer_aliases:
             new = self.writer_aliases[new.lower()]
         self.writer_factory = import_item(new)
 
     # Post-processor specific variables
     post_processor = Instance('IPython.nbconvert.post_processors.base.PostProcessorBase',  
                       help="""Instance of the PostProcessor class used to write the 
                       results of the conversion.""")
 
     post_processor_class = DottedOrNone(config=True, 
                                     help="""PostProcessor class used to write the 
                                     results of the conversion""")
     post_processor_aliases = {'pdf': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor',
                               'serve': 'IPython.nbconvert.post_processors.serve.ServePostProcessor'}
     post_processor_factory = Type()
 
     def _post_processor_class_changed(self, name, old, new):
         if new.lower() in self.post_processor_aliases:
             new = self.post_processor_aliases[new.lower()]
         if new:
             self.post_processor_factory = import_item(new)
 
 
     # Other configurable variables
     export_format = CaselessStrEnum(get_export_names(),
         default_value="html",
         config=True,
         help="""The export format to be used."""
     )
 
     notebooks = List([], config=True, help="""List of notebooks to convert.
                      Wildcards are supported.
                      Filenames passed positionally will be added to the list.
                      """)
 
     @catch_config_error
     def initialize(self, argv=None):
         super(NbConvertApp, self).initialize(argv)
         self.init_syspath()
         self.init_notebooks()
         self.init_writer()
         self.init_post_processor()
 
 
 
     def init_syspath(self):
         """
         Add the cwd to the sys.path ($PYTHONPATH)
         """
         sys.path.insert(0, os.getcwd())
         
 
     def init_notebooks(self):
         """Construct the list of notebooks.
         If notebooks are passed on the command-line,
         they override notebooks specified in config files.
         Glob each notebook to replace notebook patterns with filenames.
         """
 
         # Specifying notebooks on the command-line overrides (rather than adds)
         # the notebook list
         if self.extra_args:
             patterns = self.extra_args
         else:
             patterns = self.notebooks
 
         # Use glob to replace all the notebook patterns with filenames.
         filenames = []
         for pattern in patterns:
             
             # Use glob to find matching filenames.  Allow the user to convert 
             # notebooks without having to type the extension.
             globbed_files = glob.glob(pattern)
             globbed_files.extend(glob.glob(pattern + '.ipynb'))
             if not globbed_files:
                 self.log.warn("pattern %r matched no files", pattern)
 
             for filename in globbed_files:
                 if not filename in filenames:
                     filenames.append(filename)
         self.notebooks = filenames
 
     def init_writer(self):
         """
         Initialize the writer (which is stateless)
         """
         self._writer_class_changed(None, self.writer_class, self.writer_class)
         self.writer = self.writer_factory(parent=self)
 
     def init_post_processor(self):
         """
         Initialize the post_processor (which is stateless)
         """
         self._post_processor_class_changed(None, self.post_processor_class, 
             self.post_processor_class)
         if self.post_processor_factory:
             self.post_processor = self.post_processor_factory(parent=self)
 
     def start(self):
         """
         Ran after initialization completed
         """
         super(NbConvertApp, self).start()
         self.convert_notebooks()
 
     def convert_notebooks(self):
         """
         Convert the notebooks in the self.notebook traitlet
         """
         # Export each notebook
         conversion_success = 0
 
         if self.output_base != '' and len(self.notebooks) > 1:
             self.log.error(
             """UsageError: --output flag or `NbConvertApp.output_base` config option
             cannot be used when converting multiple notebooks.
             """)
             self.exit(1)
         
         exporter = exporter_map[self.export_format](config=self.config)
 
         for notebook_filename in self.notebooks:
             self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format)
 
             # Get a unique key for the notebook and set it in the resources object.
             basename = os.path.basename(notebook_filename)
             notebook_name = basename[:basename.rfind('.')]
             if self.output_base:
                 notebook_name = self.output_base
             resources = {}
             resources['unique_key'] = notebook_name
             resources['output_files_dir'] = '%s_files' % notebook_name
             self.log.info("Support files will be in %s", os.path.join(resources['output_files_dir'], ''))
 
             # Try to export
             try:
                 output, resources = exporter.from_filename(notebook_filename, resources=resources)
             except ConversionException as e:
                 self.log.error("Error while converting '%s'", notebook_filename,
                       exc_info=True)
                 self.exit(1)
             else:
                 write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)
 
                 #Post-process if post processor has been defined.
                 if hasattr(self, 'post_processor') and self.post_processor:
                     self.post_processor(write_resultes)
                 conversion_success += 1
 
         # If nothing was converted successfully, help the user.
         if conversion_success == 0:
             self.print_help()
             sys.exit(-1)
             
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 launch_new_instance = NbConvertApp.launch_instance

IPython/nbconvert/tests/base.py

@@ -1,143 +1,155 b''
 """
 Contains base test class for nbconvert
 """
 #-----------------------------------------------------------------------------
 #Copyright (c) 2013, the IPython Development Team.
 #
 #Distributed under the terms of the Modified BSD License.
 #
 #The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import glob
 import shutil
 
 import IPython
 from IPython.utils.tempdir import TemporaryWorkingDirectory
 from IPython.utils.process import get_output_error_code
 from IPython.testing.tools import get_ipython_cmd
 from IPython.testing.ipunittest import ParametricTestCase
 
 # a trailing space allows for simpler concatenation with the other arguments
 ipy_cmd = get_ipython_cmd(as_string=True) + " "
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 class TestsBase(ParametricTestCase):
     """Base tests class.  Contains useful fuzzy comparison and nbconvert
     functions."""
 
 
     def fuzzy_compare(self, a, b, newlines_are_spaces=True, tabs_are_spaces=True, 
                       fuzzy_spacing=True, ignore_spaces=False, 
                       ignore_newlines=False, case_sensitive=False, leave_padding=False):
         """
         Performs a fuzzy comparison of two strings.  A fuzzy comparison is a
         comparison that ignores insignificant differences in the two comparands.
         The significance of certain differences can be specified via the keyword
         parameters of this method.
         """
 
         if not leave_padding:
             a = a.strip()
             b = b.strip()
 
         if ignore_newlines:
             a = a.replace('\n', '')
             b = b.replace('\n', '')
 
         if newlines_are_spaces:
             a = a.replace('\n', ' ')
             b = b.replace('\n', ' ')
 
         if tabs_are_spaces:
             a = a.replace('\t', ' ')
             b = b.replace('\t', ' ')
 
         if ignore_spaces:
             a = a.replace(' ', '')
             b = b.replace(' ', '')
 
         if fuzzy_spacing:
             a = self.recursive_replace(a, '  ', ' ')
             b = self.recursive_replace(b, '  ', ' ')
 
         if not case_sensitive:
             a = a.lower()
             b = b.lower()
 
         self.assertEqual(a, b)
 
 
     def recursive_replace(self, text, search, replacement):
         """
         Performs a recursive replacement operation.  Replaces all instances
         of a search string in a text string with a replacement string until
         the search string no longer exists.  Recursion is needed because the
         replacement string may generate additional search strings.
 
         For example:
            Replace "ii" with "i" in the string "Hiiii" yields "Hii"
            Another replacement yields "Hi" (the desired output)
 
         Parameters:
         -----------
         text : string
             Text to replace in.
         search : string
             String to search for within "text"
         replacement : string
             String to replace "search" with
         """
         while search in text:
             text = text.replace(search, replacement)
         return text
 
     def create_temp_cwd(self, copy_filenames=None):
         temp_dir = TemporaryWorkingDirectory()
 
         #Copy the files if requested.
         if copy_filenames is not None:
             self.copy_files_to(copy_filenames)
 
         #Return directory handler
         return temp_dir
 
 
     def copy_files_to(self, copy_filenames, dest='.'):
         "Copy test files into the destination directory"
         if not os.path.isdir(dest):
             os.makedirs(dest)
         files_path = self._get_files_path()
         for pattern in copy_filenames:
             for match in glob.glob(os.path.join(files_path, pattern)):
                 shutil.copyfile(match, os.path.join(dest, os.path.basename(match)))
 
 
     def _get_files_path(self):
 
         #Get the relative path to this module in the IPython directory.
         names = self.__module__.split('.')[1:-1]
         names.append('files')
         
         #Build a path using the IPython directory and the relative path we just
         #found.
         path = IPython.__path__[0]
         for name in names:
             path = os.path.join(path, name)
         return path
 
 
-    def call(self, parameters, raise_on_error=True):
+    def call(self, parameters, ignore_return_code=False):
+        """
+        Execute a, IPython shell command, listening for both Errors and non-zero
+        return codes.
+
+        PARAMETERS:
+        -----------
+        parameters : str
+            List of parameters to pass to IPython.
+        ignore_return_code : optional bool (default False)
+            Throw an OSError if the return code
+        """
+
         stdout, stderr, retcode = get_output_error_code(ipy_cmd + parameters)
-        if retcode != 0 and raise_on_error:
+        if not (retcode == 0 or ignore_return_code):
             raise OSError(stderr)
         return stdout, stderr

IPython/nbconvert/tests/test_nbconvertapp.py

@@ -1,160 +1,160 b''
 """
 Contains tests for the nbconvertapp
 """
 #-----------------------------------------------------------------------------
 #Copyright (c) 2013, the IPython Development Team.
 #
 #Distributed under the terms of the Modified BSD License.
 #
 #The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import glob
 
 from .base import TestsBase
 
 from IPython.testing import decorators as dec
 
     
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class TestNbConvertApp(TestsBase):
     """Collection of NbConvertApp tests"""
 
 
     def test_notebook_help(self):
         """
         Will help show if no notebooks are specified?
         """
         with self.create_temp_cwd():
-            out, err = self.call('nbconvert --log-level=0', raise_on_error=False)
+            out, err = self.call('nbconvert --log-level 0', ignore_return_code=True)
             assert "see '--help-all'" in out
 
 
     def test_glob(self):
         """
         Do search patterns work for notebook names?
         """
         with self.create_temp_cwd(['notebook*.ipynb']):
-            self.call('nbconvert --to="python" --notebooks=\'["*.ipynb"]\' --log-level=0')
+            self.call('nbconvert --to python *.ipynb --log-level 0')
             assert os.path.isfile('notebook1.py')
             assert os.path.isfile('notebook2.py')
 
 
     def test_glob_subdir(self):
         """
         Do search patterns work for subdirectory notebook names?
         """
         with self.create_temp_cwd():
             self.copy_files_to(['notebook*.ipynb'], 'subdir/')
-            self.call('nbconvert --to="python" --log-level=0 --notebooks='
-                      '\'["%s"]\'' % os.path.join('subdir', '*.ipynb'))
+            self.call('nbconvert --to python --log-level 0 ' +
+                      os.path.join('subdir', '*.ipynb'))
             assert os.path.isfile('notebook1.py')
             assert os.path.isfile('notebook2.py')
 
 
     def test_explicit(self):
         """
         Do explicit notebook names work?
         """
         with self.create_temp_cwd(['notebook*.ipynb']):
-            self.call('nbconvert --log-level=0 --to="python" --notebooks='
-                      '\'["notebook2.ipynb"]\'')
+            self.call('nbconvert --log-level 0 --to python notebook2')
             assert not os.path.isfile('notebook1.py')
             assert os.path.isfile('notebook2.py')
 
 
     @dec.onlyif_cmds_exist('pdflatex')
     @dec.onlyif_cmds_exist('pandoc')
     def test_filename_spaces(self):
         """
         Generate PDFs with graphics if notebooks have spaces in the name?
         """
         with self.create_temp_cwd(['notebook2.ipynb']):
             os.rename('notebook2.ipynb', 'notebook with spaces.ipynb')
-            o,e = self.call('nbconvert --log-level=0 --to="latex" "notebook with spaces"'
-                            ' --post="PDF" --PDFPostProcessor.verbose=True')
+            o,e = self.call('nbconvert --log-level 0 --to latex '
+                            '"notebook with spaces" --post PDF '
+                            '--PDFPostProcessor.verbose=True')
             assert os.path.isfile('notebook with spaces.tex')
             assert os.path.isdir('notebook with spaces_files')
             assert os.path.isfile('notebook with spaces.pdf')
 
     @dec.onlyif_cmds_exist('pdflatex')
     @dec.onlyif_cmds_exist('pandoc')
     def test_post_processor(self):
         """
         Do post processors work?
         """
         with self.create_temp_cwd(['notebook1.ipynb']):
-            self.call('nbconvert --log-level=0 --to="latex" notebook1'
-                      ' --post="PDF" --PDFPostProcessor.verbose=True')
+            self.call('nbconvert --log-level 0 --to latex notebook1 '
+                      '--post PDF --PDFPostProcessor.verbose=True')
             assert os.path.isfile('notebook1.tex')
             assert os.path.isfile('notebook1.pdf')
 
 
     @dec.onlyif_cmds_exist('pandoc')
     def test_template(self):
         """
         Do export templates work?
         """
         with self.create_temp_cwd(['notebook2.ipynb']):
-            self.call('nbconvert --log-level=0 --to=slides --notebooks='
-                      '\'["notebook2.ipynb"]\' --template=reveal')
+            self.call('nbconvert --log-level 0 --to slides '
+                      'notebook2.ipynb --template reveal')
             assert os.path.isfile('notebook2.slides.html')
             with open('notebook2.slides.html') as f:
                 assert '/reveal.css' in f.read()
 
 
     def test_glob_explicit(self):
         """
         Can a search pattern be used along with matching explicit notebook names?
         """
         with self.create_temp_cwd(['notebook*.ipynb']):
-            self.call('nbconvert --log-level=0 --to="python" --notebooks='
-                      '\'["*.ipynb","notebook1.ipynb","notebook2.ipynb"]\'')
+            self.call('nbconvert --log-level 0 --to python '
+                      '*.ipynb notebook1.ipynb notebook2.ipynb')
             assert os.path.isfile('notebook1.py')
             assert os.path.isfile('notebook2.py')
 
 
     def test_explicit_glob(self):
         """
         Can explicit notebook names be used and then a matching search pattern?
         """
         with self.create_temp_cwd(['notebook*.ipynb']):
-            self.call('nbconvert --log-level=0 --to="python" --notebooks='
-                      '\'["notebook1.ipynb","notebook2.ipynb","*.ipynb"]\'')
+            self.call('nbconvert --log-level 0 --to=python '
+                      'notebook1.ipynb notebook2.ipynb *.ipynb')
             assert os.path.isfile('notebook1.py')
             assert os.path.isfile('notebook2.py')
 
 
     def test_default_config(self):
         """
         Does the default config work?
         """
         with self.create_temp_cwd(['notebook*.ipynb', 'ipython_nbconvert_config.py']):
-            self.call('nbconvert --log-level=0')
+            self.call('nbconvert --log-level 0')
             assert os.path.isfile('notebook1.py')
             assert not os.path.isfile('notebook2.py')
 
 
     def test_override_config(self):
         """
         Can the default config be overriden?
         """
         with self.create_temp_cwd(['notebook*.ipynb',
                                    'ipython_nbconvert_config.py',
                                    'override.py']):
-            self.call('nbconvert --log-level=0 --config="override.py"')
+            self.call('nbconvert --log-level 0 --config="override.py"')
             assert not os.path.isfile('notebook1.py')
             assert os.path.isfile('notebook2.py')