Merge pull request #1146 from minrk/savescript

Allow saving notebook.py next to notebook.ipynb.  This allows notebooks to be imported or %run via their .py companion file.

IPython/frontend/html/notebook/notebookmanager.py

@@ -1,233 +1,255 b''
 """A notebook manager that uses the local file system for storage.
 
 Authors:
 
 * Brian Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import datetime
 import os
 import uuid
 import glob
 
 from tornado import web
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.nbformat import current
-from IPython.utils.traitlets import Unicode, List, Dict
+from IPython.utils.traitlets import Unicode, List, Dict, Bool
 
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class NotebookManager(LoggingConfigurable):
 
     notebook_dir = Unicode(os.getcwd(), config=True, help="""
         The directory to use for notebooks.
     """)
+    
+    save_script = Bool(False, config=True,
+        help="""Also save notebooks as a Python script.
+        
+        For easier use of import/%loadpy across notebooks, a <notebook-name>.py
+        script will be created next to any <notebook-name>.ipynb on each save.
+        """
+    )
+    
     filename_ext = Unicode(u'.ipynb')
     allowed_formats = List([u'json',u'py'])
 
     # Map notebook_ids to notebook names
     mapping = Dict()
     # Map notebook names to notebook_ids
     rev_mapping = Dict()
 
     def list_notebooks(self):
         """List all notebooks in the notebook dir.
 
         This returns a list of dicts of the form::
 
             dict(notebook_id=notebook,name=name)
         """
         names = glob.glob(os.path.join(self.notebook_dir,
                                        '*' + self.filename_ext))
         names = [os.path.splitext(os.path.basename(name))[0]
                  for name in names]
 
         data = []
         for name in names:
             if name not in self.rev_mapping:
                 notebook_id = self.new_notebook_id(name)
             else:
                 notebook_id = self.rev_mapping[name]
             data.append(dict(notebook_id=notebook_id,name=name))
         data = sorted(data, key=lambda item: item['name'])
         return data
 
     def new_notebook_id(self, name):
         """Generate a new notebook_id for a name and store its mappings."""
         # TODO: the following will give stable urls for notebooks, but unless
         # the notebooks are immediately redirected to their new urls when their
         # filemname changes, nasty inconsistencies result.  So for now it's
         # disabled and instead we use a random uuid4() call.  But we leave the
         # logic here so that we can later reactivate it, whhen the necessary
         # url redirection code is written.
         #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,
         #                 'file://'+self.get_path_by_name(name).encode('utf-8')))
         
         notebook_id = unicode(uuid.uuid4())
         
         self.mapping[notebook_id] = name
         self.rev_mapping[name] = notebook_id
         return notebook_id
 
     def delete_notebook_id(self, notebook_id):
         """Delete a notebook's id only. This doesn't delete the actual notebook."""
         name = self.mapping[notebook_id]
         del self.mapping[notebook_id]
         del self.rev_mapping[name]
 
     def notebook_exists(self, notebook_id):
         """Does a notebook exist?"""
         if notebook_id not in self.mapping:
             return False
         path = self.get_path_by_name(self.mapping[notebook_id])
         return os.path.isfile(path)
 
     def find_path(self, notebook_id):
         """Return a full path to a notebook given its notebook_id."""
         try:
             name = self.mapping[notebook_id]
         except KeyError:
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         return self.get_path_by_name(name)
 
     def get_path_by_name(self, name):
         """Return a full path to a notebook given its name."""
         filename = name + self.filename_ext
         path = os.path.join(self.notebook_dir, filename)
         return path       
 
     def get_notebook(self, notebook_id, format=u'json'):
         """Get the representation of a notebook in format by notebook_id."""
         format = unicode(format)
         if format not in self.allowed_formats:
             raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
         last_modified, nb = self.get_notebook_object(notebook_id)
         kwargs = {}
         if format == 'json':
             # don't split lines for sending over the wire, because it
             # should match the Python in-memory format.
             kwargs['split_lines'] = False
         data = current.writes(nb, format, **kwargs)
         name = nb.get('name','notebook')
         return last_modified, name, data
 
     def get_notebook_object(self, notebook_id):
         """Get the NotebookNode representation of a notebook by notebook_id."""
         path = self.find_path(notebook_id)
         if not os.path.isfile(path):
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         info = os.stat(path)
         last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime)
         with open(path,'r') as f:
             s = f.read()
             try:
                 # v1 and v2 and json in the .ipynb files.
                 nb = current.reads(s, u'json')
             except:
                 raise web.HTTPError(500, u'Unreadable JSON notebook.')
         if 'name' not in nb:
             nb.name = os.path.split(path)[-1].split(u'.')[0]
         return last_modified, nb
 
     def save_new_notebook(self, data, name=None, format=u'json'):
         """Save a new notebook and return its notebook_id.
 
         If a name is passed in, it overrides any values in the notebook data
         and the value in the data is updated to use that value.
         """
         if format not in self.allowed_formats:
             raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
 
         try:
             nb = current.reads(data.decode('utf-8'), format)
         except:
             raise web.HTTPError(400, u'Invalid JSON data')
 
         if name is None:
             try:
                 name = nb.metadata.name
             except AttributeError:
                 raise web.HTTPError(400, u'Missing notebook name')
         nb.metadata.name = name
 
         notebook_id = self.new_notebook_id(name)
         self.save_notebook_object(notebook_id, nb)
         return notebook_id
 
     def save_notebook(self, notebook_id, data, name=None, format=u'json'):
         """Save an existing notebook by notebook_id."""
         if format not in self.allowed_formats:
             raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
 
         try:
             nb = current.reads(data.decode('utf-8'), format)
         except:
             raise web.HTTPError(400, u'Invalid JSON data')
 
         if name is not None:
             nb.metadata.name = name
         self.save_notebook_object(notebook_id, nb)
 
     def save_notebook_object(self, notebook_id, nb):
         """Save an existing notebook object by notebook_id."""
         if notebook_id not in self.mapping:
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         old_name = self.mapping[notebook_id]
         try:
             new_name = nb.metadata.name
         except AttributeError:
             raise web.HTTPError(400, u'Missing notebook name')
         path = self.get_path_by_name(new_name)
         try:
             with open(path,'w') as f:
                 current.write(nb, f, u'json')
-        except:
-            raise web.HTTPError(400, u'Unexpected error while saving notebook')
+        except Exception as e:
+            raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)
+        # save .py script as well
+        if self.save_script:
+            pypath = os.path.splitext(path)[0] + '.py'
+            try:
+                with open(pypath,'w') as f:
+                    current.write(nb, f, u'py')
+            except Exception as e:
+                raise web.HTTPError(400, u'Unexpected error while saving notebook as script: %s' % e)
+        
         if old_name != new_name:
             old_path = self.get_path_by_name(old_name)
             if os.path.isfile(old_path):
                 os.unlink(old_path)
+            if self.save_script:
+                old_pypath = os.path.splitext(old_path)[0] + '.py'
+                if os.path.isfile(old_pypath):
+                    os.unlink(old_pypath)
             self.mapping[notebook_id] = new_name
             self.rev_mapping[new_name] = notebook_id
 
     def delete_notebook(self, notebook_id):
         """Delete notebook by notebook_id."""
         path = self.find_path(notebook_id)
         if not os.path.isfile(path):
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         os.unlink(path)
         self.delete_notebook_id(notebook_id)
 
     def new_notebook(self):
         """Create a new notebook and returns its notebook_id."""
         i = 0
         while True:
             name = u'Untitled%i' % i
             path = self.get_path_by_name(name)
             if not os.path.isfile(path):
                 break
             else:
                 i = i+1
         notebook_id = self.new_notebook_id(name)
         metadata = current.new_metadata(name=name)
         nb = current.new_notebook(metadata=metadata)
         with open(path,'w') as f:
             current.write(nb, f, u'json')
         return notebook_id
 

docs/source/interactive/htmlnotebook.txt

@@ -1,375 +1,383 b''
 .. _htmlnotebook:
 
 =========================
 An HTML Notebook IPython
 =========================
 
 .. seealso::
 
     :ref:`Installation requirements <installnotebook>` for the Notebook.
 
 The IPython Notebook consists of two related components:
 
 * An JSON based Notebook document format for recording and distributing
   Python code and rich text.
 * A web-based user interface for authoring and running notebook documents.
 
 The Notebook can be used by starting the Notebook server with the 
 command::
 
     $ ipython notebook
 
 Note that by default, the notebook doesn't load pylab, it's just a normal
 IPython session like any other.  If you want pylab support, you must use::
     
     $ ipython notebook --pylab
 
 which will behave similar to the terminal and Qt console versions, using your
 default matplotlib backend and providing floating interactive plot windows.  If
 you want inline figures, you must manually select the ``inline`` backend::
 
     $ ipython notebook --pylab inline
 
 This server uses the same ZeroMQ-based two process kernel architecture as
 the QT Console as well Tornado for serving HTTP/S requests. Some of the main
 features of the Notebook include:
 
 * Display rich data (png/html/latex/svg) in the browser as a result of
   computations.
 * Compose text cells using HTML and Markdown.
 * Import and export notebook documents in range of formats (.ipynb, .py).
 * In browser syntax highlighting, tab completion and autoindentation.
 * Inline matplotlib plots that can be stored in Notebook documents and opened
   later.
 
 See :ref:`our installation documentation <install_index>` for directions on
 how to install the notebook and its dependencies.
 
 .. note::
 
    You can start more than one notebook server at the same time, if you want to
    work on notebooks in different directories.  By default the first notebook
    server starts in port 8888, later notebooks search for random ports near
    that one.  You can also manually specify the port with the ``--port``
    option.
    
 
 Basic Usage
 ===========
 
 The landing page of the notebook server application, which we call the IPython
 Notebook *dashboard*, shows the notebooks currently available in the directory
 in which the application was started, and allows you to create new notebooks.
 
 A notebook is a combination of two things:
 
 1. An interactive session connected to an IPython kernel, controlled by a web
    application that can send input to the console and display many types of
    output (text, graphics, mathematics and more).  This is the same kernel used
    by the :ref:`Qt console <qtconsole>`, but in this case the web console sends
    input in persistent cells that you can edit in-place instead of the
    vertically scrolling terminal style used by the Qt console.
 
 2. A document that can save the inputs and outputs of the session as well as
    additional text that accompanies the code but is not meant for execution.
    In this way, notebook files serve as a complete computational record of a
    session including explanatory text and mathematics, code and resulting
    figures.  These documents are internally JSON files and are saved with the
    ``.ipynb`` extension.
 
 If you have ever used the Mathematica or Sage notebooks (the latter is also
 web-based__) you should feel right at home.  If you have not, you should be
 able to learn how to use it in just a few minutes.
 
 .. __: http://sagenb.org
 
 
 Creating and editing notebooks
 ------------------------------
 
 You can create new notebooks from the dashboard with the ``New Notebook``
 button or open existing ones by clicking on their name.  Once in a notebook,
 your browser tab will reflect the name of that notebook (prefixed with "IPy:").
 The URL for that notebook is not meant to be human-readable and is *not*
 persistent across invocations of the notebook server.
 
 You can also drag and drop into the area listing files any python file: it
 will be imported into a notebook with the same name (but ``.ipynb`` extension)
 located in the directory where the notebook server was started.  This notebook
 will consist of a single cell with all the code in the file, which you can
 later manually partition into individual cells for gradual execution, add text
 and graphics, etc.
 
 
 Workflow and limitations
 ------------------------
 
 The normal workflow in a notebook is quite similar to a normal IPython session,
 with the difference that you can edit a cell in-place multiple times until you
 obtain the desired results rather than having to rerun separate scripts with
 the ``%run`` magic (though magics also work in the notebook).   Typically
 you'll work on a problem in pieces, organizing related pieces into cells and
 moving forward as previous parts work correctly.  This is much more convenient
 for interactive exploration than breaking up a computation into scripts that
 must be executed together, especially if parts of them take a long time to run
 (In the traditional terminal-based IPython, you can use tricks with namespaces
 and ``%run -i`` to achieve this capability, but we think the notebook is a more
 natural solution for  that kind of problem).
 
 The only significant limitation the notebook currently has, compared to the qt
 console, is that it can not run any code that expects input from the kernel
 (such as scripts that call :func:`raw_input`).  Very importantly, this means
 that the ``%debug`` magic does *not* work in the notebook!  We intend to
 correct this limitation, but in the meantime, there is a way to debug problems
 in the notebook: you can attach a Qt console to your existing notebook kernel,
 and run ``%debug`` from the Qt console.  If your notebook is running on a local
 computer (i.e. if you are accessing it via your localhost address at
 127.0.0.1), you can just type ``%qtconsole`` in the notebook and a Qt console
 will open up connected to that same kernel.
 
 In general, the notebook server prints the full details of how to connect to
 each kernel at the terminal, with lines like::
 
     [IPKernelApp] To connect another client to this kernel, use:
     [IPKernelApp] --existing kernel-3bb93edd-6b5a-455c-99c8-3b658f45dde5.json
 
 This is the name of a JSON file that contains all the port and validation
 information necessary to connect to the kernel.  You can manually start a
 qt console with::
 
     ipython qtconsole --existing kernel-3bb93edd-6b5a-455c-99c8-3b658f45dde5.json
 
 and if you only have a single kernel running, simply typing::
 
     ipython qtconsole --existing
 
 will automatically find it (it will always find the most recently started
 kernel if there is more than one).  You can also request this connection data
 by typing ``%connect_info``; this will print the same file information as well
 as the content of the JSON data structure it contains.
 
     
 Text input
 ----------
 
 In addition to code cells and the output they produce (such as figures), you
 can also type text not meant for execution.  To type text, change the type of a
 cell from ``Code`` to ``Markdown`` by using the button or the :kbd:`Ctrl-m m`
 keybinding (see below).  You can then type any text in Markdown_ syntax, as
 well as mathematical expressions if you use ``$...$`` for inline math or
 ``$$...$$`` for displayed math.
 
 
 Exporting a notebook and importing existing scripts
 ---------------------------------------------------
 
 If you want to provide others with a static HTML or PDF view of your notebook,
 use the ``Print`` button.  This opens a static view of the document, which you
 can print to PDF using your operating system's facilities, or save to a file
 with your web browser's 'Save' option (note that typically, this will create
 both an html file *and* a directory called `notebook_name_files` next to it
 that contains all the necessary style information, so if you intend to share
 this, you must send the directory along with the main html file).
 
 The `Download` button lets you save a notebook file to the Download area
 configured by your web browser (particularly useful if you are running the
 notebook server on a remote host and need a file locally).  The notebook is
 saved by default with the ``.ipynb`` extension and the files contain JSON data
 that is not meant for human editing or consumption.  But you can always export
 the input part of a notebook to a plain python script by choosing Python format
 in the `Download` drop list.  This removes all output and saves the text cells
 in comment areas.  See ref:`below <notebook_format>` for more details on the
 notebook format.
 
 The notebook can also *import* ``.py`` files as notebooks, by dragging and
 dropping the file into the notebook dashboard file list area.  By default, the
 entire contents of the file will be loaded into a single code cell.  But if
 prior to import, you manually add the ``# <nbformat>2</nbformat>`` marker at
 the start and then add separators for text/code cells, you can get a cleaner
 import with the file broken into individual cells.
 
+If you want use notebooks as scripts a lot, then you can set::
+
+    c.NotebookManager.save_script=True
+
+which will instruct the notebook server to save the ``.py`` export of each
+notebook adjacent to the ``.ipynb`` at every save.  Then these can be ``%run``
+or imported from regular IPython sessions or other notebooks.
+
 .. warning::
 
    While in simple cases you can roundtrip a notebook to Python, edit the
    python file and import it back without loss of main content, this is in
    general *not guaranteed to work at all*.  First, there is extra metadata
    saved in the notebook that may not be saved to the ``.py`` format.  And as
    the notebook format evolves in complexity, there will be attributes of the
    notebook that will not survive a roundtrip through the Python form.  You
    should think of the Python format as a way to output a script version of a
    notebook and the import capabilities as a way to load existing code to get a
    notebook started.  But the Python version is *not* an alternate notebook
    format.
 
     
 Keyboard use
 ------------
 
 All actions in the notebook can be achieved with the mouse, but we have also
 added keyboard shortcuts for the most common ones, so that productive use of
 the notebook can be achieved with minimal mouse intervention.  The main
 key bindings you need to remember are:
 
 * :kbd:`Shift-Enter`: execute the current cell (similar to the Qt console),
   show output (if any) and create a new cell below.  Note that in the notebook,
   simply using :kbd:`Enter` *never* forces execution, it simply inserts a new
   line in the current cell.  Therefore, in the notebook you must always use
   :kbd:`Shift-Enter` to get execution (or use the mouse and click on the ``Run
   Selected`` button).
 
 * :kbd:`Ctrl-Enter`: execute the current cell in "terminal mode", where any
   output is shown but the cursor stays in the current cell, whose input
   area is flushed empty.  This is convenient to do quick in-place experiments
   or query things like filesystem content without creating additional cells you
   may not want saved in your notebook.
 
 * :kbd:`Ctrl-m`: this is the prefix for all other keybindings, which consist
   of an additional single letter.  Type :kbd:`Ctrl-m h` (that is, the sole
   letter :kbd:`h` after :kbd:`Ctrl-m`) and IPython will show you the remaining
   available keybindings.
 
 
 .. _notebook_security:
 
 Security
 ========
 
 You can protect your notebook server with a simple single-password by
 setting the :attr:`NotebookApp.password` configurable. You can prepare a
 hashed password using the function :func:`IPython.lib.security.passwd`:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.lib import passwd
     In [2]: passwd()
     Enter password: 
     Verify password: 
     Out[2]: 'sha1:67c9e60bb8b6:9ffede0825894254b2e042ea597d771089e11aed'
     
 .. note::
 
   :func:`~IPython.lib.security.passwd` can also take the password as a string
   argument. **Do not** pass it as an argument inside an IPython session, as it
   will be saved in your input history.
 
 You can then add this to your :file:`ipython_notebook_config.py`, e.g.::
 
     # Password to use for web authentication
     c.NotebookApp.password = u'sha1:67c9e60bb8b6:9ffede0825894254b2e042ea597d771089e11aed'
 
 When using a password, it is a good idea to also use SSL, so that your password
 is not sent unencrypted by your browser. You can start the notebook to
 communicate via a secure protocol mode using a self-signed certificate by
 typing::
 
     $ ipython notebook --certfile=mycert.pem
 
 .. note::
 
     A self-signed certificate can be generated with openssl.  For example, the
     following command will create a certificate valid for 365 days with both
     the key and certificate data written to the same file::
 
         $ openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.pem -out mycert.pem
 
 Your browser will warn you of a dangerous certificate because it is
 self-signed.  If you want to have a fully compliant certificate that will not
 raise warnings, it is possible (but rather involved) to obtain one for free,
 `as explained in detailed in this tutorial`__.
 
 .. __: http://arstechnica.com/security/news/2009/12/how-to-get-set-with-a-secure-sertificate-for-free.ars
 	
 Keep in mind that when you enable SSL support, you'll need to access the
 notebook server over ``https://``, not over plain ``http://``.  The startup
 message from the server prints this, but it's easy to overlook and think the
 server is for some reason non-responsive.
 
 
 Quick Howto: running a public notebook server
 =============================================
 
 If you want to access your notebook server remotely with just a web browser,
 here is a quick set of instructions.  Start by creating a certificate file and
 a hashed password as explained above.  Then, create a custom profile for the
 notebook.  At the command line, type::
 
   ipython profile create nbserver
 
 In the profile directory, edit the file ``ipython_notebook_config.py``.  By
 default the file has all fields commented, the minimum set you need to
 uncomment and edit is here::
 
      c = get_config()
 
      # Kernel config
      c.IPKernelApp.pylab = 'inline'  # if you want plotting support always
 
      # Notebook config
      c.NotebookApp.certfile = u'/absolute/path/to/your/certificate/mycert.pem'
      c.NotebookApp.ip = '*'
      c.NotebookApp.open_browser = False
      c.NotebookApp.password = u'sha1:bcd259ccf...your hashed password here'
      # It's a good idea to put it on a known, fixed port
      c.NotebookApp.port = 9999
 
 You can then start the notebook and access it later by pointing your browser to
 ``https://your.host.com:9999``.
 
 .. _notebook_format:
 
 The notebook format
 ===================
 
 The notebooks themselves are JSON files with an ``ipynb`` extension, formatted
 as legibly as possible with minimal extra indentation and cell content broken
 across lines to make them reasonably friendly to use in version-control
 workflows.  You should be very careful if you ever edit manually this JSON
 data, as it is extremely easy to corrupt its internal structure and make the
 file impossible to load.  In general, you should consider the notebook as a
 file meant only to be edited by IPython itself, not for hand-editing.
 
 .. note::
 
      Binary data such as figures are directly saved in the JSON file.  This
      provides convenient single-file portability but means the files can be
      large and diffs of binary data aren't very meaningful.  Since the binary
      blobs are encoded in a single line they only affect one line of the diff
      output, but they are typically very long lines.  You can use the
      'ClearAll' button to remove all output from a notebook prior to
      committing it to version control, if this is a concern.
 
 The notebook server can also generate a pure-python version of your notebook,
 by clicking on the 'Download' button and selecting ``py`` as the format.  This
 file will contain all the code cells from your notebook verbatim, and all text
 cells prepended with a comment marker.  The separation between code and text
 cells is indicated with special comments and there is a header indicating the
 format version.  All output is stripped out when exporting to python.
 
 Here is an example of a simple notebook with one text cell and one code input
 cell, when exported to python format::
 
     # <nbformat>2</nbformat>
 
     # <markdowncell>
 
     # A text cell
 
     # <codecell>
 
     print "hello IPython"
 
 
 Known Issues
 ============
 
 When behind a proxy, especially if your system or browser is set to autodetect
 the proxy, the html notebook might fail to connect to the server's websockets,
 and present you with a warning at startup. In this case, you need to configure
 your system not to use the proxy for the server's address.
 
 In Firefox, for example, go to the Preferences panel, Advanced section,
 Network tab, click 'Settings...', and add the address of the notebook server
 to the 'No proxy for' field.
 
     
 .. _Markdown: http://daringfireball.net/projects/markdown/basics