jupyter-rendering: add required packaging to handle rendering of jupyter notebooks.

- added nbgenerate, and other related libs.

pkgs/python-packages-overrides.nix

@@ -48,6 +48,13 b' self: super: {'
     ];
   });
 
+  nbconvert = super.nbconvert.override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      # marcink: plug in jupyter-client for notebook rendering
+      self.jupyter-client
+    ];
+  });
+
   ipython = super.ipython.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       self.gnureadline

pkgs/python-packages.nix

@@ -431,6 +431,19 b''
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  bleach = super.buildPythonPackage {
+    name = "bleach-1.5.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [six html5lib];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/99/00/25a8fce4de102bf6e3cc76bc4ea60685b2fee33bde1b34830c70cacc26a7/bleach-1.5.0.tar.gz";
+      md5 = "b663300efdf421b3b727b19d7be9c7e7";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.asl20 ];
+    };
+  };
   bottle = super.buildPythonPackage {
     name = "bottle-0.12.8";
     buildInputs = with self; [];
@@ -522,6 +535,19 b''
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  configparser = super.buildPythonPackage {
+    name = "configparser-3.5.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/7c/69/c2ce7e91c89dc073eb1aa74c0621c3eefbffe8216b3f9af9d3885265c01c/configparser-3.5.0.tar.gz";
+      md5 = "cfdd915a5b7a6c09917a64a573140538";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   cov-core = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     buildInputs = with self; [];
@@ -665,6 +691,19 b''
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
+  entrypoints = super.buildPythonPackage {
+    name = "entrypoints-0.2.2";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [configparser];
+    src = fetchurl {
+      url = "https://code.rhodecode.com/upstream/entrypoints/archive/96e6d645684e1af3d7df5b5272f3fe85a546b233.tar.gz?md5=7db37771aea9ac9fefe093e5d6987313";
+      md5 = "7db37771aea9ac9fefe093e5d6987313";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   enum34 = super.buildPythonPackage {
     name = "enum34-1.1.6";
     buildInputs = with self; [];
@@ -678,6 +717,19 b''
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  functools32 = super.buildPythonPackage {
+    name = "functools32-3.2.3.post2";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/5e/1a/0aa2c8195a204a9f51284018562dea77e25511f02fe924fac202fc012172/functools32-3.2.3-2.zip";
+      md5 = "d55232eb132ec779e6893c902a0bc5ad";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.psfl ];
+    };
+  };
   future = super.buildPythonPackage {
     name = "future-0.14.3";
     buildInputs = with self; [];
@@ -769,6 +821,19 b''
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  html5lib = super.buildPythonPackage {
+    name = "html5lib-0.9999999";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [six];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/ae/ae/bcb60402c60932b32dfaf19bb53870b29eda2cd17551ba5639219fb5ebf9/html5lib-0.9999999.tar.gz";
+      md5 = "ef43cb05e9e799f25d65d1135838a96f";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   infrae.cache = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     buildInputs = with self; [];
@@ -860,6 +925,45 b''
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  jsonschema = super.buildPythonPackage {
+    name = "jsonschema-2.6.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [functools32];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/58/b9/171dbb07e18c6346090a37f03c7e74410a1a56123f847efed59af260a298/jsonschema-2.6.0.tar.gz";
+      md5 = "50c6b69a373a8b55ff1e0ec6e78f13f4";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
+  jupyter-client = super.buildPythonPackage {
+    name = "jupyter-client-5.0.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [traitlets jupyter-core pyzmq python-dateutil];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/e5/6f/65412ed462202b90134b7e761b0b7e7f949e07a549c1755475333727b3d0/jupyter_client-5.0.0.tar.gz";
+      md5 = "1acd331b5c9fb4d79dae9939e79f2426";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  jupyter-core = super.buildPythonPackage {
+    name = "jupyter-core-4.3.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [traitlets];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/2f/39/5138f975100ce14d150938df48a83cd852a3fd8e24b1244f4113848e69e2/jupyter_core-4.3.0.tar.gz";
+      md5 = "18819511a809afdeed9a995a9c27bcfb";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   kombu = super.buildPythonPackage {
     name = "kombu-1.5.1";
     buildInputs = with self; [];
@@ -899,6 +1003,19 b''
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
+  mistune = super.buildPythonPackage {
+    name = "mistune-0.7.3";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/88/1e/be99791262b3a794332fda598a07c2749a433b9378586361ba9d8e824607/mistune-0.7.3.tar.gz";
+      md5 = "4eba50bd121b83716fa4be6a4049004b";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   mock = super.buildPythonPackage {
     name = "mock-1.0.1";
     buildInputs = with self; [];
@@ -925,6 +1042,32 b''
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
+  nbconvert = super.buildPythonPackage {
+    name = "nbconvert-5.1.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [mistune Jinja2 Pygments traitlets jupyter-core nbformat entrypoints bleach pandocfilters testpath];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/95/58/df1c91f1658ee5df19097f915a1e71c91fc824a708d82d2b2e35f8b80e9a/nbconvert-5.1.1.tar.gz";
+      md5 = "d0263fb03a44db2f94eea09a608ed813";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  nbformat = super.buildPythonPackage {
+    name = "nbformat-4.3.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [ipython-genutils traitlets jsonschema jupyter-core];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/f9/c5/89df4abf906f766727f976e170caa85b4f1c1d1feb1f45d716016e68e19f/nbformat-4.3.0.tar.gz";
+      md5 = "9a00d20425914cd5ba5f97769d9963ca";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   nose = super.buildPythonPackage {
     name = "nose-1.3.6";
     buildInputs = with self; [];
@@ -964,6 +1107,19 b''
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
+  pandocfilters = super.buildPythonPackage {
+    name = "pandocfilters-1.4.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/e3/1f/21d1b7e8ca571e80b796c758d361fdf5554335ff138158654684bc5401d8/pandocfilters-1.4.1.tar.gz";
+      md5 = "7680d9f9ec07397dd17f380ee3818b9d";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   paramiko = super.buildPythonPackage {
     name = "paramiko-1.15.1";
     buildInputs = with self; [];
@@ -1030,13 +1186,13 b''
     };
   };
   prompt-toolkit = super.buildPythonPackage {
-    name = "prompt-toolkit-1.0.9";
+    name = "prompt-toolkit-1.0.13";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six wcwidth];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/83/14/5ac258da6c530eca02852ee25c7a9ff3ca78287bb4c198d0d0055845d856/prompt_toolkit-1.0.9.tar.gz";
-      md5 = "a39f91a54308fb7446b1a421c11f227c";
+      url = "https://pypi.python.org/packages/23/be/4876b52d5cc159cbd4b0ff6e7aa419a26470849a43a8f647857a4a24467b/prompt_toolkit-1.0.13.tar.gz";
+      md5 = "427b496d2c147bd3819bc3a7f6e0d493";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1355,16 +1511,16 b''
     };
   };
   python-dateutil = super.buildPythonPackage {
-    name = "python-dateutil-1.5";
+    name = "python-dateutil-2.1";
     buildInputs = with self; [];
     doCheck = false;
-    propagatedBuildInputs = with self; [];
+    propagatedBuildInputs = with self; [six];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/b4/7c/df59c89a753eb33c7c44e1dd42de0e9bc2ccdd5a4d576e0bfad97cc280cb/python-dateutil-1.5.tar.gz";
-      md5 = "0dcb1de5e5cad69490a3b6ab63f0cfa5";
+      url = "https://pypi.python.org/packages/65/52/9c18dac21f174ad31b65e22d24297864a954e6fe65876eba3f5773d2da43/python-dateutil-2.1.tar.gz";
+      md5 = "1534bb15cf311f07afaa3aacba1c028b";
     };
     meta = {
-      license = [ pkgs.lib.licenses.psfl ];
+      license = [ { fullName = "Simplified BSD"; } ];
     };
   };
   python-editor = super.buildPythonPackage {
@@ -1488,7 +1644,7 b''
     name = "rhodecode-enterprise-ce-4.7.0";
     buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage cssselect lxml configobj];
     doCheck = true;
-    propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
+    propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python nbconvert packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
     src = ./.;
     meta = {
       license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
@@ -1624,14 +1780,27 b''
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  testpath = super.buildPythonPackage {
+    name = "testpath-0.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/f9/c4/c0b22f35138bc26a6058c39cb61db1e8977e5e9550b12cd2cb02ef56fc51/testpath-0.1.tar.gz";
+      md5 = "401918bcd0b0e5b71a9b909835117bc6";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   traitlets = super.buildPythonPackage {
-    name = "traitlets-4.3.1";
+    name = "traitlets-4.3.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ipython-genutils six decorator enum34];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/b1/d6/5b5aa6d5c474691909b91493da1e8972e309c9f01ecfe4aeafd272eb3234/traitlets-4.3.1.tar.gz";
-      md5 = "dd0b1b6e5d31ce446d55a4b5e5083c98";
+      url = "https://pypi.python.org/packages/a5/98/7f5ef2fe9e9e071813aaf9cb91d1a732e0a68b6c44a32b38cb8e14c3f069/traitlets-4.3.2.tar.gz";
+      md5 = "3068663f2f38fd939a9eb3a500ccc154";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];

requirements.txt

@@ -42,6 +42,7 b' paramiko==1.15.1'
 Paste==2.0.3
 PasteDeploy==1.5.2
 PasteScript==1.7.5
+pathlib2==2.1.0
 psutil==4.3.1
 psycopg2==2.6.1
 py-bcrypt==0.4
@@ -57,7 +58,7 b' pyramid-jinja2==2.5'
 pyramid-mako==1.0.2
 pyramid==1.7.4
 pysqlite==2.6.3
-python-dateutil==1.5
+python-dateutil==2.1
 python-ldap==2.4.19
 python-memcached==1.57
 python-pam==1.8.2
@@ -97,6 +98,12 b' https://code.rhodecode.com/upstream/pylo'
 # not released py-gfm==0.1.3
 https://code.rhodecode.com/upstream/py-gfm/archive/0d66a19bc16e3d49de273c0f797d4e4781e8c0f2.tar.gz?md5=0d0d5385bfb629eea636a80b9c2bfd16#egg=py-gfm==0.1.3.rhodecode-upstream1
 
+# IPYTHON RENDERING
+# entrypoints backport, pypi version doesn't support egg installs
+https://code.rhodecode.com/upstream/entrypoints/archive/96e6d645684e1af3d7df5b5272f3fe85a546b233.tar.gz?md5=7db37771aea9ac9fefe093e5d6987313#egg=entrypoints==0.2.2.rhodecode-upstream1
+nbconvert==5.1.1
+nbformat==4.3.0
+jupyter_client==5.0.0
 
 ## cli tools
 alembic==0.8.4

rhodecode/lib/helpers.py

@@ -1821,15 +1821,48 b' def markdown(source, mentions=False):'
                    MarkupRenderer.markdown(source, flavored=True,
                                            mentions=mentions))
 
+
 def renderer_from_filename(filename, exclude=None):
-    return MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
+    """
+    choose a renderer based on filename
+    """
+
+    # images
+
+    # ipython
+    if filename.endswith('.ipynb'):
+        return 'ipython'
+
+    is_markup = MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
+    if is_markup:
+        return is_markup
+    return None
 
 
 def render(source, renderer='rst', mentions=False):
     if renderer == 'rst':
         return rst(source, mentions=mentions)
-    if renderer == 'markdown':
+    elif renderer == 'markdown':
         return markdown(source, mentions=mentions)
+    elif renderer == 'ipython':
+        def ipython_renderer(source):
+            import nbformat
+            from nbconvert import HTMLExporter
+            notebook = nbformat.reads(source, as_version=4)
+
+            # 2. Instantiate the exporter. We use the `basic` template for now; we'll get into more details
+            # later about how to customize the exporter further.
+            html_exporter = HTMLExporter()
+            html_exporter.template_file = 'basic'
+
+            # 3. Process the notebook we loaded earlier
+            (body, resources) = html_exporter.from_notebook_node(notebook)
+
+            return body
+
+        return ipython_renderer(source)
+    # None means just show the file-source
+    return None
 
 
 def commit_status(repo, commit_id):

setup.py

@@ -113,6 +113,7 b' install_requirements = ['
     'iso8601',
     'kombu',
     'msgpack-python',
+    'nbconvert',
     'packaging',
     'psycopg2',
     'py-gfm',