Merge pull request #4655 from minrk/marked-nbconvert

prefer marked to pandoc for markdown2html

IPython/nbconvert/filters/marked.js

@@ -0,0 +1,54 b''
+// Node.js script for markdown to html conversion
+// This applies the same math extraction and marked settings
+// that we use in the live notebook.
+
+// IPython static_path dir relative to here:
+var static_path = __dirname + "/../../html/static/";
+
+var fs = require('fs');
+var IPython;
+// marked can be loaded with require,
+// the others must be execfiled
+var marked = require(static_path + 'components/marked/lib/marked.js');
+
+eval(fs.readFileSync(static_path + "components/highlight.js/build/highlight.pack.js", 'utf8'));
+eval(fs.readFileSync(static_path + "base/js/namespace.js", 'utf8'));
+
+eval(fs.readFileSync(static_path + "base/js/utils.js", 'utf8'));
+eval(fs.readFileSync(static_path + "notebook/js/mathjaxutils.js", 'utf8'));
+
+// this is copied from notebook.main. Should it be moved somewhere we can reuse it?
+marked.setOptions({
+    gfm : true,
+    tables: true,
+    langPrefix: "language-",
+    highlight: function(code, lang) {
+        if (!lang) {
+            // no language, no highlight
+            return code;
+        }
+        var highlighted;
+        try {
+            highlighted = hljs.highlight(lang, code, false);
+        } catch(err) {
+            highlighted = hljs.highlightAuto(code);
+        }
+        return highlighted.value;
+    }
+});
+
+// read the markdown from stdin
+var md='';
+process.stdin.on("data", function (data) {
+    md += data;
+});
+
+// perform the md2html transform once stdin is complete
+process.stdin.on("end", function () {
+    var text_and_math = IPython.mathjaxutils.remove_math(md);
+    var text = text_and_math[0];
+    var math = text_and_math[1];
+    var html = marked.parser(marked.lexer(text));
+    html = IPython.mathjaxutils.replace_math(html, math);
+    process.stdout.write(html);
+});

IPython/html/static/base/js/utils.js

@@ -1,504 +1,508 b''
 //----------------------------------------------------------------------------
 //  Copyright (C) 2008-2012  The IPython Development Team
 //
 //  Distributed under the terms of the BSD License.  The full license is in
 //  the file COPYING, distributed as part of this software.
 //----------------------------------------------------------------------------
 
 //============================================================================
 // Utilities
 //============================================================================
 IPython.namespace('IPython.utils');
 
 IPython.utils = (function (IPython) {
     "use strict";
 
     //============================================================================
     // Cross-browser RegEx Split
     //============================================================================
 
     // This code has been MODIFIED from the code licensed below to not replace the
     // default browser split.  The license is reproduced here.
 
     // see http://blog.stevenlevithan.com/archives/cross-browser-split for more info:
     /*!
      * Cross-Browser Split 1.1.1
      * Copyright 2007-2012 Steven Levithan <stevenlevithan.com>
      * Available under the MIT License
      * ECMAScript compliant, uniform cross-browser split method
      */
 
     /**
      * Splits a string into an array of strings using a regex or string
      * separator. Matches of the separator are not included in the result array.
      * However, if `separator` is a regex that contains capturing groups,
      * backreferences are spliced into the result each time `separator` is
      * matched. Fixes browser bugs compared to the native
      * `String.prototype.split` and can be used reliably cross-browser.
      * @param {String} str String to split.
      * @param {RegExp|String} separator Regex or string to use for separating
      *     the string.
      * @param {Number} [limit] Maximum number of items to include in the result
      *     array.
      * @returns {Array} Array of substrings.
      * @example
      *
      * // Basic use
      * regex_split('a b c d', ' ');
      * // -> ['a', 'b', 'c', 'd']
      *
      * // With limit
      * regex_split('a b c d', ' ', 2);
      * // -> ['a', 'b']
      *
      * // Backreferences in result array
      * regex_split('..word1 word2..', /([a-z]+)(\d+)/i);
      * // -> ['..', 'word', '1', ' ', 'word', '2', '..']
      */
     var regex_split = function (str, separator, limit) {
         // If `separator` is not a regex, use `split`
         if (Object.prototype.toString.call(separator) !== "[object RegExp]") {
             return split.call(str, separator, limit);
         }
         var output = [],
             flags = (separator.ignoreCase ? "i" : "") +
                     (separator.multiline  ? "m" : "") +
                     (separator.extended   ? "x" : "") + // Proposed for ES6
                     (separator.sticky     ? "y" : ""), // Firefox 3+
             lastLastIndex = 0,
             // Make `global` and avoid `lastIndex` issues by working with a copy
             separator = new RegExp(separator.source, flags + "g"),
             separator2, match, lastIndex, lastLength;
         str += ""; // Type-convert
 
         var compliantExecNpcg = typeof(/()??/.exec("")[1]) === "undefined";
         if (!compliantExecNpcg) {
             // Doesn't need flags gy, but they don't hurt
             separator2 = new RegExp("^" + separator.source + "$(?!\\s)", flags);
         }
         /* Values for `limit`, per the spec:
          * If undefined: 4294967295 // Math.pow(2, 32) - 1
          * If 0, Infinity, or NaN: 0
          * If positive number: limit = Math.floor(limit); if (limit > 4294967295) limit -= 4294967296;
          * If negative number: 4294967296 - Math.floor(Math.abs(limit))
          * If other: Type-convert, then use the above rules
          */
         limit = typeof(limit) === "undefined" ?
             -1 >>> 0 : // Math.pow(2, 32) - 1
             limit >>> 0; // ToUint32(limit)
         while (match = separator.exec(str)) {
             // `separator.lastIndex` is not reliable cross-browser
             lastIndex = match.index + match[0].length;
             if (lastIndex > lastLastIndex) {
                 output.push(str.slice(lastLastIndex, match.index));
                 // Fix browsers whose `exec` methods don't consistently return `undefined` for
                 // nonparticipating capturing groups
                 if (!compliantExecNpcg && match.length > 1) {
                     match[0].replace(separator2, function () {
                         for (var i = 1; i < arguments.length - 2; i++) {
                             if (typeof(arguments[i]) === "undefined") {
                                 match[i] = undefined;
                             }
                         }
                     });
                 }
                 if (match.length > 1 && match.index < str.length) {
                     Array.prototype.push.apply(output, match.slice(1));
                 }
                 lastLength = match[0].length;
                 lastLastIndex = lastIndex;
                 if (output.length >= limit) {
                     break;
                 }
             }
             if (separator.lastIndex === match.index) {
                 separator.lastIndex++; // Avoid an infinite loop
             }
         }
         if (lastLastIndex === str.length) {
             if (lastLength || !separator.test("")) {
                 output.push("");
             }
         } else {
             output.push(str.slice(lastLastIndex));
         }
         return output.length > limit ? output.slice(0, limit) : output;
     };
 
     //============================================================================
     // End contributed Cross-browser RegEx Split
     //============================================================================
 
 
     var uuid = function () {
         // http://www.ietf.org/rfc/rfc4122.txt
         var s = [];
         var hexDigits = "0123456789ABCDEF";
         for (var i = 0; i < 32; i++) {
             s[i] = hexDigits.substr(Math.floor(Math.random() * 0x10), 1);
         }
         s[12] = "4";  // bits 12-15 of the time_hi_and_version field to 0010
         s[16] = hexDigits.substr((s[16] & 0x3) | 0x8, 1);  // bits 6-7 of the clock_seq_hi_and_reserved to 01
 
         var uuid = s.join("");
         return uuid;
     };
 
 
     //Fix raw text to parse correctly in crazy XML
     function xmlencode(string) {
         return string.replace(/\&/g,'&'+'amp;')
             .replace(/</g,'&'+'lt;')
             .replace(/>/g,'&'+'gt;')
             .replace(/\'/g,'&'+'apos;')
             .replace(/\"/g,'&'+'quot;')
             .replace(/`/g,'&'+'#96;');
     }
 
 
     //Map from terminal commands to CSS classes
     var ansi_colormap = {
         "01":"ansibold",
         
         "30":"ansiblack",
         "31":"ansired",
         "32":"ansigreen",
         "33":"ansiyellow",
         "34":"ansiblue",
         "35":"ansipurple",
         "36":"ansicyan",
         "37":"ansigray",
         
         "40":"ansibgblack",
         "41":"ansibgred",
         "42":"ansibggreen",
         "43":"ansibgyellow",
         "44":"ansibgblue",
         "45":"ansibgpurple",
         "46":"ansibgcyan",
         "47":"ansibggray"
     };
     
     function _process_numbers(attrs, numbers) {
         // process ansi escapes
         var n = numbers.shift();
         if (ansi_colormap[n]) {
             if ( ! attrs["class"] ) {
                 attrs["class"] = ansi_colormap[n];
             } else {
                 attrs["class"] += " " + ansi_colormap[n];
             }
         } else if (n == "38" || n == "48") {
             // VT100 256 color or 24 bit RGB
             if (numbers.length < 2) {
                 console.log("Not enough fields for VT100 color", numbers);
                 return;
             }
             
             var index_or_rgb = numbers.shift();
             var r,g,b;
             if (index_or_rgb == "5") {
                 // 256 color
                 var idx = parseInt(numbers.shift());
                 if (idx < 16) {
                     // indexed ANSI
                     // ignore bright / non-bright distinction
                     idx = idx % 8;
                     var ansiclass = ansi_colormap[n[0] + (idx % 8).toString()];
                     if ( ! attrs["class"] ) {
                         attrs["class"] = ansiclass;
                     } else {
                         attrs["class"] += " " + ansiclass;
                     }
                     return;
                 } else if (idx < 232) {
                     // 216 color 6x6x6 RGB
                     idx = idx - 16;
                     b = idx % 6;
                     g = Math.floor(idx / 6) % 6;
                     r = Math.floor(idx / 36) % 6;
                     // convert to rgb
                     r = (r * 51);
                     g = (g * 51);
                     b = (b * 51);
                 } else {
                     // grayscale
                     idx = idx - 231;
                     // it's 1-24 and should *not* include black or white,
                     // so a 26 point scale
                     r = g = b = Math.floor(idx * 256 / 26);
                 }
             } else if (index_or_rgb == "2") {
                 // Simple 24 bit RGB
                 if (numbers.length > 3) {
                     console.log("Not enough fields for RGB", numbers);
                     return;
                 }
                 r = numbers.shift();
                 g = numbers.shift();
                 b = numbers.shift();
             } else {
                 console.log("unrecognized control", numbers);
                 return;
             }
             if (r !== undefined) {
                 // apply the rgb color
                 var line;
                 if (n == "38") {
                     line = "color: ";
                 } else {
                     line = "background-color: ";
                 }
                 line = line + "rgb(" + r + "," + g + "," + b + ");"
                 if ( !attrs["style"] ) {
                     attrs["style"] = line;
                 } else {
                     attrs["style"] += " " + line;
                 }
             }
         }
     }
 
     function ansispan(str) {
         // ansispan function adapted from github.com/mmalecki/ansispan (MIT License)
         // regular ansi escapes (using the table above)
         return str.replace(/\033\[(0?[01]|22|39)?([;\d]+)?m/g, function(match, prefix, pattern) {
             if (!pattern) {
                 // [(01|22|39|)m close spans
                 return "</span>";
             }
             // consume sequence of color escapes
             var numbers = pattern.match(/\d+/g);
             var attrs = {};
             while (numbers.length > 0) {
                 _process_numbers(attrs, numbers);
             }
             
             var span = "<span ";
             for (var attr in attrs) {
                 var value = attrs[attr];
                 span = span + " " + attr + '="' + attrs[attr] + '"';
             }
             return span + ">";
         });
     };
     
     // Transform ANSI color escape codes into HTML <span> tags with css
     // classes listed in the above ansi_colormap object. The actual color used
     // are set in the css file.
     function fixConsole(txt) {
         txt = xmlencode(txt);
         var re = /\033\[([\dA-Fa-f;]*?)m/;
         var opened = false;
         var cmds = [];
         var opener = "";
         var closer = "";
 
         // Strip all ANSI codes that are not color related.  Matches
         // all ANSI codes that do not end with "m".
         var ignored_re = /(?=(\033\[[\d;=]*[a-ln-zA-Z]{1}))\1(?!m)/g;
         txt = txt.replace(ignored_re, "");
         
         // color ansi codes
         txt = ansispan(txt);
         return txt;
     }
 
     // Remove chunks that should be overridden by the effect of
     // carriage return characters
     function fixCarriageReturn(txt) {
         var tmp = txt;
         do {
             txt = tmp;
             tmp = txt.replace(/\r+\n/gm, '\n'); // \r followed by \n --> newline
             tmp = tmp.replace(/^.*\r+/gm, '');  // Other \r --> clear line
         } while (tmp.length < txt.length);
         return txt;
     }
 
     // Locate any URLs and convert them to a anchor tag
     function autoLinkUrls(txt) {
         return txt.replace(/(^|\s)(https?|ftp)(:[^'">\s]+)/gi,
             "$1<a target=\"_blank\" href=\"$2$3\">$2$3</a>");
     }
 
     // some keycodes that seem to be platform/browser independent
     var keycodes = {
                 BACKSPACE:  8,
                 TAB      :  9,
                 ENTER    : 13,
                 SHIFT    : 16,
                 CTRL     : 17,
                 CONTROL  : 17,
                 ALT      : 18,
                 CAPS_LOCK: 20,
                 ESC      : 27,
                 SPACE    : 32,
                 PGUP     : 33,
                 PGDOWN   : 34,
                 END      : 35,
                 HOME     : 36,
                 LEFT_ARROW: 37,
                 LEFTARROW: 37,
                 LEFT     : 37,
                 UP_ARROW : 38,
                 UPARROW  : 38,
                 UP       : 38,
                 RIGHT_ARROW:39,
                 RIGHTARROW:39,
                 RIGHT    : 39,
                 DOWN_ARROW: 40,
                 DOWNARROW: 40,
                 DOWN     : 40,
                 I        : 73,
                 M        : 77,
                 // all three of these keys may be COMMAND on OS X:
                 LEFT_SUPER : 91,
                 RIGHT_SUPER : 92,
                 COMMAND  : 93,
     };
     
     // trigger a key press event
     var press = function (key) {
         var key_press =  $.Event('keydown', {which: key});
         $(document).trigger(key_press);
     }
 
     var press_up = function() { press(keycodes.UP); };
     var press_down = function() { press(keycodes.DOWN); };
 
     var press_ctrl_enter = function() {
         $(document).trigger($.Event('keydown', {which: keycodes.ENTER, ctrlKey: true}));
     };
 
     var press_shift_enter = function() {
         $(document).trigger($.Event('keydown', {which: keycodes.ENTER, shiftKey: true}));
     };
 
     // trigger the ctrl-m shortcut followed by one of our keys
     var press_ghetto = function(key) {
         $(document).trigger($.Event('keydown', {which: keycodes.M, ctrlKey: true}));
         press(key);
     };
 
 
     var points_to_pixels = function (points) {
         // A reasonably good way of converting between points and pixels.
         var test = $('<div style="display: none; width: 10000pt; padding:0; border:0;"></div>');
         $(body).append(test);
         var pixel_per_point = test.width()/10000;
         test.remove();
         return Math.floor(points*pixel_per_point);
     };
     
     var always_new = function (constructor) {
         // wrapper around contructor to avoid requiring `var a = new constructor()`
         // useful for passing constructors as callbacks,
         // not for programmer laziness.
         // from http://programmers.stackexchange.com/questions/118798
         return function () {
             var obj = Object.create(constructor.prototype);
             constructor.apply(obj, arguments);
             return obj;
         };
     };
 
 
     var url_path_join = function () {
         // join a sequence of url components with '/'
         var url = '';
         for (var i = 0; i < arguments.length; i++) {
             if (arguments[i] === '') {
                 continue;
             }
             if (url.length > 0 && url[url.length-1] != '/') {
                 url = url + '/' + arguments[i];
             } else {
                 url = url + arguments[i];
             }
         }
         return url;
     };
     
     
     var encode_uri_components = function (uri) {
         // encode just the components of a multi-segment uri,
         // leaving '/' separators
         return uri.split('/').map(encodeURIComponent).join('/');
     }
     
     var url_join_encode = function () {
         // join a sequence of url components with '/',
         // encoding each component with encodeURIComponent
         return encode_uri_components(url_path_join.apply(null, arguments));
     };
 
 
     var splitext = function (filename) {
         // mimic Python os.path.splitext
         // Returns ['base', '.ext']
         var idx = filename.lastIndexOf('.');
         if (idx > 0) {
             return [filename.slice(0, idx), filename.slice(idx)];
         } else {
             return [filename, ''];
         }
     }
 
 
     // http://stackoverflow.com/questions/2400935/browser-detection-in-javascript
     var browser = (function() {
+        if (typeof navigator === 'undefined') {
+            // navigator undefined in node
+            return 'None';
+        }
         var N= navigator.appName, ua= navigator.userAgent, tem;
         var M= ua.match(/(opera|chrome|safari|firefox|msie)\/?\s*(\.?\d+(\.\d+)*)/i);
         if (M && (tem= ua.match(/version\/([\.\d]+)/i))!= null) M[2]= tem[1];
         M= M? [M[1], M[2]]: [N, navigator.appVersion,'-?'];
         return M;
     })();
 
     var is_or_has = function (a, b) {
         // Is b a child of a or a itself?
         return a.has(b).length !==0 || a.is(b);
     }
 
     var is_focused = function (e) {
         // Is element e, or one of its children focused?
         e = $(e);
         var target = $(document.activeElement);
         if (target.length > 0) {
             if (is_or_has(e, target)) {
                 return true;
             } else {
                 return false;
             }
         } else {
             return false;
         }
     }
 
 
     return {
         regex_split : regex_split,
         uuid : uuid,
         fixConsole : fixConsole,
         keycodes : keycodes,
         press : press,
         press_up : press_up,
         press_down : press_down,
         press_ctrl_enter : press_ctrl_enter,
         press_shift_enter : press_shift_enter,
         press_ghetto : press_ghetto,
         fixCarriageReturn : fixCarriageReturn,
         autoLinkUrls : autoLinkUrls,
         points_to_pixels : points_to_pixels,
         url_path_join : url_path_join,
         url_join_encode : url_join_encode,
         encode_uri_components : encode_uri_components,
         splitext : splitext,
         always_new : always_new,
         browser : browser,
         is_or_has : is_or_has,
         is_focused : is_focused
     };
 
 }(IPython));
 

IPython/html/static/notebook/js/mathjaxutils.js

@@ -1,259 +1,251 b''
 //----------------------------------------------------------------------------
 //  Copyright (C) 2008-2012  The IPython Development Team
 //
 //  Distributed under the terms of the BSD License.  The full license is in
 //  the file COPYING, distributed as part of this software.
 //----------------------------------------------------------------------------
 
 //============================================================================
 // MathJax utility functions
 //============================================================================
 
 
 IPython.namespace('IPython.mathjaxutils');
 
 IPython.mathjaxutils = (function (IPython) {
     "use strict";
 
     var init = function () {
         if (window.MathJax) {
             // MathJax loaded
             MathJax.Hub.Config({
                 tex2jax: {
                     inlineMath: [ ['$','$'], ["\\(","\\)"] ],
                     displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
                     processEscapes: true,
                     processEnvironments: true
                 },
                 // Center justify equations in code and markdown cells. Elsewhere
                 // we use CSS to left justify single line equations in code cells.
                 displayAlign: 'center',
                 "HTML-CSS": {
                     styles: {'.MathJax_Display': {"margin": 0}},
                     linebreaks: { automatic: true }
                 }
             });
             MathJax.Hub.Configured();
         } else if (window.mathjax_url !== "") {
             // Don't have MathJax, but should. Show dialog.
             var message = $('<div/>')
                 .append(
                     $("<p/></p>").addClass('dialog').text(
                         "Math/LaTeX rendering will be disabled."
                     )
                 ).append(
                     $("<p></p>").addClass('dialog').text(
                         "If you have administrative access to the notebook server and" +
                         " a working internet connection, you can install a local copy" +
                         " of MathJax for offline use with the following command on the server" +
                         " at a Python or IPython prompt:"
                     )
                 ).append(
                     $("<pre></pre>").addClass('dialog').text(
                         ">>> from IPython.external import mathjax; mathjax.install_mathjax()"
                     )
                 ).append(
                     $("<p></p>").addClass('dialog').text(
                         "This will try to install MathJax into the IPython source directory."
                     )
                 ).append(
                     $("<p></p>").addClass('dialog').text(
                         "If IPython is installed to a location that requires" +
                         " administrative privileges to write, you will need to make this call as" +
                         " an administrator, via 'sudo'."
                     )
                 ).append(
                     $("<p></p>").addClass('dialog').text(
                         "When you start the notebook server, you can instruct it to disable MathJax support altogether:"
                     )
                 ).append(
                     $("<pre></pre>").addClass('dialog').text(
                         "$ ipython notebook --no-mathjax"
                     )
                 ).append(
                     $("<p></p>").addClass('dialog').text(
                         "which will prevent this dialog from appearing."
                     )
                 );
             IPython.dialog.modal({
                 title : "Failed to retrieve MathJax from '" + window.mathjax_url + "'",
                 body : message,
                 buttons : {
                     OK : {class: "btn-danger"}
                 }
             });
         }
     };
 
     // Some magic for deferring mathematical expressions to MathJax
     // by hiding them from the Markdown parser.
     // Some of the code here is adapted with permission from Davide Cervone
     // under the terms of the Apache2 license governing the MathJax project.
     // Other minor modifications are also due to StackExchange and are used with
     // permission.
 
     var inline = "$"; // the inline math delimiter
 
     // MATHSPLIT contains the pattern for math delimiters and special symbols
     // needed for searching for math in the text input.
     var MATHSPLIT = /(\$\$?|\\(?:begin|end)\{[a-z]*\*?\}|\\[\\{}$]|[{}]|(?:\n\s*)+|@@\d+@@)/i;
 
     //  The math is in blocks i through j, so
     //    collect it into one block and clear the others.
     //  Replace &, <, and > by named entities.
     //  For IE, put <br> at the ends of comments since IE removes \n.
     //  Clear the current math positions and store the index of the
     //    math, then push the math string onto the storage array.
     //  The preProcess function is called on all blocks if it has been passed in
     var process_math = function (i, j, pre_process, math, blocks) {
-        var hub = MathJax.Hub;
         var block = blocks.slice(i, j + 1).join("").replace(/&/g, "&amp;") // use HTML entity for &
         .replace(/</g, "&lt;") // use HTML entity for <
         .replace(/>/g, "&gt;") // use HTML entity for >
         ;
-        if (hub.Browser.isMSIE) {
+        if (IPython.utils.browser === 'msie') {
             block = block.replace(/(%[^\n]*)\n/g, "$1<br/>\n");
         }
         while (j > i) {
             blocks[j] = "";
             j--;
         }
         blocks[i] = "@@" + math.length + "@@"; // replace the current block text with a unique tag to find later
         if (pre_process){
             block = pre_process(block);
         }
         math.push(block);
         return blocks;
     };
 
     //  Break up the text into its component parts and search
     //    through them for math delimiters, braces, linebreaks, etc.
     //  Math delimiters must match and braces must balance.
     //  Don't allow math to pass through a double linebreak
     //    (which will be a paragraph).
     //
     var remove_math = function (text) {
-        if (!window.MathJax) {
-            return [text, null];
-        }
-
         var math = []; // stores math strings for later
         var start;
         var end;
         var last;
         var braces;
 
         // Except for extreme edge cases, this should catch precisely those pieces of the markdown
         // source that will later be turned into code spans. While MathJax will not TeXify code spans,
         // we still have to consider them at this point; the following issue has happened several times:
         //
         //     `$foo` and `$bar` are varibales.  -->  <code>$foo ` and `$bar</code> are variables.
 
         var hasCodeSpans = /`/.test(text),
             de_tilde;
         if (hasCodeSpans) {
             text = text.replace(/~/g, "~T").replace(/(^|[^\\])(`+)([^\n]*?[^`\n])\2(?!`)/gm, function (wholematch) {
                 return wholematch.replace(/\$/g, "~D");
             });
             de_tilde = function (text) {
                 return text.replace(/~([TD])/g, function (wholematch, character) {
                                                     return { T: "~", D: "$" }[character];
                                                 });
             };
         } else {
             de_tilde = function (text) { return text; };
         }
 
         var blocks = IPython.utils.regex_split(text.replace(/\r\n?/g, "\n"),MATHSPLIT);
 
         for (var i = 1, m = blocks.length; i < m; i += 2) {
             var block = blocks[i];
             if (block.charAt(0) === "@") {
                 //
                 //  Things that look like our math markers will get
                 //  stored and then retrieved along with the math.
                 //
                 blocks[i] = "@@" + math.length + "@@";
                 math.push(block);
             }
             else if (start) {
                 //
                 //  If we are in math, look for the end delimiter,
                 //    but don't go past double line breaks, and
                 //    and balance braces within the math.
                 //
                 if (block === end) {
                     if (braces) {
                         last = i;
                     }
                     else {
                         blocks = process_math(start, i, de_tilde, math, blocks);
                         start  = null;
                         end    = null;
                         last   = null;
                     }
                 }
                 else if (block.match(/\n.*\n/)) {
                     if (last) {
                         i = last;
                         blocks = process_math(start, i, de_tilde, math, blocks);
                     }
                     start = null;
                     end = null;
                     last = null;
                     braces = 0;
                 }
                 else if (block === "{") {
                     braces++;
                 }
                 else if (block === "}" && braces) {
                     braces--;
                 }
             }
             else {
                 //
                 //  Look for math start delimiters and when
                 //    found, set up the end delimiter.
                 //
                 if (block === inline || block === "$$") {
                     start = i;
                     end = block;
                     braces = 0;
                 }
                 else if (block.substr(1, 5) === "begin") {
                     start = i;
                     end = "\\end" + block.substr(6);
                     braces = 0;
                 }
             }
         }
         if (last) {
             blocks = process_math(start, last, de_tilde, math, blocks);
             start  = null;
             end    = null;
             last   = null;
         }
         return [de_tilde(blocks.join("")), math];
     };
 
     //
     //  Put back the math strings that were saved,
     //    and clear the math array (no need to keep it around).
     //
     var replace_math = function (text, math) {
-        if (!window.MathJax) {
-            return text;
-        }
         text = text.replace(/@@(\d+)@@/g, function (match, n) {
             return math[n];
         });
         return text;
     };
 
     return {
         init : init,
         remove_math : remove_math,
         replace_math : replace_math
     };
 
 }(IPython));

IPython/nbconvert/filters/markdown.py

@@ -1,74 +1,107 b''
 """Markdown filters
 This file contains a collection of utility filters for dealing with 
 markdown within Jinja templates.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
-import sys
+import os
 import subprocess
+from io import TextIOWrapper, BytesIO
 
+# IPython imports
 from IPython.nbconvert.utils.pandoc import pandoc
+from IPython.nbconvert.utils.exceptions import ConversionException
+from IPython.utils.process import find_cmd, FindCmdError
+from IPython.utils.py3compat import cast_bytes
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
+marked = os.path.join(os.path.dirname(__file__), "marked.js")
 
 __all__ = [
     'markdown2html',
+    'markdown2html_pandoc',
+    'markdown2html_marked',
     'markdown2latex',
-    'markdown2rst'
+    'markdown2rst',
 ]
 
+class NodeJSMissing(ConversionException):
+    """Exception raised when node.js is missing."""
+    pass
+
 def markdown2latex(source):
     """Convert a markdown string to LaTeX via pandoc.
 
     This function will raise an error if pandoc is not installed.
     Any error messages generated by pandoc are printed to stderr.
 
     Parameters
     ----------
     source : string
       Input string, assumed to be valid markdown.
 
     Returns
     -------
     out : string
       Output as returned by pandoc.
     """
     return pandoc(source, 'markdown', 'latex')
 
-
-def markdown2html(source):
+def markdown2html_pandoc(source):
     """Convert a markdown string to HTML via pandoc"""
     return pandoc(source, 'markdown', 'html', extra_args=['--mathjax'])
 
+def markdown2html_marked(source, encoding='utf-8'):
+    """Convert a markdown string to HTML via marked"""
+    command = ['node', marked]
+    try:
+        p = subprocess.Popen(command,
+                             stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        )
+    except OSError as e:
+        raise NodeJSMissing(
+            "The command '%s' returned an error: %s.\n" % (" ".join(command), e) +
+            "Please check that Node.js is installed."
+        )
+    out, _ = p.communicate(cast_bytes(source, encoding))
+    out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
+    return out.rstrip('\n')
+
 def markdown2rst(source):
     """Convert a markdown string to LaTeX via pandoc.
 
     This function will raise an error if pandoc is not installed.
     Any error messages generated by pandoc are printed to stderr.
 
     Parameters
     ----------
     source : string
       Input string, assumed to be valid markdown.
 
     Returns
     -------
     out : string
       Output as returned by pandoc.
     """
     return pandoc(source, 'markdown', 'rst')
 
+try:
+    find_cmd('node')
+except FindCmdError:
+    markdown2html = markdown2html_pandoc
+else:
+    markdown2html = markdown2html_marked

setupbase.py

@@ -1,600 +1,601 b''
 # encoding: utf-8
 """
 This module defines the things that are used in setup.py for building IPython
 
 This includes:
 
     * The basic arguments to setup
     * Functions for finding things like packages, package data, etc.
     * A function for checking dependencies.
 """
 from __future__ import print_function
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import errno
 import os
 import sys
 
 from distutils.command.build_py import build_py
 from distutils.command.build_scripts import build_scripts
 from distutils.command.install import install
 from distutils.command.install_scripts import install_scripts
 from distutils.cmd import Command
 from glob import glob
 from subprocess import call
 
 from setupext import install_data_ext
 
 #-------------------------------------------------------------------------------
 # Useful globals and utility functions
 #-------------------------------------------------------------------------------
 
 # A few handy globals
 isfile = os.path.isfile
 pjoin = os.path.join
 repo_root = os.path.dirname(os.path.abspath(__file__))
 
 def oscmd(s):
     print(">", s)
     os.system(s)
 
 # Py3 compatibility hacks, without assuming IPython itself is installed with
 # the full py3compat machinery.
 
 try:
     execfile
 except NameError:
     def execfile(fname, globs, locs=None):
         locs = locs or globs
         exec(compile(open(fname).read(), fname, "exec"), globs, locs)
 
 # A little utility we'll need below, since glob() does NOT allow you to do
 # exclusion on multiple endings!
 def file_doesnt_endwith(test,endings):
     """Return true if test is a file and its name does NOT end with any
     of the strings listed in endings."""
     if not isfile(test):
         return False
     for e in endings:
         if test.endswith(e):
             return False
     return True
 
 #---------------------------------------------------------------------------
 # Basic project information
 #---------------------------------------------------------------------------
 
 # release.py contains version, authors, license, url, keywords, etc.
 execfile(pjoin(repo_root, 'IPython','core','release.py'), globals())
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
 setup_args = dict(
       name             = name,
       version          = version,
       description      = description,
       long_description = long_description,
       author           = author,
       author_email     = author_email,
       url              = url,
       download_url     = download_url,
       license          = license,
       platforms        = platforms,
       keywords         = keywords,
       classifiers      = classifiers,
       cmdclass         = {'install_data': install_data_ext},
       )
 
 
 #---------------------------------------------------------------------------
 # Find packages
 #---------------------------------------------------------------------------
 
 def find_packages():
     """
     Find all of IPython's packages.
     """
     excludes = ['deathrow', 'quarantine']
     packages = []
     for dir,subdirs,files in os.walk('IPython'):
         package = dir.replace(os.path.sep, '.')
         if any(package.startswith('IPython.'+exc) for exc in excludes):
             # package is to be excluded (e.g. deathrow)
             continue
         if '__init__.py' not in files:
             # not a package
             continue
         packages.append(package)
     return packages
 
 #---------------------------------------------------------------------------
 # Find package data
 #---------------------------------------------------------------------------
 
 def find_package_data():
     """
     Find IPython's package_data.
     """
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     
     # exclude static things that we don't ship (e.g. mathjax)
     excludes = ['mathjax']
     
     # add 'static/' prefix to exclusions, and tuplify for use in startswith
     excludes = tuple([os.path.join('static', ex) for ex in excludes])
     
     # walk notebook resources:
     cwd = os.getcwd()
     os.chdir(os.path.join('IPython', 'html'))
     static_walk = list(os.walk('static'))
     static_data = []
     for parent, dirs, files in static_walk:
         if parent.startswith(excludes):
             continue
         for f in files:
             static_data.append(os.path.join(parent, f))
 
     os.chdir(os.path.join('tests',))
     js_tests = glob('casperjs/*.*') +  glob('casperjs/*/*')
 
     os.chdir(os.path.join(cwd, 'IPython', 'nbconvert'))
     nbconvert_templates = [os.path.join(dirpath, '*.*')
                             for dirpath, _, _ in os.walk('templates')]
 
     os.chdir(cwd)
 
     package_data = {
         'IPython.config.profile' : ['README*', '*/*.py'],
         'IPython.core.tests' : ['*.png', '*.jpg'],
         'IPython.lib.tests' : ['*.wav'],
         'IPython.testing' : ['*.txt'],
         'IPython.testing.plugin' : ['*.txt'],
         'IPython.html' : ['templates/*'] + static_data,
         'IPython.html.tests' : js_tests,
         'IPython.qt.console' : ['resources/icon/*.svg'],
         'IPython.nbconvert' : nbconvert_templates +
             ['tests/files/*.*', 'exporters/tests/files/*.*'],
+        'IPython.nbconvert.filters' : ['marked.js'],
         'IPython.nbformat' : ['tests/*.ipynb']
     }
     return package_data
 
 
 #---------------------------------------------------------------------------
 # Find data files
 #---------------------------------------------------------------------------
 
 def make_dir_struct(tag,base,out_base):
     """Make the directory structure of all files below a starting dir.
 
     This is just a convenience routine to help build a nested directory
     hierarchy because distutils is too stupid to do this by itself.
 
     XXX - this needs a proper docstring!
     """
 
     # we'll use these a lot below
     lbase = len(base)
     pathsep = os.path.sep
     lpathsep = len(pathsep)
 
     out = []
     for (dirpath,dirnames,filenames) in os.walk(base):
         # we need to strip out the dirpath from the base to map it to the
         # output (installation) path.  This requires possibly stripping the
         # path separator, because otherwise pjoin will not work correctly
         # (pjoin('foo/','/bar') returns '/bar').
 
         dp_eff = dirpath[lbase:]
         if dp_eff.startswith(pathsep):
             dp_eff = dp_eff[lpathsep:]
         # The output path must be anchored at the out_base marker
         out_path = pjoin(out_base,dp_eff)
         # Now we can generate the final filenames. Since os.walk only produces
         # filenames, we must join back with the dirpath to get full valid file
         # paths:
         pfiles = [pjoin(dirpath,f) for f in filenames]
         # Finally, generate the entry we need, which is a pari of (output
         # path, files) for use as a data_files parameter in install_data.
         out.append((out_path, pfiles))
 
     return out
 
 
 def find_data_files():
     """
     Find IPython's data_files.
 
     Most of these are docs.
     """
 
     docdirbase  = pjoin('share', 'doc', 'ipython')
     manpagebase = pjoin('share', 'man', 'man1')
 
     # Simple file lists can be made by hand
     manpages = [f for f in glob(pjoin('docs','man','*.1.gz')) if isfile(f)]
     if not manpages:
         # When running from a source tree, the manpages aren't gzipped
         manpages = [f for f in glob(pjoin('docs','man','*.1')) if isfile(f)]
 
     igridhelpfiles = [f for f in glob(pjoin('IPython','extensions','igrid_help.*')) if isfile(f)]
 
     # For nested structures, use the utility above
     example_files = make_dir_struct(
         'data',
         pjoin('docs','examples'),
         pjoin(docdirbase,'examples')
     )
     manual_files = make_dir_struct(
         'data',
         pjoin('docs','html'),
         pjoin(docdirbase,'manual')
     )
 
     # And assemble the entire output list
     data_files = [ (manpagebase, manpages),
                    (pjoin(docdirbase, 'extensions'), igridhelpfiles),
                    ] + manual_files + example_files
 
     return data_files
 
 
 def make_man_update_target(manpage):
     """Return a target_update-compliant tuple for the given manpage.
 
     Parameters
     ----------
     manpage : string
       Name of the manpage, must include the section number (trailing number).
 
     Example
     -------
 
     >>> make_man_update_target('ipython.1') #doctest: +NORMALIZE_WHITESPACE
     ('docs/man/ipython.1.gz',
      ['docs/man/ipython.1'],
      'cd docs/man && gzip -9c ipython.1 > ipython.1.gz')
     """
     man_dir = pjoin('docs', 'man')
     manpage_gz = manpage + '.gz'
     manpath = pjoin(man_dir, manpage)
     manpath_gz = pjoin(man_dir, manpage_gz)
     gz_cmd = ( "cd %(man_dir)s && gzip -9c %(manpage)s > %(manpage_gz)s" %
                locals() )
     return (manpath_gz, [manpath], gz_cmd)
 
 # The two functions below are copied from IPython.utils.path, so we don't need
 # to import IPython during setup, which fails on Python 3.
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         os.system(cmd)
 
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def find_entry_points():
     """Find IPython's scripts.
 
     if entry_points is True:
         return setuptools entry_point-style definitions
     else:
         return file paths of plain scripts [default]
 
     suffix is appended to script names if entry_points is True, so that the
     Python 3 scripts get named "ipython3" etc.
     """
     ep = [
             'ipython%s = IPython:start_ipython',
             'ipcontroller%s = IPython.parallel.apps.ipcontrollerapp:launch_new_instance',
             'ipengine%s = IPython.parallel.apps.ipengineapp:launch_new_instance',
             'ipcluster%s = IPython.parallel.apps.ipclusterapp:launch_new_instance',
             'iptest%s = IPython.testing.iptestcontroller:main',
         ]
     suffix = str(sys.version_info[0])
     return [e % '' for e in ep] + [e % suffix for e in ep]
 
 script_src = """#!{executable}
 # This script was automatically generated by setup.py
 if __name__ == '__main__':
     from {mod} import {func}
     {func}()
 """
 
 class build_scripts_entrypt(build_scripts):
     def run(self):
         self.mkpath(self.build_dir)
         outfiles = []
         for script in find_entry_points():
             name, entrypt = script.split('=')
             name = name.strip()
             entrypt = entrypt.strip()
             outfile = os.path.join(self.build_dir, name)
             outfiles.append(outfile)
             print('Writing script to', outfile)
 
             mod, func = entrypt.split(':')
             with open(outfile, 'w') as f:
                 f.write(script_src.format(executable=sys.executable,
                                           mod=mod, func=func))
 
         return outfiles, outfiles
 
 class install_lib_symlink(Command):
     user_options = [
         ('install-dir=', 'd', "directory to install to"),
         ]
 
     def initialize_options(self):
         self.install_dir = None
 
     def finalize_options(self):
         self.set_undefined_options('symlink',
                                    ('install_lib', 'install_dir'),
                                   )
 
     def run(self):
         if sys.platform == 'win32':
             raise Exception("This doesn't work on Windows.")
         pkg = os.path.join(os.getcwd(), 'IPython')
         dest = os.path.join(self.install_dir, 'IPython')
         if os.path.islink(dest):
             print('removing existing symlink at %s' % dest)
             os.unlink(dest)
         print('symlinking %s -> %s' % (pkg, dest))
         os.symlink(pkg, dest)
 
 class unsymlink(install):
     def run(self):
         dest = os.path.join(self.install_lib, 'IPython')
         if os.path.islink(dest):
             print('removing symlink at %s' % dest)
             os.unlink(dest)
         else:
             print('No symlink exists at %s' % dest)
 
 class install_symlinked(install):
     def run(self):
         if sys.platform == 'win32':
             raise Exception("This doesn't work on Windows.")
 
         # Run all sub-commands (at least those that need to be run)
         for cmd_name in self.get_sub_commands():
             self.run_command(cmd_name)
     
     # 'sub_commands': a list of commands this command might have to run to
     # get its work done.  See cmd.py for more info.
     sub_commands = [('install_lib_symlink', lambda self:True),
                     ('install_scripts_sym', lambda self:True),
                    ]
 
 class install_scripts_for_symlink(install_scripts):
     """Redefined to get options from 'symlink' instead of 'install'.
     
     I love distutils almost as much as I love setuptools.
     """
     def finalize_options(self):
         self.set_undefined_options('build', ('build_scripts', 'build_dir'))
         self.set_undefined_options('symlink',
                                    ('install_scripts', 'install_dir'),
                                    ('force', 'force'),
                                    ('skip_build', 'skip_build'),
                                   )
 
 #---------------------------------------------------------------------------
 # Verify all dependencies
 #---------------------------------------------------------------------------
 
 def check_for_dependencies():
     """Check for IPython's dependencies.
 
     This function should NOT be called if running under setuptools!
     """
     from setupext.setupext import (
         print_line, print_raw, print_status,
         check_for_sphinx, check_for_pygments,
         check_for_nose, check_for_pexpect,
         check_for_pyzmq, check_for_readline,
         check_for_jinja2, check_for_tornado
     )
     print_line()
     print_raw("BUILDING IPYTHON")
     print_status('python', sys.version)
     print_status('platform', sys.platform)
     if sys.platform == 'win32':
         print_status('Windows version', sys.getwindowsversion())
 
     print_raw("")
     print_raw("OPTIONAL DEPENDENCIES")
 
     check_for_sphinx()
     check_for_pygments()
     check_for_nose()
     check_for_pexpect()
     check_for_pyzmq()
     check_for_tornado()
     check_for_readline()
     check_for_jinja2()
 
 #---------------------------------------------------------------------------
 # VCS related
 #---------------------------------------------------------------------------
 
 # utils.submodule has checks for submodule status
 execfile(pjoin('IPython','utils','submodule.py'), globals())
 
 class UpdateSubmodules(Command):
     """Update git submodules
     
     IPython's external javascript dependencies live in a separate repo.
     """
     description = "Update git submodules"
     user_options = []
     
     def initialize_options(self):
         pass
     
     def finalize_options(self):
         pass
     
     def run(self):
         failure = False
         try:
             self.spawn('git submodule init'.split())
             self.spawn('git submodule update --recursive'.split())
         except Exception as e:
             failure = e
             print(e)
         
         if not check_submodule_status(repo_root) == 'clean':
             print("submodules could not be checked out")
             sys.exit(1)
 
 
 def git_prebuild(pkg_dir, build_cmd=build_py):
     """Return extended build or sdist command class for recording commit
     
     records git commit in IPython.utils._sysinfo.commit
     
     for use in IPython.utils.sysinfo.sys_info() calls after installation.
     
     Also ensures that submodules exist prior to running
     """
     
     class MyBuildPy(build_cmd):
         ''' Subclass to write commit data into installation tree '''
         def run(self):
             build_cmd.run(self)
             # this one will only fire for build commands
             if hasattr(self, 'build_lib'):
                 self._record_commit(self.build_lib)
         
         def make_release_tree(self, base_dir, files):
             # this one will fire for sdist
             build_cmd.make_release_tree(self, base_dir, files)
             self._record_commit(base_dir)
         
         def _record_commit(self, base_dir):
             import subprocess
             proc = subprocess.Popen('git rev-parse --short HEAD',
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE,
                                     shell=True)
             repo_commit, _ = proc.communicate()
             repo_commit = repo_commit.strip().decode("ascii")
             
             out_pth = pjoin(base_dir, pkg_dir, 'utils', '_sysinfo.py')
             if os.path.isfile(out_pth) and not repo_commit:
                 # nothing to write, don't clobber
                 return
             
             print("writing git commit '%s' to %s" % (repo_commit, out_pth))
             
             # remove to avoid overwriting original via hard link
             try:
                 os.remove(out_pth)
             except (IOError, OSError):
                 pass
             with open(out_pth, 'w') as out_file:
                 out_file.writelines([
                     '# GENERATED BY setup.py\n',
                     'commit = "%s"\n' % repo_commit,
                 ])
     return require_submodules(MyBuildPy)
 
 
 def require_submodules(command):
     """decorator for instructing a command to check for submodules before running"""
     class DecoratedCommand(command):
         def run(self):
             if not check_submodule_status(repo_root) == 'clean':
                 print("submodules missing! Run `setup.py submodule` and try again")
                 sys.exit(1)
             command.run(self)
     return DecoratedCommand
 
 #---------------------------------------------------------------------------
 # Notebook related
 #---------------------------------------------------------------------------
 
 class CompileCSS(Command):
     """Recompile Notebook CSS
     
     Regenerate the compiled CSS from LESS sources.
     
     Requires various dev dependencies, such as fabric and lessc.
     """
     description = "Recompile Notebook CSS"
     user_options = []
     
     def initialize_options(self):
         pass
     
     def finalize_options(self):
         pass
     
     def run(self):
         call("fab css", shell=True, cwd=pjoin(repo_root, "IPython", "html"))
 
 class JavascriptVersion(Command):
     """write the javascript version to notebook javascript"""
     description = "Write IPython version to javascript"
     user_options = []
     
     def initialize_options(self):
         pass
     
     def finalize_options(self):
         pass
     
     def run(self):
         nsfile = pjoin(repo_root, "IPython", "html", "static", "base", "js", "namespace.js")
         with open(nsfile) as f:
             lines = f.readlines()
         with open(nsfile, 'w') as f:
             for line in lines:
                 if line.startswith("IPython.version"):
                     line = 'IPython.version = "{0}";\n'.format(version)
                 f.write(line)