diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index c577618..6966ad5 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2017,7 +2017,8 @@ class InteractiveShell(SingletonConfigurable): self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics, m.ConfigMagics, m.DeprecatedMagics, m.ExecutionMagics, m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics, - m.NamespaceMagics, m.OSMagics, m.PylabMagics ) + m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics, + ) # FIXME: Move the color initialization to the DisplayHook, which # should be split into a prompt manager and displayhook. We probably diff --git a/IPython/core/magic.py b/IPython/core/magic.py index f5d4c79..c9a723d 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -30,7 +30,7 @@ from IPython.external.decorator import decorator from IPython.utils.ipstruct import Struct from IPython.utils.process import arg_split from IPython.utils.text import dedent -from IPython.utils.traitlets import Bool, Dict, Instance +from IPython.utils.traitlets import Bool, Dict, Instance, MetaHasTraits from IPython.utils.warn import error, warn #----------------------------------------------------------------------------- @@ -365,10 +365,10 @@ class MagicsManager(Configurable): for m in magic_objects: if not m.registered: raise ValueError("Class of magics %r was constructed without " - "the @register_macics class decorator") - if type(m) is type: + "the @register_magics class decorator") + if type(m) in (type, MetaHasTraits): # If we're given an uninstantiated class - m = m(self.shell) + m = m(shell=self.shell) # Now that we have an instance, we can register it and update the # table of callables diff --git a/IPython/core/magics/__init__.py b/IPython/core/magics/__init__.py index 23479b5..3b72f15 100644 --- a/IPython/core/magics/__init__.py +++ b/IPython/core/magics/__init__.py @@ -25,6 +25,7 @@ from .logging import LoggingMagics from .namespace import NamespaceMagics from .osm import OSMagics from .pylab import PylabMagics +from .script import ScriptMagics #----------------------------------------------------------------------------- # Magic implementation classes diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index c2b64bd..f761513 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -16,17 +16,20 @@ builtin. #----------------------------------------------------------------------------- # Stdlib +import io import os import re import sys from pprint import pformat # Our own packages +from IPython.core import magic_arguments from IPython.core import oinspect from IPython.core import page -from IPython.core.error import UsageError -from IPython.core.magic import (Magics, compress_dhist, magics_class, - line_magic) +from IPython.core.error import UsageError, StdinNotImplementedError +from IPython.core.magic import ( + Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic +) from IPython.testing.skipdoctest import skip_doctest from IPython.utils.io import file_read, nlprint from IPython.utils.path import get_py_filename, unquote_filename @@ -432,7 +435,7 @@ class OSMagics(Magics): @skip_doctest @line_magic def sc(self, parameter_s=''): - """Shell capture - execute a shell command and capture its output. + """Shell capture - run shell command and capture output (DEPRECATED use !). DEPRECATED. Suboptimal, retained for backwards compatibility. @@ -545,9 +548,9 @@ class OSMagics(Magics): else: return out - @line_magic - def sx(self, parameter_s=''): - """Shell execute - run a shell command and capture its output. + @line_cell_magic + def sx(self, line='', cell=None): + """Shell execute - run shell command and capture output (!! is short-hand). %sx command @@ -586,10 +589,21 @@ class OSMagics(Magics): This is very useful when trying to use such lists as arguments to system commands.""" + + if cell is None: + # line magic + return self.shell.getoutput(line) + else: + opts,args = self.parse_options(line, '', 'out=') + output = self.shell.getoutput(cell) + out_name = opts.get('out', opts.get('o')) + if out_name: + self.shell.user_ns[out_name] = output + else: + return output - if parameter_s: - return self.shell.getoutput(parameter_s) - + system = line_cell_magic('system')(sx) + bang = cell_magic('!')(sx) @line_magic def bookmark(self, parameter_s=''): @@ -675,3 +689,33 @@ class OSMagics(Magics): return page.page(self.shell.pycolorize(cont)) + + @magic_arguments.magic_arguments() + @magic_arguments.argument( + '-a', '--amend', action='store_true', default=False, + help='Open file for amending if it exists' + ) + @magic_arguments.argument( + 'filename', type=unicode, + help='file to write' + ) + @cell_magic + def file(self, line, cell): + """Write the contents of the cell to a file. + + For frontends that do not support stdin (Notebook), -f is implied. + """ + args = magic_arguments.parse_argstring(self.file, line) + filename = unquote_filename(args.filename) + + if os.path.exists(filename): + if args.amend: + print "Amending to %s" % filename + else: + print "Overwriting %s" % filename + else: + print "Writing %s" % filename + + mode = 'a' if args.amend else 'w' + with io.open(filename, mode, encoding='utf-8') as f: + f.write(cell) diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py new file mode 100644 index 0000000..907352b --- /dev/null +++ b/IPython/core/magics/script.py @@ -0,0 +1,215 @@ +"""Magic functions for running cells in various scripts.""" +#----------------------------------------------------------------------------- +# Copyright (c) 2012 The IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib +import os +import re +import sys +from subprocess import Popen, PIPE + +# Our own packages +from IPython.config.configurable import Configurable +from IPython.core import magic_arguments +from IPython.core.error import UsageError +from IPython.core.magic import ( + Magics, magics_class, line_magic, cell_magic +) +from IPython.lib.backgroundjobs import BackgroundJobManager +from IPython.testing.skipdoctest import skip_doctest +from IPython.utils import py3compat +from IPython.utils.process import find_cmd, FindCmdError, arg_split +from IPython.utils.traitlets import List, Dict + +#----------------------------------------------------------------------------- +# Magic implementation classes +#----------------------------------------------------------------------------- + +def script_args(f): + """single decorator for adding script args""" + args = [ + magic_arguments.argument( + '--out', type=str, + help="""The variable in which to store stdout from the script. + If the script is backgrounded, this will be the stdout *pipe*, + instead of the stderr text itself. + """ + ), + magic_arguments.argument( + '--err', type=str, + help="""The variable in which to store stderr from the script. + If the script is backgrounded, this will be the stderr *pipe*, + instead of the stderr text itself. + """ + ), + magic_arguments.argument( + '--bg', action="store_true", + help="""Whether to run the script in the background. + If given, the only way to see the output of the command is + with --out/err. + """ + ), + ] + for arg in args: + f = arg(f) + return f + +@magics_class +class ScriptMagics(Magics, Configurable): + """Magics for talking to scripts + + This defines a base `%%script` cell magic for running a cell + with a program in a subprocess, and registers a few top-level + magics that call %%script with common interpreters. + """ + script_magics = List(config=True, + help="""Extra script cell magics to define + + This generates simple wrappers of `%%script foo` as `%%foo`. + + If you want to add script magics that aren't on your path, + specify them in script_paths + """, + ) + def _script_magics_default(self): + """default to a common list of programs if we find them""" + + defaults = [] + to_try = [] + if os.name == 'nt': + defaults.append('cmd') + to_try.append('powershell') + to_try.extend([ + 'sh', + 'bash', + 'perl', + 'ruby', + 'python3', + 'pypy', + ]) + + for cmd in to_try: + if cmd in self.script_paths: + defaults.append(cmd) + else: + try: + find_cmd(cmd) + except FindCmdError: + # command not found, ignore it + pass + except ImportError: + # Windows without pywin32, find_cmd doesn't work + pass + else: + defaults.append(cmd) + return defaults + + script_paths = Dict(config=True, + help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby' + + Only necessary for items in script_magics where the default path will not + find the right interpreter. + """ + ) + + def __init__(self, shell=None): + Configurable.__init__(self, config=shell.config) + self._generate_script_magics() + Magics.__init__(self, shell=shell) + self.job_manager = BackgroundJobManager() + + def _generate_script_magics(self): + cell_magics = self.magics['cell'] + for name in self.script_magics: + cell_magics[name] = self._make_script_magic(name) + + def _make_script_magic(self, name): + """make a named magic, that calls %%script with a particular program""" + # expand to explicit path if necessary: + script = self.script_paths.get(name, name) + + @magic_arguments.magic_arguments() + @script_args + def named_script_magic(line, cell): + # if line, add it as cl-flags + if line: + line = "%s %s" % (script, line) + else: + line = script + return self.shebang(line, cell) + + # write a basic docstring: + named_script_magic.__doc__ = \ + """%%{name} script magic + + Run cells with {script} in a subprocess. + + This is a shortcut for `%%script {script}` + """.format(**locals()) + + return named_script_magic + + @magic_arguments.magic_arguments() + @script_args + @cell_magic("script") + def shebang(self, line, cell): + """Run a cell via a shell command + + The `%%script` line is like the #! line of script, + specifying a program (bash, perl, ruby, etc.) with which to run. + + The rest of the cell is run by that program. + + Examples + -------- + :: + + In [1]: %%script bash + ...: for i in 1 2 3; do + ...: echo $i + ...: done + 1 + 2 + 3 + """ + argv = arg_split(line, posix = not sys.platform.startswith('win')) + args, cmd = self.shebang.parser.parse_known_args(argv) + + p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) + + if args.bg: + if args.out: + self.shell.user_ns[args.out] = p.stdout + if args.err: + self.shell.user_ns[args.err] = p.stderr + self.job_manager.new(self._run_script, p, cell) + return + + out, err = p.communicate(cell) + out = py3compat.bytes_to_str(out) + err = py3compat.bytes_to_str(err) + if args.out: + self.shell.user_ns[args.out] = out + else: + sys.stdout.write(out) + sys.stdout.flush() + if args.err: + self.shell.user_ns[args.err] = err + else: + sys.stderr.write(err) + sys.stderr.flush() + + def _run_script(self, p, cell): + """callback for running the script in the background""" + p.stdin.write(cell) + p.stdin.close() + p.wait() diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 09d82b1..f22141e 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -28,13 +28,14 @@ from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic, line_cell_magic, register_line_magic, register_cell_magic, register_line_cell_magic) -from IPython.core.magics import execution +from IPython.core.magics import execution, script from IPython.nbformat.v3.tests.nbexamples import nb0 from IPython.nbformat import current from IPython.testing import decorators as dec from IPython.testing import tools as tt from IPython.utils import py3compat from IPython.utils.tempdir import TemporaryDirectory +from IPython.utils.process import find_cmd #----------------------------------------------------------------------------- # Test functions begin @@ -560,4 +561,104 @@ class CellMagicTestCase(TestCase): # Check that nothing is registered as 'cellm33' c33 = _ip.find_cell_magic('cellm33') nt.assert_equals(c33, None) + +def test_file(): + """Basic %%file""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, 'file1') + ip.run_cell_magic("file", fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) + +def test_file_unicode(): + """%%file with unicode cell""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, 'file1') + ip.run_cell_magic("file", fname, u'\n'.join([ + u'liné1', + u'liné2', + ])) + with io.open(fname, encoding='utf-8') as f: + s = f.read() + nt.assert_in(u'liné1\n', s) + nt.assert_in(u'liné2', s) + +def test_file_amend(): + """%%file -a amends files""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, 'file2') + ip.run_cell_magic("file", fname, u'\n'.join([ + 'line1', + 'line2', + ])) + ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([ + 'line3', + 'line4', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line3\n', s) + +def test_script_config(): + ip = get_ipython() + ip.config.ScriptMagics.script_magics = ['whoda'] + sm = script.ScriptMagics(shell=ip) + nt.assert_in('whoda', sm.magics['cell']) + +@dec.skip_win32 +def test_script_out(): + ip = get_ipython() + ip.run_cell_magic("script", "--out output sh", "echo 'hi'") + nt.assert_equals(ip.user_ns['output'], 'hi\n') + +@dec.skip_win32 +def test_script_err(): + ip = get_ipython() + ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2") + nt.assert_equals(ip.user_ns['error'], 'hello\n') + +@dec.skip_win32 +def test_script_out_err(): + ip = get_ipython() + ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2") + nt.assert_equals(ip.user_ns['output'], 'hi\n') + nt.assert_equals(ip.user_ns['error'], 'hello\n') + +@dec.skip_win32 +def test_script_bg_out(): + ip = get_ipython() + ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'") + nt.assert_equals(ip.user_ns['output'].read(), 'hi\n') + +@dec.skip_win32 +def test_script_bg_err(): + ip = get_ipython() + ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2") + nt.assert_equals(ip.user_ns['error'].read(), 'hello\n') + +@dec.skip_win32 +def test_script_bg_out_err(): + ip = get_ipython() + ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2") + nt.assert_equals(ip.user_ns['output'].read(), 'hi\n') + nt.assert_equals(ip.user_ns['error'].read(), 'hello\n') + +def test_script_defaults(): + ip = get_ipython() + for cmd in ['sh', 'bash', 'perl', 'ruby']: + try: + find_cmd(cmd) + except Exception: + pass + else: + nt.assert_in(cmd, ip.magics_manager.magics['cell']) diff --git a/docs/examples/notebooks/Script Magics.ipynb b/docs/examples/notebooks/Script Magics.ipynb new file mode 100644 index 0000000..4c53e0b --- /dev/null +++ b/docs/examples/notebooks/Script Magics.ipynb @@ -0,0 +1,448 @@ +{ + "metadata": { + "name": "Script Magics" + }, + "nbformat": 3, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "level": 1, + "source": [ + "Running Scripts from IPython" + ] + }, + { + "cell_type": "markdown", + "source": [ + "IPython has a `%%script` cell magic, which lets you run a cell in", + "a subprocess of any interpreter on your system, such as: bash, ruby, perl, zsh, R, etc.", + "", + "It can even be a script of your own, which expects input on stdin." + ] + }, + { + "cell_type": "code", + "input": [ + "import sys" + ], + "language": "python", + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "source": [ + "To use it, simply pass a path or shell command to the program you want to run on the `%%script` line,", + "and the rest of the cell will be run by that script, and stdout/err from the subprocess are captured and displayed." + ] + }, + { + "cell_type": "code", + "input": [ + "%%script python", + "import sys", + "print 'hello from Python %s' % sys.version" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "hello from Python 2.7.1 (r271:86832, Jul 31 2011, 19:30:53) ", + "[GCC 4.2.1 (Based on Apple Inc. build 5658) (LLVM build 2335.15.00)]", + "" + ] + } + ], + "prompt_number": 2 + }, + { + "cell_type": "code", + "input": [ + "%%script python3", + "import sys", + "print('hello from Python: %s' % sys.version)" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "hello from Python: 3.2.3 (v3.2.3:3d0686d90f55, Apr 10 2012, 11:25:50) ", + "[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]", + "" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "source": [ + "IPython also creates aliases for a few common interpreters, such as bash, ruby, perl, etc.", + "", + "These are all equivalent to `%%script `" + ] + }, + { + "cell_type": "code", + "input": [ + "%%ruby", + "puts \"Hello from Ruby #{RUBY_VERSION}\"" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Hello from Ruby 1.8.7", + "" + ] + } + ], + "prompt_number": 4 + }, + { + "cell_type": "code", + "input": [ + "%%bash", + "echo \"hello from $BASH\"" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "hello from /usr/local/bin/bash", + "" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "heading", + "level": 2, + "source": [ + "Capturing output" + ] + }, + { + "cell_type": "markdown", + "source": [ + "You can also capture stdout/err from these subprocesses into Python variables, instead of letting them go directly to stdout/err" + ] + }, + { + "cell_type": "code", + "input": [ + "%%bash", + "echo \"hi, stdout\"", + "echo \"hello, stderr\" >&2", + "" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "hi, stdout", + "" + ] + }, + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "hello, stderr", + "" + ] + } + ], + "prompt_number": 6 + }, + { + "cell_type": "code", + "input": [ + "%%bash --out output --err error", + "echo \"hi, stdout\"", + "echo \"hello, stderr\" >&2" + ], + "language": "python", + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "code", + "input": [ + "print error", + "print output" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "hello, stderr", + "", + "hi, stdout", + "", + "" + ] + } + ], + "prompt_number": 8 + }, + { + "cell_type": "heading", + "level": 2, + "source": [ + "Background Scripts" + ] + }, + { + "cell_type": "markdown", + "source": [ + "These scripts can be run in the background, by adding the `--bg` flag.", + "", + "When you do this, output is discarded unless you use the `--out/err`", + "flags to store output as above." + ] + }, + { + "cell_type": "code", + "input": [ + "%%ruby --bg --out ruby_lines", + "for n in 1...10", + " sleep 1", + " puts \"line #{n}\"", + " STDOUT.flush", + "end" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Starting job # 0 in a separate thread.", + "" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "markdown", + "source": [ + "When you do store output of a background thread, these are the stdout/err *pipes*,", + "rather than the text of the output." + ] + }, + { + "cell_type": "code", + "input": [ + "ruby_lines" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 10, + "text": [ + "', mode 'rb' at 0x10dc651e0>" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "code", + "input": [ + "print ruby_lines.read()" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "line 1", + "line 2", + "line 3", + "line 4", + "line 5", + "line 6", + "line 7", + "line 8", + "line 9", + "", + "" + ] + } + ], + "prompt_number": 11 + }, + { + "cell_type": "heading", + "level": 2, + "source": [ + "Arguments to subcommand" + ] + }, + { + "cell_type": "markdown", + "source": [ + "You can pass arguments the subcommand as well,", + "such as this example instructing Python to use integer division from Python 3:" + ] + }, + { + "cell_type": "code", + "input": [ + "%%script python -Qnew", + "print 1/3" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0.333333333333", + "" + ] + } + ], + "prompt_number": 12 + }, + { + "cell_type": "markdown", + "source": [ + "You can really specify *any* program for `%%script`,", + "for instance here is a 'program' that echos the lines of stdin, with delays between each line." + ] + }, + { + "cell_type": "code", + "input": [ + "%%script --bg --out bashout bash -c \"while read line; do echo $line; sleep 1; done\"", + "line 1", + "line 2", + "line 3", + "line 4", + "line 5", + "" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Starting job # 2 in a separate thread.", + "" + ] + } + ], + "prompt_number": 13 + }, + { + "cell_type": "markdown", + "source": [ + "Remember, since the output of a background script is just the stdout pipe,", + "you can read it as lines become available:" + ] + }, + { + "cell_type": "code", + "input": [ + "import time", + "tic = time.time()", + "line = True", + "while True:", + " line = bashout.readline()", + " if not line:", + " break", + " sys.stdout.write(\"%.1fs: %s\" %(time.time()-tic, line))", + " sys.stdout.flush()", + "" + ], + "language": "python", + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0.0s: line 1", + "" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "1.0s: line 2", + "" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "2.0s: line 3", + "" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "3.0s: line 4", + "" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "4.0s: line 5", + "" + ] + } + ], + "prompt_number": 14 + }, + { + "cell_type": "heading", + "level": 2, + "source": [ + "Configuring the default ScriptMagics" + ] + }, + { + "cell_type": "markdown", + "source": [ + "The list of aliased script magics is configurable.", + "", + "The default is to pick from a few common interpreters, and use them if found, but you can specify your own in ipython_config.py:", + "", + " c.ScriptMagics.scripts = ['R', 'pypy', 'myprogram']", + "", + "And if any of these programs do not apear on your default PATH, then you would also need to specify their location with:", + "", + " c.ScriptMagics.script_paths = {'myprogram': '/opt/path/to/myprogram'}" + ] + } + ] + } + ] +} \ No newline at end of file