Show More
@@ -1,463 +1,534 b'' | |||||
1 | { |
|
1 | { | |
2 | "metadata": { |
|
2 | "metadata": { | |
3 | "gist_id": "5241793", |
|
3 | "gist_id": "5241793", | |
4 | "name": "" |
|
4 | "name": "" | |
5 | }, |
|
5 | }, | |
6 | "nbformat": 3, |
|
6 | "nbformat": 3, | |
7 | "nbformat_minor": 0, |
|
7 | "nbformat_minor": 0, | |
8 | "worksheets": [ |
|
8 | "worksheets": [ | |
9 | { |
|
9 | { | |
10 | "cells": [ |
|
10 | "cells": [ | |
11 | { |
|
11 | { | |
12 | "cell_type": "heading", |
|
12 | "cell_type": "heading", | |
13 | "level": 1, |
|
13 | "level": 1, | |
14 | "metadata": {}, |
|
14 | "metadata": {}, | |
15 | "source": [ |
|
15 | "source": [ | |
16 | "Using dill to pickle anything" |
|
16 | "Using dill to pickle anything" | |
17 | ] |
|
17 | ] | |
18 | }, |
|
18 | }, | |
19 | { |
|
19 | { | |
20 | "cell_type": "markdown", |
|
20 | "cell_type": "markdown", | |
21 | "metadata": {}, |
|
21 | "metadata": {}, | |
22 | "source": [ |
|
22 | "source": [ | |
23 | "IPython.parallel doesn't do much in the way of serialization.\n", |
|
23 | "IPython.parallel doesn't do much in the way of serialization.\n", | |
24 | "It has custom zero-copy handling of numpy arrays,\n", |
|
24 | "It has custom zero-copy handling of numpy arrays,\n", | |
25 | "but other than that, it doesn't do anything other than the bare minimum to make basic interactively defined functions and classes sendable.\n", |
|
25 | "but other than that, it doesn't do anything other than the bare minimum to make basic interactively defined functions and classes sendable.\n", | |
26 | "\n", |
|
26 | "\n", | |
27 | "There are a few projects that extend pickle to make just about anything sendable, and one of these is [dill](http://www.cacr.caltech.edu/~mmckerns/dill).\n", |
|
27 | "There are a few projects that extend pickle to make just about anything sendable, and one of these is [dill](http://www.cacr.caltech.edu/~mmckerns/dill).\n", | |
28 | "\n", |
|
28 | "\n", | |
29 | "To install dill:\n", |
|
29 | "To install dill:\n", | |
30 | " \n", |
|
30 | " \n", | |
31 | " pip install dill" |
|
31 | " pip install --pre dill" | |
32 | ] |
|
32 | ] | |
33 | }, |
|
33 | }, | |
34 | { |
|
34 | { | |
35 | "cell_type": "markdown", |
|
35 | "cell_type": "markdown", | |
36 | "metadata": {}, |
|
36 | "metadata": {}, | |
37 | "source": [ |
|
37 | "source": [ | |
38 | "First, as always, we create a task function, this time with a closure" |
|
38 | "First, as always, we create a task function, this time with a closure" | |
39 | ] |
|
39 | ] | |
40 | }, |
|
40 | }, | |
41 | { |
|
41 | { | |
42 | "cell_type": "code", |
|
42 | "cell_type": "code", | |
43 | "collapsed": false, |
|
43 | "collapsed": false, | |
44 | "input": [ |
|
44 | "input": [ | |
45 | "def make_closure(a):\n", |
|
45 | "def make_closure(a):\n", | |
46 | " \"\"\"make a function with a closure, and return it\"\"\"\n", |
|
46 | " \"\"\"make a function with a closure, and return it\"\"\"\n", | |
47 | " def has_closure(b):\n", |
|
47 | " def has_closure(b):\n", | |
48 | " return a * b\n", |
|
48 | " return a * b\n", | |
49 | " return has_closure" |
|
49 | " return has_closure" | |
50 | ], |
|
50 | ], | |
51 | "language": "python", |
|
51 | "language": "python", | |
52 | "metadata": {}, |
|
52 | "metadata": {}, | |
53 | "outputs": [], |
|
53 | "outputs": [], | |
54 | "prompt_number": 1 |
|
54 | "prompt_number": 1 | |
55 | }, |
|
55 | }, | |
56 | { |
|
56 | { | |
57 | "cell_type": "code", |
|
57 | "cell_type": "code", | |
58 | "collapsed": false, |
|
58 | "collapsed": false, | |
59 | "input": [ |
|
59 | "input": [ | |
60 | "closed = make_closure(5)" |
|
60 | "closed = make_closure(5)" | |
61 | ], |
|
61 | ], | |
62 | "language": "python", |
|
62 | "language": "python", | |
63 | "metadata": {}, |
|
63 | "metadata": {}, | |
64 | "outputs": [], |
|
64 | "outputs": [], | |
65 | "prompt_number": 2 |
|
65 | "prompt_number": 2 | |
66 | }, |
|
66 | }, | |
67 | { |
|
67 | { | |
68 | "cell_type": "code", |
|
68 | "cell_type": "code", | |
69 | "collapsed": false, |
|
69 | "collapsed": false, | |
70 | "input": [ |
|
70 | "input": [ | |
71 | "closed(2)" |
|
71 | "closed(2)" | |
72 | ], |
|
72 | ], | |
73 | "language": "python", |
|
73 | "language": "python", | |
74 | "metadata": {}, |
|
74 | "metadata": {}, | |
75 | "outputs": [ |
|
75 | "outputs": [ | |
76 | { |
|
76 | { | |
77 | "metadata": {}, |
|
77 | "metadata": {}, | |
78 | "output_type": "pyout", |
|
78 | "output_type": "pyout", | |
79 | "prompt_number": 3, |
|
79 | "prompt_number": 3, | |
80 | "text": [ |
|
80 | "text": [ | |
81 | "10" |
|
81 | "10" | |
82 | ] |
|
82 | ] | |
83 | } |
|
83 | } | |
84 | ], |
|
84 | ], | |
85 | "prompt_number": 3 |
|
85 | "prompt_number": 3 | |
86 | }, |
|
86 | }, | |
87 | { |
|
87 | { | |
88 | "cell_type": "code", |
|
88 | "cell_type": "code", | |
89 | "collapsed": false, |
|
89 | "collapsed": false, | |
90 | "input": [ |
|
90 | "input": [ | |
91 | "import pickle" |
|
91 | "import pickle" | |
92 | ], |
|
92 | ], | |
93 | "language": "python", |
|
93 | "language": "python", | |
94 | "metadata": {}, |
|
94 | "metadata": {}, | |
95 | "outputs": [], |
|
95 | "outputs": [], | |
96 | "prompt_number": 4 |
|
96 | "prompt_number": 4 | |
97 | }, |
|
97 | }, | |
98 | { |
|
98 | { | |
99 | "cell_type": "markdown", |
|
99 | "cell_type": "markdown", | |
100 | "metadata": {}, |
|
100 | "metadata": {}, | |
101 | "source": [ |
|
101 | "source": [ | |
102 | "Without help, pickle can't deal with closures" |
|
102 | "Without help, pickle can't deal with closures" | |
103 | ] |
|
103 | ] | |
104 | }, |
|
104 | }, | |
105 | { |
|
105 | { | |
106 | "cell_type": "code", |
|
106 | "cell_type": "code", | |
107 | "collapsed": false, |
|
107 | "collapsed": false, | |
108 | "input": [ |
|
108 | "input": [ | |
109 | "pickle.dumps(closed)" |
|
109 | "pickle.dumps(closed)" | |
110 | ], |
|
110 | ], | |
111 | "language": "python", |
|
111 | "language": "python", | |
112 | "metadata": {}, |
|
112 | "metadata": {}, | |
113 | "outputs": [ |
|
113 | "outputs": [ | |
114 | { |
|
114 | { | |
115 | "ename": "PicklingError", |
|
115 | "ename": "PicklingError", | |
116 |
"evalue": "Can't pickle <function has_closure at 0x10 |
|
116 | "evalue": "Can't pickle <function has_closure at 0x10412c6e0>: it's not found as __main__.has_closure", | |
117 | "output_type": "pyerr", |
|
117 | "output_type": "pyerr", | |
118 | "traceback": [ |
|
118 | "traceback": [ | |
119 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mPicklingError\u001b[0m Traceback (most recent call last)", |
|
119 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mPicklingError\u001b[0m Traceback (most recent call last)", | |
120 | "\u001b[1;32m<ipython-input-5-0f1f376cfea0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclosed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", |
|
120 | "\u001b[1;32m<ipython-input-5-0f1f376cfea0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclosed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", | |
121 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36mdumps\u001b[1;34m(obj, protocol)\u001b[0m\n\u001b[0;32m 1372\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1373\u001b[0m \u001b[0mfile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mStringIO\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1374\u001b[1;33m \u001b[0mPickler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1375\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1376\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", |
|
121 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36mdumps\u001b[1;34m(obj, protocol)\u001b[0m\n\u001b[0;32m 1372\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1373\u001b[0m \u001b[0mfile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mStringIO\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1374\u001b[1;33m \u001b[0mPickler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1375\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1376\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", | |
122 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36mdump\u001b[1;34m(self, obj)\u001b[0m\n\u001b[0;32m 222\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 223\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mPROTO\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mchr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 224\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 225\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSTOP\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 226\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", |
|
122 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36mdump\u001b[1;34m(self, obj)\u001b[0m\n\u001b[0;32m 222\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 223\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mPROTO\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mchr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 224\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 225\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSTOP\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 226\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", | |
123 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36msave\u001b[1;34m(self, obj)\u001b[0m\n\u001b[0;32m 284\u001b[0m \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 285\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 286\u001b[1;33m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# Call unbound method with explicit self\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 287\u001b[0m \u001b[1;32mreturn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 288\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", |
|
123 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36msave\u001b[1;34m(self, obj)\u001b[0m\n\u001b[0;32m 284\u001b[0m \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 285\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 286\u001b[1;33m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# Call unbound method with explicit self\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 287\u001b[0m \u001b[1;32mreturn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 288\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", | |
124 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36msave_global\u001b[1;34m(self, obj, name, pack)\u001b[0m\n\u001b[0;32m 746\u001b[0m raise PicklingError(\n\u001b[0;32m 747\u001b[0m \u001b[1;34m\"Can't pickle %r: it's not found as %s.%s\"\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 748\u001b[1;33m (obj, module, name))\n\u001b[0m\u001b[0;32m 749\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 750\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mklass\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", |
|
124 | "\u001b[1;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc\u001b[0m in \u001b[0;36msave_global\u001b[1;34m(self, obj, name, pack)\u001b[0m\n\u001b[0;32m 746\u001b[0m raise PicklingError(\n\u001b[0;32m 747\u001b[0m \u001b[1;34m\"Can't pickle %r: it's not found as %s.%s\"\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 748\u001b[1;33m (obj, module, name))\n\u001b[0m\u001b[0;32m 749\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 750\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mklass\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
125 |
"\u001b[1;31mPicklingError\u001b[0m: Can't pickle <function has_closure at 0x10 |
|
125 | "\u001b[1;31mPicklingError\u001b[0m: Can't pickle <function has_closure at 0x10412c6e0>: it's not found as __main__.has_closure" | |
126 | ] |
|
126 | ] | |
127 | } |
|
127 | } | |
128 | ], |
|
128 | ], | |
129 | "prompt_number": 5 |
|
129 | "prompt_number": 5 | |
130 | }, |
|
130 | }, | |
131 | { |
|
131 | { | |
132 | "cell_type": "markdown", |
|
132 | "cell_type": "markdown", | |
133 | "metadata": {}, |
|
133 | "metadata": {}, | |
134 | "source": [ |
|
134 | "source": [ | |
135 | "But after we import dill, magic happens" |
|
135 | "But after we import dill, magic happens" | |
136 | ] |
|
136 | ] | |
137 | }, |
|
137 | }, | |
138 | { |
|
138 | { | |
139 | "cell_type": "code", |
|
139 | "cell_type": "code", | |
140 | "collapsed": false, |
|
140 | "collapsed": false, | |
141 | "input": [ |
|
141 | "input": [ | |
142 | "import dill" |
|
142 | "import dill" | |
143 | ], |
|
143 | ], | |
144 | "language": "python", |
|
144 | "language": "python", | |
145 | "metadata": {}, |
|
145 | "metadata": {}, | |
146 | "outputs": [], |
|
146 | "outputs": [], | |
147 | "prompt_number": 6 |
|
147 | "prompt_number": 6 | |
148 | }, |
|
148 | }, | |
149 | { |
|
149 | { | |
150 | "cell_type": "code", |
|
150 | "cell_type": "code", | |
151 | "collapsed": false, |
|
151 | "collapsed": false, | |
152 | "input": [ |
|
152 | "input": [ | |
153 | "pickle.dumps(closed)[:64] + '...'" |
|
153 | "pickle.dumps(closed)[:64] + '...'" | |
154 | ], |
|
154 | ], | |
155 | "language": "python", |
|
155 | "language": "python", | |
156 | "metadata": {}, |
|
156 | "metadata": {}, | |
157 | "outputs": [ |
|
157 | "outputs": [ | |
158 | { |
|
158 | { | |
159 | "metadata": {}, |
|
159 | "metadata": {}, | |
160 | "output_type": "pyout", |
|
160 | "output_type": "pyout", | |
161 | "prompt_number": 7, |
|
161 | "prompt_number": 7, | |
162 | "text": [ |
|
162 | "text": [ | |
163 | "\"cdill.dill\\n_load_type\\np0\\n(S'FunctionType'\\np1\\ntp2\\nRp3\\n(cdill.dill...\"" |
|
163 | "\"cdill.dill\\n_load_type\\np0\\n(S'FunctionType'\\np1\\ntp2\\nRp3\\n(cdill.dill...\"" | |
164 | ] |
|
164 | ] | |
165 | } |
|
165 | } | |
166 | ], |
|
166 | ], | |
167 | "prompt_number": 7 |
|
167 | "prompt_number": 7 | |
168 | }, |
|
168 | }, | |
169 | { |
|
169 | { | |
170 | "cell_type": "markdown", |
|
170 | "cell_type": "markdown", | |
171 | "metadata": {}, |
|
171 | "metadata": {}, | |
172 | "source": [ |
|
172 | "source": [ | |
173 | "So from now on, pretty much everything is pickleable." |
|
173 | "So from now on, pretty much everything is pickleable." | |
174 | ] |
|
174 | ] | |
175 | }, |
|
175 | }, | |
176 | { |
|
176 | { | |
177 | "cell_type": "heading", |
|
177 | "cell_type": "heading", | |
178 | "level": 2, |
|
178 | "level": 2, | |
179 | "metadata": {}, |
|
179 | "metadata": {}, | |
180 | "source": [ |
|
180 | "source": [ | |
181 | "Now use this in IPython.parallel" |
|
181 | "Now use this in IPython.parallel" | |
182 | ] |
|
182 | ] | |
183 | }, |
|
183 | }, | |
184 | { |
|
184 | { | |
185 | "cell_type": "markdown", |
|
185 | "cell_type": "markdown", | |
186 | "metadata": {}, |
|
186 | "metadata": {}, | |
187 | "source": [ |
|
187 | "source": [ | |
188 | "As usual, we start by creating our Client and View" |
|
188 | "As usual, we start by creating our Client and View" | |
189 | ] |
|
189 | ] | |
190 | }, |
|
190 | }, | |
191 | { |
|
191 | { | |
192 | "cell_type": "code", |
|
192 | "cell_type": "code", | |
193 | "collapsed": false, |
|
193 | "collapsed": false, | |
194 | "input": [ |
|
194 | "input": [ | |
195 | "from IPython import parallel\n", |
|
195 | "from IPython import parallel\n", | |
196 | "rc = parallel.Client()\n", |
|
196 | "rc = parallel.Client()\n", | |
197 | "view = rc.load_balanced_view()" |
|
197 | "view = rc.load_balanced_view()" | |
198 | ], |
|
198 | ], | |
199 | "language": "python", |
|
199 | "language": "python", | |
200 | "metadata": {}, |
|
200 | "metadata": {}, | |
201 | "outputs": [], |
|
201 | "outputs": [], | |
202 | "prompt_number": 8 |
|
202 | "prompt_number": 8 | |
203 | }, |
|
203 | }, | |
204 | { |
|
204 | { | |
205 | "cell_type": "markdown", |
|
205 | "cell_type": "markdown", | |
206 | "metadata": {}, |
|
206 | "metadata": {}, | |
207 | "source": [ |
|
207 | "source": [ | |
208 | "Now let's try sending our function with a closure:" |
|
208 | "Now let's try sending our function with a closure:" | |
209 | ] |
|
209 | ] | |
210 | }, |
|
210 | }, | |
211 | { |
|
211 | { | |
212 | "cell_type": "code", |
|
212 | "cell_type": "code", | |
213 | "collapsed": false, |
|
213 | "collapsed": false, | |
214 | "input": [ |
|
214 | "input": [ | |
215 | "view.apply_sync(closed, 3)" |
|
215 | "view.apply_sync(closed, 3)" | |
216 | ], |
|
216 | ], | |
217 | "language": "python", |
|
217 | "language": "python", | |
218 | "metadata": {}, |
|
218 | "metadata": {}, | |
219 | "outputs": [ |
|
219 | "outputs": [ | |
220 | { |
|
220 | { | |
221 | "ename": "ValueError", |
|
221 | "ename": "ValueError", | |
222 | "evalue": "Sorry, cannot pickle code objects with closures", |
|
222 | "evalue": "Sorry, cannot pickle code objects with closures", | |
223 | "output_type": "pyerr", |
|
223 | "output_type": "pyerr", | |
224 | "traceback": [ |
|
224 | "traceback": [ | |
225 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", |
|
225 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", | |
226 | "\u001b[1;32m<ipython-input-9-23a646829fdc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mview\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_sync\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclosed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", |
|
226 | "\u001b[1;32m<ipython-input-9-23a646829fdc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mview\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_sync\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclosed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", | |
227 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36mapply_sync\u001b[1;34m(self, f, *args, **kwargs)\u001b[0m\n", |
|
227 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36mapply_sync\u001b[1;34m(self, f, *args, **kwargs)\u001b[0m\n", | |
228 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36mspin_after\u001b[1;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 73\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mspin_after\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 74\u001b[0m \u001b[1;34m\"\"\"call spin after the method.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 75\u001b[1;33m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 76\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mspin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 77\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", |
|
228 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36mspin_after\u001b[1;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 73\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mspin_after\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 74\u001b[0m \u001b[1;34m\"\"\"call spin after the method.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 75\u001b[1;33m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 76\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mspin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 77\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
229 |
"\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36mapply_sync\u001b[1;34m(self, f, *args, **kwargs)\u001b[0m\n\u001b[0;32m 24 |
|
229 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36mapply_sync\u001b[1;34m(self, f, *args, **kwargs)\u001b[0m\n\u001b[0;32m 245\u001b[0m \u001b[0mreturning\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 246\u001b[0m \"\"\"\n\u001b[1;32m--> 247\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_really_apply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mblock\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 248\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 249\u001b[0m \u001b[1;31m#----------------------------------------------------------------\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
230 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36m_really_apply\u001b[1;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n", |
|
230 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36m_really_apply\u001b[1;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n", | |
231 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36msync_results\u001b[1;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 64\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_sync_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 65\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 67\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_sync_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", |
|
231 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36msync_results\u001b[1;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 64\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_sync_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 65\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 67\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_sync_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
232 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36m_really_apply\u001b[1;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n", |
|
232 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36m_really_apply\u001b[1;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n", | |
233 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36msave_ids\u001b[1;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[0mn_previous\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 50\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 52\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[0mnmsgs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mn_previous\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", |
|
233 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36msave_ids\u001b[1;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[0mn_previous\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 50\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 52\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[0mnmsgs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mn_previous\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
234 |
"\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36m_really_apply\u001b[1;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n\u001b[0;32m 10 |
|
234 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/view.pyc\u001b[0m in \u001b[0;36m_really_apply\u001b[1;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n\u001b[0;32m 1049\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1050\u001b[0m msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,\n\u001b[1;32m-> 1051\u001b[1;33m metadata=metadata)\n\u001b[0m\u001b[0;32m 1052\u001b[0m \u001b[0mtracker\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mtrack\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mFalse\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tracker'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1053\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", | |
235 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/client.pyc\u001b[0m in \u001b[0;36msend_apply_request\u001b[1;34m(self, socket, f, args, kwargs, metadata, track, ident)\u001b[0m\n\u001b[0;32m 1252\u001b[0m bufs = serialize.pack_apply_message(f, args, kwargs,\n\u001b[0;32m 1253\u001b[0m \u001b[0mbuffer_threshold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuffer_threshold\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1254\u001b[1;33m \u001b[0mitem_threshold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem_threshold\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1255\u001b[0m )\n\u001b[0;32m 1256\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", |
|
235 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/parallel/client/client.pyc\u001b[0m in \u001b[0;36msend_apply_request\u001b[1;34m(self, socket, f, args, kwargs, metadata, track, ident)\u001b[0m\n\u001b[0;32m 1252\u001b[0m bufs = serialize.pack_apply_message(f, args, kwargs,\n\u001b[0;32m 1253\u001b[0m \u001b[0mbuffer_threshold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuffer_threshold\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1254\u001b[1;33m \u001b[0mitem_threshold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem_threshold\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1255\u001b[0m )\n\u001b[0;32m 1256\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", | |
236 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/kernel/zmq/serialize.pyc\u001b[0m in \u001b[0;36mpack_apply_message\u001b[1;34m(f, args, kwargs, buffer_threshold, item_threshold)\u001b[0m\n\u001b[0;32m 163\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnargs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnarg_bufs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg_bufs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkw_keys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkw_keys\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 164\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 165\u001b[1;33m \u001b[0mmsg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcan\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 166\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 167\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg_bufs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", |
|
236 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/kernel/zmq/serialize.pyc\u001b[0m in \u001b[0;36mpack_apply_message\u001b[1;34m(f, args, kwargs, buffer_threshold, item_threshold)\u001b[0m\n\u001b[0;32m 163\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnargs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnarg_bufs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg_bufs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkw_keys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkw_keys\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 164\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 165\u001b[1;33m \u001b[0mmsg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcan\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 166\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 167\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg_bufs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
237 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/utils/codeutil.pyc\u001b[0m in \u001b[0;36mreduce_code\u001b[1;34m(co)\u001b[0m\n\u001b[0;32m 36\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mreduce_code\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mco\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 37\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_freevars\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_cellvars\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Sorry, cannot pickle code objects with closures\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 39\u001b[0m args = [co.co_argcount, co.co_nlocals, co.co_stacksize,\n\u001b[0;32m 40\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_flags\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_consts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_names\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", |
|
237 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/utils/codeutil.pyc\u001b[0m in \u001b[0;36mreduce_code\u001b[1;34m(co)\u001b[0m\n\u001b[0;32m 36\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mreduce_code\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mco\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 37\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_freevars\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_cellvars\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Sorry, cannot pickle code objects with closures\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 39\u001b[0m args = [co.co_argcount, co.co_nlocals, co.co_stacksize,\n\u001b[0;32m 40\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_flags\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_consts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_names\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", | |
238 | "\u001b[1;31mValueError\u001b[0m: Sorry, cannot pickle code objects with closures" |
|
238 | "\u001b[1;31mValueError\u001b[0m: Sorry, cannot pickle code objects with closures" | |
239 | ] |
|
239 | ] | |
240 | } |
|
240 | } | |
241 | ], |
|
241 | ], | |
242 | "prompt_number": 9 |
|
242 | "prompt_number": 9 | |
243 | }, |
|
243 | }, | |
244 | { |
|
244 | { | |
245 | "cell_type": "markdown", |
|
245 | "cell_type": "markdown", | |
246 | "metadata": {}, |
|
246 | "metadata": {}, | |
247 | "source": [ |
|
247 | "source": [ | |
248 | "Oops, no dice. For IPython to work with dill,\n", |
|
248 | "Oops, no dice. For IPython to work with dill,\n", | |
249 |
"there are one or two more steps. IPython will do these for you if you call ` |
|
249 | "there are one or two more steps. IPython will do these for you if you call `DirectView.use_dill`:" | |
250 | ] |
|
250 | ] | |
251 | }, |
|
251 | }, | |
252 | { |
|
252 | { | |
253 | "cell_type": "code", |
|
253 | "cell_type": "code", | |
254 | "collapsed": false, |
|
254 | "collapsed": false, | |
255 | "input": [ |
|
255 | "input": [ | |
256 | "from IPython.utils import pickleutil\n", |
|
256 | "rc[:].use_dill()" | |
257 | "pickleutil.use_dill()" |
|
|||
258 | ], |
|
257 | ], | |
259 | "language": "python", |
|
258 | "language": "python", | |
260 | "metadata": {}, |
|
259 | "metadata": {}, | |
261 |
"outputs": [ |
|
260 | "outputs": [ | |
|
261 | { | |||
|
262 | "metadata": {}, | |||
|
263 | "output_type": "pyout", | |||
|
264 | "prompt_number": 10, | |||
|
265 | "text": [ | |||
|
266 | "<AsyncResult: use_dill>" | |||
|
267 | ] | |||
|
268 | } | |||
|
269 | ], | |||
262 | "prompt_number": 10 |
|
270 | "prompt_number": 10 | |
263 | }, |
|
271 | }, | |
264 | { |
|
272 | { | |
265 | "cell_type": "markdown", |
|
273 | "cell_type": "markdown", | |
266 | "metadata": {}, |
|
274 | "metadata": {}, | |
267 | "source": [ |
|
275 | "source": [ | |
|
276 | "This is equivalent to\n", | |||
|
277 | "\n", | |||
|
278 | "```python\n", | |||
|
279 | "from IPython.utils.pickleutil import use_dill\n", | |||
|
280 | "use_dill()\n", | |||
|
281 | "rc[:].apply(use_dill)\n", | |||
|
282 | "```" | |||
|
283 | ] | |||
|
284 | }, | |||
|
285 | { | |||
|
286 | "cell_type": "markdown", | |||
|
287 | "metadata": {}, | |||
|
288 | "source": [ | |||
268 | "Now let's try again" |
|
289 | "Now let's try again" | |
269 | ] |
|
290 | ] | |
270 | }, |
|
291 | }, | |
271 | { |
|
292 | { | |
272 | "cell_type": "code", |
|
293 | "cell_type": "code", | |
273 | "collapsed": false, |
|
294 | "collapsed": false, | |
274 | "input": [ |
|
295 | "input": [ | |
275 | "view.apply_sync(closed, 3)" |
|
296 | "view.apply_sync(closed, 3)" | |
276 | ], |
|
297 | ], | |
277 | "language": "python", |
|
298 | "language": "python", | |
278 | "metadata": {}, |
|
299 | "metadata": {}, | |
279 | "outputs": [ |
|
300 | "outputs": [ | |
280 | { |
|
301 | { | |
281 | "metadata": {}, |
|
302 | "metadata": {}, | |
282 | "output_type": "pyout", |
|
303 | "output_type": "pyout", | |
283 | "prompt_number": 11, |
|
304 | "prompt_number": 11, | |
284 | "text": [ |
|
305 | "text": [ | |
285 | "15" |
|
306 | "15" | |
286 | ] |
|
307 | ] | |
287 | } |
|
308 | } | |
288 | ], |
|
309 | ], | |
289 | "prompt_number": 11 |
|
310 | "prompt_number": 11 | |
290 | }, |
|
311 | }, | |
291 | { |
|
312 | { | |
292 | "cell_type": "markdown", |
|
313 | "cell_type": "markdown", | |
293 | "metadata": {}, |
|
314 | "metadata": {}, | |
294 | "source": [ |
|
315 | "source": [ | |
295 | "Yay! Now we can use dill to allow IPython.parallel to send anything.\n", |
|
316 | "Yay! Now we can use dill to allow IPython.parallel to send anything.\n", | |
296 | "\n", |
|
317 | "\n", | |
297 | "And that's it! We can send closures and other previously non-pickleables to our engines.\n", |
|
318 | "And that's it! We can send closures and other previously non-pickleables to our engines.\n", | |
298 | "\n", |
|
319 | "\n", | |
299 | "But wait, there's more!" |
|
320 | "Let's give it a try now:" | |
300 | ] |
|
321 | ] | |
301 | }, |
|
322 | }, | |
302 | { |
|
323 | { | |
303 | "cell_type": "code", |
|
324 | "cell_type": "code", | |
304 | "collapsed": false, |
|
325 | "collapsed": false, | |
305 | "input": [ |
|
326 | "input": [ | |
306 |
"view.apply_sync(make_closure, |
|
327 | "remote_closure = view.apply_sync(make_closure, 4)\n", | |
|
328 | "remote_closure(5)" | |||
307 | ], |
|
329 | ], | |
308 | "language": "python", |
|
330 | "language": "python", | |
309 | "metadata": {}, |
|
331 | "metadata": {}, | |
310 | "outputs": [ |
|
332 | "outputs": [ | |
311 | { |
|
333 | { | |
312 | "ename": "RemoteError", |
|
334 | "metadata": {}, | |
313 | "evalue": "ValueError(Sorry, cannot pickle code objects with closures)", |
|
335 | "output_type": "pyout", | |
314 | "output_type": "pyerr", |
|
336 | "prompt_number": 12, | |
315 |
"t |
|
337 | "text": [ | |
316 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", |
|
338 | "20" | |
317 | "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/kernel/zmq/serialize.pyc\u001b[0m in \u001b[0;36mserialize_object\u001b[1;34m(obj, buffer_threshold, item_threshold)\u001b[0m", |
|
|||
318 | "\u001b[0;32m 100\u001b[0m \u001b[0mbuffers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_extract_buffers\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer_threshold\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
319 | "\u001b[0;32m 101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
320 | "\u001b[1;32m--> 102\u001b[1;33m \u001b[0mbuffers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcobj\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
321 | "\u001b[0m\u001b[0;32m 103\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mbuffers\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
322 | "\u001b[0;32m 104\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
323 | "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/utils/codeutil.pyc\u001b[0m in \u001b[0;36mreduce_code\u001b[1;34m(co)\u001b[0m", |
|
|||
324 | "\u001b[0;32m 36\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mreduce_code\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mco\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
325 | "\u001b[0;32m 37\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_freevars\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_cellvars\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
326 | "\u001b[1;32m---> 38\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Sorry, cannot pickle code objects with closures\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
327 | "\u001b[0m\u001b[0;32m 39\u001b[0m args = [co.co_argcount, co.co_nlocals, co.co_stacksize,", |
|
|||
328 | "\u001b[0;32m 40\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_flags\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_consts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mco\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mco_names\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m", |
|
|||
329 | "\u001b[1;31mValueError\u001b[0m: Sorry, cannot pickle code objects with closures" |
|
|||
330 | ] |
|
339 | ] | |
331 | } |
|
340 | } | |
332 | ], |
|
341 | ], | |
333 | "prompt_number": 12 |
|
342 | "prompt_number": 12 | |
334 | }, |
|
343 | }, | |
335 | { |
|
344 | { | |
336 | "cell_type": "markdown", |
|
345 | "cell_type": "markdown", | |
337 | "metadata": {}, |
|
346 | "metadata": {}, | |
338 | "source": [ |
|
347 | "source": [ | |
339 | "If we want dill support for objects coming *from* the engines,\n", |
|
348 | "But wait, there's more!\n", | |
340 | "then we need to call `use_dill()` there as well.\n", |
|
|||
341 | "\n", |
|
349 | "\n", | |
342 | "`DirectView` objects have a method to call `use_dill` locally and on every engine:" |
|
350 | "At this point, we can send/recv all kinds of stuff" | |
343 | ] |
|
351 | ] | |
344 | }, |
|
352 | }, | |
345 | { |
|
353 | { | |
346 | "cell_type": "code", |
|
354 | "cell_type": "code", | |
347 | "collapsed": false, |
|
355 | "collapsed": false, | |
348 | "input": [ |
|
356 | "input": [ | |
349 | "rc[:].use_dill()" |
|
357 | "def outer(a):\n", | |
|
358 | " def inner(b):\n", | |||
|
359 | " def inner_again(c):\n", | |||
|
360 | " return c * b * a\n", | |||
|
361 | " return inner_again\n", | |||
|
362 | " return inner" | |||
350 | ], |
|
363 | ], | |
351 | "language": "python", |
|
364 | "language": "python", | |
352 | "metadata": {}, |
|
365 | "metadata": {}, | |
353 | "outputs": [ |
|
366 | "outputs": [], | |
354 | { |
|
|||
355 | "metadata": {}, |
|
|||
356 | "output_type": "pyout", |
|
|||
357 | "prompt_number": 13, |
|
|||
358 | "text": [ |
|
|||
359 | "<AsyncResult: use_dill>" |
|
|||
360 | ] |
|
|||
361 | } |
|
|||
362 | ], |
|
|||
363 | "prompt_number": 13 |
|
367 | "prompt_number": 13 | |
364 | }, |
|
368 | }, | |
365 | { |
|
369 | { | |
366 | "cell_type": "markdown", |
|
370 | "cell_type": "markdown", | |
367 | "metadata": {}, |
|
371 | "metadata": {}, | |
368 | "source": [ |
|
372 | "source": [ | |
369 | "This is equivalent to\n", |
|
373 | "So outer returns a function with a closure, which returns a function with a closure.\n", | |
370 | "\n", |
|
374 | "\n", | |
371 | "```python\n", |
|
375 | "Now, we can resolve the first closure on the engine, the second here, and the third on a different engine,\n", | |
372 | "from IPython.utils.pickleutil import use_dill\n", |
|
376 | "after passing through a lambda we define here and call there, just for good measure." | |
373 | "use_dill()\n", |
|
|||
374 | "rc[:].apply(use_dill)\n", |
|
|||
375 | "```" |
|
|||
376 | ] |
|
377 | ] | |
377 | }, |
|
378 | }, | |
378 | { |
|
379 | { | |
|
380 | "cell_type": "code", | |||
|
381 | "collapsed": false, | |||
|
382 | "input": [ | |||
|
383 | "view.apply_sync(lambda f: f(3),view.apply_sync(outer, 1)(2))" | |||
|
384 | ], | |||
|
385 | "language": "python", | |||
|
386 | "metadata": {}, | |||
|
387 | "outputs": [ | |||
|
388 | { | |||
|
389 | "metadata": {}, | |||
|
390 | "output_type": "pyout", | |||
|
391 | "prompt_number": 14, | |||
|
392 | "text": [ | |||
|
393 | "6" | |||
|
394 | ] | |||
|
395 | } | |||
|
396 | ], | |||
|
397 | "prompt_number": 14 | |||
|
398 | }, | |||
|
399 | { | |||
379 | "cell_type": "markdown", |
|
400 | "cell_type": "markdown", | |
380 | "metadata": {}, |
|
401 | "metadata": {}, | |
381 | "source": [ |
|
402 | "source": [ | |
382 | "Let's give it a try now:" |
|
403 | "And for good measure, let's test that normal execution still works:" | |
383 | ] |
|
404 | ] | |
384 | }, |
|
405 | }, | |
385 | { |
|
406 | { | |
386 | "cell_type": "code", |
|
407 | "cell_type": "code", | |
387 | "collapsed": false, |
|
408 | "collapsed": false, | |
388 | "input": [ |
|
409 | "input": [ | |
389 | "remote_closure = view.apply_sync(make_closure, 4)\n", |
|
410 | "%px foo = 5\n", | |
390 | "remote_closure(5)" |
|
411 | "\n", | |
|
412 | "print(rc[:]['foo'])\n", | |||
|
413 | "rc[:]['bar'] = lambda : 2 * foo\n", | |||
|
414 | "rc[:].apply_sync(parallel.Reference('bar'))" | |||
391 | ], |
|
415 | ], | |
392 | "language": "python", |
|
416 | "language": "python", | |
393 | "metadata": {}, |
|
417 | "metadata": {}, | |
394 | "outputs": [ |
|
418 | "outputs": [ | |
395 | { |
|
419 | { | |
|
420 | "output_type": "stream", | |||
|
421 | "stream": "stdout", | |||
|
422 | "text": [ | |||
|
423 | "[5, 5, 5, 5, 5, 5, 5, 5]\n" | |||
|
424 | ] | |||
|
425 | }, | |||
|
426 | { | |||
396 | "metadata": {}, |
|
427 | "metadata": {}, | |
397 | "output_type": "pyout", |
|
428 | "output_type": "pyout", | |
398 |
"prompt_number": 1 |
|
429 | "prompt_number": 15, | |
399 | "text": [ |
|
430 | "text": [ | |
400 | "20" |
|
431 | "[10, 10, 10, 10, 10, 10, 10, 10]" | |
401 | ] |
|
432 | ] | |
402 | } |
|
433 | } | |
403 | ], |
|
434 | ], | |
404 |
"prompt_number": 1 |
|
435 | "prompt_number": 15 | |
405 | }, |
|
436 | }, | |
406 | { |
|
437 | { | |
407 | "cell_type": "markdown", |
|
438 | "cell_type": "markdown", | |
408 | "metadata": {}, |
|
439 | "metadata": {}, | |
409 | "source": [ |
|
440 | "source": [ | |
410 | "At this point, we can send/recv all kinds of stuff" |
|
441 | "And test that the `@interactive` decorator works" | |
411 | ] |
|
442 | ] | |
412 | }, |
|
443 | }, | |
413 | { |
|
444 | { | |
414 | "cell_type": "code", |
|
445 | "cell_type": "code", | |
415 | "collapsed": false, |
|
446 | "collapsed": false, | |
416 | "input": [ |
|
447 | "input": [ | |
417 | "def outer(a):\n", |
|
448 | "%%file testdill.py\n", | |
418 | " def inner(b):\n", |
|
449 | "from IPython.parallel import interactive\n", | |
419 | " def inner_again(c):\n", |
|
450 | "\n", | |
420 | " return c * b * a\n", |
|
451 | "@interactive\n", | |
421 | " return inner_again\n", |
|
452 | "class C(object):\n", | |
422 | " return inner" |
|
453 | " a = 5\n", | |
|
454 | "\n", | |||
|
455 | "@interactive\n", | |||
|
456 | "class D(C):\n", | |||
|
457 | " b = 10\n", | |||
|
458 | "\n", | |||
|
459 | "@interactive\n", | |||
|
460 | "def foo(a):\n", | |||
|
461 | " return a * b\n" | |||
|
462 | ], | |||
|
463 | "language": "python", | |||
|
464 | "metadata": {}, | |||
|
465 | "outputs": [ | |||
|
466 | { | |||
|
467 | "output_type": "stream", | |||
|
468 | "stream": "stdout", | |||
|
469 | "text": [ | |||
|
470 | "Overwriting testdill.py\n" | |||
|
471 | ] | |||
|
472 | } | |||
|
473 | ], | |||
|
474 | "prompt_number": 16 | |||
|
475 | }, | |||
|
476 | { | |||
|
477 | "cell_type": "code", | |||
|
478 | "collapsed": false, | |||
|
479 | "input": [ | |||
|
480 | "import testdill" | |||
423 | ], |
|
481 | ], | |
424 | "language": "python", |
|
482 | "language": "python", | |
425 | "metadata": {}, |
|
483 | "metadata": {}, | |
426 | "outputs": [], |
|
484 | "outputs": [], | |
427 |
"prompt_number": 1 |
|
485 | "prompt_number": 17 | |
428 | }, |
|
486 | }, | |
429 | { |
|
487 | { | |
430 |
"cell_type": " |
|
488 | "cell_type": "code", | |
|
489 | "collapsed": false, | |||
|
490 | "input": [ | |||
|
491 | "v = rc[-1]\n", | |||
|
492 | "v['D'] = testdill.D\n", | |||
|
493 | "d = v.apply_sync(lambda : D())\n", | |||
|
494 | "print d.a, d.b" | |||
|
495 | ], | |||
|
496 | "language": "python", | |||
431 | "metadata": {}, |
|
497 | "metadata": {}, | |
432 |
" |
|
498 | "outputs": [ | |
433 | "So outer returns a function with a closure, which returns a function with a closure.\n", |
|
499 | { | |
434 | "\n", |
|
500 | "output_type": "stream", | |
435 | "Now, we can resolve the first closure on the engine, the second here, and the third on a different engine,\n", |
|
501 | "stream": "stdout", | |
436 | "after passing through a lambda we define here and call there, just for good measure." |
|
502 | "text": [ | |
437 | ] |
|
503 | "5 10\n" | |
|
504 | ] | |||
|
505 | } | |||
|
506 | ], | |||
|
507 | "prompt_number": 18 | |||
438 | }, |
|
508 | }, | |
439 | { |
|
509 | { | |
440 | "cell_type": "code", |
|
510 | "cell_type": "code", | |
441 | "collapsed": false, |
|
511 | "collapsed": false, | |
442 | "input": [ |
|
512 | "input": [ | |
443 | "view.apply_sync(lambda f: f(3),view.apply_sync(outer, 1)(2))" |
|
513 | "v['b'] = 10\n", | |
|
514 | "v.apply_sync(testdill.foo, 5)" | |||
444 | ], |
|
515 | ], | |
445 | "language": "python", |
|
516 | "language": "python", | |
446 | "metadata": {}, |
|
517 | "metadata": {}, | |
447 | "outputs": [ |
|
518 | "outputs": [ | |
448 | { |
|
519 | { | |
449 | "metadata": {}, |
|
520 | "metadata": {}, | |
450 | "output_type": "pyout", |
|
521 | "output_type": "pyout", | |
451 |
"prompt_number": 1 |
|
522 | "prompt_number": 19, | |
452 | "text": [ |
|
523 | "text": [ | |
453 |
" |
|
524 | "50" | |
454 | ] |
|
525 | ] | |
455 | } |
|
526 | } | |
456 | ], |
|
527 | ], | |
457 |
"prompt_number": 1 |
|
528 | "prompt_number": 19 | |
458 | } |
|
529 | } | |
459 | ], |
|
530 | ], | |
460 | "metadata": {} |
|
531 | "metadata": {} | |
461 | } |
|
532 | } | |
462 | ] |
|
533 | ] | |
463 | } No newline at end of file |
|
534 | } |
General Comments 0
You need to be logged in to leave comments.
Login now