Show More
@@ -0,0 +1,90 b'' | |||
|
1 | from numpy import * | |
|
2 | ||
|
3 | def mandel(n, m, itermax, xmin, xmax, ymin, ymax): | |
|
4 | ''' | |
|
5 | Fast mandelbrot computation using numpy. | |
|
6 | ||
|
7 | (n, m) are the output image dimensions | |
|
8 | itermax is the maximum number of iterations to do | |
|
9 | xmin, xmax, ymin, ymax specify the region of the | |
|
10 | set to compute. | |
|
11 | ''' | |
|
12 | # The point of ix and iy is that they are 2D arrays | |
|
13 | # giving the x-coord and y-coord at each point in | |
|
14 | # the array. The reason for doing this will become | |
|
15 | # clear below... | |
|
16 | ix, iy = mgrid[0:n, 0:m] | |
|
17 | # Now x and y are the x-values and y-values at each | |
|
18 | # point in the array, linspace(start, end, n) | |
|
19 | # is an array of n linearly spaced points between | |
|
20 | # start and end, and we then index this array using | |
|
21 | # numpy fancy indexing. If A is an array and I is | |
|
22 | # an array of indices, then A[I] has the same shape | |
|
23 | # as I and at each place i in I has the value A[i]. | |
|
24 | x = linspace(xmin, xmax, n)[ix] | |
|
25 | y = linspace(ymin, ymax, m)[iy] | |
|
26 | # c is the complex number with the given x, y coords | |
|
27 | c = x+complex(0,1)*y | |
|
28 | del x, y # save a bit of memory, we only need z | |
|
29 | # the output image coloured according to the number | |
|
30 | # of iterations it takes to get to the boundary | |
|
31 | # abs(z)>2 | |
|
32 | img = zeros(c.shape, dtype=int) | |
|
33 | # Here is where the improvement over the standard | |
|
34 | # algorithm for drawing fractals in numpy comes in. | |
|
35 | # We flatten all the arrays ix, iy and c. This | |
|
36 | # flattening doesn't use any more memory because | |
|
37 | # we are just changing the shape of the array, the | |
|
38 | # data in memory stays the same. It also affects | |
|
39 | # each array in the same way, so that index i in | |
|
40 | # array c has x, y coords ix[i], iy[i]. The way the | |
|
41 | # algorithm works is that whenever abs(z)>2 we | |
|
42 | # remove the corresponding index from each of the | |
|
43 | # arrays ix, iy and c. Since we do the same thing | |
|
44 | # to each array, the correspondence between c and | |
|
45 | # the x, y coords stored in ix and iy is kept. | |
|
46 | ix.shape = n*m | |
|
47 | iy.shape = n*m | |
|
48 | c.shape = n*m | |
|
49 | # we iterate z->z^2+c with z starting at 0, but the | |
|
50 | # first iteration makes z=c so we just start there. | |
|
51 | # We need to copy c because otherwise the operation | |
|
52 | # z->z^2 will send c->c^2. | |
|
53 | z = copy(c) | |
|
54 | for i in xrange(itermax): | |
|
55 | if not len(z): break # all points have escaped | |
|
56 | # equivalent to z = z*z+c but quicker and uses | |
|
57 | # less memory | |
|
58 | multiply(z, z, z) | |
|
59 | add(z, c, z) | |
|
60 | # these are the points that have escaped | |
|
61 | rem = abs(z)>2.0 | |
|
62 | # colour them with the iteration number, we | |
|
63 | # add one so that points which haven't | |
|
64 | # escaped have 0 as their iteration number, | |
|
65 | # this is why we keep the arrays ix and iy | |
|
66 | # because we need to know which point in img | |
|
67 | # to colour | |
|
68 | img[ix[rem], iy[rem]] = i+1 | |
|
69 | # -rem is the array of points which haven't | |
|
70 | # escaped, in numpy -A for a boolean array A | |
|
71 | # is the NOT operation. | |
|
72 | rem = -rem | |
|
73 | # So we select out the points in | |
|
74 | # z, ix, iy and c which are still to be | |
|
75 | # iterated on in the next step | |
|
76 | z = z[rem] | |
|
77 | ix, iy = ix[rem], iy[rem] | |
|
78 | c = c[rem] | |
|
79 | return img | |
|
80 | ||
|
81 | if __name__=='__main__': | |
|
82 | from pylab import * | |
|
83 | import time | |
|
84 | start = time.time() | |
|
85 | I = mandel(400, 400, 100, -2, .5, -1.25, 1.25) | |
|
86 | print 'Time taken:', time.time()-start | |
|
87 | I[I==0] = 101 | |
|
88 | img = imshow(I.T, origin='lower left') | |
|
89 | img.write_png('mandel.png', noscale=True) | |
|
90 | show() |
@@ -1,394 +1,458 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | # encoding: utf-8 |
|
3 | 3 | """ |
|
4 | 4 | The IPython cluster directory |
|
5 | 5 | """ |
|
6 | 6 | |
|
7 | 7 | #----------------------------------------------------------------------------- |
|
8 | 8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
9 | 9 | # |
|
10 | 10 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 11 | # the file COPYING, distributed as part of this software. |
|
12 | 12 | #----------------------------------------------------------------------------- |
|
13 | 13 | |
|
14 | 14 | #----------------------------------------------------------------------------- |
|
15 | 15 | # Imports |
|
16 | 16 | #----------------------------------------------------------------------------- |
|
17 | 17 | |
|
18 | from __future__ import with_statement | |
|
19 | ||
|
18 | 20 | import os |
|
19 | 21 | import shutil |
|
20 | 22 | import sys |
|
21 | 23 | |
|
22 | 24 | from twisted.python import log |
|
23 | 25 | |
|
24 | 26 | from IPython.core import release |
|
25 | 27 | from IPython.config.loader import PyFileConfigLoader |
|
26 | 28 | from IPython.core.application import Application |
|
27 | 29 | from IPython.core.component import Component |
|
28 | 30 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault |
|
29 | 31 | from IPython.utils.traitlets import Unicode, Bool |
|
30 | 32 | |
|
31 | 33 | #----------------------------------------------------------------------------- |
|
32 | 34 | # Imports |
|
33 | 35 | #----------------------------------------------------------------------------- |
|
34 | 36 | |
|
35 | 37 | |
|
36 | 38 | class ClusterDirError(Exception): |
|
37 | 39 | pass |
|
38 | 40 | |
|
39 | 41 | |
|
42 | class PIDFileError(Exception): | |
|
43 | pass | |
|
44 | ||
|
45 | ||
|
40 | 46 | class ClusterDir(Component): |
|
41 | 47 | """An object to manage the cluster directory and its resources. |
|
42 | 48 | |
|
43 | 49 | The cluster directory is used by :command:`ipcontroller`, |
|
44 | 50 | :command:`ipcontroller` and :command:`ipcontroller` to manage the |
|
45 | 51 | configuration, logging and security of these applications. |
|
46 | 52 | |
|
47 | 53 | This object knows how to find, create and manage these directories. This |
|
48 | 54 | should be used by any code that want's to handle cluster directories. |
|
49 | 55 | """ |
|
50 | 56 | |
|
51 | 57 | security_dir_name = Unicode('security') |
|
52 | 58 | log_dir_name = Unicode('log') |
|
53 |
|
|
|
54 |
|
|
|
55 |
lo |
|
|
59 | pid_dir_name = Unicode('pid') | |
|
60 | security_dir = Unicode(u'') | |
|
61 | log_dir = Unicode(u'') | |
|
62 | pid_dir = Unicode(u'') | |
|
63 | location = Unicode(u'') | |
|
56 | 64 | |
|
57 | 65 | def __init__(self, location): |
|
58 | 66 | super(ClusterDir, self).__init__(None) |
|
59 | 67 | self.location = location |
|
60 | 68 | |
|
61 | 69 | def _location_changed(self, name, old, new): |
|
62 | 70 | if not os.path.isdir(new): |
|
63 | 71 | os.makedirs(new, mode=0777) |
|
64 | 72 | else: |
|
65 | 73 | os.chmod(new, 0777) |
|
66 | 74 | self.security_dir = os.path.join(new, self.security_dir_name) |
|
67 | 75 | self.log_dir = os.path.join(new, self.log_dir_name) |
|
76 | self.pid_dir = os.path.join(new, self.pid_dir_name) | |
|
68 | 77 | self.check_dirs() |
|
69 | 78 | |
|
70 | 79 | def _log_dir_changed(self, name, old, new): |
|
71 | 80 | self.check_log_dir() |
|
72 | 81 | |
|
73 | 82 | def check_log_dir(self): |
|
74 | 83 | if not os.path.isdir(self.log_dir): |
|
75 | 84 | os.mkdir(self.log_dir, 0777) |
|
76 | 85 | else: |
|
77 | 86 | os.chmod(self.log_dir, 0777) |
|
78 | 87 | |
|
79 | 88 | def _security_dir_changed(self, name, old, new): |
|
80 | 89 | self.check_security_dir() |
|
81 | 90 | |
|
82 | 91 | def check_security_dir(self): |
|
83 | 92 | if not os.path.isdir(self.security_dir): |
|
84 | 93 | os.mkdir(self.security_dir, 0700) |
|
85 | 94 | else: |
|
86 | 95 | os.chmod(self.security_dir, 0700) |
|
87 | 96 | |
|
97 | def _pid_dir_changed(self, name, old, new): | |
|
98 | self.check_pid_dir() | |
|
99 | ||
|
100 | def check_pid_dir(self): | |
|
101 | if not os.path.isdir(self.pid_dir): | |
|
102 | os.mkdir(self.pid_dir, 0700) | |
|
103 | else: | |
|
104 | os.chmod(self.pid_dir, 0700) | |
|
105 | ||
|
88 | 106 | def check_dirs(self): |
|
89 | 107 | self.check_security_dir() |
|
90 | 108 | self.check_log_dir() |
|
109 | self.check_pid_dir() | |
|
91 | 110 | |
|
92 | 111 | def load_config_file(self, filename): |
|
93 | 112 | """Load a config file from the top level of the cluster dir. |
|
94 | 113 | |
|
95 | 114 | Parameters |
|
96 | 115 | ---------- |
|
97 | 116 | filename : unicode or str |
|
98 | 117 | The filename only of the config file that must be located in |
|
99 | 118 | the top-level of the cluster directory. |
|
100 | 119 | """ |
|
101 | 120 | loader = PyFileConfigLoader(filename, self.location) |
|
102 | 121 | return loader.load_config() |
|
103 | 122 | |
|
104 | 123 | def copy_config_file(self, config_file, path=None, overwrite=False): |
|
105 | 124 | """Copy a default config file into the active cluster directory. |
|
106 | 125 | |
|
107 | 126 | Default configuration files are kept in :mod:`IPython.config.default`. |
|
108 | 127 | This function moves these from that location to the working cluster |
|
109 | 128 | directory. |
|
110 | 129 | """ |
|
111 | 130 | if path is None: |
|
112 | 131 | import IPython.config.default |
|
113 | 132 | path = IPython.config.default.__file__.split(os.path.sep)[:-1] |
|
114 | 133 | path = os.path.sep.join(path) |
|
115 | 134 | src = os.path.join(path, config_file) |
|
116 | 135 | dst = os.path.join(self.location, config_file) |
|
117 | 136 | if not os.path.isfile(dst) or overwrite: |
|
118 | 137 | shutil.copy(src, dst) |
|
119 | 138 | |
|
120 | 139 | def copy_all_config_files(self, path=None, overwrite=False): |
|
121 | 140 | """Copy all config files into the active cluster directory.""" |
|
122 | 141 | for f in ['ipcontroller_config.py', 'ipengine_config.py', |
|
123 | 142 | 'ipcluster_config.py']: |
|
124 | 143 | self.copy_config_file(f, path=path, overwrite=overwrite) |
|
125 | 144 | |
|
126 | 145 | @classmethod |
|
127 | 146 | def create_cluster_dir(csl, cluster_dir): |
|
128 | 147 | """Create a new cluster directory given a full path. |
|
129 | 148 | |
|
130 | 149 | Parameters |
|
131 | 150 | ---------- |
|
132 | 151 | cluster_dir : str |
|
133 | 152 | The full path to the cluster directory. If it does exist, it will |
|
134 | 153 | be used. If not, it will be created. |
|
135 | 154 | """ |
|
136 | 155 | return ClusterDir(cluster_dir) |
|
137 | 156 | |
|
138 | 157 | @classmethod |
|
139 | 158 | def create_cluster_dir_by_profile(cls, path, profile='default'): |
|
140 | 159 | """Create a cluster dir by profile name and path. |
|
141 | 160 | |
|
142 | 161 | Parameters |
|
143 | 162 | ---------- |
|
144 | 163 | path : str |
|
145 | 164 | The path (directory) to put the cluster directory in. |
|
146 | 165 | profile : str |
|
147 | 166 | The name of the profile. The name of the cluster directory will |
|
148 | 167 | be "cluster_<profile>". |
|
149 | 168 | """ |
|
150 | 169 | if not os.path.isdir(path): |
|
151 | 170 | raise ClusterDirError('Directory not found: %s' % path) |
|
152 | 171 | cluster_dir = os.path.join(path, 'cluster_' + profile) |
|
153 | 172 | return ClusterDir(cluster_dir) |
|
154 | 173 | |
|
155 | 174 | @classmethod |
|
156 | 175 | def find_cluster_dir_by_profile(cls, ipythondir, profile='default'): |
|
157 | 176 | """Find an existing cluster dir by profile name, return its ClusterDir. |
|
158 | 177 | |
|
159 | 178 | This searches through a sequence of paths for a cluster dir. If it |
|
160 | 179 | is not found, a :class:`ClusterDirError` exception will be raised. |
|
161 | 180 | |
|
162 | 181 | The search path algorithm is: |
|
163 | 182 | 1. ``os.getcwd()`` |
|
164 | 183 | 2. ``ipythondir`` |
|
165 | 184 | 3. The directories found in the ":" separated |
|
166 | 185 | :env:`IPCLUSTERDIR_PATH` environment variable. |
|
167 | 186 | |
|
168 | 187 | Parameters |
|
169 | 188 | ---------- |
|
170 | 189 | ipythondir : unicode or str |
|
171 | 190 | The IPython directory to use. |
|
172 | 191 | profile : unicode or str |
|
173 | 192 | The name of the profile. The name of the cluster directory |
|
174 | 193 | will be "cluster_<profile>". |
|
175 | 194 | """ |
|
176 | 195 | dirname = 'cluster_' + profile |
|
177 | 196 | cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','') |
|
178 | 197 | if cluster_dir_paths: |
|
179 | 198 | cluster_dir_paths = cluster_dir_paths.split(':') |
|
180 | 199 | else: |
|
181 | 200 | cluster_dir_paths = [] |
|
182 | 201 | paths = [os.getcwd(), ipythondir] + cluster_dir_paths |
|
183 | 202 | for p in paths: |
|
184 | 203 | cluster_dir = os.path.join(p, dirname) |
|
185 | 204 | if os.path.isdir(cluster_dir): |
|
186 | 205 | return ClusterDir(cluster_dir) |
|
187 | 206 | else: |
|
188 | 207 | raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) |
|
189 | 208 | |
|
190 | 209 | @classmethod |
|
191 | 210 | def find_cluster_dir(cls, cluster_dir): |
|
192 | 211 | """Find/create a cluster dir and return its ClusterDir. |
|
193 | 212 | |
|
194 | 213 | This will create the cluster directory if it doesn't exist. |
|
195 | 214 | |
|
196 | 215 | Parameters |
|
197 | 216 | ---------- |
|
198 | 217 | cluster_dir : unicode or str |
|
199 | 218 | The path of the cluster directory. This is expanded using |
|
200 | 219 | :func:`os.path.expandvars` and :func:`os.path.expanduser`. |
|
201 | 220 | """ |
|
202 | 221 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) |
|
203 | 222 | if not os.path.isdir(cluster_dir): |
|
204 | 223 | raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) |
|
205 | 224 | return ClusterDir(cluster_dir) |
|
206 | 225 | |
|
207 | 226 | |
|
208 | 227 | class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader): |
|
209 | 228 | """Default command line options for IPython cluster applications.""" |
|
210 | 229 | |
|
211 | 230 | def _add_other_arguments(self): |
|
212 |
self.parser.add_argument(' |
|
|
231 | self.parser.add_argument('--ipython-dir', | |
|
213 | 232 | dest='Global.ipythondir',type=str, |
|
214 | 233 | help='Set to override default location of Global.ipythondir.', |
|
215 | 234 | default=NoConfigDefault, |
|
216 | 235 | metavar='Global.ipythondir' |
|
217 | 236 | ) |
|
218 |
self.parser.add_argument('-p', |
|
|
237 | self.parser.add_argument('-p', '--profile', | |
|
219 | 238 | dest='Global.profile',type=str, |
|
220 | 239 | help='The string name of the profile to be used. This determines ' |
|
221 | 240 | 'the name of the cluster dir as: cluster_<profile>. The default profile ' |
|
222 | 241 | 'is named "default". The cluster directory is resolve this way ' |
|
223 | 242 | 'if the --cluster-dir option is not used.', |
|
224 | 243 | default=NoConfigDefault, |
|
225 | 244 | metavar='Global.profile' |
|
226 | 245 | ) |
|
227 |
self.parser.add_argument(' |
|
|
246 | self.parser.add_argument('--log-level', | |
|
228 | 247 | dest="Global.log_level",type=int, |
|
229 | 248 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
230 | 249 | default=NoConfigDefault, |
|
231 | 250 | metavar="Global.log_level" |
|
232 | 251 | ) |
|
233 |
self.parser.add_argument(' |
|
|
252 | self.parser.add_argument('--cluster-dir', | |
|
234 | 253 | dest='Global.cluster_dir',type=str, |
|
235 | 254 | help='Set the cluster dir. This overrides the logic used by the ' |
|
236 | 255 | '--profile option.', |
|
237 | 256 | default=NoConfigDefault, |
|
238 | 257 | metavar='Global.cluster_dir' |
|
239 | 258 | ) |
|
240 |
self.parser.add_argument(' |
|
|
259 | self.parser.add_argument('--clean-logs', | |
|
241 | 260 | dest='Global.clean_logs', action='store_true', |
|
242 | 261 | help='Delete old log flies before starting.', |
|
243 | 262 | default=NoConfigDefault |
|
244 | 263 | ) |
|
245 |
self.parser.add_argument(' |
|
|
264 | self.parser.add_argument('--no-clean-logs', | |
|
246 | 265 | dest='Global.clean_logs', action='store_false', |
|
247 | 266 | help="Don't Delete old log flies before starting.", |
|
248 | 267 | default=NoConfigDefault |
|
249 | 268 | ) |
|
250 | 269 | |
|
251 | 270 | class ApplicationWithClusterDir(Application): |
|
252 | 271 | """An application that puts everything into a cluster directory. |
|
253 | 272 | |
|
254 | 273 | Instead of looking for things in the ipythondir, this type of application |
|
255 | 274 | will use its own private directory called the "cluster directory" |
|
256 | 275 | for things like config files, log files, etc. |
|
257 | 276 | |
|
258 | 277 | The cluster directory is resolved as follows: |
|
259 | 278 | |
|
260 | 279 | * If the ``--cluster-dir`` option is given, it is used. |
|
261 | 280 | * If ``--cluster-dir`` is not given, the application directory is |
|
262 | 281 | resolve using the profile name as ``cluster_<profile>``. The search |
|
263 | 282 | path for this directory is then i) cwd if it is found there |
|
264 | 283 | and ii) in ipythondir otherwise. |
|
265 | 284 | |
|
266 | 285 | The config file for the application is to be put in the cluster |
|
267 | 286 | dir and named the value of the ``config_file_name`` class attribute. |
|
268 | 287 | """ |
|
269 | 288 | |
|
270 | 289 | auto_create_cluster_dir = True |
|
271 | 290 | |
|
272 | 291 | def create_default_config(self): |
|
273 | 292 | super(ApplicationWithClusterDir, self).create_default_config() |
|
274 | 293 | self.default_config.Global.profile = 'default' |
|
275 | 294 | self.default_config.Global.cluster_dir = '' |
|
276 | 295 | self.default_config.Global.log_to_file = False |
|
277 | 296 | self.default_config.Global.clean_logs = False |
|
278 | 297 | |
|
279 | 298 | def create_command_line_config(self): |
|
280 | 299 | """Create and return a command line config loader.""" |
|
281 | 300 | return AppWithClusterDirArgParseConfigLoader( |
|
282 | 301 | description=self.description, |
|
283 | 302 | version=release.version |
|
284 | 303 | ) |
|
285 | 304 | |
|
286 | 305 | def find_resources(self): |
|
287 | 306 | """This resolves the cluster directory. |
|
288 | 307 | |
|
289 | 308 | This tries to find the cluster directory and if successful, it will |
|
290 | 309 | have done: |
|
291 | 310 | * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for |
|
292 | 311 | the application. |
|
293 | 312 | * Sets ``self.cluster_dir`` attribute of the application and config |
|
294 | 313 | objects. |
|
295 | 314 | |
|
296 | 315 | The algorithm used for this is as follows: |
|
297 | 316 | 1. Try ``Global.cluster_dir``. |
|
298 | 317 | 2. Try using ``Global.profile``. |
|
299 | 318 | 3. If both of these fail and ``self.auto_create_cluster_dir`` is |
|
300 | 319 | ``True``, then create the new cluster dir in the IPython directory. |
|
301 | 320 | 4. If all fails, then raise :class:`ClusterDirError`. |
|
302 | 321 | """ |
|
303 | 322 | |
|
304 | 323 | try: |
|
305 | 324 | cluster_dir = self.command_line_config.Global.cluster_dir |
|
306 | 325 | except AttributeError: |
|
307 | 326 | cluster_dir = self.default_config.Global.cluster_dir |
|
308 | 327 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) |
|
309 | 328 | try: |
|
310 | 329 | self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) |
|
311 | 330 | except ClusterDirError: |
|
312 | 331 | pass |
|
313 | 332 | else: |
|
314 | 333 | self.log.info('Using existing cluster dir: %s' % \ |
|
315 | 334 | self.cluster_dir_obj.location |
|
316 | 335 | ) |
|
317 | 336 | self.finish_cluster_dir() |
|
318 | 337 | return |
|
319 | 338 | |
|
320 | 339 | try: |
|
321 | 340 | self.profile = self.command_line_config.Global.profile |
|
322 | 341 | except AttributeError: |
|
323 | 342 | self.profile = self.default_config.Global.profile |
|
324 | 343 | try: |
|
325 | 344 | self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( |
|
326 | 345 | self.ipythondir, self.profile) |
|
327 | 346 | except ClusterDirError: |
|
328 | 347 | pass |
|
329 | 348 | else: |
|
330 | 349 | self.log.info('Using existing cluster dir: %s' % \ |
|
331 | 350 | self.cluster_dir_obj.location |
|
332 | 351 | ) |
|
333 | 352 | self.finish_cluster_dir() |
|
334 | 353 | return |
|
335 | 354 | |
|
336 | 355 | if self.auto_create_cluster_dir: |
|
337 | 356 | self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( |
|
338 | 357 | self.ipythondir, self.profile |
|
339 | 358 | ) |
|
340 | 359 | self.log.info('Creating new cluster dir: %s' % \ |
|
341 | 360 | self.cluster_dir_obj.location |
|
342 | 361 | ) |
|
343 | 362 | self.finish_cluster_dir() |
|
344 | 363 | else: |
|
345 | 364 | raise ClusterDirError('Could not find a valid cluster directory.') |
|
346 | 365 | |
|
347 | 366 | def finish_cluster_dir(self): |
|
348 | 367 | # Set the cluster directory |
|
349 | 368 | self.cluster_dir = self.cluster_dir_obj.location |
|
350 | 369 | |
|
351 | 370 | # These have to be set because they could be different from the one |
|
352 | 371 | # that we just computed. Because command line has the highest |
|
353 | 372 | # priority, this will always end up in the master_config. |
|
354 | 373 | self.default_config.Global.cluster_dir = self.cluster_dir |
|
355 | 374 | self.command_line_config.Global.cluster_dir = self.cluster_dir |
|
356 | 375 | |
|
357 | 376 | # Set the search path to the cluster directory |
|
358 | 377 | self.config_file_paths = (self.cluster_dir,) |
|
359 | 378 | |
|
360 | 379 | def find_config_file_name(self): |
|
361 | 380 | """Find the config file name for this application.""" |
|
362 | 381 | # For this type of Application it should be set as a class attribute. |
|
363 | 382 | if not hasattr(self, 'config_file_name'): |
|
364 | 383 | self.log.critical("No config filename found") |
|
365 | 384 | |
|
366 | 385 | def find_config_file_paths(self): |
|
367 | 386 | # Set the search path to the cluster directory |
|
368 | 387 | self.config_file_paths = (self.cluster_dir,) |
|
369 | 388 | |
|
370 | 389 | def pre_construct(self): |
|
371 | 390 | # The log and security dirs were set earlier, but here we put them |
|
372 | 391 | # into the config and log them. |
|
373 | 392 | config = self.master_config |
|
374 | 393 | sdir = self.cluster_dir_obj.security_dir |
|
375 | 394 | self.security_dir = config.Global.security_dir = sdir |
|
376 | 395 | ldir = self.cluster_dir_obj.log_dir |
|
377 | 396 | self.log_dir = config.Global.log_dir = ldir |
|
397 | pdir = self.cluster_dir_obj.pid_dir | |
|
398 | self.pid_dir = config.Global.pid_dir = pdir | |
|
378 | 399 | self.log.info("Cluster directory set to: %s" % self.cluster_dir) |
|
379 | 400 | |
|
380 | 401 | def start_logging(self): |
|
381 | 402 | # Remove old log files |
|
382 | 403 | if self.master_config.Global.clean_logs: |
|
383 | 404 | log_dir = self.master_config.Global.log_dir |
|
384 | 405 | for f in os.listdir(log_dir): |
|
385 | 406 | if f.startswith(self.name + '-') and f.endswith('.log'): |
|
386 | 407 | os.remove(os.path.join(log_dir, f)) |
|
387 | 408 | # Start logging to the new log file |
|
388 | 409 | if self.master_config.Global.log_to_file: |
|
389 | 410 | log_filename = self.name + '-' + str(os.getpid()) + '.log' |
|
390 | 411 | logfile = os.path.join(self.log_dir, log_filename) |
|
391 | 412 | open_log_file = open(logfile, 'w') |
|
392 | 413 | else: |
|
393 | 414 | open_log_file = sys.stdout |
|
394 | 415 | log.startLogging(open_log_file) |
|
416 | ||
|
417 | def write_pid_file(self): | |
|
418 | """Create a .pid file in the pid_dir with my pid. | |
|
419 | ||
|
420 | This must be called after pre_construct, which sets `self.pid_dir`. | |
|
421 | This raises :exc:`PIDFileError` if the pid file exists already. | |
|
422 | """ | |
|
423 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |
|
424 | if os.path.isfile(pid_file): | |
|
425 | pid = self.get_pid_from_file() | |
|
426 | raise PIDFileError( | |
|
427 | 'The pid file [%s] already exists. \nThis could mean that this ' | |
|
428 | 'server is already running with [pid=%s].' % (pid_file, pid)) | |
|
429 | with open(pid_file, 'w') as f: | |
|
430 | self.log.info("Creating pid file: %s" % pid_file) | |
|
431 | f.write(repr(os.getpid())+'\n') | |
|
432 | ||
|
433 | def remove_pid_file(self): | |
|
434 | """Remove the pid file. | |
|
435 | ||
|
436 | This should be called at shutdown by registering a callback with | |
|
437 | :func:`reactor.addSystemEventTrigger`. | |
|
438 | """ | |
|
439 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |
|
440 | if os.path.isfile(pid_file): | |
|
441 | try: | |
|
442 | self.log.info("Removing pid file: %s" % pid_file) | |
|
443 | os.remove(pid_file) | |
|
444 | except: | |
|
445 | pass | |
|
446 | ||
|
447 | def get_pid_from_file(self): | |
|
448 | """Get the pid from the pid file. | |
|
449 | ||
|
450 | If the pid file doesn't exist a :exc:`PIDFileError` is raised. | |
|
451 | """ | |
|
452 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |
|
453 | if os.path.isfile(pid_file): | |
|
454 | with open(pid_file, 'r') as f: | |
|
455 | pid = int(f.read().strip()) | |
|
456 | return pid | |
|
457 | else: | |
|
458 | raise PIDFileError('pid file not found: %s' % pid_file) No newline at end of file |
@@ -1,306 +1,383 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | # encoding: utf-8 |
|
3 | 3 | """ |
|
4 | 4 | The ipcluster application. |
|
5 | 5 | """ |
|
6 | 6 | |
|
7 | 7 | #----------------------------------------------------------------------------- |
|
8 | 8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
9 | 9 | # |
|
10 | 10 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 11 | # the file COPYING, distributed as part of this software. |
|
12 | 12 | #----------------------------------------------------------------------------- |
|
13 | 13 | |
|
14 | 14 | #----------------------------------------------------------------------------- |
|
15 | 15 | # Imports |
|
16 | 16 | #----------------------------------------------------------------------------- |
|
17 | 17 | |
|
18 | 18 | import logging |
|
19 | 19 | import os |
|
20 | 20 | import signal |
|
21 | 21 | import sys |
|
22 | 22 | |
|
23 | from twisted.scripts._twistd_unix import daemonize | |
|
24 | ||
|
23 | 25 | from IPython.core import release |
|
24 | 26 | from IPython.external import argparse |
|
25 | 27 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault |
|
26 | 28 | from IPython.utils.importstring import import_item |
|
27 | 29 | |
|
28 | 30 | from IPython.kernel.clusterdir import ( |
|
29 | ApplicationWithClusterDir, ClusterDirError | |
|
31 | ApplicationWithClusterDir, ClusterDirError, PIDFileError | |
|
30 | 32 | ) |
|
31 | 33 | |
|
32 | 34 | from twisted.internet import reactor, defer |
|
33 | 35 | from twisted.python import log |
|
34 | 36 | |
|
35 | 37 | #----------------------------------------------------------------------------- |
|
36 | 38 | # Code for launchers |
|
37 | 39 | #----------------------------------------------------------------------------- |
|
38 | 40 | |
|
39 | 41 | |
|
40 | 42 | |
|
41 | 43 | #----------------------------------------------------------------------------- |
|
42 | 44 | # The ipcluster application |
|
43 | 45 | #----------------------------------------------------------------------------- |
|
44 | 46 | |
|
45 | 47 | |
|
46 | 48 | class IPClusterCLLoader(ArgParseConfigLoader): |
|
47 | 49 | |
|
48 | 50 | def _add_arguments(self): |
|
49 | 51 | # This has all the common options that all subcommands use |
|
50 | 52 | parent_parser1 = argparse.ArgumentParser(add_help=False) |
|
51 |
parent_parser1.add_argument(' |
|
|
53 | parent_parser1.add_argument('--ipython-dir', | |
|
52 | 54 | dest='Global.ipythondir',type=str, |
|
53 | 55 | help='Set to override default location of Global.ipythondir.', |
|
54 | 56 | default=NoConfigDefault, |
|
55 | 57 | metavar='Global.ipythondir') |
|
56 |
parent_parser1.add_argument(' |
|
|
58 | parent_parser1.add_argument('--log-level', | |
|
57 | 59 | dest="Global.log_level",type=int, |
|
58 | 60 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
59 | 61 | default=NoConfigDefault, |
|
60 | 62 | metavar='Global.log_level') |
|
61 | 63 | |
|
62 | 64 | # This has all the common options that other subcommands use |
|
63 | 65 | parent_parser2 = argparse.ArgumentParser(add_help=False) |
|
64 |
parent_parser2.add_argument('-p',' |
|
|
66 | parent_parser2.add_argument('-p','--profile', | |
|
65 | 67 | dest='Global.profile',type=str, |
|
66 | 68 | default=NoConfigDefault, |
|
67 | 69 | help='The string name of the profile to be used. This determines ' |
|
68 | 70 | 'the name of the cluster dir as: cluster_<profile>. The default profile ' |
|
69 | 71 | 'is named "default". The cluster directory is resolve this way ' |
|
70 | 72 | 'if the --cluster-dir option is not used.', |
|
71 | 73 | default=NoConfigDefault, |
|
72 | 74 | metavar='Global.profile') |
|
73 |
parent_parser2.add_argument(' |
|
|
75 | parent_parser2.add_argument('--cluster-dir', | |
|
74 | 76 | dest='Global.cluster_dir',type=str, |
|
75 | 77 | default=NoConfigDefault, |
|
76 | 78 | help='Set the cluster dir. This overrides the logic used by the ' |
|
77 | 79 | '--profile option.', |
|
78 | 80 | default=NoConfigDefault, |
|
79 | 81 | metavar='Global.cluster_dir') |
|
80 | 82 | parent_parser2.add_argument('--log-to-file', |
|
81 | 83 | action='store_true', dest='Global.log_to_file', |
|
82 | 84 | default=NoConfigDefault, |
|
83 | 85 | help='Log to a file in the log directory (default is stdout)' |
|
84 | 86 | ) |
|
85 | 87 | |
|
86 | 88 | subparsers = self.parser.add_subparsers( |
|
87 | 89 | dest='Global.subcommand', |
|
88 | 90 | title='ipcluster subcommands', |
|
89 | 91 | description='ipcluster has a variety of subcommands. ' |
|
90 | 92 | 'The general way of running ipcluster is "ipcluster <cmd> ' |
|
91 | 93 | ' [options]""', |
|
92 | 94 | help='For more help, type "ipcluster <cmd> -h"') |
|
93 | 95 | |
|
94 | 96 | parser_list = subparsers.add_parser( |
|
95 | 97 | 'list', |
|
96 | 98 | help='List all clusters in cwd and ipythondir.', |
|
97 | 99 | parents=[parent_parser1] |
|
98 | 100 | ) |
|
99 | 101 | |
|
100 | 102 | parser_create = subparsers.add_parser( |
|
101 | 103 | 'create', |
|
102 | 104 | help='Create a new cluster directory.', |
|
103 | 105 | parents=[parent_parser1, parent_parser2] |
|
104 | 106 | ) |
|
105 | 107 | parser_create.add_argument( |
|
106 | 108 | '--reset-config', |
|
107 | 109 | dest='Global.reset_config', action='store_true', |
|
108 | 110 | default=NoConfigDefault, |
|
109 | 111 | help='Recopy the default config files to the cluster directory. ' |
|
110 | 112 | 'You will loose any modifications you have made to these files.' |
|
111 | 113 | ) |
|
112 | 114 | |
|
113 | 115 | parser_start = subparsers.add_parser( |
|
114 | 116 | 'start', |
|
115 | 117 | help='Start a cluster.', |
|
116 | 118 | parents=[parent_parser1, parent_parser2] |
|
117 | 119 | ) |
|
118 | 120 | parser_start.add_argument( |
|
119 | 121 | '-n', '--number', |
|
120 | 122 | type=int, dest='Global.n', |
|
121 | 123 | default=NoConfigDefault, |
|
122 | 124 | help='The number of engines to start.', |
|
123 | 125 | metavar='Global.n' |
|
124 | 126 | ) |
|
125 |
parser_start.add_argument(' |
|
|
127 | parser_start.add_argument('--clean-logs', | |
|
126 | 128 | dest='Global.clean_logs', action='store_true', |
|
127 | 129 | help='Delete old log flies before starting.', |
|
128 | 130 | default=NoConfigDefault |
|
129 | 131 | ) |
|
130 |
parser_start.add_argument(' |
|
|
132 | parser_start.add_argument('--no-clean-logs', | |
|
131 | 133 | dest='Global.clean_logs', action='store_false', |
|
132 | 134 | help="Don't delete old log flies before starting.", |
|
133 | 135 | default=NoConfigDefault |
|
134 | 136 | ) |
|
137 | parser_start.add_argument('--daemon', | |
|
138 | dest='Global.daemonize', action='store_true', | |
|
139 | help='Daemonize the ipcluster program. This implies --log-to-file', | |
|
140 | default=NoConfigDefault | |
|
141 | ) | |
|
142 | parser_start.add_argument('--nodaemon', | |
|
143 | dest='Global.daemonize', action='store_false', | |
|
144 | help="Dont't daemonize the ipcluster program.", | |
|
145 | default=NoConfigDefault | |
|
146 | ) | |
|
147 | ||
|
148 | parser_start = subparsers.add_parser( | |
|
149 | 'stop', | |
|
150 | help='Stop a cluster.', | |
|
151 | parents=[parent_parser1, parent_parser2] | |
|
152 | ) | |
|
153 | parser_start.add_argument('--signal-number', | |
|
154 | dest='Global.stop_signal', type=int, | |
|
155 | help="The signal number to use in stopping the cluster (default=2).", | |
|
156 | metavar="Global.stop_signal", | |
|
157 | default=NoConfigDefault | |
|
158 | ) | |
|
135 | 159 | |
|
136 | 160 | default_config_file_name = 'ipcluster_config.py' |
|
137 | 161 | |
|
138 | 162 | |
|
139 | 163 | class IPClusterApp(ApplicationWithClusterDir): |
|
140 | 164 | |
|
141 | 165 | name = 'ipcluster' |
|
142 | 166 | description = 'Start an IPython cluster (controller and engines).' |
|
143 | 167 | config_file_name = default_config_file_name |
|
144 | 168 | default_log_level = logging.INFO |
|
145 | 169 | auto_create_cluster_dir = False |
|
146 | 170 | |
|
147 | 171 | def create_default_config(self): |
|
148 | 172 | super(IPClusterApp, self).create_default_config() |
|
149 | 173 | self.default_config.Global.controller_launcher = \ |
|
150 | 174 | 'IPython.kernel.launcher.LocalControllerLauncher' |
|
151 | 175 | self.default_config.Global.engine_launcher = \ |
|
152 | 176 | 'IPython.kernel.launcher.LocalEngineSetLauncher' |
|
153 | 177 | self.default_config.Global.n = 2 |
|
154 | 178 | self.default_config.Global.reset_config = False |
|
155 | 179 | self.default_config.Global.clean_logs = True |
|
180 | self.default_config.Global.stop_signal = 2 | |
|
181 | self.default_config.Global.daemonize = False | |
|
156 | 182 | |
|
157 | 183 | def create_command_line_config(self): |
|
158 | 184 | """Create and return a command line config loader.""" |
|
159 | 185 | return IPClusterCLLoader( |
|
160 | 186 | description=self.description, |
|
161 | 187 | version=release.version |
|
162 | 188 | ) |
|
163 | 189 | |
|
164 | 190 | def find_resources(self): |
|
165 | 191 | subcommand = self.command_line_config.Global.subcommand |
|
166 | 192 | if subcommand=='list': |
|
167 | 193 | self.list_cluster_dirs() |
|
168 | 194 | # Exit immediately because there is nothing left to do. |
|
169 | 195 | self.exit() |
|
170 | 196 | elif subcommand=='create': |
|
171 | 197 | self.auto_create_cluster_dir = True |
|
172 | 198 | super(IPClusterApp, self).find_resources() |
|
173 | elif subcommand=='start': | |
|
199 | elif subcommand=='start' or subcommand=='stop': | |
|
174 | 200 | self.auto_create_cluster_dir = False |
|
175 | 201 | try: |
|
176 | 202 | super(IPClusterApp, self).find_resources() |
|
177 | 203 | except ClusterDirError: |
|
178 | 204 | raise ClusterDirError( |
|
179 | 205 | "Could not find a cluster directory. A cluster dir must " |
|
180 | 206 | "be created before running 'ipcluster start'. Do " |
|
181 | 207 | "'ipcluster create -h' or 'ipcluster list -h' for more " |
|
182 | 208 | "information about creating and listing cluster dirs." |
|
183 | 209 | ) |
|
184 | 210 | |
|
211 | def pre_construct(self): | |
|
212 | super(IPClusterApp, self).pre_construct() | |
|
213 | config = self.master_config | |
|
214 | try: | |
|
215 | daemon = config.Global.daemonize | |
|
216 | if daemon: | |
|
217 | config.Global.log_to_file = True | |
|
218 | except AttributeError: | |
|
219 | pass | |
|
220 | ||
|
185 | 221 | def construct(self): |
|
186 | 222 | config = self.master_config |
|
187 | 223 | if config.Global.subcommand=='list': |
|
188 | 224 | pass |
|
189 | 225 | elif config.Global.subcommand=='create': |
|
190 | 226 | self.log.info('Copying default config files to cluster directory ' |
|
191 | 227 | '[overwrite=%r]' % (config.Global.reset_config,)) |
|
192 | 228 | self.cluster_dir_obj.copy_all_config_files(overwrite=config.Global.reset_config) |
|
193 | 229 | elif config.Global.subcommand=='start': |
|
194 | 230 | self.start_logging() |
|
195 | 231 | reactor.callWhenRunning(self.start_launchers) |
|
196 | 232 | |
|
197 | 233 | def list_cluster_dirs(self): |
|
198 | 234 | # Find the search paths |
|
199 | 235 | cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','') |
|
200 | 236 | if cluster_dir_paths: |
|
201 | 237 | cluster_dir_paths = cluster_dir_paths.split(':') |
|
202 | 238 | else: |
|
203 | 239 | cluster_dir_paths = [] |
|
204 | 240 | try: |
|
205 | 241 | ipythondir = self.command_line_config.Global.ipythondir |
|
206 | 242 | except AttributeError: |
|
207 | 243 | ipythondir = self.default_config.Global.ipythondir |
|
208 | 244 | paths = [os.getcwd(), ipythondir] + \ |
|
209 | 245 | cluster_dir_paths |
|
210 | 246 | paths = list(set(paths)) |
|
211 | 247 | |
|
212 | 248 | self.log.info('Searching for cluster dirs in paths: %r' % paths) |
|
213 | 249 | for path in paths: |
|
214 | 250 | files = os.listdir(path) |
|
215 | 251 | for f in files: |
|
216 | 252 | full_path = os.path.join(path, f) |
|
217 | 253 | if os.path.isdir(full_path) and f.startswith('cluster_'): |
|
218 | 254 | profile = full_path.split('_')[-1] |
|
219 | 255 | start_cmd = '"ipcluster start -n 4 -p %s"' % profile |
|
220 | 256 | print start_cmd + " ==> " + full_path |
|
221 | 257 | |
|
222 | 258 | def start_launchers(self): |
|
223 | 259 | config = self.master_config |
|
224 | 260 | |
|
225 | 261 | # Create the launchers |
|
226 | 262 | el_class = import_item(config.Global.engine_launcher) |
|
227 | 263 | self.engine_launcher = el_class( |
|
228 | 264 | self.cluster_dir, config=config |
|
229 | 265 | ) |
|
230 | 266 | cl_class = import_item(config.Global.controller_launcher) |
|
231 | 267 | self.controller_launcher = cl_class( |
|
232 | 268 | self.cluster_dir, config=config |
|
233 | 269 | ) |
|
234 | 270 | |
|
235 | 271 | # Setup signals |
|
236 | 272 | signal.signal(signal.SIGINT, self.stop_launchers) |
|
237 | 273 | # signal.signal(signal.SIGKILL, self.stop_launchers) |
|
238 | 274 | |
|
239 | 275 | # Setup the observing of stopping |
|
240 | 276 | d1 = self.controller_launcher.observe_stop() |
|
241 | 277 | d1.addCallback(self.stop_engines) |
|
242 | 278 | d1.addErrback(self.err_and_stop) |
|
243 | 279 | # If this triggers, just let them die |
|
244 | 280 | # d2 = self.engine_launcher.observe_stop() |
|
245 | 281 | |
|
246 | 282 | # Start the controller and engines |
|
247 | 283 | d = self.controller_launcher.start( |
|
248 | 284 | profile=None, cluster_dir=config.Global.cluster_dir |
|
249 | 285 | ) |
|
250 | 286 | d.addCallback(lambda _: self.start_engines()) |
|
251 | 287 | d.addErrback(self.err_and_stop) |
|
252 | 288 | |
|
253 | 289 | def err_and_stop(self, f): |
|
254 | 290 | log.msg('Unexpected error in ipcluster:') |
|
255 | 291 | log.err(f) |
|
256 | 292 | reactor.stop() |
|
257 | 293 | |
|
258 | 294 | def stop_engines(self, r): |
|
259 | 295 | return self.engine_launcher.stop() |
|
260 | 296 | |
|
261 | 297 | def start_engines(self): |
|
262 | 298 | config = self.master_config |
|
263 | 299 | d = self.engine_launcher.start( |
|
264 | 300 | config.Global.n, |
|
265 | 301 | profile=None, cluster_dir=config.Global.cluster_dir |
|
266 | 302 | ) |
|
267 | 303 | return d |
|
268 | 304 | |
|
269 | 305 | def stop_launchers(self, signum, frame): |
|
270 | 306 | log.msg("Stopping cluster") |
|
271 | 307 | d1 = self.engine_launcher.stop() |
|
272 | 308 | d2 = self.controller_launcher.stop() |
|
273 | 309 | # d1.addCallback(lambda _: self.controller_launcher.stop) |
|
274 | 310 | d1.addErrback(self.err_and_stop) |
|
275 | 311 | d2.addErrback(self.err_and_stop) |
|
276 | 312 | reactor.callLater(2.0, reactor.stop) |
|
277 | 313 | |
|
278 | 314 | def start_logging(self): |
|
279 | 315 | # Remove old log files |
|
280 | 316 | if self.master_config.Global.clean_logs: |
|
281 | 317 | log_dir = self.master_config.Global.log_dir |
|
282 | 318 | for f in os.listdir(log_dir): |
|
283 | 319 | if f.startswith('ipengine' + '-') and f.endswith('.log'): |
|
284 | 320 | os.remove(os.path.join(log_dir, f)) |
|
285 | 321 | for f in os.listdir(log_dir): |
|
286 | 322 | if f.startswith('ipcontroller' + '-') and f.endswith('.log'): |
|
287 | 323 | os.remove(os.path.join(log_dir, f)) |
|
288 | 324 | super(IPClusterApp, self).start_logging() |
|
289 | 325 | |
|
290 | 326 | def start_app(self): |
|
327 | """Start the application, depending on what subcommand is used.""" | |
|
291 | 328 | config = self.master_config |
|
292 | if config.Global.subcommand=='create' or config.Global.subcommand=='list': | |
|
329 | subcmd = config.Global.subcommand | |
|
330 | if subcmd=='create' or subcmd=='list': | |
|
293 | 331 | return |
|
294 |
elif |
|
|
332 | elif subcmd=='start': | |
|
333 | # First see if the cluster is already running | |
|
334 | try: | |
|
335 | pid = self.get_pid_from_file() | |
|
336 | except: | |
|
337 | pass | |
|
338 | else: | |
|
339 | self.log.critical( | |
|
340 | 'Cluster is already running with [pid=%s]. ' | |
|
341 | 'use "ipcluster stop" to stop the cluster.' % pid | |
|
342 | ) | |
|
343 | # Here I exit with a unusual exit status that other processes | |
|
344 | # can watch for to learn how I existed. | |
|
345 | sys.exit(10) | |
|
346 | # Now log and daemonize | |
|
347 | self.log.info('Starting ipcluster with [daemon=%r]' % config.Global.daemonize) | |
|
348 | if config.Global.daemonize: | |
|
349 | if os.name=='posix': | |
|
350 | os.chdir(config.Global.cluster_dir) | |
|
351 | self.log_level = 40 | |
|
352 | daemonize() | |
|
353 | ||
|
354 | # Now write the new pid file after our new forked pid is active. | |
|
355 | self.write_pid_file() | |
|
356 | reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file) | |
|
295 | 357 | reactor.run() |
|
358 | elif subcmd=='stop': | |
|
359 | try: | |
|
360 | pid = self.get_pid_from_file() | |
|
361 | except PIDFileError: | |
|
362 | self.log.critical( | |
|
363 | 'Problem reading pid file, cluster is probably not running.' | |
|
364 | ) | |
|
365 | # Here I exit with a unusual exit status that other processes | |
|
366 | # can watch for to learn how I existed. | |
|
367 | sys.exit(11) | |
|
368 | sig = config.Global.stop_signal | |
|
369 | self.log.info( | |
|
370 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) | |
|
371 | ) | |
|
372 | os.kill(pid, sig) | |
|
296 | 373 | |
|
297 | 374 | |
|
298 | 375 | def launch_new_instance(): |
|
299 | 376 | """Create and run the IPython cluster.""" |
|
300 | 377 | app = IPClusterApp() |
|
301 | 378 | app.start() |
|
302 | 379 | |
|
303 | 380 | |
|
304 | 381 | if __name__ == '__main__': |
|
305 | 382 | launch_new_instance() |
|
306 | 383 |
@@ -1,254 +1,262 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | # encoding: utf-8 |
|
3 | 3 | """ |
|
4 | 4 | The IPython controller application. |
|
5 | 5 | """ |
|
6 | 6 | |
|
7 | 7 | #----------------------------------------------------------------------------- |
|
8 | 8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
9 | 9 | # |
|
10 | 10 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 11 | # the file COPYING, distributed as part of this software. |
|
12 | 12 | #----------------------------------------------------------------------------- |
|
13 | 13 | |
|
14 | 14 | #----------------------------------------------------------------------------- |
|
15 | 15 | # Imports |
|
16 | 16 | #----------------------------------------------------------------------------- |
|
17 | 17 | |
|
18 | from __future__ import with_statement | |
|
19 | ||
|
18 | 20 | import copy |
|
19 | 21 | import os |
|
20 | 22 | import sys |
|
21 | 23 | |
|
22 | 24 | from twisted.application import service |
|
23 | 25 | from twisted.internet import reactor |
|
24 | 26 | from twisted.python import log |
|
25 | 27 | |
|
26 | 28 | from IPython.config.loader import Config, NoConfigDefault |
|
27 | 29 | |
|
28 | 30 | from IPython.kernel.clusterdir import ( |
|
29 | 31 | ApplicationWithClusterDir, |
|
30 | 32 | AppWithClusterDirArgParseConfigLoader |
|
31 | 33 | ) |
|
32 | 34 | |
|
33 | 35 | from IPython.core import release |
|
34 | 36 | |
|
35 | 37 | from IPython.utils.traitlets import Str, Instance |
|
36 | 38 | |
|
37 | 39 | from IPython.kernel import controllerservice |
|
38 | 40 | |
|
39 | 41 | from IPython.kernel.fcutil import FCServiceFactory |
|
40 | 42 | |
|
41 | 43 | #----------------------------------------------------------------------------- |
|
42 | 44 | # Default interfaces |
|
43 | 45 | #----------------------------------------------------------------------------- |
|
44 | 46 | |
|
45 | 47 | |
|
46 | 48 | # The default client interfaces for FCClientServiceFactory.interfaces |
|
47 | 49 | default_client_interfaces = Config() |
|
48 | 50 | default_client_interfaces.Task.interface_chain = [ |
|
49 | 51 | 'IPython.kernel.task.ITaskController', |
|
50 | 52 | 'IPython.kernel.taskfc.IFCTaskController' |
|
51 | 53 | ] |
|
52 | 54 | |
|
53 | 55 | default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' |
|
54 | 56 | |
|
55 | 57 | default_client_interfaces.MultiEngine.interface_chain = [ |
|
56 | 58 | 'IPython.kernel.multiengine.IMultiEngine', |
|
57 | 59 | 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' |
|
58 | 60 | ] |
|
59 | 61 | |
|
60 | 62 | default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' |
|
61 | 63 | |
|
62 | 64 | # Make this a dict we can pass to Config.__init__ for the default |
|
63 | 65 | default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items())) |
|
64 | 66 | |
|
65 | 67 | |
|
66 | 68 | |
|
67 | 69 | # The default engine interfaces for FCEngineServiceFactory.interfaces |
|
68 | 70 | default_engine_interfaces = Config() |
|
69 | 71 | default_engine_interfaces.Default.interface_chain = [ |
|
70 | 72 | 'IPython.kernel.enginefc.IFCControllerBase' |
|
71 | 73 | ] |
|
72 | 74 | |
|
73 | 75 | default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' |
|
74 | 76 | |
|
75 | 77 | # Make this a dict we can pass to Config.__init__ for the default |
|
76 | 78 | default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) |
|
77 | 79 | |
|
78 | 80 | |
|
79 | 81 | #----------------------------------------------------------------------------- |
|
80 | 82 | # Service factories |
|
81 | 83 | #----------------------------------------------------------------------------- |
|
82 | 84 | |
|
83 | 85 | |
|
84 | 86 | class FCClientServiceFactory(FCServiceFactory): |
|
85 | 87 | """A Foolscap implementation of the client services.""" |
|
86 | 88 | |
|
87 | 89 | cert_file = Str('ipcontroller-client.pem', config=True) |
|
88 | 90 | interfaces = Instance(klass=Config, kw=default_client_interfaces, |
|
89 | 91 | allow_none=False, config=True) |
|
90 | 92 | |
|
91 | 93 | |
|
92 | 94 | class FCEngineServiceFactory(FCServiceFactory): |
|
93 | 95 | """A Foolscap implementation of the engine services.""" |
|
94 | 96 | |
|
95 | 97 | cert_file = Str('ipcontroller-engine.pem', config=True) |
|
96 | 98 | interfaces = Instance(klass=dict, kw=default_engine_interfaces, |
|
97 | 99 | allow_none=False, config=True) |
|
98 | 100 | |
|
99 | 101 | |
|
100 | 102 | #----------------------------------------------------------------------------- |
|
101 | 103 | # The main application |
|
102 | 104 | #----------------------------------------------------------------------------- |
|
103 | 105 | |
|
104 | 106 | |
|
105 | 107 | cl_args = ( |
|
106 | 108 | # Client config |
|
107 | 109 | (('--client-ip',), dict( |
|
108 | 110 | type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault, |
|
109 | 111 | help='The IP address or hostname the controller will listen on for ' |
|
110 | 112 | 'client connections.', |
|
111 | 113 | metavar='FCClientServiceFactory.ip') |
|
112 | 114 | ), |
|
113 | 115 | (('--client-port',), dict( |
|
114 | 116 | type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault, |
|
115 | 117 | help='The port the controller will listen on for client connections. ' |
|
116 | 118 | 'The default is to use 0, which will autoselect an open port.', |
|
117 | 119 | metavar='FCClientServiceFactory.port') |
|
118 | 120 | ), |
|
119 | 121 | (('--client-location',), dict( |
|
120 | 122 | type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault, |
|
121 | 123 | help='The hostname or IP that clients should connect to. This does ' |
|
122 | 124 | 'not control which interface the controller listens on. Instead, this ' |
|
123 | 125 | 'determines the hostname/IP that is listed in the FURL, which is how ' |
|
124 | 126 | 'clients know where to connect. Useful if the controller is listening ' |
|
125 | 127 | 'on multiple interfaces.', |
|
126 | 128 | metavar='FCClientServiceFactory.location') |
|
127 | 129 | ), |
|
128 | 130 | # Engine config |
|
129 | 131 | (('--engine-ip',), dict( |
|
130 | 132 | type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, |
|
131 | 133 | help='The IP address or hostname the controller will listen on for ' |
|
132 | 134 | 'engine connections.', |
|
133 | 135 | metavar='FCEngineServiceFactory.ip') |
|
134 | 136 | ), |
|
135 | 137 | (('--engine-port',), dict( |
|
136 | 138 | type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault, |
|
137 | 139 | help='The port the controller will listen on for engine connections. ' |
|
138 | 140 | 'The default is to use 0, which will autoselect an open port.', |
|
139 | 141 | metavar='FCEngineServiceFactory.port') |
|
140 | 142 | ), |
|
141 | 143 | (('--engine-location',), dict( |
|
142 | 144 | type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault, |
|
143 | 145 | help='The hostname or IP that engines should connect to. This does ' |
|
144 | 146 | 'not control which interface the controller listens on. Instead, this ' |
|
145 | 147 | 'determines the hostname/IP that is listed in the FURL, which is how ' |
|
146 | 148 | 'engines know where to connect. Useful if the controller is listening ' |
|
147 | 149 | 'on multiple interfaces.', |
|
148 | 150 | metavar='FCEngineServiceFactory.location') |
|
149 | 151 | ), |
|
150 | 152 | # Global config |
|
151 | 153 | (('--log-to-file',), dict( |
|
152 | 154 | action='store_true', dest='Global.log_to_file', default=NoConfigDefault, |
|
153 | 155 | help='Log to a file in the log directory (default is stdout)') |
|
154 | 156 | ), |
|
155 | 157 | (('-r','--reuse-furls'), dict( |
|
156 | 158 | action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, |
|
157 | 159 | help='Try to reuse all FURL files. If this is not set all FURL files ' |
|
158 | 160 | 'are deleted before the controller starts. This must be set if ' |
|
159 | 161 | 'specific ports are specified by --engine-port or --client-port.') |
|
160 | 162 | ), |
|
161 |
(('-ns', |
|
|
163 | (('--no-secure',), dict( | |
|
162 | 164 | action='store_false', dest='Global.secure', default=NoConfigDefault, |
|
163 | 165 | help='Turn off SSL encryption for all connections.') |
|
166 | ), | |
|
167 | (('--secure',), dict( | |
|
168 | action='store_true', dest='Global.secure', default=NoConfigDefault, | |
|
169 | help='Turn off SSL encryption for all connections.') | |
|
164 | 170 | ) |
|
165 | 171 | ) |
|
166 | 172 | |
|
167 | 173 | |
|
168 | 174 | class IPControllerAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader): |
|
169 | 175 | |
|
170 | 176 | arguments = cl_args |
|
171 | 177 | |
|
172 | 178 | |
|
173 | 179 | default_config_file_name = 'ipcontroller_config.py' |
|
174 | 180 | |
|
175 | 181 | |
|
176 | 182 | class IPControllerApp(ApplicationWithClusterDir): |
|
177 | 183 | |
|
178 | 184 | name = 'ipcontroller' |
|
179 | 185 | description = 'Start the IPython controller for parallel computing.' |
|
180 | 186 | config_file_name = default_config_file_name |
|
181 | 187 | auto_create_cluster_dir = True |
|
182 | 188 | |
|
183 | 189 | def create_default_config(self): |
|
184 | 190 | super(IPControllerApp, self).create_default_config() |
|
185 | 191 | self.default_config.Global.reuse_furls = False |
|
186 | 192 | self.default_config.Global.secure = True |
|
187 | 193 | self.default_config.Global.import_statements = [] |
|
188 | 194 | self.default_config.Global.clean_logs = True |
|
189 | 195 | |
|
190 | 196 | def create_command_line_config(self): |
|
191 | 197 | """Create and return a command line config loader.""" |
|
192 | 198 | return IPControllerAppCLConfigLoader( |
|
193 | 199 | description=self.description, |
|
194 | 200 | version=release.version |
|
195 | 201 | ) |
|
196 | 202 | |
|
197 | 203 | def post_load_command_line_config(self): |
|
198 | 204 | # Now setup reuse_furls |
|
199 | 205 | c = self.command_line_config |
|
200 | 206 | if hasattr(c.Global, 'reuse_furls'): |
|
201 | 207 | c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls |
|
202 | 208 | c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls |
|
203 | 209 | del c.Global.reuse_furls |
|
204 | 210 | if hasattr(c.Global, 'secure'): |
|
205 | 211 | c.FCClientServiceFactory.secure = c.Global.secure |
|
206 | 212 | c.FCEngineServiceFactory.secure = c.Global.secure |
|
207 | 213 | del c.Global.secure |
|
208 | 214 | |
|
209 | 215 | def construct(self): |
|
210 | 216 | # I am a little hesitant to put these into InteractiveShell itself. |
|
211 | 217 | # But that might be the place for them |
|
212 | 218 | sys.path.insert(0, '') |
|
213 | 219 | |
|
214 | 220 | self.start_logging() |
|
215 | 221 | self.import_statements() |
|
216 | ||
|
222 | ||
|
217 | 223 | # Create the service hierarchy |
|
218 | 224 | self.main_service = service.MultiService() |
|
219 | 225 | # The controller service |
|
220 | 226 | controller_service = controllerservice.ControllerService() |
|
221 | 227 | controller_service.setServiceParent(self.main_service) |
|
222 | 228 | # The client tub and all its refereceables |
|
223 | 229 | csfactory = FCClientServiceFactory(self.master_config, controller_service) |
|
224 | 230 | client_service = csfactory.create() |
|
225 | 231 | client_service.setServiceParent(self.main_service) |
|
226 | 232 | # The engine tub |
|
227 | 233 | esfactory = FCEngineServiceFactory(self.master_config, controller_service) |
|
228 | 234 | engine_service = esfactory.create() |
|
229 | 235 | engine_service.setServiceParent(self.main_service) |
|
230 | 236 | |
|
231 | 237 | def import_statements(self): |
|
232 | 238 | statements = self.master_config.Global.import_statements |
|
233 | 239 | for s in statements: |
|
234 | 240 | try: |
|
235 | 241 | log.msg("Executing statement: '%s'" % s) |
|
236 | 242 | exec s in globals(), locals() |
|
237 | 243 | except: |
|
238 | 244 | log.msg("Error running statement: %s" % s) |
|
239 | 245 | |
|
240 | 246 | def start_app(self): |
|
241 | 247 | # Start the controller service and set things running |
|
242 | 248 | self.main_service.startService() |
|
249 | self.write_pid_file() | |
|
250 | reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file) | |
|
243 | 251 | reactor.run() |
|
244 | 252 | |
|
245 | 253 | |
|
246 | 254 | def launch_new_instance(): |
|
247 | 255 | """Create and run the IPython controller""" |
|
248 | 256 | app = IPControllerApp() |
|
249 | 257 | app.start() |
|
250 | 258 | |
|
251 | 259 | |
|
252 | 260 | if __name__ == '__main__': |
|
253 | 261 | launch_new_instance() |
|
254 | 262 |
@@ -1,23 +1,66 b'' | |||
|
1 | 1 | """Count the frequencies of words in a string""" |
|
2 | 2 | |
|
3 | from __future__ import division | |
|
4 | ||
|
5 | import cmath as math | |
|
6 | ||
|
7 | ||
|
3 | 8 | def wordfreq(text): |
|
4 | 9 | """Return a dictionary of words and word counts in a string.""" |
|
5 | 10 | |
|
6 | 11 | freqs = {} |
|
7 | 12 | for word in text.split(): |
|
8 | freqs[word] = freqs.get(word, 0) + 1 | |
|
13 | lword = word.lower() | |
|
14 | freqs[lword] = freqs.get(lword, 0) + 1 | |
|
9 | 15 | return freqs |
|
10 | 16 | |
|
17 | ||
|
11 | 18 | def print_wordfreq(freqs, n=10): |
|
12 | 19 | """Print the n most common words and counts in the freqs dict.""" |
|
13 | 20 | |
|
14 | 21 | words, counts = freqs.keys(), freqs.values() |
|
15 | 22 | items = zip(counts, words) |
|
16 | 23 | items.sort(reverse=True) |
|
17 | 24 | for (count, word) in items[:n]: |
|
18 | 25 | print word, count |
|
19 | 26 | |
|
20 | if __name__ == '__main__': | |
|
21 | import gzip | |
|
22 | text = gzip.open('HISTORY.gz').read() | |
|
23 | freqs = wordfreq(text) No newline at end of file | |
|
27 | ||
|
28 | def wordfreq_to_weightsize(worddict, minsize=10, maxsize=50, minalpha=0.4, maxalpha=1.0): | |
|
29 | mincount = min(worddict.itervalues()) | |
|
30 | maxcount = max(worddict.itervalues()) | |
|
31 | weights = {} | |
|
32 | for k, v in worddict.iteritems(): | |
|
33 | w = (v-mincount)/(maxcount-mincount) | |
|
34 | alpha = minalpha + (maxalpha-minalpha)*w | |
|
35 | size = minsize + (maxsize-minsize)*w | |
|
36 | weights[k] = (alpha, size) | |
|
37 | return weights | |
|
38 | ||
|
39 | ||
|
40 | def tagcloud(worddict, n=10, minsize=10, maxsize=50, minalpha=0.4, maxalpha=1.0): | |
|
41 | from matplotlib import pyplot as plt | |
|
42 | import random | |
|
43 | ||
|
44 | worddict = wordfreq_to_weightsize(worddict, minsize, maxsize, minalpha, maxalpha) | |
|
45 | ||
|
46 | fig = plt.figure() | |
|
47 | ax = fig.add_subplot(111) | |
|
48 | ax.set_position([0.0,0.0,1.0,1.0]) | |
|
49 | plt.xticks([]) | |
|
50 | plt.yticks([]) | |
|
51 | ||
|
52 | words = worddict.keys() | |
|
53 | alphas = [v[0] for v in worddict.values()] | |
|
54 | sizes = [v[1] for v in worddict.values()] | |
|
55 | items = zip(alphas, sizes, words) | |
|
56 | items.sort(reverse=True) | |
|
57 | for alpha, size, word in items[:n]: | |
|
58 | xpos = random.normalvariate(0.5, 0.3) | |
|
59 | ypos = random.normalvariate(0.5, 0.3) | |
|
60 | # xpos = random.uniform(0.0,1.0) | |
|
61 | # ypos = random.uniform(0.0,1.0) | |
|
62 | ax.text(xpos, ypos, word.lower(), alpha=alpha, fontsize=size) | |
|
63 | ax.autoscale_view() | |
|
64 | return ax | |
|
65 | ||
|
66 | No newline at end of file |
General Comments 0
You need to be logged in to leave comments.
Login now