##// END OF EJS Templates
remove path from external
MinRK -
Show More
@@ -1,325 +1,326 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2
2
3 """ PickleShare - a small 'shelve' like datastore with concurrency support
3 """ PickleShare - a small 'shelve' like datastore with concurrency support
4
4
5 Like shelve, a PickleShareDB object acts like a normal dictionary. Unlike
5 Like shelve, a PickleShareDB object acts like a normal dictionary. Unlike
6 shelve, many processes can access the database simultaneously. Changing a
6 shelve, many processes can access the database simultaneously. Changing a
7 value in database is immediately visible to other processes accessing the
7 value in database is immediately visible to other processes accessing the
8 same database.
8 same database.
9
9
10 Concurrency is possible because the values are stored in separate files. Hence
10 Concurrency is possible because the values are stored in separate files. Hence
11 the "database" is a directory where *all* files are governed by PickleShare.
11 the "database" is a directory where *all* files are governed by PickleShare.
12
12
13 Example usage::
13 Example usage::
14
14
15 from pickleshare import *
15 from pickleshare import *
16 db = PickleShareDB('~/testpickleshare')
16 db = PickleShareDB('~/testpickleshare')
17 db.clear()
17 db.clear()
18 print "Should be empty:",db.items()
18 print "Should be empty:",db.items()
19 db['hello'] = 15
19 db['hello'] = 15
20 db['aku ankka'] = [1,2,313]
20 db['aku ankka'] = [1,2,313]
21 db['paths/are/ok/key'] = [1,(5,46)]
21 db['paths/are/ok/key'] = [1,(5,46)]
22 print db.keys()
22 print db.keys()
23 del db['aku ankka']
23 del db['aku ankka']
24
24
25 This module is certainly not ZODB, but can be used for low-load
25 This module is certainly not ZODB, but can be used for low-load
26 (non-mission-critical) situations where tiny code size trumps the
26 (non-mission-critical) situations where tiny code size trumps the
27 advanced features of a "real" object database.
27 advanced features of a "real" object database.
28
28
29 Installation guide: easy_install pickleshare
29 Installation guide: easy_install pickleshare
30
30
31 Author: Ville Vainio <vivainio@gmail.com>
31 Author: Ville Vainio <vivainio@gmail.com>
32 License: MIT open source license.
32 License: MIT open source license.
33
33
34 """
34 """
35 from __future__ import print_function
35 from __future__ import print_function
36
36
37 from IPython.external.path import path as Path
38 import stat, time
37 import stat, time
39 import collections
38 import collections
40 try:
39 try:
41 import cPickle as pickle
40 import cPickle as pickle
42 except ImportError:
41 except ImportError:
43 import pickle
42 import pickle
44 import glob
43 import glob
45
44
45 from path import path as Path
46
46 def gethashfile(key):
47 def gethashfile(key):
47 return ("%02x" % abs(hash(key) % 256))[-2:]
48 return ("%02x" % abs(hash(key) % 256))[-2:]
48
49
49 _sentinel = object()
50 _sentinel = object()
50
51
51 class PickleShareDB(collections.MutableMapping):
52 class PickleShareDB(collections.MutableMapping):
52 """ The main 'connection' object for PickleShare database """
53 """ The main 'connection' object for PickleShare database """
53 def __init__(self,root):
54 def __init__(self,root):
54 """ Return a db object that will manage the specied directory"""
55 """ Return a db object that will manage the specied directory"""
55 self.root = Path(root).expanduser().abspath()
56 self.root = Path(root).expanduser().abspath()
56 if not self.root.isdir():
57 if not self.root.isdir():
57 self.root.makedirs_p()
58 self.root.makedirs_p()
58 # cache has { 'key' : (obj, orig_mod_time) }
59 # cache has { 'key' : (obj, orig_mod_time) }
59 self.cache = {}
60 self.cache = {}
60
61
61
62
62 def __getitem__(self,key):
63 def __getitem__(self,key):
63 """ db['key'] reading """
64 """ db['key'] reading """
64 fil = self.root / key
65 fil = self.root / key
65 try:
66 try:
66 mtime = (fil.stat()[stat.ST_MTIME])
67 mtime = (fil.stat()[stat.ST_MTIME])
67 except OSError:
68 except OSError:
68 raise KeyError(key)
69 raise KeyError(key)
69
70
70 if fil in self.cache and mtime == self.cache[fil][1]:
71 if fil in self.cache and mtime == self.cache[fil][1]:
71 return self.cache[fil][0]
72 return self.cache[fil][0]
72 try:
73 try:
73 # The cached item has expired, need to read
74 # The cached item has expired, need to read
74 with fil.open("rb") as f:
75 with fil.open("rb") as f:
75 obj = pickle.loads(f.read())
76 obj = pickle.loads(f.read())
76 except:
77 except:
77 raise KeyError(key)
78 raise KeyError(key)
78
79
79 self.cache[fil] = (obj,mtime)
80 self.cache[fil] = (obj,mtime)
80 return obj
81 return obj
81
82
82 def __setitem__(self,key,value):
83 def __setitem__(self,key,value):
83 """ db['key'] = 5 """
84 """ db['key'] = 5 """
84 fil = self.root / key
85 fil = self.root / key
85 parent = fil.parent
86 parent = fil.parent
86 if parent and not parent.isdir():
87 if parent and not parent.isdir():
87 parent.makedirs()
88 parent.makedirs()
88 # We specify protocol 2, so that we can mostly go between Python 2
89 # We specify protocol 2, so that we can mostly go between Python 2
89 # and Python 3. We can upgrade to protocol 3 when Python 2 is obsolete.
90 # and Python 3. We can upgrade to protocol 3 when Python 2 is obsolete.
90 with fil.open('wb') as f:
91 with fil.open('wb') as f:
91 pickled = pickle.dump(value, f, protocol=2)
92 pickled = pickle.dump(value, f, protocol=2)
92 try:
93 try:
93 self.cache[fil] = (value,fil.mtime)
94 self.cache[fil] = (value,fil.mtime)
94 except OSError as e:
95 except OSError as e:
95 if e.errno != 2:
96 if e.errno != 2:
96 raise
97 raise
97
98
98 def hset(self, hashroot, key, value):
99 def hset(self, hashroot, key, value):
99 """ hashed set """
100 """ hashed set """
100 hroot = self.root / hashroot
101 hroot = self.root / hashroot
101 if not hroot.isdir():
102 if not hroot.isdir():
102 hroot.makedirs()
103 hroot.makedirs()
103 hfile = hroot / gethashfile(key)
104 hfile = hroot / gethashfile(key)
104 d = self.get(hfile, {})
105 d = self.get(hfile, {})
105 d.update( {key : value})
106 d.update( {key : value})
106 self[hfile] = d
107 self[hfile] = d
107
108
108
109
109
110
110 def hget(self, hashroot, key, default = _sentinel, fast_only = True):
111 def hget(self, hashroot, key, default = _sentinel, fast_only = True):
111 """ hashed get """
112 """ hashed get """
112 hroot = self.root / hashroot
113 hroot = self.root / hashroot
113 hfile = hroot / gethashfile(key)
114 hfile = hroot / gethashfile(key)
114
115
115 d = self.get(hfile, _sentinel )
116 d = self.get(hfile, _sentinel )
116 #print "got dict",d,"from",hfile
117 #print "got dict",d,"from",hfile
117 if d is _sentinel:
118 if d is _sentinel:
118 if fast_only:
119 if fast_only:
119 if default is _sentinel:
120 if default is _sentinel:
120 raise KeyError(key)
121 raise KeyError(key)
121
122
122 return default
123 return default
123
124
124 # slow mode ok, works even after hcompress()
125 # slow mode ok, works even after hcompress()
125 d = self.hdict(hashroot)
126 d = self.hdict(hashroot)
126
127
127 return d.get(key, default)
128 return d.get(key, default)
128
129
129 def hdict(self, hashroot):
130 def hdict(self, hashroot):
130 """ Get all data contained in hashed category 'hashroot' as dict """
131 """ Get all data contained in hashed category 'hashroot' as dict """
131 hfiles = self.keys(hashroot + "/*")
132 hfiles = self.keys(hashroot + "/*")
132 hfiles.sort()
133 hfiles.sort()
133 last = len(hfiles) and hfiles[-1] or ''
134 last = len(hfiles) and hfiles[-1] or ''
134 if last.endswith('xx'):
135 if last.endswith('xx'):
135 # print "using xx"
136 # print "using xx"
136 hfiles = [last] + hfiles[:-1]
137 hfiles = [last] + hfiles[:-1]
137
138
138 all = {}
139 all = {}
139
140
140 for f in hfiles:
141 for f in hfiles:
141 # print "using",f
142 # print "using",f
142 try:
143 try:
143 all.update(self[f])
144 all.update(self[f])
144 except KeyError:
145 except KeyError:
145 print("Corrupt",f,"deleted - hset is not threadsafe!")
146 print("Corrupt",f,"deleted - hset is not threadsafe!")
146 del self[f]
147 del self[f]
147
148
148 self.uncache(f)
149 self.uncache(f)
149
150
150 return all
151 return all
151
152
152 def hcompress(self, hashroot):
153 def hcompress(self, hashroot):
153 """ Compress category 'hashroot', so hset is fast again
154 """ Compress category 'hashroot', so hset is fast again
154
155
155 hget will fail if fast_only is True for compressed items (that were
156 hget will fail if fast_only is True for compressed items (that were
156 hset before hcompress).
157 hset before hcompress).
157
158
158 """
159 """
159 hfiles = self.keys(hashroot + "/*")
160 hfiles = self.keys(hashroot + "/*")
160 all = {}
161 all = {}
161 for f in hfiles:
162 for f in hfiles:
162 # print "using",f
163 # print "using",f
163 all.update(self[f])
164 all.update(self[f])
164 self.uncache(f)
165 self.uncache(f)
165
166
166 self[hashroot + '/xx'] = all
167 self[hashroot + '/xx'] = all
167 for f in hfiles:
168 for f in hfiles:
168 p = self.root / f
169 p = self.root / f
169 if p.basename() == 'xx':
170 if p.basename() == 'xx':
170 continue
171 continue
171 p.remove()
172 p.remove()
172
173
173
174
174
175
175 def __delitem__(self,key):
176 def __delitem__(self,key):
176 """ del db["key"] """
177 """ del db["key"] """
177 fil = self.root / key
178 fil = self.root / key
178 self.cache.pop(fil,None)
179 self.cache.pop(fil,None)
179 try:
180 try:
180 fil.remove()
181 fil.remove()
181 except OSError:
182 except OSError:
182 # notfound and permission denied are ok - we
183 # notfound and permission denied are ok - we
183 # lost, the other process wins the conflict
184 # lost, the other process wins the conflict
184 pass
185 pass
185
186
186 def _normalized(self, p):
187 def _normalized(self, p):
187 """ Make a key suitable for user's eyes """
188 """ Make a key suitable for user's eyes """
188 return str(self.root.relpathto(p)).replace('\\','/')
189 return str(self.root.relpathto(p)).replace('\\','/')
189
190
190 def keys(self, globpat = None):
191 def keys(self, globpat = None):
191 """ All keys in DB, or all keys matching a glob"""
192 """ All keys in DB, or all keys matching a glob"""
192
193
193 if globpat is None:
194 if globpat is None:
194 files = self.root.walkfiles()
195 files = self.root.walkfiles()
195 else:
196 else:
196 files = [Path(p) for p in glob.glob(self.root/globpat)]
197 files = [Path(p) for p in glob.glob(self.root/globpat)]
197 return [self._normalized(p) for p in files if p.isfile()]
198 return [self._normalized(p) for p in files if p.isfile()]
198
199
199 def __iter__(self):
200 def __iter__(self):
200 return iter(self.keys())
201 return iter(self.keys())
201
202
202 def __len__(self):
203 def __len__(self):
203 return len(self.keys())
204 return len(self.keys())
204
205
205 def uncache(self,*items):
206 def uncache(self,*items):
206 """ Removes all, or specified items from cache
207 """ Removes all, or specified items from cache
207
208
208 Use this after reading a large amount of large objects
209 Use this after reading a large amount of large objects
209 to free up memory, when you won't be needing the objects
210 to free up memory, when you won't be needing the objects
210 for a while.
211 for a while.
211
212
212 """
213 """
213 if not items:
214 if not items:
214 self.cache = {}
215 self.cache = {}
215 for it in items:
216 for it in items:
216 self.cache.pop(it,None)
217 self.cache.pop(it,None)
217
218
218 def waitget(self,key, maxwaittime = 60 ):
219 def waitget(self,key, maxwaittime = 60 ):
219 """ Wait (poll) for a key to get a value
220 """ Wait (poll) for a key to get a value
220
221
221 Will wait for `maxwaittime` seconds before raising a KeyError.
222 Will wait for `maxwaittime` seconds before raising a KeyError.
222 The call exits normally if the `key` field in db gets a value
223 The call exits normally if the `key` field in db gets a value
223 within the timeout period.
224 within the timeout period.
224
225
225 Use this for synchronizing different processes or for ensuring
226 Use this for synchronizing different processes or for ensuring
226 that an unfortunately timed "db['key'] = newvalue" operation
227 that an unfortunately timed "db['key'] = newvalue" operation
227 in another process (which causes all 'get' operation to cause a
228 in another process (which causes all 'get' operation to cause a
228 KeyError for the duration of pickling) won't screw up your program
229 KeyError for the duration of pickling) won't screw up your program
229 logic.
230 logic.
230 """
231 """
231
232
232 wtimes = [0.2] * 3 + [0.5] * 2 + [1]
233 wtimes = [0.2] * 3 + [0.5] * 2 + [1]
233 tries = 0
234 tries = 0
234 waited = 0
235 waited = 0
235 while 1:
236 while 1:
236 try:
237 try:
237 val = self[key]
238 val = self[key]
238 return val
239 return val
239 except KeyError:
240 except KeyError:
240 pass
241 pass
241
242
242 if waited > maxwaittime:
243 if waited > maxwaittime:
243 raise KeyError(key)
244 raise KeyError(key)
244
245
245 time.sleep(wtimes[tries])
246 time.sleep(wtimes[tries])
246 waited+=wtimes[tries]
247 waited+=wtimes[tries]
247 if tries < len(wtimes) -1:
248 if tries < len(wtimes) -1:
248 tries+=1
249 tries+=1
249
250
250 def getlink(self,folder):
251 def getlink(self,folder):
251 """ Get a convenient link for accessing items """
252 """ Get a convenient link for accessing items """
252 return PickleShareLink(self, folder)
253 return PickleShareLink(self, folder)
253
254
254 def __repr__(self):
255 def __repr__(self):
255 return "PickleShareDB('%s')" % self.root
256 return "PickleShareDB('%s')" % self.root
256
257
257
258
258
259
259 class PickleShareLink:
260 class PickleShareLink:
260 """ A shortdand for accessing nested PickleShare data conveniently.
261 """ A shortdand for accessing nested PickleShare data conveniently.
261
262
262 Created through PickleShareDB.getlink(), example::
263 Created through PickleShareDB.getlink(), example::
263
264
264 lnk = db.getlink('myobjects/test')
265 lnk = db.getlink('myobjects/test')
265 lnk.foo = 2
266 lnk.foo = 2
266 lnk.bar = lnk.foo + 5
267 lnk.bar = lnk.foo + 5
267
268
268 """
269 """
269 def __init__(self, db, keydir ):
270 def __init__(self, db, keydir ):
270 self.__dict__.update(locals())
271 self.__dict__.update(locals())
271
272
272 def __getattr__(self,key):
273 def __getattr__(self,key):
273 return self.__dict__['db'][self.__dict__['keydir']+'/' + key]
274 return self.__dict__['db'][self.__dict__['keydir']+'/' + key]
274 def __setattr__(self,key,val):
275 def __setattr__(self,key,val):
275 self.db[self.keydir+'/' + key] = val
276 self.db[self.keydir+'/' + key] = val
276 def __repr__(self):
277 def __repr__(self):
277 db = self.__dict__['db']
278 db = self.__dict__['db']
278 keys = db.keys( self.__dict__['keydir'] +"/*")
279 keys = db.keys( self.__dict__['keydir'] +"/*")
279 return "<PickleShareLink '%s': %s>" % (
280 return "<PickleShareLink '%s': %s>" % (
280 self.__dict__['keydir'],
281 self.__dict__['keydir'],
281 ";".join([Path(k).basename() for k in keys]))
282 ";".join([Path(k).basename() for k in keys]))
282
283
283 def main():
284 def main():
284 import textwrap
285 import textwrap
285 usage = textwrap.dedent("""\
286 usage = textwrap.dedent("""\
286 pickleshare - manage PickleShare databases
287 pickleshare - manage PickleShare databases
287
288
288 Usage:
289 Usage:
289
290
290 pickleshare dump /path/to/db > dump.txt
291 pickleshare dump /path/to/db > dump.txt
291 pickleshare load /path/to/db < dump.txt
292 pickleshare load /path/to/db < dump.txt
292 pickleshare test /path/to/db
293 pickleshare test /path/to/db
293 """)
294 """)
294 DB = PickleShareDB
295 DB = PickleShareDB
295 import sys
296 import sys
296 if len(sys.argv) < 2:
297 if len(sys.argv) < 2:
297 print(usage)
298 print(usage)
298 return
299 return
299
300
300 cmd = sys.argv[1]
301 cmd = sys.argv[1]
301 args = sys.argv[2:]
302 args = sys.argv[2:]
302 if cmd == 'dump':
303 if cmd == 'dump':
303 if not args: args= ['.']
304 if not args: args= ['.']
304 db = DB(args[0])
305 db = DB(args[0])
305 import pprint
306 import pprint
306 pprint.pprint(db.items())
307 pprint.pprint(db.items())
307 elif cmd == 'load':
308 elif cmd == 'load':
308 cont = sys.stdin.read()
309 cont = sys.stdin.read()
309 db = DB(args[0])
310 db = DB(args[0])
310 data = eval(cont)
311 data = eval(cont)
311 db.clear()
312 db.clear()
312 for k,v in db.items():
313 for k,v in db.items():
313 db[k] = v
314 db[k] = v
314 elif cmd == 'testwait':
315 elif cmd == 'testwait':
315 db = DB(args[0])
316 db = DB(args[0])
316 db.clear()
317 db.clear()
317 print(db.waitget('250'))
318 print(db.waitget('250'))
318 elif cmd == 'test':
319 elif cmd == 'test':
319 test()
320 test()
320 stress()
321 stress()
321
322
322 if __name__== "__main__":
323 if __name__== "__main__":
323 main()
324 main()
324
325
325
326
@@ -1,784 +1,765 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 Utilities for working with strings and text.
3 Utilities for working with strings and text.
4
4
5 Inheritance diagram:
5 Inheritance diagram:
6
6
7 .. inheritance-diagram:: IPython.utils.text
7 .. inheritance-diagram:: IPython.utils.text
8 :parts: 3
8 :parts: 3
9 """
9 """
10
10
11 #-----------------------------------------------------------------------------
12 # Copyright (C) 2008-2011 The IPython Development Team
13 #
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
16 #-----------------------------------------------------------------------------
17
18 #-----------------------------------------------------------------------------
19 # Imports
20 #-----------------------------------------------------------------------------
21
22 import os
11 import os
23 import re
12 import re
24 import sys
13 import sys
25 import textwrap
14 import textwrap
26 from string import Formatter
15 from string import Formatter
27
16
28 from IPython.external.path import path
29 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
17 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
30 from IPython.utils import py3compat
18 from IPython.utils import py3compat
31
19
32 #-----------------------------------------------------------------------------
33 # Declarations
34 #-----------------------------------------------------------------------------
35
36 # datetime.strftime date format for ipython
20 # datetime.strftime date format for ipython
37 if sys.platform == 'win32':
21 if sys.platform == 'win32':
38 date_format = "%B %d, %Y"
22 date_format = "%B %d, %Y"
39 else:
23 else:
40 date_format = "%B %-d, %Y"
24 date_format = "%B %-d, %Y"
41
25
42
43 #-----------------------------------------------------------------------------
44 # Code
45 #-----------------------------------------------------------------------------
46
47 class LSString(str):
26 class LSString(str):
48 """String derivative with a special access attributes.
27 """String derivative with a special access attributes.
49
28
50 These are normal strings, but with the special attributes:
29 These are normal strings, but with the special attributes:
51
30
52 .l (or .list) : value as list (split on newlines).
31 .l (or .list) : value as list (split on newlines).
53 .n (or .nlstr): original value (the string itself).
32 .n (or .nlstr): original value (the string itself).
54 .s (or .spstr): value as whitespace-separated string.
33 .s (or .spstr): value as whitespace-separated string.
55 .p (or .paths): list of path objects
34 .p (or .paths): list of path objects (requires path.py package)
56
35
57 Any values which require transformations are computed only once and
36 Any values which require transformations are computed only once and
58 cached.
37 cached.
59
38
60 Such strings are very useful to efficiently interact with the shell, which
39 Such strings are very useful to efficiently interact with the shell, which
61 typically only understands whitespace-separated options for commands."""
40 typically only understands whitespace-separated options for commands."""
62
41
63 def get_list(self):
42 def get_list(self):
64 try:
43 try:
65 return self.__list
44 return self.__list
66 except AttributeError:
45 except AttributeError:
67 self.__list = self.split('\n')
46 self.__list = self.split('\n')
68 return self.__list
47 return self.__list
69
48
70 l = list = property(get_list)
49 l = list = property(get_list)
71
50
72 def get_spstr(self):
51 def get_spstr(self):
73 try:
52 try:
74 return self.__spstr
53 return self.__spstr
75 except AttributeError:
54 except AttributeError:
76 self.__spstr = self.replace('\n',' ')
55 self.__spstr = self.replace('\n',' ')
77 return self.__spstr
56 return self.__spstr
78
57
79 s = spstr = property(get_spstr)
58 s = spstr = property(get_spstr)
80
59
81 def get_nlstr(self):
60 def get_nlstr(self):
82 return self
61 return self
83
62
84 n = nlstr = property(get_nlstr)
63 n = nlstr = property(get_nlstr)
85
64
86 def get_paths(self):
65 def get_paths(self):
66 from path import path
87 try:
67 try:
88 return self.__paths
68 return self.__paths
89 except AttributeError:
69 except AttributeError:
90 self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
70 self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
91 return self.__paths
71 return self.__paths
92
72
93 p = paths = property(get_paths)
73 p = paths = property(get_paths)
94
74
95 # FIXME: We need to reimplement type specific displayhook and then add this
75 # FIXME: We need to reimplement type specific displayhook and then add this
96 # back as a custom printer. This should also be moved outside utils into the
76 # back as a custom printer. This should also be moved outside utils into the
97 # core.
77 # core.
98
78
99 # def print_lsstring(arg):
79 # def print_lsstring(arg):
100 # """ Prettier (non-repr-like) and more informative printer for LSString """
80 # """ Prettier (non-repr-like) and more informative printer for LSString """
101 # print "LSString (.p, .n, .l, .s available). Value:"
81 # print "LSString (.p, .n, .l, .s available). Value:"
102 # print arg
82 # print arg
103 #
83 #
104 #
84 #
105 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
85 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
106
86
107
87
108 class SList(list):
88 class SList(list):
109 """List derivative with a special access attributes.
89 """List derivative with a special access attributes.
110
90
111 These are normal lists, but with the special attributes:
91 These are normal lists, but with the special attributes:
112
92
113 * .l (or .list) : value as list (the list itself).
93 * .l (or .list) : value as list (the list itself).
114 * .n (or .nlstr): value as a string, joined on newlines.
94 * .n (or .nlstr): value as a string, joined on newlines.
115 * .s (or .spstr): value as a string, joined on spaces.
95 * .s (or .spstr): value as a string, joined on spaces.
116 * .p (or .paths): list of path objects
96 * .p (or .paths): list of path objects (requires path.py package)
117
97
118 Any values which require transformations are computed only once and
98 Any values which require transformations are computed only once and
119 cached."""
99 cached."""
120
100
121 def get_list(self):
101 def get_list(self):
122 return self
102 return self
123
103
124 l = list = property(get_list)
104 l = list = property(get_list)
125
105
126 def get_spstr(self):
106 def get_spstr(self):
127 try:
107 try:
128 return self.__spstr
108 return self.__spstr
129 except AttributeError:
109 except AttributeError:
130 self.__spstr = ' '.join(self)
110 self.__spstr = ' '.join(self)
131 return self.__spstr
111 return self.__spstr
132
112
133 s = spstr = property(get_spstr)
113 s = spstr = property(get_spstr)
134
114
135 def get_nlstr(self):
115 def get_nlstr(self):
136 try:
116 try:
137 return self.__nlstr
117 return self.__nlstr
138 except AttributeError:
118 except AttributeError:
139 self.__nlstr = '\n'.join(self)
119 self.__nlstr = '\n'.join(self)
140 return self.__nlstr
120 return self.__nlstr
141
121
142 n = nlstr = property(get_nlstr)
122 n = nlstr = property(get_nlstr)
143
123
144 def get_paths(self):
124 def get_paths(self):
125 from path import path
145 try:
126 try:
146 return self.__paths
127 return self.__paths
147 except AttributeError:
128 except AttributeError:
148 self.__paths = [path(p) for p in self if os.path.exists(p)]
129 self.__paths = [path(p) for p in self if os.path.exists(p)]
149 return self.__paths
130 return self.__paths
150
131
151 p = paths = property(get_paths)
132 p = paths = property(get_paths)
152
133
153 def grep(self, pattern, prune = False, field = None):
134 def grep(self, pattern, prune = False, field = None):
154 """ Return all strings matching 'pattern' (a regex or callable)
135 """ Return all strings matching 'pattern' (a regex or callable)
155
136
156 This is case-insensitive. If prune is true, return all items
137 This is case-insensitive. If prune is true, return all items
157 NOT matching the pattern.
138 NOT matching the pattern.
158
139
159 If field is specified, the match must occur in the specified
140 If field is specified, the match must occur in the specified
160 whitespace-separated field.
141 whitespace-separated field.
161
142
162 Examples::
143 Examples::
163
144
164 a.grep( lambda x: x.startswith('C') )
145 a.grep( lambda x: x.startswith('C') )
165 a.grep('Cha.*log', prune=1)
146 a.grep('Cha.*log', prune=1)
166 a.grep('chm', field=-1)
147 a.grep('chm', field=-1)
167 """
148 """
168
149
169 def match_target(s):
150 def match_target(s):
170 if field is None:
151 if field is None:
171 return s
152 return s
172 parts = s.split()
153 parts = s.split()
173 try:
154 try:
174 tgt = parts[field]
155 tgt = parts[field]
175 return tgt
156 return tgt
176 except IndexError:
157 except IndexError:
177 return ""
158 return ""
178
159
179 if isinstance(pattern, py3compat.string_types):
160 if isinstance(pattern, py3compat.string_types):
180 pred = lambda x : re.search(pattern, x, re.IGNORECASE)
161 pred = lambda x : re.search(pattern, x, re.IGNORECASE)
181 else:
162 else:
182 pred = pattern
163 pred = pattern
183 if not prune:
164 if not prune:
184 return SList([el for el in self if pred(match_target(el))])
165 return SList([el for el in self if pred(match_target(el))])
185 else:
166 else:
186 return SList([el for el in self if not pred(match_target(el))])
167 return SList([el for el in self if not pred(match_target(el))])
187
168
188 def fields(self, *fields):
169 def fields(self, *fields):
189 """ Collect whitespace-separated fields from string list
170 """ Collect whitespace-separated fields from string list
190
171
191 Allows quick awk-like usage of string lists.
172 Allows quick awk-like usage of string lists.
192
173
193 Example data (in var a, created by 'a = !ls -l')::
174 Example data (in var a, created by 'a = !ls -l')::
194
175
195 -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog
176 -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog
196 drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython
177 drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython
197
178
198 * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
179 * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
199 * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
180 * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
200 (note the joining by space).
181 (note the joining by space).
201 * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
182 * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
202
183
203 IndexErrors are ignored.
184 IndexErrors are ignored.
204
185
205 Without args, fields() just split()'s the strings.
186 Without args, fields() just split()'s the strings.
206 """
187 """
207 if len(fields) == 0:
188 if len(fields) == 0:
208 return [el.split() for el in self]
189 return [el.split() for el in self]
209
190
210 res = SList()
191 res = SList()
211 for el in [f.split() for f in self]:
192 for el in [f.split() for f in self]:
212 lineparts = []
193 lineparts = []
213
194
214 for fd in fields:
195 for fd in fields:
215 try:
196 try:
216 lineparts.append(el[fd])
197 lineparts.append(el[fd])
217 except IndexError:
198 except IndexError:
218 pass
199 pass
219 if lineparts:
200 if lineparts:
220 res.append(" ".join(lineparts))
201 res.append(" ".join(lineparts))
221
202
222 return res
203 return res
223
204
224 def sort(self,field= None, nums = False):
205 def sort(self,field= None, nums = False):
225 """ sort by specified fields (see fields())
206 """ sort by specified fields (see fields())
226
207
227 Example::
208 Example::
228
209
229 a.sort(1, nums = True)
210 a.sort(1, nums = True)
230
211
231 Sorts a by second field, in numerical order (so that 21 > 3)
212 Sorts a by second field, in numerical order (so that 21 > 3)
232
213
233 """
214 """
234
215
235 #decorate, sort, undecorate
216 #decorate, sort, undecorate
236 if field is not None:
217 if field is not None:
237 dsu = [[SList([line]).fields(field), line] for line in self]
218 dsu = [[SList([line]).fields(field), line] for line in self]
238 else:
219 else:
239 dsu = [[line, line] for line in self]
220 dsu = [[line, line] for line in self]
240 if nums:
221 if nums:
241 for i in range(len(dsu)):
222 for i in range(len(dsu)):
242 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
223 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
243 try:
224 try:
244 n = int(numstr)
225 n = int(numstr)
245 except ValueError:
226 except ValueError:
246 n = 0;
227 n = 0;
247 dsu[i][0] = n
228 dsu[i][0] = n
248
229
249
230
250 dsu.sort()
231 dsu.sort()
251 return SList([t[1] for t in dsu])
232 return SList([t[1] for t in dsu])
252
233
253
234
254 # FIXME: We need to reimplement type specific displayhook and then add this
235 # FIXME: We need to reimplement type specific displayhook and then add this
255 # back as a custom printer. This should also be moved outside utils into the
236 # back as a custom printer. This should also be moved outside utils into the
256 # core.
237 # core.
257
238
258 # def print_slist(arg):
239 # def print_slist(arg):
259 # """ Prettier (non-repr-like) and more informative printer for SList """
240 # """ Prettier (non-repr-like) and more informative printer for SList """
260 # print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
241 # print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
261 # if hasattr(arg, 'hideonce') and arg.hideonce:
242 # if hasattr(arg, 'hideonce') and arg.hideonce:
262 # arg.hideonce = False
243 # arg.hideonce = False
263 # return
244 # return
264 #
245 #
265 # nlprint(arg) # This was a nested list printer, now removed.
246 # nlprint(arg) # This was a nested list printer, now removed.
266 #
247 #
267 # print_slist = result_display.when_type(SList)(print_slist)
248 # print_slist = result_display.when_type(SList)(print_slist)
268
249
269
250
270 def indent(instr,nspaces=4, ntabs=0, flatten=False):
251 def indent(instr,nspaces=4, ntabs=0, flatten=False):
271 """Indent a string a given number of spaces or tabstops.
252 """Indent a string a given number of spaces or tabstops.
272
253
273 indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
254 indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
274
255
275 Parameters
256 Parameters
276 ----------
257 ----------
277
258
278 instr : basestring
259 instr : basestring
279 The string to be indented.
260 The string to be indented.
280 nspaces : int (default: 4)
261 nspaces : int (default: 4)
281 The number of spaces to be indented.
262 The number of spaces to be indented.
282 ntabs : int (default: 0)
263 ntabs : int (default: 0)
283 The number of tabs to be indented.
264 The number of tabs to be indented.
284 flatten : bool (default: False)
265 flatten : bool (default: False)
285 Whether to scrub existing indentation. If True, all lines will be
266 Whether to scrub existing indentation. If True, all lines will be
286 aligned to the same indentation. If False, existing indentation will
267 aligned to the same indentation. If False, existing indentation will
287 be strictly increased.
268 be strictly increased.
288
269
289 Returns
270 Returns
290 -------
271 -------
291
272
292 str|unicode : string indented by ntabs and nspaces.
273 str|unicode : string indented by ntabs and nspaces.
293
274
294 """
275 """
295 if instr is None:
276 if instr is None:
296 return
277 return
297 ind = '\t'*ntabs+' '*nspaces
278 ind = '\t'*ntabs+' '*nspaces
298 if flatten:
279 if flatten:
299 pat = re.compile(r'^\s*', re.MULTILINE)
280 pat = re.compile(r'^\s*', re.MULTILINE)
300 else:
281 else:
301 pat = re.compile(r'^', re.MULTILINE)
282 pat = re.compile(r'^', re.MULTILINE)
302 outstr = re.sub(pat, ind, instr)
283 outstr = re.sub(pat, ind, instr)
303 if outstr.endswith(os.linesep+ind):
284 if outstr.endswith(os.linesep+ind):
304 return outstr[:-len(ind)]
285 return outstr[:-len(ind)]
305 else:
286 else:
306 return outstr
287 return outstr
307
288
308
289
309 def list_strings(arg):
290 def list_strings(arg):
310 """Always return a list of strings, given a string or list of strings
291 """Always return a list of strings, given a string or list of strings
311 as input.
292 as input.
312
293
313 Examples
294 Examples
314 --------
295 --------
315 ::
296 ::
316
297
317 In [7]: list_strings('A single string')
298 In [7]: list_strings('A single string')
318 Out[7]: ['A single string']
299 Out[7]: ['A single string']
319
300
320 In [8]: list_strings(['A single string in a list'])
301 In [8]: list_strings(['A single string in a list'])
321 Out[8]: ['A single string in a list']
302 Out[8]: ['A single string in a list']
322
303
323 In [9]: list_strings(['A','list','of','strings'])
304 In [9]: list_strings(['A','list','of','strings'])
324 Out[9]: ['A', 'list', 'of', 'strings']
305 Out[9]: ['A', 'list', 'of', 'strings']
325 """
306 """
326
307
327 if isinstance(arg, py3compat.string_types): return [arg]
308 if isinstance(arg, py3compat.string_types): return [arg]
328 else: return arg
309 else: return arg
329
310
330
311
331 def marquee(txt='',width=78,mark='*'):
312 def marquee(txt='',width=78,mark='*'):
332 """Return the input string centered in a 'marquee'.
313 """Return the input string centered in a 'marquee'.
333
314
334 Examples
315 Examples
335 --------
316 --------
336 ::
317 ::
337
318
338 In [16]: marquee('A test',40)
319 In [16]: marquee('A test',40)
339 Out[16]: '**************** A test ****************'
320 Out[16]: '**************** A test ****************'
340
321
341 In [17]: marquee('A test',40,'-')
322 In [17]: marquee('A test',40,'-')
342 Out[17]: '---------------- A test ----------------'
323 Out[17]: '---------------- A test ----------------'
343
324
344 In [18]: marquee('A test',40,' ')
325 In [18]: marquee('A test',40,' ')
345 Out[18]: ' A test '
326 Out[18]: ' A test '
346
327
347 """
328 """
348 if not txt:
329 if not txt:
349 return (mark*width)[:width]
330 return (mark*width)[:width]
350 nmark = (width-len(txt)-2)//len(mark)//2
331 nmark = (width-len(txt)-2)//len(mark)//2
351 if nmark < 0: nmark =0
332 if nmark < 0: nmark =0
352 marks = mark*nmark
333 marks = mark*nmark
353 return '%s %s %s' % (marks,txt,marks)
334 return '%s %s %s' % (marks,txt,marks)
354
335
355
336
356 ini_spaces_re = re.compile(r'^(\s+)')
337 ini_spaces_re = re.compile(r'^(\s+)')
357
338
358 def num_ini_spaces(strng):
339 def num_ini_spaces(strng):
359 """Return the number of initial spaces in a string"""
340 """Return the number of initial spaces in a string"""
360
341
361 ini_spaces = ini_spaces_re.match(strng)
342 ini_spaces = ini_spaces_re.match(strng)
362 if ini_spaces:
343 if ini_spaces:
363 return ini_spaces.end()
344 return ini_spaces.end()
364 else:
345 else:
365 return 0
346 return 0
366
347
367
348
368 def format_screen(strng):
349 def format_screen(strng):
369 """Format a string for screen printing.
350 """Format a string for screen printing.
370
351
371 This removes some latex-type format codes."""
352 This removes some latex-type format codes."""
372 # Paragraph continue
353 # Paragraph continue
373 par_re = re.compile(r'\\$',re.MULTILINE)
354 par_re = re.compile(r'\\$',re.MULTILINE)
374 strng = par_re.sub('',strng)
355 strng = par_re.sub('',strng)
375 return strng
356 return strng
376
357
377
358
378 def dedent(text):
359 def dedent(text):
379 """Equivalent of textwrap.dedent that ignores unindented first line.
360 """Equivalent of textwrap.dedent that ignores unindented first line.
380
361
381 This means it will still dedent strings like:
362 This means it will still dedent strings like:
382 '''foo
363 '''foo
383 is a bar
364 is a bar
384 '''
365 '''
385
366
386 For use in wrap_paragraphs.
367 For use in wrap_paragraphs.
387 """
368 """
388
369
389 if text.startswith('\n'):
370 if text.startswith('\n'):
390 # text starts with blank line, don't ignore the first line
371 # text starts with blank line, don't ignore the first line
391 return textwrap.dedent(text)
372 return textwrap.dedent(text)
392
373
393 # split first line
374 # split first line
394 splits = text.split('\n',1)
375 splits = text.split('\n',1)
395 if len(splits) == 1:
376 if len(splits) == 1:
396 # only one line
377 # only one line
397 return textwrap.dedent(text)
378 return textwrap.dedent(text)
398
379
399 first, rest = splits
380 first, rest = splits
400 # dedent everything but the first line
381 # dedent everything but the first line
401 rest = textwrap.dedent(rest)
382 rest = textwrap.dedent(rest)
402 return '\n'.join([first, rest])
383 return '\n'.join([first, rest])
403
384
404
385
405 def wrap_paragraphs(text, ncols=80):
386 def wrap_paragraphs(text, ncols=80):
406 """Wrap multiple paragraphs to fit a specified width.
387 """Wrap multiple paragraphs to fit a specified width.
407
388
408 This is equivalent to textwrap.wrap, but with support for multiple
389 This is equivalent to textwrap.wrap, but with support for multiple
409 paragraphs, as separated by empty lines.
390 paragraphs, as separated by empty lines.
410
391
411 Returns
392 Returns
412 -------
393 -------
413
394
414 list of complete paragraphs, wrapped to fill `ncols` columns.
395 list of complete paragraphs, wrapped to fill `ncols` columns.
415 """
396 """
416 paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
397 paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
417 text = dedent(text).strip()
398 text = dedent(text).strip()
418 paragraphs = paragraph_re.split(text)[::2] # every other entry is space
399 paragraphs = paragraph_re.split(text)[::2] # every other entry is space
419 out_ps = []
400 out_ps = []
420 indent_re = re.compile(r'\n\s+', re.MULTILINE)
401 indent_re = re.compile(r'\n\s+', re.MULTILINE)
421 for p in paragraphs:
402 for p in paragraphs:
422 # presume indentation that survives dedent is meaningful formatting,
403 # presume indentation that survives dedent is meaningful formatting,
423 # so don't fill unless text is flush.
404 # so don't fill unless text is flush.
424 if indent_re.search(p) is None:
405 if indent_re.search(p) is None:
425 # wrap paragraph
406 # wrap paragraph
426 p = textwrap.fill(p, ncols)
407 p = textwrap.fill(p, ncols)
427 out_ps.append(p)
408 out_ps.append(p)
428 return out_ps
409 return out_ps
429
410
430
411
431 def long_substr(data):
412 def long_substr(data):
432 """Return the longest common substring in a list of strings.
413 """Return the longest common substring in a list of strings.
433
414
434 Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
415 Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
435 """
416 """
436 substr = ''
417 substr = ''
437 if len(data) > 1 and len(data[0]) > 0:
418 if len(data) > 1 and len(data[0]) > 0:
438 for i in range(len(data[0])):
419 for i in range(len(data[0])):
439 for j in range(len(data[0])-i+1):
420 for j in range(len(data[0])-i+1):
440 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
421 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
441 substr = data[0][i:i+j]
422 substr = data[0][i:i+j]
442 elif len(data) == 1:
423 elif len(data) == 1:
443 substr = data[0]
424 substr = data[0]
444 return substr
425 return substr
445
426
446
427
447 def strip_email_quotes(text):
428 def strip_email_quotes(text):
448 """Strip leading email quotation characters ('>').
429 """Strip leading email quotation characters ('>').
449
430
450 Removes any combination of leading '>' interspersed with whitespace that
431 Removes any combination of leading '>' interspersed with whitespace that
451 appears *identically* in all lines of the input text.
432 appears *identically* in all lines of the input text.
452
433
453 Parameters
434 Parameters
454 ----------
435 ----------
455 text : str
436 text : str
456
437
457 Examples
438 Examples
458 --------
439 --------
459
440
460 Simple uses::
441 Simple uses::
461
442
462 In [2]: strip_email_quotes('> > text')
443 In [2]: strip_email_quotes('> > text')
463 Out[2]: 'text'
444 Out[2]: 'text'
464
445
465 In [3]: strip_email_quotes('> > text\\n> > more')
446 In [3]: strip_email_quotes('> > text\\n> > more')
466 Out[3]: 'text\\nmore'
447 Out[3]: 'text\\nmore'
467
448
468 Note how only the common prefix that appears in all lines is stripped::
449 Note how only the common prefix that appears in all lines is stripped::
469
450
470 In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
451 In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
471 Out[4]: '> text\\n> more\\nmore...'
452 Out[4]: '> text\\n> more\\nmore...'
472
453
473 So if any line has no quote marks ('>') , then none are stripped from any
454 So if any line has no quote marks ('>') , then none are stripped from any
474 of them ::
455 of them ::
475
456
476 In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
457 In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
477 Out[5]: '> > text\\n> > more\\nlast different'
458 Out[5]: '> > text\\n> > more\\nlast different'
478 """
459 """
479 lines = text.splitlines()
460 lines = text.splitlines()
480 matches = set()
461 matches = set()
481 for line in lines:
462 for line in lines:
482 prefix = re.match(r'^(\s*>[ >]*)', line)
463 prefix = re.match(r'^(\s*>[ >]*)', line)
483 if prefix:
464 if prefix:
484 matches.add(prefix.group(1))
465 matches.add(prefix.group(1))
485 else:
466 else:
486 break
467 break
487 else:
468 else:
488 prefix = long_substr(list(matches))
469 prefix = long_substr(list(matches))
489 if prefix:
470 if prefix:
490 strip = len(prefix)
471 strip = len(prefix)
491 text = '\n'.join([ ln[strip:] for ln in lines])
472 text = '\n'.join([ ln[strip:] for ln in lines])
492 return text
473 return text
493
474
494 def strip_ansi(source):
475 def strip_ansi(source):
495 """
476 """
496 Remove ansi escape codes from text.
477 Remove ansi escape codes from text.
497
478
498 Parameters
479 Parameters
499 ----------
480 ----------
500 source : str
481 source : str
501 Source to remove the ansi from
482 Source to remove the ansi from
502 """
483 """
503 return re.sub(r'\033\[(\d|;)+?m', '', source)
484 return re.sub(r'\033\[(\d|;)+?m', '', source)
504
485
505
486
506 class EvalFormatter(Formatter):
487 class EvalFormatter(Formatter):
507 """A String Formatter that allows evaluation of simple expressions.
488 """A String Formatter that allows evaluation of simple expressions.
508
489
509 Note that this version interprets a : as specifying a format string (as per
490 Note that this version interprets a : as specifying a format string (as per
510 standard string formatting), so if slicing is required, you must explicitly
491 standard string formatting), so if slicing is required, you must explicitly
511 create a slice.
492 create a slice.
512
493
513 This is to be used in templating cases, such as the parallel batch
494 This is to be used in templating cases, such as the parallel batch
514 script templates, where simple arithmetic on arguments is useful.
495 script templates, where simple arithmetic on arguments is useful.
515
496
516 Examples
497 Examples
517 --------
498 --------
518 ::
499 ::
519
500
520 In [1]: f = EvalFormatter()
501 In [1]: f = EvalFormatter()
521 In [2]: f.format('{n//4}', n=8)
502 In [2]: f.format('{n//4}', n=8)
522 Out[2]: '2'
503 Out[2]: '2'
523
504
524 In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
505 In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
525 Out[3]: 'll'
506 Out[3]: 'll'
526 """
507 """
527 def get_field(self, name, args, kwargs):
508 def get_field(self, name, args, kwargs):
528 v = eval(name, kwargs)
509 v = eval(name, kwargs)
529 return v, name
510 return v, name
530
511
531 #XXX: As of Python 3.4, the format string parsing no longer splits on a colon
512 #XXX: As of Python 3.4, the format string parsing no longer splits on a colon
532 # inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
513 # inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
533 # above, it should be possible to remove FullEvalFormatter.
514 # above, it should be possible to remove FullEvalFormatter.
534
515
535 @skip_doctest_py3
516 @skip_doctest_py3
536 class FullEvalFormatter(Formatter):
517 class FullEvalFormatter(Formatter):
537 """A String Formatter that allows evaluation of simple expressions.
518 """A String Formatter that allows evaluation of simple expressions.
538
519
539 Any time a format key is not found in the kwargs,
520 Any time a format key is not found in the kwargs,
540 it will be tried as an expression in the kwargs namespace.
521 it will be tried as an expression in the kwargs namespace.
541
522
542 Note that this version allows slicing using [1:2], so you cannot specify
523 Note that this version allows slicing using [1:2], so you cannot specify
543 a format string. Use :class:`EvalFormatter` to permit format strings.
524 a format string. Use :class:`EvalFormatter` to permit format strings.
544
525
545 Examples
526 Examples
546 --------
527 --------
547 ::
528 ::
548
529
549 In [1]: f = FullEvalFormatter()
530 In [1]: f = FullEvalFormatter()
550 In [2]: f.format('{n//4}', n=8)
531 In [2]: f.format('{n//4}', n=8)
551 Out[2]: u'2'
532 Out[2]: u'2'
552
533
553 In [3]: f.format('{list(range(5))[2:4]}')
534 In [3]: f.format('{list(range(5))[2:4]}')
554 Out[3]: u'[2, 3]'
535 Out[3]: u'[2, 3]'
555
536
556 In [4]: f.format('{3*2}')
537 In [4]: f.format('{3*2}')
557 Out[4]: u'6'
538 Out[4]: u'6'
558 """
539 """
559 # copied from Formatter._vformat with minor changes to allow eval
540 # copied from Formatter._vformat with minor changes to allow eval
560 # and replace the format_spec code with slicing
541 # and replace the format_spec code with slicing
561 def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
542 def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
562 if recursion_depth < 0:
543 if recursion_depth < 0:
563 raise ValueError('Max string recursion exceeded')
544 raise ValueError('Max string recursion exceeded')
564 result = []
545 result = []
565 for literal_text, field_name, format_spec, conversion in \
546 for literal_text, field_name, format_spec, conversion in \
566 self.parse(format_string):
547 self.parse(format_string):
567
548
568 # output the literal text
549 # output the literal text
569 if literal_text:
550 if literal_text:
570 result.append(literal_text)
551 result.append(literal_text)
571
552
572 # if there's a field, output it
553 # if there's a field, output it
573 if field_name is not None:
554 if field_name is not None:
574 # this is some markup, find the object and do
555 # this is some markup, find the object and do
575 # the formatting
556 # the formatting
576
557
577 if format_spec:
558 if format_spec:
578 # override format spec, to allow slicing:
559 # override format spec, to allow slicing:
579 field_name = ':'.join([field_name, format_spec])
560 field_name = ':'.join([field_name, format_spec])
580
561
581 # eval the contents of the field for the object
562 # eval the contents of the field for the object
582 # to be formatted
563 # to be formatted
583 obj = eval(field_name, kwargs)
564 obj = eval(field_name, kwargs)
584
565
585 # do any conversion on the resulting object
566 # do any conversion on the resulting object
586 obj = self.convert_field(obj, conversion)
567 obj = self.convert_field(obj, conversion)
587
568
588 # format the object and append to the result
569 # format the object and append to the result
589 result.append(self.format_field(obj, ''))
570 result.append(self.format_field(obj, ''))
590
571
591 return u''.join(py3compat.cast_unicode(s) for s in result)
572 return u''.join(py3compat.cast_unicode(s) for s in result)
592
573
593
574
594 @skip_doctest_py3
575 @skip_doctest_py3
595 class DollarFormatter(FullEvalFormatter):
576 class DollarFormatter(FullEvalFormatter):
596 """Formatter allowing Itpl style $foo replacement, for names and attribute
577 """Formatter allowing Itpl style $foo replacement, for names and attribute
597 access only. Standard {foo} replacement also works, and allows full
578 access only. Standard {foo} replacement also works, and allows full
598 evaluation of its arguments.
579 evaluation of its arguments.
599
580
600 Examples
581 Examples
601 --------
582 --------
602 ::
583 ::
603
584
604 In [1]: f = DollarFormatter()
585 In [1]: f = DollarFormatter()
605 In [2]: f.format('{n//4}', n=8)
586 In [2]: f.format('{n//4}', n=8)
606 Out[2]: u'2'
587 Out[2]: u'2'
607
588
608 In [3]: f.format('23 * 76 is $result', result=23*76)
589 In [3]: f.format('23 * 76 is $result', result=23*76)
609 Out[3]: u'23 * 76 is 1748'
590 Out[3]: u'23 * 76 is 1748'
610
591
611 In [4]: f.format('$a or {b}', a=1, b=2)
592 In [4]: f.format('$a or {b}', a=1, b=2)
612 Out[4]: u'1 or 2'
593 Out[4]: u'1 or 2'
613 """
594 """
614 _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
595 _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
615 def parse(self, fmt_string):
596 def parse(self, fmt_string):
616 for literal_txt, field_name, format_spec, conversion \
597 for literal_txt, field_name, format_spec, conversion \
617 in Formatter.parse(self, fmt_string):
598 in Formatter.parse(self, fmt_string):
618
599
619 # Find $foo patterns in the literal text.
600 # Find $foo patterns in the literal text.
620 continue_from = 0
601 continue_from = 0
621 txt = ""
602 txt = ""
622 for m in self._dollar_pattern.finditer(literal_txt):
603 for m in self._dollar_pattern.finditer(literal_txt):
623 new_txt, new_field = m.group(1,2)
604 new_txt, new_field = m.group(1,2)
624 # $$foo --> $foo
605 # $$foo --> $foo
625 if new_field.startswith("$"):
606 if new_field.startswith("$"):
626 txt += new_txt + new_field
607 txt += new_txt + new_field
627 else:
608 else:
628 yield (txt + new_txt, new_field, "", None)
609 yield (txt + new_txt, new_field, "", None)
629 txt = ""
610 txt = ""
630 continue_from = m.end()
611 continue_from = m.end()
631
612
632 # Re-yield the {foo} style pattern
613 # Re-yield the {foo} style pattern
633 yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
614 yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
634
615
635 #-----------------------------------------------------------------------------
616 #-----------------------------------------------------------------------------
636 # Utils to columnize a list of string
617 # Utils to columnize a list of string
637 #-----------------------------------------------------------------------------
618 #-----------------------------------------------------------------------------
638
619
639 def _chunks(l, n):
620 def _chunks(l, n):
640 """Yield successive n-sized chunks from l."""
621 """Yield successive n-sized chunks from l."""
641 for i in py3compat.xrange(0, len(l), n):
622 for i in py3compat.xrange(0, len(l), n):
642 yield l[i:i+n]
623 yield l[i:i+n]
643
624
644
625
645 def _find_optimal(rlist , separator_size=2 , displaywidth=80):
626 def _find_optimal(rlist , separator_size=2 , displaywidth=80):
646 """Calculate optimal info to columnize a list of string"""
627 """Calculate optimal info to columnize a list of string"""
647 for nrow in range(1, len(rlist)+1) :
628 for nrow in range(1, len(rlist)+1) :
648 chk = list(map(max,_chunks(rlist, nrow)))
629 chk = list(map(max,_chunks(rlist, nrow)))
649 sumlength = sum(chk)
630 sumlength = sum(chk)
650 ncols = len(chk)
631 ncols = len(chk)
651 if sumlength+separator_size*(ncols-1) <= displaywidth :
632 if sumlength+separator_size*(ncols-1) <= displaywidth :
652 break;
633 break;
653 return {'columns_numbers' : ncols,
634 return {'columns_numbers' : ncols,
654 'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
635 'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
655 'rows_numbers' : nrow,
636 'rows_numbers' : nrow,
656 'columns_width' : chk
637 'columns_width' : chk
657 }
638 }
658
639
659
640
660 def _get_or_default(mylist, i, default=None):
641 def _get_or_default(mylist, i, default=None):
661 """return list item number, or default if don't exist"""
642 """return list item number, or default if don't exist"""
662 if i >= len(mylist):
643 if i >= len(mylist):
663 return default
644 return default
664 else :
645 else :
665 return mylist[i]
646 return mylist[i]
666
647
667
648
668 @skip_doctest
649 @skip_doctest
669 def compute_item_matrix(items, empty=None, *args, **kwargs) :
650 def compute_item_matrix(items, empty=None, *args, **kwargs) :
670 """Returns a nested list, and info to columnize items
651 """Returns a nested list, and info to columnize items
671
652
672 Parameters
653 Parameters
673 ----------
654 ----------
674
655
675 items
656 items
676 list of strings to columize
657 list of strings to columize
677 empty : (default None)
658 empty : (default None)
678 default value to fill list if needed
659 default value to fill list if needed
679 separator_size : int (default=2)
660 separator_size : int (default=2)
680 How much caracters will be used as a separation between each columns.
661 How much caracters will be used as a separation between each columns.
681 displaywidth : int (default=80)
662 displaywidth : int (default=80)
682 The width of the area onto wich the columns should enter
663 The width of the area onto wich the columns should enter
683
664
684 Returns
665 Returns
685 -------
666 -------
686
667
687 strings_matrix
668 strings_matrix
688
669
689 nested list of string, the outer most list contains as many list as
670 nested list of string, the outer most list contains as many list as
690 rows, the innermost lists have each as many element as colums. If the
671 rows, the innermost lists have each as many element as colums. If the
691 total number of elements in `items` does not equal the product of
672 total number of elements in `items` does not equal the product of
692 rows*columns, the last element of some lists are filled with `None`.
673 rows*columns, the last element of some lists are filled with `None`.
693
674
694 dict_info
675 dict_info
695 some info to make columnize easier:
676 some info to make columnize easier:
696
677
697 columns_numbers
678 columns_numbers
698 number of columns
679 number of columns
699 rows_numbers
680 rows_numbers
700 number of rows
681 number of rows
701 columns_width
682 columns_width
702 list of with of each columns
683 list of with of each columns
703 optimal_separator_width
684 optimal_separator_width
704 best separator width between columns
685 best separator width between columns
705
686
706 Examples
687 Examples
707 --------
688 --------
708 ::
689 ::
709
690
710 In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
691 In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
711 ...: compute_item_matrix(l,displaywidth=12)
692 ...: compute_item_matrix(l,displaywidth=12)
712 Out[1]:
693 Out[1]:
713 ([['aaa', 'f', 'k'],
694 ([['aaa', 'f', 'k'],
714 ['b', 'g', 'l'],
695 ['b', 'g', 'l'],
715 ['cc', 'h', None],
696 ['cc', 'h', None],
716 ['d', 'i', None],
697 ['d', 'i', None],
717 ['eeeee', 'j', None]],
698 ['eeeee', 'j', None]],
718 {'columns_numbers': 3,
699 {'columns_numbers': 3,
719 'columns_width': [5, 1, 1],
700 'columns_width': [5, 1, 1],
720 'optimal_separator_width': 2,
701 'optimal_separator_width': 2,
721 'rows_numbers': 5})
702 'rows_numbers': 5})
722 """
703 """
723 info = _find_optimal(list(map(len, items)), *args, **kwargs)
704 info = _find_optimal(list(map(len, items)), *args, **kwargs)
724 nrow, ncol = info['rows_numbers'], info['columns_numbers']
705 nrow, ncol = info['rows_numbers'], info['columns_numbers']
725 return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)
706 return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)
726
707
727
708
728 def columnize(items, separator=' ', displaywidth=80):
709 def columnize(items, separator=' ', displaywidth=80):
729 """ Transform a list of strings into a single string with columns.
710 """ Transform a list of strings into a single string with columns.
730
711
731 Parameters
712 Parameters
732 ----------
713 ----------
733 items : sequence of strings
714 items : sequence of strings
734 The strings to process.
715 The strings to process.
735
716
736 separator : str, optional [default is two spaces]
717 separator : str, optional [default is two spaces]
737 The string that separates columns.
718 The string that separates columns.
738
719
739 displaywidth : int, optional [default is 80]
720 displaywidth : int, optional [default is 80]
740 Width of the display in number of characters.
721 Width of the display in number of characters.
741
722
742 Returns
723 Returns
743 -------
724 -------
744 The formatted string.
725 The formatted string.
745 """
726 """
746 if not items :
727 if not items :
747 return '\n'
728 return '\n'
748 matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
729 matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
749 fmatrix = [filter(None, x) for x in matrix]
730 fmatrix = [filter(None, x) for x in matrix]
750 sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
731 sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
751 return '\n'.join(map(sjoin, fmatrix))+'\n'
732 return '\n'.join(map(sjoin, fmatrix))+'\n'
752
733
753
734
754 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
735 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
755 """
736 """
756 Return a string with a natural enumeration of items
737 Return a string with a natural enumeration of items
757
738
758 >>> get_text_list(['a', 'b', 'c', 'd'])
739 >>> get_text_list(['a', 'b', 'c', 'd'])
759 'a, b, c and d'
740 'a, b, c and d'
760 >>> get_text_list(['a', 'b', 'c'], ' or ')
741 >>> get_text_list(['a', 'b', 'c'], ' or ')
761 'a, b or c'
742 'a, b or c'
762 >>> get_text_list(['a', 'b', 'c'], ', ')
743 >>> get_text_list(['a', 'b', 'c'], ', ')
763 'a, b, c'
744 'a, b, c'
764 >>> get_text_list(['a', 'b'], ' or ')
745 >>> get_text_list(['a', 'b'], ' or ')
765 'a or b'
746 'a or b'
766 >>> get_text_list(['a'])
747 >>> get_text_list(['a'])
767 'a'
748 'a'
768 >>> get_text_list([])
749 >>> get_text_list([])
769 ''
750 ''
770 >>> get_text_list(['a', 'b'], wrap_item_with="`")
751 >>> get_text_list(['a', 'b'], wrap_item_with="`")
771 '`a` and `b`'
752 '`a` and `b`'
772 >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
753 >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
773 'a + b + c = d'
754 'a + b + c = d'
774 """
755 """
775 if len(list_) == 0:
756 if len(list_) == 0:
776 return ''
757 return ''
777 if wrap_item_with:
758 if wrap_item_with:
778 list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
759 list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
779 item in list_]
760 item in list_]
780 if len(list_) == 1:
761 if len(list_) == 1:
781 return list_[0]
762 return list_[0]
782 return '%s%s%s' % (
763 return '%s%s%s' % (
783 sep.join(i for i in list_[:-1]),
764 sep.join(i for i in list_[:-1]),
784 last_sep, list_[-1]) No newline at end of file
765 last_sep, list_[-1])
@@ -1,342 +1,344 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
2 # -*- coding: utf-8 -*-
3 """Setup script for IPython.
3 """Setup script for IPython.
4
4
5 Under Posix environments it works like a typical setup.py script.
5 Under Posix environments it works like a typical setup.py script.
6 Under Windows, the command sdist is not supported, since IPython
6 Under Windows, the command sdist is not supported, since IPython
7 requires utilities which are not available under Windows."""
7 requires utilities which are not available under Windows."""
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (c) 2008-2011, IPython Development Team.
10 # Copyright (c) 2008-2011, IPython Development Team.
11 # Copyright (c) 2001-2007, Fernando Perez <fernando.perez@colorado.edu>
11 # Copyright (c) 2001-2007, Fernando Perez <fernando.perez@colorado.edu>
12 # Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
12 # Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
13 # Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
13 # Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
14 #
14 #
15 # Distributed under the terms of the Modified BSD License.
15 # Distributed under the terms of the Modified BSD License.
16 #
16 #
17 # The full license is in the file COPYING.rst, distributed with this software.
17 # The full license is in the file COPYING.rst, distributed with this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Minimal Python version sanity check
21 # Minimal Python version sanity check
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23 from __future__ import print_function
23 from __future__ import print_function
24
24
25 import sys
25 import sys
26
26
27 # This check is also made in IPython/__init__, don't forget to update both when
27 # This check is also made in IPython/__init__, don't forget to update both when
28 # changing Python version requirements.
28 # changing Python version requirements.
29 v = sys.version_info
29 v = sys.version_info
30 if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,3)):
30 if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,3)):
31 error = "ERROR: IPython requires Python version 2.7 or 3.3 or above."
31 error = "ERROR: IPython requires Python version 2.7 or 3.3 or above."
32 print(error, file=sys.stderr)
32 print(error, file=sys.stderr)
33 sys.exit(1)
33 sys.exit(1)
34
34
35 PY3 = (sys.version_info[0] >= 3)
35 PY3 = (sys.version_info[0] >= 3)
36
36
37 # At least we're on the python version we need, move on.
37 # At least we're on the python version we need, move on.
38
38
39 #-------------------------------------------------------------------------------
39 #-------------------------------------------------------------------------------
40 # Imports
40 # Imports
41 #-------------------------------------------------------------------------------
41 #-------------------------------------------------------------------------------
42
42
43 # Stdlib imports
43 # Stdlib imports
44 import os
44 import os
45 import shutil
45 import shutil
46
46
47 from glob import glob
47 from glob import glob
48
48
49 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
49 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
50 # update it when the contents of directories change.
50 # update it when the contents of directories change.
51 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
51 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
52
52
53 from distutils.core import setup
53 from distutils.core import setup
54
54
55 # Our own imports
55 # Our own imports
56 from setupbase import target_update
56 from setupbase import target_update
57
57
58 from setupbase import (
58 from setupbase import (
59 setup_args,
59 setup_args,
60 find_packages,
60 find_packages,
61 find_package_data,
61 find_package_data,
62 check_package_data_first,
62 check_package_data_first,
63 find_entry_points,
63 find_entry_points,
64 build_scripts_entrypt,
64 build_scripts_entrypt,
65 find_data_files,
65 find_data_files,
66 check_for_dependencies,
66 check_for_dependencies,
67 git_prebuild,
67 git_prebuild,
68 check_submodule_status,
68 check_submodule_status,
69 update_submodules,
69 update_submodules,
70 require_submodules,
70 require_submodules,
71 UpdateSubmodules,
71 UpdateSubmodules,
72 get_bdist_wheel,
72 get_bdist_wheel,
73 CompileCSS,
73 CompileCSS,
74 JavascriptVersion,
74 JavascriptVersion,
75 css_js_prerelease,
75 css_js_prerelease,
76 install_symlinked,
76 install_symlinked,
77 install_lib_symlink,
77 install_lib_symlink,
78 install_scripts_for_symlink,
78 install_scripts_for_symlink,
79 unsymlink,
79 unsymlink,
80 )
80 )
81 from setupext import setupext
81 from setupext import setupext
82
82
83 isfile = os.path.isfile
83 isfile = os.path.isfile
84 pjoin = os.path.join
84 pjoin = os.path.join
85
85
86 #-------------------------------------------------------------------------------
86 #-------------------------------------------------------------------------------
87 # Handle OS specific things
87 # Handle OS specific things
88 #-------------------------------------------------------------------------------
88 #-------------------------------------------------------------------------------
89
89
90 if os.name in ('nt','dos'):
90 if os.name in ('nt','dos'):
91 os_name = 'windows'
91 os_name = 'windows'
92 else:
92 else:
93 os_name = os.name
93 os_name = os.name
94
94
95 # Under Windows, 'sdist' has not been supported. Now that the docs build with
95 # Under Windows, 'sdist' has not been supported. Now that the docs build with
96 # Sphinx it might work, but let's not turn it on until someone confirms that it
96 # Sphinx it might work, but let's not turn it on until someone confirms that it
97 # actually works.
97 # actually works.
98 if os_name == 'windows' and 'sdist' in sys.argv:
98 if os_name == 'windows' and 'sdist' in sys.argv:
99 print('The sdist command is not available under Windows. Exiting.')
99 print('The sdist command is not available under Windows. Exiting.')
100 sys.exit(1)
100 sys.exit(1)
101
101
102 #-------------------------------------------------------------------------------
102 #-------------------------------------------------------------------------------
103 # Make sure we aren't trying to run without submodules
103 # Make sure we aren't trying to run without submodules
104 #-------------------------------------------------------------------------------
104 #-------------------------------------------------------------------------------
105 here = os.path.abspath(os.path.dirname(__file__))
105 here = os.path.abspath(os.path.dirname(__file__))
106
106
107 def require_clean_submodules():
107 def require_clean_submodules():
108 """Check on git submodules before distutils can do anything
108 """Check on git submodules before distutils can do anything
109
109
110 Since distutils cannot be trusted to update the tree
110 Since distutils cannot be trusted to update the tree
111 after everything has been set in motion,
111 after everything has been set in motion,
112 this is not a distutils command.
112 this is not a distutils command.
113 """
113 """
114 # PACKAGERS: Add a return here to skip checks for git submodules
114 # PACKAGERS: Add a return here to skip checks for git submodules
115
115
116 # don't do anything if nothing is actually supposed to happen
116 # don't do anything if nothing is actually supposed to happen
117 for do_nothing in ('-h', '--help', '--help-commands', 'clean', 'submodule'):
117 for do_nothing in ('-h', '--help', '--help-commands', 'clean', 'submodule'):
118 if do_nothing in sys.argv:
118 if do_nothing in sys.argv:
119 return
119 return
120
120
121 status = check_submodule_status(here)
121 status = check_submodule_status(here)
122
122
123 if status == "missing":
123 if status == "missing":
124 print("checking out submodules for the first time")
124 print("checking out submodules for the first time")
125 update_submodules(here)
125 update_submodules(here)
126 elif status == "unclean":
126 elif status == "unclean":
127 print('\n'.join([
127 print('\n'.join([
128 "Cannot build / install IPython with unclean submodules",
128 "Cannot build / install IPython with unclean submodules",
129 "Please update submodules with",
129 "Please update submodules with",
130 " python setup.py submodule",
130 " python setup.py submodule",
131 "or",
131 "or",
132 " git submodule update",
132 " git submodule update",
133 "or commit any submodule changes you have made."
133 "or commit any submodule changes you have made."
134 ]))
134 ]))
135 sys.exit(1)
135 sys.exit(1)
136
136
137 require_clean_submodules()
137 require_clean_submodules()
138
138
139 #-------------------------------------------------------------------------------
139 #-------------------------------------------------------------------------------
140 # Things related to the IPython documentation
140 # Things related to the IPython documentation
141 #-------------------------------------------------------------------------------
141 #-------------------------------------------------------------------------------
142
142
143 # update the manuals when building a source dist
143 # update the manuals when building a source dist
144 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
144 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
145
145
146 # List of things to be updated. Each entry is a triplet of args for
146 # List of things to be updated. Each entry is a triplet of args for
147 # target_update()
147 # target_update()
148 to_update = [
148 to_update = [
149 # FIXME - Disabled for now: we need to redo an automatic way
149 # FIXME - Disabled for now: we need to redo an automatic way
150 # of generating the magic info inside the rst.
150 # of generating the magic info inside the rst.
151 #('docs/magic.tex',
151 #('docs/magic.tex',
152 #['IPython/Magic.py'],
152 #['IPython/Magic.py'],
153 #"cd doc && ./update_magic.sh" ),
153 #"cd doc && ./update_magic.sh" ),
154
154
155 ('docs/man/ipcluster.1.gz',
155 ('docs/man/ipcluster.1.gz',
156 ['docs/man/ipcluster.1'],
156 ['docs/man/ipcluster.1'],
157 'cd docs/man && gzip -9c ipcluster.1 > ipcluster.1.gz'),
157 'cd docs/man && gzip -9c ipcluster.1 > ipcluster.1.gz'),
158
158
159 ('docs/man/ipcontroller.1.gz',
159 ('docs/man/ipcontroller.1.gz',
160 ['docs/man/ipcontroller.1'],
160 ['docs/man/ipcontroller.1'],
161 'cd docs/man && gzip -9c ipcontroller.1 > ipcontroller.1.gz'),
161 'cd docs/man && gzip -9c ipcontroller.1 > ipcontroller.1.gz'),
162
162
163 ('docs/man/ipengine.1.gz',
163 ('docs/man/ipengine.1.gz',
164 ['docs/man/ipengine.1'],
164 ['docs/man/ipengine.1'],
165 'cd docs/man && gzip -9c ipengine.1 > ipengine.1.gz'),
165 'cd docs/man && gzip -9c ipengine.1 > ipengine.1.gz'),
166
166
167 ('docs/man/ipython.1.gz',
167 ('docs/man/ipython.1.gz',
168 ['docs/man/ipython.1'],
168 ['docs/man/ipython.1'],
169 'cd docs/man && gzip -9c ipython.1 > ipython.1.gz'),
169 'cd docs/man && gzip -9c ipython.1 > ipython.1.gz'),
170
170
171 ]
171 ]
172
172
173
173
174 [ target_update(*t) for t in to_update ]
174 [ target_update(*t) for t in to_update ]
175
175
176 #---------------------------------------------------------------------------
176 #---------------------------------------------------------------------------
177 # Find all the packages, package data, and data_files
177 # Find all the packages, package data, and data_files
178 #---------------------------------------------------------------------------
178 #---------------------------------------------------------------------------
179
179
180 packages = find_packages()
180 packages = find_packages()
181 package_data = find_package_data()
181 package_data = find_package_data()
182
182
183 data_files = find_data_files()
183 data_files = find_data_files()
184
184
185 setup_args['packages'] = packages
185 setup_args['packages'] = packages
186 setup_args['package_data'] = package_data
186 setup_args['package_data'] = package_data
187 setup_args['data_files'] = data_files
187 setup_args['data_files'] = data_files
188
188
189 #---------------------------------------------------------------------------
189 #---------------------------------------------------------------------------
190 # custom distutils commands
190 # custom distutils commands
191 #---------------------------------------------------------------------------
191 #---------------------------------------------------------------------------
192 # imports here, so they are after setuptools import if there was one
192 # imports here, so they are after setuptools import if there was one
193 from distutils.command.sdist import sdist
193 from distutils.command.sdist import sdist
194 from distutils.command.upload import upload
194 from distutils.command.upload import upload
195
195
196 class UploadWindowsInstallers(upload):
196 class UploadWindowsInstallers(upload):
197
197
198 description = "Upload Windows installers to PyPI (only used from tools/release_windows.py)"
198 description = "Upload Windows installers to PyPI (only used from tools/release_windows.py)"
199 user_options = upload.user_options + [
199 user_options = upload.user_options + [
200 ('files=', 'f', 'exe file (or glob) to upload')
200 ('files=', 'f', 'exe file (or glob) to upload')
201 ]
201 ]
202 def initialize_options(self):
202 def initialize_options(self):
203 upload.initialize_options(self)
203 upload.initialize_options(self)
204 meta = self.distribution.metadata
204 meta = self.distribution.metadata
205 base = '{name}-{version}'.format(
205 base = '{name}-{version}'.format(
206 name=meta.get_name(),
206 name=meta.get_name(),
207 version=meta.get_version()
207 version=meta.get_version()
208 )
208 )
209 self.files = os.path.join('dist', '%s.*.exe' % base)
209 self.files = os.path.join('dist', '%s.*.exe' % base)
210
210
211 def run(self):
211 def run(self):
212 for dist_file in glob(self.files):
212 for dist_file in glob(self.files):
213 self.upload_file('bdist_wininst', 'any', dist_file)
213 self.upload_file('bdist_wininst', 'any', dist_file)
214
214
215 setup_args['cmdclass'] = {
215 setup_args['cmdclass'] = {
216 'build_py': css_js_prerelease(
216 'build_py': css_js_prerelease(
217 check_package_data_first(git_prebuild('IPython'))),
217 check_package_data_first(git_prebuild('IPython'))),
218 'sdist' : css_js_prerelease(git_prebuild('IPython', sdist)),
218 'sdist' : css_js_prerelease(git_prebuild('IPython', sdist)),
219 'upload_wininst' : UploadWindowsInstallers,
219 'upload_wininst' : UploadWindowsInstallers,
220 'submodule' : UpdateSubmodules,
220 'submodule' : UpdateSubmodules,
221 'css' : CompileCSS,
221 'css' : CompileCSS,
222 'symlink': install_symlinked,
222 'symlink': install_symlinked,
223 'install_lib_symlink': install_lib_symlink,
223 'install_lib_symlink': install_lib_symlink,
224 'install_scripts_sym': install_scripts_for_symlink,
224 'install_scripts_sym': install_scripts_for_symlink,
225 'unsymlink': unsymlink,
225 'unsymlink': unsymlink,
226 'jsversion' : JavascriptVersion,
226 'jsversion' : JavascriptVersion,
227 }
227 }
228
228
229 #---------------------------------------------------------------------------
229 #---------------------------------------------------------------------------
230 # Handle scripts, dependencies, and setuptools specific things
230 # Handle scripts, dependencies, and setuptools specific things
231 #---------------------------------------------------------------------------
231 #---------------------------------------------------------------------------
232
232
233 # For some commands, use setuptools. Note that we do NOT list install here!
233 # For some commands, use setuptools. Note that we do NOT list install here!
234 # If you want a setuptools-enhanced install, just run 'setupegg.py install'
234 # If you want a setuptools-enhanced install, just run 'setupegg.py install'
235 needs_setuptools = set(('develop', 'release', 'bdist_egg', 'bdist_rpm',
235 needs_setuptools = set(('develop', 'release', 'bdist_egg', 'bdist_rpm',
236 'bdist', 'bdist_dumb', 'bdist_wininst', 'bdist_wheel',
236 'bdist', 'bdist_dumb', 'bdist_wininst', 'bdist_wheel',
237 'egg_info', 'easy_install', 'upload', 'install_egg_info',
237 'egg_info', 'easy_install', 'upload', 'install_egg_info',
238 ))
238 ))
239
239
240 if len(needs_setuptools.intersection(sys.argv)) > 0:
240 if len(needs_setuptools.intersection(sys.argv)) > 0:
241 import setuptools
241 import setuptools
242
242
243 # This dict is used for passing extra arguments that are setuptools
243 # This dict is used for passing extra arguments that are setuptools
244 # specific to setup
244 # specific to setup
245 setuptools_extra_args = {}
245 setuptools_extra_args = {}
246
246
247 # setuptools requirements
247 # setuptools requirements
248
248
249 pyzmq = 'pyzmq>=13'
249 pyzmq = 'pyzmq>=13'
250
250
251 extras_require = dict(
251 extras_require = dict(
252 parallel = [pyzmq],
252 parallel = [pyzmq],
253 qtconsole = [pyzmq, 'pygments'],
253 qtconsole = [pyzmq, 'pygments'],
254 doc = ['Sphinx>=1.1', 'numpydoc'],
254 doc = ['Sphinx>=1.1', 'numpydoc'],
255 test = ['nose>=0.10.1', 'requests'],
255 test = ['nose>=0.10.1', 'requests'],
256 terminal = [],
256 terminal = [],
257 nbformat = ['jsonschema>=2.0'],
257 nbformat = ['jsonschema>=2.0'],
258 notebook = ['tornado>=4.0', pyzmq, 'jinja2', 'pygments', 'mistune>=0.5'],
258 notebook = ['tornado>=4.0', pyzmq, 'jinja2', 'pygments', 'mistune>=0.5'],
259 nbconvert = ['pygments', 'jinja2', 'mistune>=0.3.1']
259 nbconvert = ['pygments', 'jinja2', 'mistune>=0.3.1']
260 )
260 )
261
261
262 if not sys.platform.startswith('win'):
262 if not sys.platform.startswith('win'):
263 extras_require['notebook'].append('terminado>=0.3.3')
263 extras_require['notebook'].append('terminado>=0.3.3')
264
264
265 if sys.version_info < (3, 3):
265 if sys.version_info < (3, 3):
266 extras_require['test'].append('mock')
266 extras_require['test'].append('mock')
267
267
268 extras_require['notebook'].extend(extras_require['nbformat'])
268 extras_require['notebook'].extend(extras_require['nbformat'])
269 extras_require['nbconvert'].extend(extras_require['nbformat'])
269 extras_require['nbconvert'].extend(extras_require['nbformat'])
270
270
271 install_requires = []
271 install_requires = [
272 'path.py', # required by pickleshare, remove when pickleshare is added here
273 ]
272
274
273 # add readline
275 # add readline
274 if sys.platform == 'darwin':
276 if sys.platform == 'darwin':
275 if 'bdist_wheel' in sys.argv[1:] or not setupext.check_for_readline():
277 if 'bdist_wheel' in sys.argv[1:] or not setupext.check_for_readline():
276 install_requires.append('gnureadline')
278 install_requires.append('gnureadline')
277 elif sys.platform.startswith('win'):
279 elif sys.platform.startswith('win'):
278 extras_require['terminal'].append('pyreadline>=2.0')
280 extras_require['terminal'].append('pyreadline>=2.0')
279
281
280 everything = set()
282 everything = set()
281 for deps in extras_require.values():
283 for deps in extras_require.values():
282 everything.update(deps)
284 everything.update(deps)
283 extras_require['all'] = everything
285 extras_require['all'] = everything
284
286
285 if 'setuptools' in sys.modules:
287 if 'setuptools' in sys.modules:
286 # setup.py develop should check for submodules
288 # setup.py develop should check for submodules
287 from setuptools.command.develop import develop
289 from setuptools.command.develop import develop
288 setup_args['cmdclass']['develop'] = require_submodules(develop)
290 setup_args['cmdclass']['develop'] = require_submodules(develop)
289 setup_args['cmdclass']['bdist_wheel'] = css_js_prerelease(get_bdist_wheel())
291 setup_args['cmdclass']['bdist_wheel'] = css_js_prerelease(get_bdist_wheel())
290
292
291 setuptools_extra_args['zip_safe'] = False
293 setuptools_extra_args['zip_safe'] = False
292 setuptools_extra_args['entry_points'] = {
294 setuptools_extra_args['entry_points'] = {
293 'console_scripts': find_entry_points(),
295 'console_scripts': find_entry_points(),
294 'pygments.lexers': [
296 'pygments.lexers': [
295 'ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer',
297 'ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer',
296 'ipython = IPython.lib.lexers:IPythonLexer',
298 'ipython = IPython.lib.lexers:IPythonLexer',
297 'ipython3 = IPython.lib.lexers:IPython3Lexer',
299 'ipython3 = IPython.lib.lexers:IPython3Lexer',
298 ],
300 ],
299 }
301 }
300 setup_args['extras_require'] = extras_require
302 setup_args['extras_require'] = extras_require
301 requires = setup_args['install_requires'] = install_requires
303 requires = setup_args['install_requires'] = install_requires
302
304
303 # Script to be run by the windows binary installer after the default setup
305 # Script to be run by the windows binary installer after the default setup
304 # routine, to add shortcuts and similar windows-only things. Windows
306 # routine, to add shortcuts and similar windows-only things. Windows
305 # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
307 # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
306 # doesn't find them.
308 # doesn't find them.
307 if 'bdist_wininst' in sys.argv:
309 if 'bdist_wininst' in sys.argv:
308 if len(sys.argv) > 2 and \
310 if len(sys.argv) > 2 and \
309 ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
311 ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
310 print("ERROR: bdist_wininst must be run alone. Exiting.", file=sys.stderr)
312 print("ERROR: bdist_wininst must be run alone. Exiting.", file=sys.stderr)
311 sys.exit(1)
313 sys.exit(1)
312 setup_args['data_files'].append(
314 setup_args['data_files'].append(
313 ['Scripts', ('scripts/ipython.ico', 'scripts/ipython_nb.ico')])
315 ['Scripts', ('scripts/ipython.ico', 'scripts/ipython_nb.ico')])
314 setup_args['scripts'] = [pjoin('scripts','ipython_win_post_install.py')]
316 setup_args['scripts'] = [pjoin('scripts','ipython_win_post_install.py')]
315 setup_args['options'] = {"bdist_wininst":
317 setup_args['options'] = {"bdist_wininst":
316 {"install_script":
318 {"install_script":
317 "ipython_win_post_install.py"}}
319 "ipython_win_post_install.py"}}
318
320
319 else:
321 else:
320 # If we are installing without setuptools, call this function which will
322 # If we are installing without setuptools, call this function which will
321 # check for dependencies an inform the user what is needed. This is
323 # check for dependencies an inform the user what is needed. This is
322 # just to make life easy for users.
324 # just to make life easy for users.
323 for install_cmd in ('install', 'symlink'):
325 for install_cmd in ('install', 'symlink'):
324 if install_cmd in sys.argv:
326 if install_cmd in sys.argv:
325 check_for_dependencies()
327 check_for_dependencies()
326 break
328 break
327 # scripts has to be a non-empty list, or install_scripts isn't called
329 # scripts has to be a non-empty list, or install_scripts isn't called
328 setup_args['scripts'] = [e.split('=')[0].strip() for e in find_entry_points()]
330 setup_args['scripts'] = [e.split('=')[0].strip() for e in find_entry_points()]
329
331
330 setup_args['cmdclass']['build_scripts'] = build_scripts_entrypt
332 setup_args['cmdclass']['build_scripts'] = build_scripts_entrypt
331
333
332 #---------------------------------------------------------------------------
334 #---------------------------------------------------------------------------
333 # Do the actual setup now
335 # Do the actual setup now
334 #---------------------------------------------------------------------------
336 #---------------------------------------------------------------------------
335
337
336 setup_args.update(setuptools_extra_args)
338 setup_args.update(setuptools_extra_args)
337
339
338 def main():
340 def main():
339 setup(**setup_args)
341 setup(**setup_args)
340
342
341 if __name__ == '__main__':
343 if __name__ == '__main__':
342 main()
344 main()
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
This diff has been collapsed as it changes many lines, (1267 lines changed) Show them Hide them
General Comments 0
You need to be logged in to leave comments. Login now