##// END OF EJS Templates
store: unindent most of the contents of the for loop in _auxencode()...
Adrian Buehlmann -
r17572:b644287e default
parent child Browse files
Show More
@@ -1,450 +1,451
1 1 # store.py - repository store handling for Mercurial
2 2 #
3 3 # Copyright 2008 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from i18n import _
9 9 import osutil, scmutil, util
10 10 import os, stat, errno
11 11
12 12 _sha = util.sha1
13 13
14 14 # This avoids a collision between a file named foo and a dir named
15 15 # foo.i or foo.d
16 16 def encodedir(path):
17 17 '''
18 18 >>> encodedir('data/foo.i')
19 19 'data/foo.i'
20 20 >>> encodedir('data/foo.i/bla.i')
21 21 'data/foo.i.hg/bla.i'
22 22 >>> encodedir('data/foo.i.hg/bla.i')
23 23 'data/foo.i.hg.hg/bla.i'
24 24 '''
25 25 if not path.startswith('data/'):
26 26 return path
27 27 return (path
28 28 .replace(".hg/", ".hg.hg/")
29 29 .replace(".i/", ".i.hg/")
30 30 .replace(".d/", ".d.hg/"))
31 31
32 32 def decodedir(path):
33 33 '''
34 34 >>> decodedir('data/foo.i')
35 35 'data/foo.i'
36 36 >>> decodedir('data/foo.i.hg/bla.i')
37 37 'data/foo.i/bla.i'
38 38 >>> decodedir('data/foo.i.hg.hg/bla.i')
39 39 'data/foo.i.hg/bla.i'
40 40 '''
41 41 if not path.startswith('data/') or ".hg/" not in path:
42 42 return path
43 43 return (path
44 44 .replace(".d.hg/", ".d/")
45 45 .replace(".i.hg/", ".i/")
46 46 .replace(".hg.hg/", ".hg/"))
47 47
48 48 def _buildencodefun():
49 49 '''
50 50 >>> enc, dec = _buildencodefun()
51 51
52 52 >>> enc('nothing/special.txt')
53 53 'nothing/special.txt'
54 54 >>> dec('nothing/special.txt')
55 55 'nothing/special.txt'
56 56
57 57 >>> enc('HELLO')
58 58 '_h_e_l_l_o'
59 59 >>> dec('_h_e_l_l_o')
60 60 'HELLO'
61 61
62 62 >>> enc('hello:world?')
63 63 'hello~3aworld~3f'
64 64 >>> dec('hello~3aworld~3f')
65 65 'hello:world?'
66 66
67 67 >>> enc('the\x07quick\xADshot')
68 68 'the~07quick~adshot'
69 69 >>> dec('the~07quick~adshot')
70 70 'the\\x07quick\\xadshot'
71 71 '''
72 72 e = '_'
73 73 winreserved = [ord(x) for x in '\\:*?"<>|']
74 74 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
75 75 for x in (range(32) + range(126, 256) + winreserved):
76 76 cmap[chr(x)] = "~%02x" % x
77 77 for x in range(ord("A"), ord("Z")+1) + [ord(e)]:
78 78 cmap[chr(x)] = e + chr(x).lower()
79 79 dmap = {}
80 80 for k, v in cmap.iteritems():
81 81 dmap[v] = k
82 82 def decode(s):
83 83 i = 0
84 84 while i < len(s):
85 85 for l in xrange(1, 4):
86 86 try:
87 87 yield dmap[s[i:i + l]]
88 88 i += l
89 89 break
90 90 except KeyError:
91 91 pass
92 92 else:
93 93 raise KeyError
94 94 return (lambda s: "".join([cmap[c] for c in encodedir(s)]),
95 95 lambda s: decodedir("".join(list(decode(s)))))
96 96
97 97 encodefilename, decodefilename = _buildencodefun()
98 98
99 99 def _buildlowerencodefun():
100 100 '''
101 101 >>> f = _buildlowerencodefun()
102 102 >>> f('nothing/special.txt')
103 103 'nothing/special.txt'
104 104 >>> f('HELLO')
105 105 'hello'
106 106 >>> f('hello:world?')
107 107 'hello~3aworld~3f'
108 108 >>> f('the\x07quick\xADshot')
109 109 'the~07quick~adshot'
110 110 '''
111 111 winreserved = [ord(x) for x in '\\:*?"<>|']
112 112 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
113 113 for x in (range(32) + range(126, 256) + winreserved):
114 114 cmap[chr(x)] = "~%02x" % x
115 115 for x in range(ord("A"), ord("Z")+1):
116 116 cmap[chr(x)] = chr(x).lower()
117 117 return lambda s: "".join([cmap[c] for c in s])
118 118
119 119 lowerencode = _buildlowerencodefun()
120 120
121 121 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9
122 122 _winres3 = ('aux', 'con', 'prn', 'nul') # length 3
123 123 _winres4 = ('com', 'lpt') # length 4 (with trailing 1..9)
124 124 def _auxencode(path, dotencode):
125 125 '''
126 126 Encodes filenames containing names reserved by Windows or which end in
127 127 period or space. Does not touch other single reserved characters c.
128 128 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here.
129 129 Additionally encodes space or period at the beginning, if dotencode is
130 130 True. Parameter path is assumed to be all lowercase.
131 131 A segment only needs encoding if a reserved name appears as a
132 132 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux"
133 133 doesn't need encoding.
134 134
135 135 >>> _auxencode('.foo/aux.txt/txt.aux/con/prn/nul/foo.', True)
136 136 '~2efoo/au~78.txt/txt.aux/co~6e/pr~6e/nu~6c/foo~2e'
137 137 >>> _auxencode('.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.', False)
138 138 '.com1com2/lp~749.lpt4.lpt1/conprn/com0/lpt0/foo~2e'
139 139 >>> _auxencode('foo. ', True)
140 140 'foo.~20'
141 141 >>> _auxencode(' .foo', True)
142 142 '~20.foo'
143 143 '''
144 144 res = path.split('/')
145 145 for i, n in enumerate(res):
146 if n:
146 if not n:
147 continue
147 148 if dotencode and n[0] in '. ':
148 149 n = "~%02x" % ord(n[0]) + n[1:]
149 150 res[i] = n
150 151 else:
151 152 l = n.find('.')
152 153 if l == -1:
153 154 l = len(n)
154 155 if ((l == 3 and n[:3] in _winres3) or
155 156 (l == 4 and n[3] <= '9' and n[3] >= '1'
156 157 and n[:3] in _winres4)):
157 158 # encode third letter ('aux' -> 'au~78')
158 159 ec = "~%02x" % ord(n[2])
159 160 n = n[0:2] + ec + n[3:]
160 161 res[i] = n
161 162 if n[-1] in '. ':
162 163 # encode last period or space ('foo...' -> 'foo..~2e')
163 164 n = n[:-1] + "~%02x" % ord(n[-1])
164 165 res[i] = n
165 166 return '/'.join(res)
166 167
167 168 _maxstorepathlen = 120
168 169 _dirprefixlen = 8
169 170 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
170 171 def _hybridencode(path, auxencode):
171 172 '''encodes path with a length limit
172 173
173 174 Encodes all paths that begin with 'data/', according to the following.
174 175
175 176 Default encoding (reversible):
176 177
177 178 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
178 179 characters are encoded as '~xx', where xx is the two digit hex code
179 180 of the character (see encodefilename).
180 181 Relevant path components consisting of Windows reserved filenames are
181 182 masked by encoding the third character ('aux' -> 'au~78', see auxencode).
182 183
183 184 Hashed encoding (not reversible):
184 185
185 186 If the default-encoded path is longer than _maxstorepathlen, a
186 187 non-reversible hybrid hashing of the path is done instead.
187 188 This encoding uses up to _dirprefixlen characters of all directory
188 189 levels of the lowerencoded path, but not more levels than can fit into
189 190 _maxshortdirslen.
190 191 Then follows the filler followed by the sha digest of the full path.
191 192 The filler is the beginning of the basename of the lowerencoded path
192 193 (the basename is everything after the last path separator). The filler
193 194 is as long as possible, filling in characters from the basename until
194 195 the encoded path has _maxstorepathlen characters (or all chars of the
195 196 basename have been taken).
196 197 The extension (e.g. '.i' or '.d') is preserved.
197 198
198 199 The string 'data/' at the beginning is replaced with 'dh/', if the hashed
199 200 encoding was used.
200 201 '''
201 202 if not path.startswith('data/'):
202 203 return path
203 204 # escape directories ending with .i and .d
204 205 path = encodedir(path)
205 206 ndpath = path[len('data/'):]
206 207 res = 'data/' + auxencode(encodefilename(ndpath))
207 208 if len(res) > _maxstorepathlen:
208 209 digest = _sha(path).hexdigest()
209 210 aep = auxencode(lowerencode(ndpath))
210 211 _root, ext = os.path.splitext(aep)
211 212 parts = aep.split('/')
212 213 basename = parts[-1]
213 214 sdirs = []
214 215 for p in parts[:-1]:
215 216 d = p[:_dirprefixlen]
216 217 if d[-1] in '. ':
217 218 # Windows can't access dirs ending in period or space
218 219 d = d[:-1] + '_'
219 220 t = '/'.join(sdirs) + '/' + d
220 221 if len(t) > _maxshortdirslen:
221 222 break
222 223 sdirs.append(d)
223 224 dirs = '/'.join(sdirs)
224 225 if len(dirs) > 0:
225 226 dirs += '/'
226 227 res = 'dh/' + dirs + digest + ext
227 228 spaceleft = _maxstorepathlen - len(res)
228 229 if spaceleft > 0:
229 230 filler = basename[:spaceleft]
230 231 res = 'dh/' + dirs + filler + digest + ext
231 232 return res
232 233
233 234 def _calcmode(path):
234 235 try:
235 236 # files in .hg/ will be created using this mode
236 237 mode = os.stat(path).st_mode
237 238 # avoid some useless chmods
238 239 if (0777 & ~util.umask) == (0777 & mode):
239 240 mode = None
240 241 except OSError:
241 242 mode = None
242 243 return mode
243 244
244 245 _data = ('data 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
245 246 ' phaseroots obsstore')
246 247
247 248 class basicstore(object):
248 249 '''base class for local repository stores'''
249 250 def __init__(self, path, openertype):
250 251 self.path = path
251 252 self.createmode = _calcmode(path)
252 253 op = openertype(self.path)
253 254 op.createmode = self.createmode
254 255 self.opener = scmutil.filteropener(op, encodedir)
255 256
256 257 def join(self, f):
257 258 return self.path + '/' + encodedir(f)
258 259
259 260 def _walk(self, relpath, recurse):
260 261 '''yields (unencoded, encoded, size)'''
261 262 path = self.path
262 263 if relpath:
263 264 path += '/' + relpath
264 265 striplen = len(self.path) + 1
265 266 l = []
266 267 if os.path.isdir(path):
267 268 visit = [path]
268 269 while visit:
269 270 p = visit.pop()
270 271 for f, kind, st in osutil.listdir(p, stat=True):
271 272 fp = p + '/' + f
272 273 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'):
273 274 n = util.pconvert(fp[striplen:])
274 275 l.append((decodedir(n), n, st.st_size))
275 276 elif kind == stat.S_IFDIR and recurse:
276 277 visit.append(fp)
277 278 l.sort()
278 279 return l
279 280
280 281 def datafiles(self):
281 282 return self._walk('data', True)
282 283
283 284 def walk(self):
284 285 '''yields (unencoded, encoded, size)'''
285 286 # yield data files first
286 287 for x in self.datafiles():
287 288 yield x
288 289 # yield manifest before changelog
289 290 for x in reversed(self._walk('', False)):
290 291 yield x
291 292
292 293 def copylist(self):
293 294 return ['requires'] + _data.split()
294 295
295 296 def write(self):
296 297 pass
297 298
298 299 class encodedstore(basicstore):
299 300 def __init__(self, path, openertype):
300 301 self.path = path + '/store'
301 302 self.createmode = _calcmode(self.path)
302 303 op = openertype(self.path)
303 304 op.createmode = self.createmode
304 305 self.opener = scmutil.filteropener(op, encodefilename)
305 306
306 307 def datafiles(self):
307 308 for a, b, size in self._walk('data', True):
308 309 try:
309 310 a = decodefilename(a)
310 311 except KeyError:
311 312 a = None
312 313 yield a, b, size
313 314
314 315 def join(self, f):
315 316 return self.path + '/' + encodefilename(f)
316 317
317 318 def copylist(self):
318 319 return (['requires', '00changelog.i'] +
319 320 ['store/' + f for f in _data.split()])
320 321
321 322 class fncache(object):
322 323 # the filename used to be partially encoded
323 324 # hence the encodedir/decodedir dance
324 325 def __init__(self, opener):
325 326 self.opener = opener
326 327 self.entries = None
327 328 self._dirty = False
328 329
329 330 def _load(self):
330 331 '''fill the entries from the fncache file'''
331 332 self._dirty = False
332 333 try:
333 334 fp = self.opener('fncache', mode='rb')
334 335 except IOError:
335 336 # skip nonexistent file
336 337 self.entries = set()
337 338 return
338 339 self.entries = set(map(decodedir, fp.read().splitlines()))
339 340 if '' in self.entries:
340 341 fp.seek(0)
341 342 for n, line in enumerate(fp):
342 343 if not line.rstrip('\n'):
343 344 t = _('invalid entry in fncache, line %s') % (n + 1)
344 345 raise util.Abort(t)
345 346 fp.close()
346 347
347 348 def _write(self, files, atomictemp):
348 349 fp = self.opener('fncache', mode='wb', atomictemp=atomictemp)
349 350 if files:
350 351 fp.write('\n'.join(map(encodedir, files)) + '\n')
351 352 fp.close()
352 353 self._dirty = False
353 354
354 355 def rewrite(self, files):
355 356 self._write(files, False)
356 357 self.entries = set(files)
357 358
358 359 def write(self):
359 360 if self._dirty:
360 361 self._write(self.entries, True)
361 362
362 363 def add(self, fn):
363 364 if self.entries is None:
364 365 self._load()
365 366 if fn not in self.entries:
366 367 self._dirty = True
367 368 self.entries.add(fn)
368 369
369 370 def __contains__(self, fn):
370 371 if self.entries is None:
371 372 self._load()
372 373 return fn in self.entries
373 374
374 375 def __iter__(self):
375 376 if self.entries is None:
376 377 self._load()
377 378 return iter(self.entries)
378 379
379 380 class _fncacheopener(scmutil.abstractopener):
380 381 def __init__(self, op, fnc, encode):
381 382 self.opener = op
382 383 self.fncache = fnc
383 384 self.encode = encode
384 385
385 386 def _getmustaudit(self):
386 387 return self.opener.mustaudit
387 388
388 389 def _setmustaudit(self, onoff):
389 390 self.opener.mustaudit = onoff
390 391
391 392 mustaudit = property(_getmustaudit, _setmustaudit)
392 393
393 394 def __call__(self, path, mode='r', *args, **kw):
394 395 if mode not in ('r', 'rb') and path.startswith('data/'):
395 396 self.fncache.add(path)
396 397 return self.opener(self.encode(path), mode, *args, **kw)
397 398
398 399 class fncachestore(basicstore):
399 400 def __init__(self, path, openertype, encode):
400 401 self.encode = encode
401 402 self.path = path + '/store'
402 403 self.pathsep = self.path + '/'
403 404 self.createmode = _calcmode(self.path)
404 405 op = openertype(self.path)
405 406 op.createmode = self.createmode
406 407 fnc = fncache(op)
407 408 self.fncache = fnc
408 409 self.opener = _fncacheopener(op, fnc, encode)
409 410
410 411 def join(self, f):
411 412 return self.pathsep + self.encode(f)
412 413
413 414 def getsize(self, path):
414 415 return os.stat(self.pathsep + path).st_size
415 416
416 417 def datafiles(self):
417 418 rewrite = False
418 419 existing = []
419 420 for f in sorted(self.fncache):
420 421 ef = self.encode(f)
421 422 try:
422 423 yield f, ef, self.getsize(ef)
423 424 existing.append(f)
424 425 except OSError, err:
425 426 if err.errno != errno.ENOENT:
426 427 raise
427 428 # nonexistent entry
428 429 rewrite = True
429 430 if rewrite:
430 431 # rewrite fncache to remove nonexistent entries
431 432 # (may be caused by rollback / strip)
432 433 self.fncache.rewrite(existing)
433 434
434 435 def copylist(self):
435 436 d = ('data dh fncache phaseroots obsstore'
436 437 ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i')
437 438 return (['requires', '00changelog.i'] +
438 439 ['store/' + f for f in d.split()])
439 440
440 441 def write(self):
441 442 self.fncache.write()
442 443
443 444 def store(requirements, path, openertype):
444 445 if 'store' in requirements:
445 446 if 'fncache' in requirements:
446 447 auxencode = lambda f: _auxencode(f, 'dotencode' in requirements)
447 448 encode = lambda f: _hybridencode(f, auxencode)
448 449 return fncachestore(path, openertype, encode)
449 450 return encodedstore(path, openertype)
450 451 return basicstore(path, openertype)
General Comments 0
You need to be logged in to leave comments. Login now