##// END OF EJS Templates
store: only add new entries to the fncache file...
Adrian Buehlmann -
r10577:d5bd1bef stable
parent child Browse files
Show More
@@ -1,333 +1,333 b''
1 # store.py - repository store handling for Mercurial
1 # store.py - repository store handling for Mercurial
2 #
2 #
3 # Copyright 2008 Matt Mackall <mpm@selenic.com>
3 # Copyright 2008 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from i18n import _
8 from i18n import _
9 import osutil, util
9 import osutil, util
10 import os, stat
10 import os, stat
11
11
12 _sha = util.sha1
12 _sha = util.sha1
13
13
14 # This avoids a collision between a file named foo and a dir named
14 # This avoids a collision between a file named foo and a dir named
15 # foo.i or foo.d
15 # foo.i or foo.d
16 def encodedir(path):
16 def encodedir(path):
17 if not path.startswith('data/'):
17 if not path.startswith('data/'):
18 return path
18 return path
19 return (path
19 return (path
20 .replace(".hg/", ".hg.hg/")
20 .replace(".hg/", ".hg.hg/")
21 .replace(".i/", ".i.hg/")
21 .replace(".i/", ".i.hg/")
22 .replace(".d/", ".d.hg/"))
22 .replace(".d/", ".d.hg/"))
23
23
24 def decodedir(path):
24 def decodedir(path):
25 if not path.startswith('data/'):
25 if not path.startswith('data/'):
26 return path
26 return path
27 return (path
27 return (path
28 .replace(".d.hg/", ".d/")
28 .replace(".d.hg/", ".d/")
29 .replace(".i.hg/", ".i/")
29 .replace(".i.hg/", ".i/")
30 .replace(".hg.hg/", ".hg/"))
30 .replace(".hg.hg/", ".hg/"))
31
31
32 def _buildencodefun():
32 def _buildencodefun():
33 e = '_'
33 e = '_'
34 win_reserved = [ord(x) for x in '\\:*?"<>|']
34 win_reserved = [ord(x) for x in '\\:*?"<>|']
35 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
35 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
36 for x in (range(32) + range(126, 256) + win_reserved):
36 for x in (range(32) + range(126, 256) + win_reserved):
37 cmap[chr(x)] = "~%02x" % x
37 cmap[chr(x)] = "~%02x" % x
38 for x in range(ord("A"), ord("Z")+1) + [ord(e)]:
38 for x in range(ord("A"), ord("Z")+1) + [ord(e)]:
39 cmap[chr(x)] = e + chr(x).lower()
39 cmap[chr(x)] = e + chr(x).lower()
40 dmap = {}
40 dmap = {}
41 for k, v in cmap.iteritems():
41 for k, v in cmap.iteritems():
42 dmap[v] = k
42 dmap[v] = k
43 def decode(s):
43 def decode(s):
44 i = 0
44 i = 0
45 while i < len(s):
45 while i < len(s):
46 for l in xrange(1, 4):
46 for l in xrange(1, 4):
47 try:
47 try:
48 yield dmap[s[i:i + l]]
48 yield dmap[s[i:i + l]]
49 i += l
49 i += l
50 break
50 break
51 except KeyError:
51 except KeyError:
52 pass
52 pass
53 else:
53 else:
54 raise KeyError
54 raise KeyError
55 return (lambda s: "".join([cmap[c] for c in encodedir(s)]),
55 return (lambda s: "".join([cmap[c] for c in encodedir(s)]),
56 lambda s: decodedir("".join(list(decode(s)))))
56 lambda s: decodedir("".join(list(decode(s)))))
57
57
58 encodefilename, decodefilename = _buildencodefun()
58 encodefilename, decodefilename = _buildencodefun()
59
59
60 def _build_lower_encodefun():
60 def _build_lower_encodefun():
61 win_reserved = [ord(x) for x in '\\:*?"<>|']
61 win_reserved = [ord(x) for x in '\\:*?"<>|']
62 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
62 cmap = dict([(chr(x), chr(x)) for x in xrange(127)])
63 for x in (range(32) + range(126, 256) + win_reserved):
63 for x in (range(32) + range(126, 256) + win_reserved):
64 cmap[chr(x)] = "~%02x" % x
64 cmap[chr(x)] = "~%02x" % x
65 for x in range(ord("A"), ord("Z")+1):
65 for x in range(ord("A"), ord("Z")+1):
66 cmap[chr(x)] = chr(x).lower()
66 cmap[chr(x)] = chr(x).lower()
67 return lambda s: "".join([cmap[c] for c in s])
67 return lambda s: "".join([cmap[c] for c in s])
68
68
69 lowerencode = _build_lower_encodefun()
69 lowerencode = _build_lower_encodefun()
70
70
71 _windows_reserved_filenames = '''con prn aux nul
71 _windows_reserved_filenames = '''con prn aux nul
72 com1 com2 com3 com4 com5 com6 com7 com8 com9
72 com1 com2 com3 com4 com5 com6 com7 com8 com9
73 lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9'''.split()
73 lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9'''.split()
74 def auxencode(path):
74 def auxencode(path):
75 res = []
75 res = []
76 for n in path.split('/'):
76 for n in path.split('/'):
77 if n:
77 if n:
78 base = n.split('.')[0]
78 base = n.split('.')[0]
79 if base and (base in _windows_reserved_filenames):
79 if base and (base in _windows_reserved_filenames):
80 # encode third letter ('aux' -> 'au~78')
80 # encode third letter ('aux' -> 'au~78')
81 ec = "~%02x" % ord(n[2])
81 ec = "~%02x" % ord(n[2])
82 n = n[0:2] + ec + n[3:]
82 n = n[0:2] + ec + n[3:]
83 if n[-1] in '. ':
83 if n[-1] in '. ':
84 # encode last period or space ('foo...' -> 'foo..~2e')
84 # encode last period or space ('foo...' -> 'foo..~2e')
85 n = n[:-1] + "~%02x" % ord(n[-1])
85 n = n[:-1] + "~%02x" % ord(n[-1])
86 res.append(n)
86 res.append(n)
87 return '/'.join(res)
87 return '/'.join(res)
88
88
89 MAX_PATH_LEN_IN_HGSTORE = 120
89 MAX_PATH_LEN_IN_HGSTORE = 120
90 DIR_PREFIX_LEN = 8
90 DIR_PREFIX_LEN = 8
91 _MAX_SHORTENED_DIRS_LEN = 8 * (DIR_PREFIX_LEN + 1) - 4
91 _MAX_SHORTENED_DIRS_LEN = 8 * (DIR_PREFIX_LEN + 1) - 4
92 def hybridencode(path):
92 def hybridencode(path):
93 '''encodes path with a length limit
93 '''encodes path with a length limit
94
94
95 Encodes all paths that begin with 'data/', according to the following.
95 Encodes all paths that begin with 'data/', according to the following.
96
96
97 Default encoding (reversible):
97 Default encoding (reversible):
98
98
99 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
99 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
100 characters are encoded as '~xx', where xx is the two digit hex code
100 characters are encoded as '~xx', where xx is the two digit hex code
101 of the character (see encodefilename).
101 of the character (see encodefilename).
102 Relevant path components consisting of Windows reserved filenames are
102 Relevant path components consisting of Windows reserved filenames are
103 masked by encoding the third character ('aux' -> 'au~78', see auxencode).
103 masked by encoding the third character ('aux' -> 'au~78', see auxencode).
104
104
105 Hashed encoding (not reversible):
105 Hashed encoding (not reversible):
106
106
107 If the default-encoded path is longer than MAX_PATH_LEN_IN_HGSTORE, a
107 If the default-encoded path is longer than MAX_PATH_LEN_IN_HGSTORE, a
108 non-reversible hybrid hashing of the path is done instead.
108 non-reversible hybrid hashing of the path is done instead.
109 This encoding uses up to DIR_PREFIX_LEN characters of all directory
109 This encoding uses up to DIR_PREFIX_LEN characters of all directory
110 levels of the lowerencoded path, but not more levels than can fit into
110 levels of the lowerencoded path, but not more levels than can fit into
111 _MAX_SHORTENED_DIRS_LEN.
111 _MAX_SHORTENED_DIRS_LEN.
112 Then follows the filler followed by the sha digest of the full path.
112 Then follows the filler followed by the sha digest of the full path.
113 The filler is the beginning of the basename of the lowerencoded path
113 The filler is the beginning of the basename of the lowerencoded path
114 (the basename is everything after the last path separator). The filler
114 (the basename is everything after the last path separator). The filler
115 is as long as possible, filling in characters from the basename until
115 is as long as possible, filling in characters from the basename until
116 the encoded path has MAX_PATH_LEN_IN_HGSTORE characters (or all chars
116 the encoded path has MAX_PATH_LEN_IN_HGSTORE characters (or all chars
117 of the basename have been taken).
117 of the basename have been taken).
118 The extension (e.g. '.i' or '.d') is preserved.
118 The extension (e.g. '.i' or '.d') is preserved.
119
119
120 The string 'data/' at the beginning is replaced with 'dh/', if the hashed
120 The string 'data/' at the beginning is replaced with 'dh/', if the hashed
121 encoding was used.
121 encoding was used.
122 '''
122 '''
123 if not path.startswith('data/'):
123 if not path.startswith('data/'):
124 return path
124 return path
125 # escape directories ending with .i and .d
125 # escape directories ending with .i and .d
126 path = encodedir(path)
126 path = encodedir(path)
127 ndpath = path[len('data/'):]
127 ndpath = path[len('data/'):]
128 res = 'data/' + auxencode(encodefilename(ndpath))
128 res = 'data/' + auxencode(encodefilename(ndpath))
129 if len(res) > MAX_PATH_LEN_IN_HGSTORE:
129 if len(res) > MAX_PATH_LEN_IN_HGSTORE:
130 digest = _sha(path).hexdigest()
130 digest = _sha(path).hexdigest()
131 aep = auxencode(lowerencode(ndpath))
131 aep = auxencode(lowerencode(ndpath))
132 _root, ext = os.path.splitext(aep)
132 _root, ext = os.path.splitext(aep)
133 parts = aep.split('/')
133 parts = aep.split('/')
134 basename = parts[-1]
134 basename = parts[-1]
135 sdirs = []
135 sdirs = []
136 for p in parts[:-1]:
136 for p in parts[:-1]:
137 d = p[:DIR_PREFIX_LEN]
137 d = p[:DIR_PREFIX_LEN]
138 if d[-1] in '. ':
138 if d[-1] in '. ':
139 # Windows can't access dirs ending in period or space
139 # Windows can't access dirs ending in period or space
140 d = d[:-1] + '_'
140 d = d[:-1] + '_'
141 t = '/'.join(sdirs) + '/' + d
141 t = '/'.join(sdirs) + '/' + d
142 if len(t) > _MAX_SHORTENED_DIRS_LEN:
142 if len(t) > _MAX_SHORTENED_DIRS_LEN:
143 break
143 break
144 sdirs.append(d)
144 sdirs.append(d)
145 dirs = '/'.join(sdirs)
145 dirs = '/'.join(sdirs)
146 if len(dirs) > 0:
146 if len(dirs) > 0:
147 dirs += '/'
147 dirs += '/'
148 res = 'dh/' + dirs + digest + ext
148 res = 'dh/' + dirs + digest + ext
149 space_left = MAX_PATH_LEN_IN_HGSTORE - len(res)
149 space_left = MAX_PATH_LEN_IN_HGSTORE - len(res)
150 if space_left > 0:
150 if space_left > 0:
151 filler = basename[:space_left]
151 filler = basename[:space_left]
152 res = 'dh/' + dirs + filler + digest + ext
152 res = 'dh/' + dirs + filler + digest + ext
153 return res
153 return res
154
154
155 def _calcmode(path):
155 def _calcmode(path):
156 try:
156 try:
157 # files in .hg/ will be created using this mode
157 # files in .hg/ will be created using this mode
158 mode = os.stat(path).st_mode
158 mode = os.stat(path).st_mode
159 # avoid some useless chmods
159 # avoid some useless chmods
160 if (0777 & ~util.umask) == (0777 & mode):
160 if (0777 & ~util.umask) == (0777 & mode):
161 mode = None
161 mode = None
162 except OSError:
162 except OSError:
163 mode = None
163 mode = None
164 return mode
164 return mode
165
165
166 _data = 'data 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
166 _data = 'data 00manifest.d 00manifest.i 00changelog.d 00changelog.i'
167
167
168 class basicstore(object):
168 class basicstore(object):
169 '''base class for local repository stores'''
169 '''base class for local repository stores'''
170 def __init__(self, path, opener, pathjoiner):
170 def __init__(self, path, opener, pathjoiner):
171 self.pathjoiner = pathjoiner
171 self.pathjoiner = pathjoiner
172 self.path = path
172 self.path = path
173 self.createmode = _calcmode(path)
173 self.createmode = _calcmode(path)
174 op = opener(self.path)
174 op = opener(self.path)
175 op.createmode = self.createmode
175 op.createmode = self.createmode
176 self.opener = lambda f, *args, **kw: op(encodedir(f), *args, **kw)
176 self.opener = lambda f, *args, **kw: op(encodedir(f), *args, **kw)
177
177
178 def join(self, f):
178 def join(self, f):
179 return self.pathjoiner(self.path, encodedir(f))
179 return self.pathjoiner(self.path, encodedir(f))
180
180
181 def _walk(self, relpath, recurse):
181 def _walk(self, relpath, recurse):
182 '''yields (unencoded, encoded, size)'''
182 '''yields (unencoded, encoded, size)'''
183 path = self.pathjoiner(self.path, relpath)
183 path = self.pathjoiner(self.path, relpath)
184 striplen = len(self.path) + len(os.sep)
184 striplen = len(self.path) + len(os.sep)
185 l = []
185 l = []
186 if os.path.isdir(path):
186 if os.path.isdir(path):
187 visit = [path]
187 visit = [path]
188 while visit:
188 while visit:
189 p = visit.pop()
189 p = visit.pop()
190 for f, kind, st in osutil.listdir(p, stat=True):
190 for f, kind, st in osutil.listdir(p, stat=True):
191 fp = self.pathjoiner(p, f)
191 fp = self.pathjoiner(p, f)
192 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'):
192 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'):
193 n = util.pconvert(fp[striplen:])
193 n = util.pconvert(fp[striplen:])
194 l.append((decodedir(n), n, st.st_size))
194 l.append((decodedir(n), n, st.st_size))
195 elif kind == stat.S_IFDIR and recurse:
195 elif kind == stat.S_IFDIR and recurse:
196 visit.append(fp)
196 visit.append(fp)
197 return sorted(l)
197 return sorted(l)
198
198
199 def datafiles(self):
199 def datafiles(self):
200 return self._walk('data', True)
200 return self._walk('data', True)
201
201
202 def walk(self):
202 def walk(self):
203 '''yields (unencoded, encoded, size)'''
203 '''yields (unencoded, encoded, size)'''
204 # yield data files first
204 # yield data files first
205 for x in self.datafiles():
205 for x in self.datafiles():
206 yield x
206 yield x
207 # yield manifest before changelog
207 # yield manifest before changelog
208 for x in reversed(self._walk('', False)):
208 for x in reversed(self._walk('', False)):
209 yield x
209 yield x
210
210
211 def copylist(self):
211 def copylist(self):
212 return ['requires'] + _data.split()
212 return ['requires'] + _data.split()
213
213
214 class encodedstore(basicstore):
214 class encodedstore(basicstore):
215 def __init__(self, path, opener, pathjoiner):
215 def __init__(self, path, opener, pathjoiner):
216 self.pathjoiner = pathjoiner
216 self.pathjoiner = pathjoiner
217 self.path = self.pathjoiner(path, 'store')
217 self.path = self.pathjoiner(path, 'store')
218 self.createmode = _calcmode(self.path)
218 self.createmode = _calcmode(self.path)
219 op = opener(self.path)
219 op = opener(self.path)
220 op.createmode = self.createmode
220 op.createmode = self.createmode
221 self.opener = lambda f, *args, **kw: op(encodefilename(f), *args, **kw)
221 self.opener = lambda f, *args, **kw: op(encodefilename(f), *args, **kw)
222
222
223 def datafiles(self):
223 def datafiles(self):
224 for a, b, size in self._walk('data', True):
224 for a, b, size in self._walk('data', True):
225 try:
225 try:
226 a = decodefilename(a)
226 a = decodefilename(a)
227 except KeyError:
227 except KeyError:
228 a = None
228 a = None
229 yield a, b, size
229 yield a, b, size
230
230
231 def join(self, f):
231 def join(self, f):
232 return self.pathjoiner(self.path, encodefilename(f))
232 return self.pathjoiner(self.path, encodefilename(f))
233
233
234 def copylist(self):
234 def copylist(self):
235 return (['requires', '00changelog.i'] +
235 return (['requires', '00changelog.i'] +
236 [self.pathjoiner('store', f) for f in _data.split()])
236 [self.pathjoiner('store', f) for f in _data.split()])
237
237
238 class fncache(object):
238 class fncache(object):
239 # the filename used to be partially encoded
239 # the filename used to be partially encoded
240 # hence the encodedir/decodedir dance
240 # hence the encodedir/decodedir dance
241 def __init__(self, opener):
241 def __init__(self, opener):
242 self.opener = opener
242 self.opener = opener
243 self.entries = None
243 self.entries = None
244
244
245 def _load(self):
245 def _load(self):
246 '''fill the entries from the fncache file'''
246 '''fill the entries from the fncache file'''
247 self.entries = set()
247 self.entries = set()
248 try:
248 try:
249 fp = self.opener('fncache', mode='rb')
249 fp = self.opener('fncache', mode='rb')
250 except IOError:
250 except IOError:
251 # skip nonexistent file
251 # skip nonexistent file
252 return
252 return
253 for n, line in enumerate(fp):
253 for n, line in enumerate(fp):
254 if (len(line) < 2) or (line[-1] != '\n'):
254 if (len(line) < 2) or (line[-1] != '\n'):
255 t = _('invalid entry in fncache, line %s') % (n + 1)
255 t = _('invalid entry in fncache, line %s') % (n + 1)
256 raise util.Abort(t)
256 raise util.Abort(t)
257 self.entries.add(decodedir(line[:-1]))
257 self.entries.add(decodedir(line[:-1]))
258 fp.close()
258 fp.close()
259
259
260 def rewrite(self, files):
260 def rewrite(self, files):
261 fp = self.opener('fncache', mode='wb')
261 fp = self.opener('fncache', mode='wb')
262 for p in files:
262 for p in files:
263 fp.write(encodedir(p) + '\n')
263 fp.write(encodedir(p) + '\n')
264 fp.close()
264 fp.close()
265 self.entries = set(files)
265 self.entries = set(files)
266
266
267 def add(self, fn):
267 def add(self, fn):
268 if self.entries is None:
268 if self.entries is None:
269 self._load()
269 self._load()
270 if fn not in self.entries:
270 self.opener('fncache', 'ab').write(encodedir(fn) + '\n')
271 self.opener('fncache', 'ab').write(encodedir(fn) + '\n')
272 self.entries.add(fn)
271
273
272 def __contains__(self, fn):
274 def __contains__(self, fn):
273 if self.entries is None:
275 if self.entries is None:
274 self._load()
276 self._load()
275 return fn in self.entries
277 return fn in self.entries
276
278
277 def __iter__(self):
279 def __iter__(self):
278 if self.entries is None:
280 if self.entries is None:
279 self._load()
281 self._load()
280 return iter(self.entries)
282 return iter(self.entries)
281
283
282 class fncachestore(basicstore):
284 class fncachestore(basicstore):
283 def __init__(self, path, opener, pathjoiner):
285 def __init__(self, path, opener, pathjoiner):
284 self.pathjoiner = pathjoiner
286 self.pathjoiner = pathjoiner
285 self.path = self.pathjoiner(path, 'store')
287 self.path = self.pathjoiner(path, 'store')
286 self.createmode = _calcmode(self.path)
288 self.createmode = _calcmode(self.path)
287 op = opener(self.path)
289 op = opener(self.path)
288 op.createmode = self.createmode
290 op.createmode = self.createmode
289 fnc = fncache(op)
291 fnc = fncache(op)
290 self.fncache = fnc
292 self.fncache = fnc
291
293
292 def fncacheopener(path, mode='r', *args, **kw):
294 def fncacheopener(path, mode='r', *args, **kw):
293 if (mode not in ('r', 'rb')
295 if mode not in ('r', 'rb') and path.startswith('data/'):
294 and path.startswith('data/')
295 and path not in fnc):
296 fnc.add(path)
296 fnc.add(path)
297 return op(hybridencode(path), mode, *args, **kw)
297 return op(hybridencode(path), mode, *args, **kw)
298 self.opener = fncacheopener
298 self.opener = fncacheopener
299
299
300 def join(self, f):
300 def join(self, f):
301 return self.pathjoiner(self.path, hybridencode(f))
301 return self.pathjoiner(self.path, hybridencode(f))
302
302
303 def datafiles(self):
303 def datafiles(self):
304 rewrite = False
304 rewrite = False
305 existing = []
305 existing = []
306 pjoin = self.pathjoiner
306 pjoin = self.pathjoiner
307 spath = self.path
307 spath = self.path
308 for f in self.fncache:
308 for f in self.fncache:
309 ef = hybridencode(f)
309 ef = hybridencode(f)
310 try:
310 try:
311 st = os.stat(pjoin(spath, ef))
311 st = os.stat(pjoin(spath, ef))
312 yield f, ef, st.st_size
312 yield f, ef, st.st_size
313 existing.append(f)
313 existing.append(f)
314 except OSError:
314 except OSError:
315 # nonexistent entry
315 # nonexistent entry
316 rewrite = True
316 rewrite = True
317 if rewrite:
317 if rewrite:
318 # rewrite fncache to remove nonexistent entries
318 # rewrite fncache to remove nonexistent entries
319 # (may be caused by rollback / strip)
319 # (may be caused by rollback / strip)
320 self.fncache.rewrite(existing)
320 self.fncache.rewrite(existing)
321
321
322 def copylist(self):
322 def copylist(self):
323 d = _data + ' dh fncache'
323 d = _data + ' dh fncache'
324 return (['requires', '00changelog.i'] +
324 return (['requires', '00changelog.i'] +
325 [self.pathjoiner('store', f) for f in d.split()])
325 [self.pathjoiner('store', f) for f in d.split()])
326
326
327 def store(requirements, path, opener, pathjoiner=None):
327 def store(requirements, path, opener, pathjoiner=None):
328 pathjoiner = pathjoiner or os.path.join
328 pathjoiner = pathjoiner or os.path.join
329 if 'store' in requirements:
329 if 'store' in requirements:
330 if 'fncache' in requirements:
330 if 'fncache' in requirements:
331 return fncachestore(path, opener, pathjoiner)
331 return fncachestore(path, opener, pathjoiner)
332 return encodedstore(path, opener, pathjoiner)
332 return encodedstore(path, opener, pathjoiner)
333 return basicstore(path, opener, pathjoiner)
333 return basicstore(path, opener, pathjoiner)
@@ -1,59 +1,65 b''
1 #!/bin/sh
1 #!/bin/sh
2
2
3 cat >> $HGRCPATH <<EOF
3 cat >> $HGRCPATH <<EOF
4 [extensions]
4 [extensions]
5 convert=
5 convert=
6 [convert]
6 [convert]
7 hg.saverev=False
7 hg.saverev=False
8 EOF
8 EOF
9
9
10 hg help convert
10 hg help convert
11
11
12 hg init a
12 hg init a
13 cd a
13 cd a
14 echo a > a
14 echo a > a
15 hg ci -d'0 0' -Ama
15 hg ci -d'0 0' -Ama
16 hg cp a b
16 hg cp a b
17 hg ci -d'1 0' -mb
17 hg ci -d'1 0' -mb
18 hg rm a
18 hg rm a
19 hg ci -d'2 0' -mc
19 hg ci -d'2 0' -mc
20 hg mv b a
20 hg mv b a
21 hg ci -d'3 0' -md
21 hg ci -d'3 0' -md
22 echo a >> a
22 echo a >> a
23 hg ci -d'4 0' -me
23 hg ci -d'4 0' -me
24
24
25 cd ..
25 cd ..
26 hg convert a 2>&1 | grep -v 'subversion python bindings could not be loaded'
26 hg convert a 2>&1 | grep -v 'subversion python bindings could not be loaded'
27 hg --cwd a-hg pull ../a
27 hg --cwd a-hg pull ../a
28
28
29 touch bogusfile
29 touch bogusfile
30 echo % should fail
30 echo % should fail
31 hg convert a bogusfile
31 hg convert a bogusfile
32
32
33 mkdir bogusdir
33 mkdir bogusdir
34 chmod 000 bogusdir
34 chmod 000 bogusdir
35
35
36 echo % should fail
36 echo % should fail
37 hg convert a bogusdir
37 hg convert a bogusdir
38
38
39 echo % should succeed
39 echo % should succeed
40 chmod 700 bogusdir
40 chmod 700 bogusdir
41 hg convert a bogusdir
41 hg convert a bogusdir
42
42
43 echo % test pre and post conversion actions
43 echo % test pre and post conversion actions
44 echo 'include b' > filemap
44 echo 'include b' > filemap
45 hg convert --debug --filemap filemap a partialb | \
45 hg convert --debug --filemap filemap a partialb | \
46 grep 'run hg'
46 grep 'run hg'
47
47
48 echo % converting empty dir should fail "nicely"
48 echo % converting empty dir should fail "nicely"
49 mkdir emptydir
49 mkdir emptydir
50 # override $PATH to ensure p4 not visible; use $PYTHON in case we're
50 # override $PATH to ensure p4 not visible; use $PYTHON in case we're
51 # running from a devel copy, not a temp installation
51 # running from a devel copy, not a temp installation
52 PATH=$BINDIR $PYTHON $BINDIR/hg convert emptydir 2>&1 | sed 's,file://.*/emptydir,.../emptydir,g'
52 PATH=$BINDIR $PYTHON $BINDIR/hg convert emptydir 2>&1 | sed 's,file://.*/emptydir,.../emptydir,g'
53
53
54 echo % convert with imaginary source type
54 echo % convert with imaginary source type
55 hg convert --source-type foo a a-foo
55 hg convert --source-type foo a a-foo
56 echo % convert with imaginary sink type
56 echo % convert with imaginary sink type
57 hg convert --dest-type foo a a-foo
57 hg convert --dest-type foo a a-foo
58
58
59 echo
60 echo % "testing: convert must not produce duplicate entries in fncache"
61 hg convert a b
62 echo % "contents of fncache file:"
63 cat b/.hg/store/fncache
64
59 true
65 true
@@ -1,275 +1,289 b''
1 hg convert [OPTION]... SOURCE [DEST [REVMAP]]
1 hg convert [OPTION]... SOURCE [DEST [REVMAP]]
2
2
3 convert a foreign SCM repository to a Mercurial one.
3 convert a foreign SCM repository to a Mercurial one.
4
4
5 Accepted source formats [identifiers]:
5 Accepted source formats [identifiers]:
6
6
7 - Mercurial [hg]
7 - Mercurial [hg]
8 - CVS [cvs]
8 - CVS [cvs]
9 - Darcs [darcs]
9 - Darcs [darcs]
10 - git [git]
10 - git [git]
11 - Subversion [svn]
11 - Subversion [svn]
12 - Monotone [mtn]
12 - Monotone [mtn]
13 - GNU Arch [gnuarch]
13 - GNU Arch [gnuarch]
14 - Bazaar [bzr]
14 - Bazaar [bzr]
15 - Perforce [p4]
15 - Perforce [p4]
16
16
17 Accepted destination formats [identifiers]:
17 Accepted destination formats [identifiers]:
18
18
19 - Mercurial [hg]
19 - Mercurial [hg]
20 - Subversion [svn] (history on branches is not preserved)
20 - Subversion [svn] (history on branches is not preserved)
21
21
22 If no revision is given, all revisions will be converted. Otherwise,
22 If no revision is given, all revisions will be converted. Otherwise,
23 convert will only import up to the named revision (given in a format
23 convert will only import up to the named revision (given in a format
24 understood by the source).
24 understood by the source).
25
25
26 If no destination directory name is specified, it defaults to the basename
26 If no destination directory name is specified, it defaults to the basename
27 of the source with '-hg' appended. If the destination repository doesn't
27 of the source with '-hg' appended. If the destination repository doesn't
28 exist, it will be created.
28 exist, it will be created.
29
29
30 By default, all sources except Mercurial will use --branchsort. Mercurial
30 By default, all sources except Mercurial will use --branchsort. Mercurial
31 uses --sourcesort to preserve original revision numbers order. Sort modes
31 uses --sourcesort to preserve original revision numbers order. Sort modes
32 have the following effects:
32 have the following effects:
33
33
34 --branchsort convert from parent to child revision when possible, which
34 --branchsort convert from parent to child revision when possible, which
35 means branches are usually converted one after the other. It
35 means branches are usually converted one after the other. It
36 generates more compact repositories.
36 generates more compact repositories.
37 --datesort sort revisions by date. Converted repositories have good-
37 --datesort sort revisions by date. Converted repositories have good-
38 looking changelogs but are often an order of magnitude
38 looking changelogs but are often an order of magnitude
39 larger than the same ones generated by --branchsort.
39 larger than the same ones generated by --branchsort.
40 --sourcesort try to preserve source revisions order, only supported by
40 --sourcesort try to preserve source revisions order, only supported by
41 Mercurial sources.
41 Mercurial sources.
42
42
43 If <REVMAP> isn't given, it will be put in a default location
43 If <REVMAP> isn't given, it will be put in a default location
44 (<dest>/.hg/shamap by default). The <REVMAP> is a simple text file that
44 (<dest>/.hg/shamap by default). The <REVMAP> is a simple text file that
45 maps each source commit ID to the destination ID for that revision, like
45 maps each source commit ID to the destination ID for that revision, like
46 so:
46 so:
47
47
48 <source ID> <destination ID>
48 <source ID> <destination ID>
49
49
50 If the file doesn't exist, it's automatically created. It's updated on
50 If the file doesn't exist, it's automatically created. It's updated on
51 each commit copied, so convert-repo can be interrupted and can be run
51 each commit copied, so convert-repo can be interrupted and can be run
52 repeatedly to copy new commits.
52 repeatedly to copy new commits.
53
53
54 The [username mapping] file is a simple text file that maps each source
54 The [username mapping] file is a simple text file that maps each source
55 commit author to a destination commit author. It is handy for source SCMs
55 commit author to a destination commit author. It is handy for source SCMs
56 that use unix logins to identify authors (eg: CVS). One line per author
56 that use unix logins to identify authors (eg: CVS). One line per author
57 mapping and the line format is: srcauthor=whatever string you want
57 mapping and the line format is: srcauthor=whatever string you want
58
58
59 The filemap is a file that allows filtering and remapping of files and
59 The filemap is a file that allows filtering and remapping of files and
60 directories. Comment lines start with '#'. Each line can contain one of
60 directories. Comment lines start with '#'. Each line can contain one of
61 the following directives:
61 the following directives:
62
62
63 include path/to/file
63 include path/to/file
64
64
65 exclude path/to/file
65 exclude path/to/file
66
66
67 rename from/file to/file
67 rename from/file to/file
68
68
69 The 'include' directive causes a file, or all files under a directory, to
69 The 'include' directive causes a file, or all files under a directory, to
70 be included in the destination repository, and the exclusion of all other
70 be included in the destination repository, and the exclusion of all other
71 files and directories not explicitly included. The 'exclude' directive
71 files and directories not explicitly included. The 'exclude' directive
72 causes files or directories to be omitted. The 'rename' directive renames
72 causes files or directories to be omitted. The 'rename' directive renames
73 a file or directory. To rename from a subdirectory into the root of the
73 a file or directory. To rename from a subdirectory into the root of the
74 repository, use '.' as the path to rename to.
74 repository, use '.' as the path to rename to.
75
75
76 The splicemap is a file that allows insertion of synthetic history,
76 The splicemap is a file that allows insertion of synthetic history,
77 letting you specify the parents of a revision. This is useful if you want
77 letting you specify the parents of a revision. This is useful if you want
78 to e.g. give a Subversion merge two parents, or graft two disconnected
78 to e.g. give a Subversion merge two parents, or graft two disconnected
79 series of history together. Each entry contains a key, followed by a
79 series of history together. Each entry contains a key, followed by a
80 space, followed by one or two comma-separated values. The key is the
80 space, followed by one or two comma-separated values. The key is the
81 revision ID in the source revision control system whose parents should be
81 revision ID in the source revision control system whose parents should be
82 modified (same format as a key in .hg/shamap). The values are the revision
82 modified (same format as a key in .hg/shamap). The values are the revision
83 IDs (in either the source or destination revision control system) that
83 IDs (in either the source or destination revision control system) that
84 should be used as the new parents for that node. For example, if you have
84 should be used as the new parents for that node. For example, if you have
85 merged "release-1.0" into "trunk", then you should specify the revision on
85 merged "release-1.0" into "trunk", then you should specify the revision on
86 "trunk" as the first parent and the one on the "release-1.0" branch as the
86 "trunk" as the first parent and the one on the "release-1.0" branch as the
87 second.
87 second.
88
88
89 The branchmap is a file that allows you to rename a branch when it is
89 The branchmap is a file that allows you to rename a branch when it is
90 being brought in from whatever external repository. When used in
90 being brought in from whatever external repository. When used in
91 conjunction with a splicemap, it allows for a powerful combination to help
91 conjunction with a splicemap, it allows for a powerful combination to help
92 fix even the most badly mismanaged repositories and turn them into nicely
92 fix even the most badly mismanaged repositories and turn them into nicely
93 structured Mercurial repositories. The branchmap contains lines of the
93 structured Mercurial repositories. The branchmap contains lines of the
94 form "original_branch_name new_branch_name". "original_branch_name" is the
94 form "original_branch_name new_branch_name". "original_branch_name" is the
95 name of the branch in the source repository, and "new_branch_name" is the
95 name of the branch in the source repository, and "new_branch_name" is the
96 name of the branch is the destination repository. This can be used to (for
96 name of the branch is the destination repository. This can be used to (for
97 instance) move code in one repository from "default" to a named branch.
97 instance) move code in one repository from "default" to a named branch.
98
98
99 Mercurial Source
99 Mercurial Source
100 ----------------
100 ----------------
101
101
102 --config convert.hg.ignoreerrors=False (boolean)
102 --config convert.hg.ignoreerrors=False (boolean)
103 ignore integrity errors when reading. Use it to fix Mercurial
103 ignore integrity errors when reading. Use it to fix Mercurial
104 repositories with missing revlogs, by converting from and to
104 repositories with missing revlogs, by converting from and to
105 Mercurial.
105 Mercurial.
106 --config convert.hg.saverev=False (boolean)
106 --config convert.hg.saverev=False (boolean)
107 store original revision ID in changeset (forces target IDs to change)
107 store original revision ID in changeset (forces target IDs to change)
108 --config convert.hg.startrev=0 (hg revision identifier)
108 --config convert.hg.startrev=0 (hg revision identifier)
109 convert start revision and its descendants
109 convert start revision and its descendants
110
110
111 CVS Source
111 CVS Source
112 ----------
112 ----------
113
113
114 CVS source will use a sandbox (i.e. a checked-out copy) from CVS to
114 CVS source will use a sandbox (i.e. a checked-out copy) from CVS to
115 indicate the starting point of what will be converted. Direct access to
115 indicate the starting point of what will be converted. Direct access to
116 the repository files is not needed, unless of course the repository is
116 the repository files is not needed, unless of course the repository is
117 :local:. The conversion uses the top level directory in the sandbox to
117 :local:. The conversion uses the top level directory in the sandbox to
118 find the CVS repository, and then uses CVS rlog commands to find files to
118 find the CVS repository, and then uses CVS rlog commands to find files to
119 convert. This means that unless a filemap is given, all files under the
119 convert. This means that unless a filemap is given, all files under the
120 starting directory will be converted, and that any directory
120 starting directory will be converted, and that any directory
121 reorganization in the CVS sandbox is ignored.
121 reorganization in the CVS sandbox is ignored.
122
122
123 The options shown are the defaults.
123 The options shown are the defaults.
124
124
125 --config convert.cvsps.cache=True (boolean)
125 --config convert.cvsps.cache=True (boolean)
126 Set to False to disable remote log caching, for testing and debugging
126 Set to False to disable remote log caching, for testing and debugging
127 purposes.
127 purposes.
128 --config convert.cvsps.fuzz=60 (integer)
128 --config convert.cvsps.fuzz=60 (integer)
129 Specify the maximum time (in seconds) that is allowed between commits
129 Specify the maximum time (in seconds) that is allowed between commits
130 with identical user and log message in a single changeset. When very
130 with identical user and log message in a single changeset. When very
131 large files were checked in as part of a changeset then the default
131 large files were checked in as part of a changeset then the default
132 may not be long enough.
132 may not be long enough.
133 --config convert.cvsps.mergeto='{{mergetobranch ([-\w]+)}}'
133 --config convert.cvsps.mergeto='{{mergetobranch ([-\w]+)}}'
134 Specify a regular expression to which commit log messages are matched.
134 Specify a regular expression to which commit log messages are matched.
135 If a match occurs, then the conversion process will insert a dummy
135 If a match occurs, then the conversion process will insert a dummy
136 revision merging the branch on which this log message occurs to the
136 revision merging the branch on which this log message occurs to the
137 branch indicated in the regex.
137 branch indicated in the regex.
138 --config convert.cvsps.mergefrom='{{mergefrombranch ([-\w]+)}}'
138 --config convert.cvsps.mergefrom='{{mergefrombranch ([-\w]+)}}'
139 Specify a regular expression to which commit log messages are matched.
139 Specify a regular expression to which commit log messages are matched.
140 If a match occurs, then the conversion process will add the most
140 If a match occurs, then the conversion process will add the most
141 recent revision on the branch indicated in the regex as the second
141 recent revision on the branch indicated in the regex as the second
142 parent of the changeset.
142 parent of the changeset.
143 --config hook.cvslog
143 --config hook.cvslog
144 Specify a Python function to be called at the end of gathering the CVS
144 Specify a Python function to be called at the end of gathering the CVS
145 log. The function is passed a list with the log entries, and can
145 log. The function is passed a list with the log entries, and can
146 modify the entries in-place, or add or delete them.
146 modify the entries in-place, or add or delete them.
147 --config hook.cvschangesets
147 --config hook.cvschangesets
148 Specify a Python function to be called after the changesets are
148 Specify a Python function to be called after the changesets are
149 calculated from the the CVS log. The function is passed a list with
149 calculated from the the CVS log. The function is passed a list with
150 the changeset entries, and can modify the changesets in-place, or add
150 the changeset entries, and can modify the changesets in-place, or add
151 or delete them.
151 or delete them.
152
152
153 An additional "debugcvsps" Mercurial command allows the builtin changeset
153 An additional "debugcvsps" Mercurial command allows the builtin changeset
154 merging code to be run without doing a conversion. Its parameters and
154 merging code to be run without doing a conversion. Its parameters and
155 output are similar to that of cvsps 2.1. Please see the command help for
155 output are similar to that of cvsps 2.1. Please see the command help for
156 more details.
156 more details.
157
157
158 Subversion Source
158 Subversion Source
159 -----------------
159 -----------------
160
160
161 Subversion source detects classical trunk/branches/tags layouts. By
161 Subversion source detects classical trunk/branches/tags layouts. By
162 default, the supplied "svn://repo/path/" source URL is converted as a
162 default, the supplied "svn://repo/path/" source URL is converted as a
163 single branch. If "svn://repo/path/trunk" exists it replaces the default
163 single branch. If "svn://repo/path/trunk" exists it replaces the default
164 branch. If "svn://repo/path/branches" exists, its subdirectories are
164 branch. If "svn://repo/path/branches" exists, its subdirectories are
165 listed as possible branches. If "svn://repo/path/tags" exists, it is
165 listed as possible branches. If "svn://repo/path/tags" exists, it is
166 looked for tags referencing converted branches. Default "trunk",
166 looked for tags referencing converted branches. Default "trunk",
167 "branches" and "tags" values can be overridden with following options. Set
167 "branches" and "tags" values can be overridden with following options. Set
168 them to paths relative to the source URL, or leave them blank to disable
168 them to paths relative to the source URL, or leave them blank to disable
169 auto detection.
169 auto detection.
170
170
171 --config convert.svn.branches=branches (directory name)
171 --config convert.svn.branches=branches (directory name)
172 specify the directory containing branches
172 specify the directory containing branches
173 --config convert.svn.tags=tags (directory name)
173 --config convert.svn.tags=tags (directory name)
174 specify the directory containing tags
174 specify the directory containing tags
175 --config convert.svn.trunk=trunk (directory name)
175 --config convert.svn.trunk=trunk (directory name)
176 specify the name of the trunk branch
176 specify the name of the trunk branch
177
177
178 Source history can be retrieved starting at a specific revision, instead
178 Source history can be retrieved starting at a specific revision, instead
179 of being integrally converted. Only single branch conversions are
179 of being integrally converted. Only single branch conversions are
180 supported.
180 supported.
181
181
182 --config convert.svn.startrev=0 (svn revision number)
182 --config convert.svn.startrev=0 (svn revision number)
183 specify start Subversion revision.
183 specify start Subversion revision.
184
184
185 Perforce Source
185 Perforce Source
186 ---------------
186 ---------------
187
187
188 The Perforce (P4) importer can be given a p4 depot path or a client
188 The Perforce (P4) importer can be given a p4 depot path or a client
189 specification as source. It will convert all files in the source to a flat
189 specification as source. It will convert all files in the source to a flat
190 Mercurial repository, ignoring labels, branches and integrations. Note
190 Mercurial repository, ignoring labels, branches and integrations. Note
191 that when a depot path is given you then usually should specify a target
191 that when a depot path is given you then usually should specify a target
192 directory, because otherwise the target may be named ...-hg.
192 directory, because otherwise the target may be named ...-hg.
193
193
194 It is possible to limit the amount of source history to be converted by
194 It is possible to limit the amount of source history to be converted by
195 specifying an initial Perforce revision.
195 specifying an initial Perforce revision.
196
196
197 --config convert.p4.startrev=0 (perforce changelist number)
197 --config convert.p4.startrev=0 (perforce changelist number)
198 specify initial Perforce revision.
198 specify initial Perforce revision.
199
199
200 Mercurial Destination
200 Mercurial Destination
201 ---------------------
201 ---------------------
202
202
203 --config convert.hg.clonebranches=False (boolean)
203 --config convert.hg.clonebranches=False (boolean)
204 dispatch source branches in separate clones.
204 dispatch source branches in separate clones.
205 --config convert.hg.tagsbranch=default (branch name)
205 --config convert.hg.tagsbranch=default (branch name)
206 tag revisions branch name
206 tag revisions branch name
207 --config convert.hg.usebranchnames=True (boolean)
207 --config convert.hg.usebranchnames=True (boolean)
208 preserve branch names
208 preserve branch names
209
209
210 options:
210 options:
211
211
212 -A --authors username mapping filename
212 -A --authors username mapping filename
213 -d --dest-type destination repository type
213 -d --dest-type destination repository type
214 --filemap remap file names using contents of file
214 --filemap remap file names using contents of file
215 -r --rev import up to target revision REV
215 -r --rev import up to target revision REV
216 -s --source-type source repository type
216 -s --source-type source repository type
217 --splicemap splice synthesized history into place
217 --splicemap splice synthesized history into place
218 --branchmap change branch names while converting
218 --branchmap change branch names while converting
219 --branchsort try to sort changesets by branches
219 --branchsort try to sort changesets by branches
220 --datesort try to sort changesets by date
220 --datesort try to sort changesets by date
221 --sourcesort preserve source changesets order
221 --sourcesort preserve source changesets order
222
222
223 use "hg -v help convert" to show global options
223 use "hg -v help convert" to show global options
224 adding a
224 adding a
225 assuming destination a-hg
225 assuming destination a-hg
226 initializing destination a-hg repository
226 initializing destination a-hg repository
227 scanning source...
227 scanning source...
228 sorting...
228 sorting...
229 converting...
229 converting...
230 4 a
230 4 a
231 3 b
231 3 b
232 2 c
232 2 c
233 1 d
233 1 d
234 0 e
234 0 e
235 pulling from ../a
235 pulling from ../a
236 searching for changes
236 searching for changes
237 no changes found
237 no changes found
238 % should fail
238 % should fail
239 initializing destination bogusfile repository
239 initializing destination bogusfile repository
240 abort: cannot create new bundle repository
240 abort: cannot create new bundle repository
241 % should fail
241 % should fail
242 abort: Permission denied: bogusdir
242 abort: Permission denied: bogusdir
243 % should succeed
243 % should succeed
244 initializing destination bogusdir repository
244 initializing destination bogusdir repository
245 scanning source...
245 scanning source...
246 sorting...
246 sorting...
247 converting...
247 converting...
248 4 a
248 4 a
249 3 b
249 3 b
250 2 c
250 2 c
251 1 d
251 1 d
252 0 e
252 0 e
253 % test pre and post conversion actions
253 % test pre and post conversion actions
254 run hg source pre-conversion action
254 run hg source pre-conversion action
255 run hg sink pre-conversion action
255 run hg sink pre-conversion action
256 run hg sink post-conversion action
256 run hg sink post-conversion action
257 run hg source post-conversion action
257 run hg source post-conversion action
258 % converting empty dir should fail nicely
258 % converting empty dir should fail nicely
259 assuming destination emptydir-hg
259 assuming destination emptydir-hg
260 initializing destination emptydir-hg repository
260 initializing destination emptydir-hg repository
261 emptydir does not look like a CVS checkout
261 emptydir does not look like a CVS checkout
262 emptydir does not look like a Git repo
262 emptydir does not look like a Git repo
263 emptydir does not look like a Subversion repo
263 emptydir does not look like a Subversion repo
264 emptydir is not a local Mercurial repo
264 emptydir is not a local Mercurial repo
265 emptydir does not look like a darcs repo
265 emptydir does not look like a darcs repo
266 emptydir does not look like a monotone repo
266 emptydir does not look like a monotone repo
267 emptydir does not look like a GNU Arch repo
267 emptydir does not look like a GNU Arch repo
268 emptydir does not look like a Bazaar repo
268 emptydir does not look like a Bazaar repo
269 cannot find required "p4" tool
269 cannot find required "p4" tool
270 abort: emptydir: missing or unsupported repository
270 abort: emptydir: missing or unsupported repository
271 % convert with imaginary source type
271 % convert with imaginary source type
272 initializing destination a-foo repository
272 initializing destination a-foo repository
273 abort: foo: invalid source repository type
273 abort: foo: invalid source repository type
274 % convert with imaginary sink type
274 % convert with imaginary sink type
275 abort: foo: invalid destination repository type
275 abort: foo: invalid destination repository type
276
277 % testing: convert must not produce duplicate entries in fncache
278 initializing destination b repository
279 scanning source...
280 sorting...
281 converting...
282 4 a
283 3 b
284 2 c
285 1 d
286 0 e
287 % contents of fncache file:
288 data/a.i
289 data/b.i
General Comments 0
You need to be logged in to leave comments. Login now