##// END OF EJS Templates
interfaces: create a new folder for interfaces and move repository.py in it...
Pulkit Goyal -
r43078:268662aa default
parent child Browse files
Show More

The requested changes are too big and content was truncated. Show full diff

1 NO CONTENT: new file 100644
NO CONTENT: new file 100644
@@ -1,748 +1,749
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2
2
3 from __future__ import absolute_import, print_function
3 from __future__ import absolute_import, print_function
4
4
5 import ast
5 import ast
6 import collections
6 import collections
7 import os
7 import os
8 import sys
8 import sys
9
9
10 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
10 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
11 # to work when run from a virtualenv. The modules were chosen empirically
11 # to work when run from a virtualenv. The modules were chosen empirically
12 # so that the return value matches the return value without virtualenv.
12 # so that the return value matches the return value without virtualenv.
13 if True: # disable lexical sorting checks
13 if True: # disable lexical sorting checks
14 try:
14 try:
15 import BaseHTTPServer as basehttpserver
15 import BaseHTTPServer as basehttpserver
16 except ImportError:
16 except ImportError:
17 basehttpserver = None
17 basehttpserver = None
18 import zlib
18 import zlib
19
19
20 import testparseutil
20 import testparseutil
21
21
22 # Whitelist of modules that symbols can be directly imported from.
22 # Whitelist of modules that symbols can be directly imported from.
23 allowsymbolimports = (
23 allowsymbolimports = (
24 '__future__',
24 '__future__',
25 'bzrlib',
25 'bzrlib',
26 'hgclient',
26 'hgclient',
27 'mercurial',
27 'mercurial',
28 'mercurial.hgweb.common',
28 'mercurial.hgweb.common',
29 'mercurial.hgweb.request',
29 'mercurial.hgweb.request',
30 'mercurial.i18n',
30 'mercurial.i18n',
31 'mercurial.interfaces',
31 'mercurial.node',
32 'mercurial.node',
32 # for revlog to re-export constant to extensions
33 # for revlog to re-export constant to extensions
33 'mercurial.revlogutils.constants',
34 'mercurial.revlogutils.constants',
34 'mercurial.revlogutils.flagutil',
35 'mercurial.revlogutils.flagutil',
35 # for cffi modules to re-export pure functions
36 # for cffi modules to re-export pure functions
36 'mercurial.pure.base85',
37 'mercurial.pure.base85',
37 'mercurial.pure.bdiff',
38 'mercurial.pure.bdiff',
38 'mercurial.pure.mpatch',
39 'mercurial.pure.mpatch',
39 'mercurial.pure.osutil',
40 'mercurial.pure.osutil',
40 'mercurial.pure.parsers',
41 'mercurial.pure.parsers',
41 # third-party imports should be directly imported
42 # third-party imports should be directly imported
42 'mercurial.thirdparty',
43 'mercurial.thirdparty',
43 'mercurial.thirdparty.attr',
44 'mercurial.thirdparty.attr',
44 'mercurial.thirdparty.zope',
45 'mercurial.thirdparty.zope',
45 'mercurial.thirdparty.zope.interface',
46 'mercurial.thirdparty.zope.interface',
46 )
47 )
47
48
48 # Whitelist of symbols that can be directly imported.
49 # Whitelist of symbols that can be directly imported.
49 directsymbols = (
50 directsymbols = (
50 'demandimport',
51 'demandimport',
51 )
52 )
52
53
53 # Modules that must be aliased because they are commonly confused with
54 # Modules that must be aliased because they are commonly confused with
54 # common variables and can create aliasing and readability issues.
55 # common variables and can create aliasing and readability issues.
55 requirealias = {
56 requirealias = {
56 'ui': 'uimod',
57 'ui': 'uimod',
57 }
58 }
58
59
59 def usingabsolute(root):
60 def usingabsolute(root):
60 """Whether absolute imports are being used."""
61 """Whether absolute imports are being used."""
61 if sys.version_info[0] >= 3:
62 if sys.version_info[0] >= 3:
62 return True
63 return True
63
64
64 for node in ast.walk(root):
65 for node in ast.walk(root):
65 if isinstance(node, ast.ImportFrom):
66 if isinstance(node, ast.ImportFrom):
66 if node.module == '__future__':
67 if node.module == '__future__':
67 for n in node.names:
68 for n in node.names:
68 if n.name == 'absolute_import':
69 if n.name == 'absolute_import':
69 return True
70 return True
70
71
71 return False
72 return False
72
73
73 def walklocal(root):
74 def walklocal(root):
74 """Recursively yield all descendant nodes but not in a different scope"""
75 """Recursively yield all descendant nodes but not in a different scope"""
75 todo = collections.deque(ast.iter_child_nodes(root))
76 todo = collections.deque(ast.iter_child_nodes(root))
76 yield root, False
77 yield root, False
77 while todo:
78 while todo:
78 node = todo.popleft()
79 node = todo.popleft()
79 newscope = isinstance(node, ast.FunctionDef)
80 newscope = isinstance(node, ast.FunctionDef)
80 if not newscope:
81 if not newscope:
81 todo.extend(ast.iter_child_nodes(node))
82 todo.extend(ast.iter_child_nodes(node))
82 yield node, newscope
83 yield node, newscope
83
84
84 def dotted_name_of_path(path):
85 def dotted_name_of_path(path):
85 """Given a relative path to a source file, return its dotted module name.
86 """Given a relative path to a source file, return its dotted module name.
86
87
87 >>> dotted_name_of_path('mercurial/error.py')
88 >>> dotted_name_of_path('mercurial/error.py')
88 'mercurial.error'
89 'mercurial.error'
89 >>> dotted_name_of_path('zlibmodule.so')
90 >>> dotted_name_of_path('zlibmodule.so')
90 'zlib'
91 'zlib'
91 """
92 """
92 parts = path.replace(os.sep, '/').split('/')
93 parts = path.replace(os.sep, '/').split('/')
93 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
94 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
94 if parts[-1].endswith('module'):
95 if parts[-1].endswith('module'):
95 parts[-1] = parts[-1][:-6]
96 parts[-1] = parts[-1][:-6]
96 return '.'.join(parts)
97 return '.'.join(parts)
97
98
98 def fromlocalfunc(modulename, localmods):
99 def fromlocalfunc(modulename, localmods):
99 """Get a function to examine which locally defined module the
100 """Get a function to examine which locally defined module the
100 target source imports via a specified name.
101 target source imports via a specified name.
101
102
102 `modulename` is an `dotted_name_of_path()`-ed source file path,
103 `modulename` is an `dotted_name_of_path()`-ed source file path,
103 which may have `.__init__` at the end of it, of the target source.
104 which may have `.__init__` at the end of it, of the target source.
104
105
105 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
106 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
106 paths of locally defined (= Mercurial specific) modules.
107 paths of locally defined (= Mercurial specific) modules.
107
108
108 This function assumes that module names not existing in
109 This function assumes that module names not existing in
109 `localmods` are from the Python standard library.
110 `localmods` are from the Python standard library.
110
111
111 This function returns the function, which takes `name` argument,
112 This function returns the function, which takes `name` argument,
112 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
113 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
113 matches against locally defined module. Otherwise, it returns
114 matches against locally defined module. Otherwise, it returns
114 False.
115 False.
115
116
116 It is assumed that `name` doesn't have `.__init__`.
117 It is assumed that `name` doesn't have `.__init__`.
117
118
118 `absname` is an absolute module name of specified `name`
119 `absname` is an absolute module name of specified `name`
119 (e.g. "hgext.convert"). This can be used to compose prefix for sub
120 (e.g. "hgext.convert"). This can be used to compose prefix for sub
120 modules or so.
121 modules or so.
121
122
122 `dottedpath` is a `dotted_name_of_path()`-ed source file path
123 `dottedpath` is a `dotted_name_of_path()`-ed source file path
123 (e.g. "hgext.convert.__init__") of `name`. This is used to look
124 (e.g. "hgext.convert.__init__") of `name`. This is used to look
124 module up in `localmods` again.
125 module up in `localmods` again.
125
126
126 `hassubmod` is whether it may have sub modules under it (for
127 `hassubmod` is whether it may have sub modules under it (for
127 convenient, even though this is also equivalent to "absname !=
128 convenient, even though this is also equivalent to "absname !=
128 dottednpath")
129 dottednpath")
129
130
130 >>> localmods = {'foo.__init__', 'foo.foo1',
131 >>> localmods = {'foo.__init__', 'foo.foo1',
131 ... 'foo.bar.__init__', 'foo.bar.bar1',
132 ... 'foo.bar.__init__', 'foo.bar.bar1',
132 ... 'baz.__init__', 'baz.baz1'}
133 ... 'baz.__init__', 'baz.baz1'}
133 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
134 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
134 >>> # relative
135 >>> # relative
135 >>> fromlocal('foo1')
136 >>> fromlocal('foo1')
136 ('foo.foo1', 'foo.foo1', False)
137 ('foo.foo1', 'foo.foo1', False)
137 >>> fromlocal('bar')
138 >>> fromlocal('bar')
138 ('foo.bar', 'foo.bar.__init__', True)
139 ('foo.bar', 'foo.bar.__init__', True)
139 >>> fromlocal('bar.bar1')
140 >>> fromlocal('bar.bar1')
140 ('foo.bar.bar1', 'foo.bar.bar1', False)
141 ('foo.bar.bar1', 'foo.bar.bar1', False)
141 >>> # absolute
142 >>> # absolute
142 >>> fromlocal('baz')
143 >>> fromlocal('baz')
143 ('baz', 'baz.__init__', True)
144 ('baz', 'baz.__init__', True)
144 >>> fromlocal('baz.baz1')
145 >>> fromlocal('baz.baz1')
145 ('baz.baz1', 'baz.baz1', False)
146 ('baz.baz1', 'baz.baz1', False)
146 >>> # unknown = maybe standard library
147 >>> # unknown = maybe standard library
147 >>> fromlocal('os')
148 >>> fromlocal('os')
148 False
149 False
149 >>> fromlocal(None, 1)
150 >>> fromlocal(None, 1)
150 ('foo', 'foo.__init__', True)
151 ('foo', 'foo.__init__', True)
151 >>> fromlocal('foo1', 1)
152 >>> fromlocal('foo1', 1)
152 ('foo.foo1', 'foo.foo1', False)
153 ('foo.foo1', 'foo.foo1', False)
153 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
154 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
154 >>> fromlocal2(None, 2)
155 >>> fromlocal2(None, 2)
155 ('foo', 'foo.__init__', True)
156 ('foo', 'foo.__init__', True)
156 >>> fromlocal2('bar2', 1)
157 >>> fromlocal2('bar2', 1)
157 False
158 False
158 >>> fromlocal2('bar', 2)
159 >>> fromlocal2('bar', 2)
159 ('foo.bar', 'foo.bar.__init__', True)
160 ('foo.bar', 'foo.bar.__init__', True)
160 """
161 """
161 if not isinstance(modulename, str):
162 if not isinstance(modulename, str):
162 modulename = modulename.decode('ascii')
163 modulename = modulename.decode('ascii')
163 prefix = '.'.join(modulename.split('.')[:-1])
164 prefix = '.'.join(modulename.split('.')[:-1])
164 if prefix:
165 if prefix:
165 prefix += '.'
166 prefix += '.'
166 def fromlocal(name, level=0):
167 def fromlocal(name, level=0):
167 # name is false value when relative imports are used.
168 # name is false value when relative imports are used.
168 if not name:
169 if not name:
169 # If relative imports are used, level must not be absolute.
170 # If relative imports are used, level must not be absolute.
170 assert level > 0
171 assert level > 0
171 candidates = ['.'.join(modulename.split('.')[:-level])]
172 candidates = ['.'.join(modulename.split('.')[:-level])]
172 else:
173 else:
173 if not level:
174 if not level:
174 # Check relative name first.
175 # Check relative name first.
175 candidates = [prefix + name, name]
176 candidates = [prefix + name, name]
176 else:
177 else:
177 candidates = ['.'.join(modulename.split('.')[:-level]) +
178 candidates = ['.'.join(modulename.split('.')[:-level]) +
178 '.' + name]
179 '.' + name]
179
180
180 for n in candidates:
181 for n in candidates:
181 if n in localmods:
182 if n in localmods:
182 return (n, n, False)
183 return (n, n, False)
183 dottedpath = n + '.__init__'
184 dottedpath = n + '.__init__'
184 if dottedpath in localmods:
185 if dottedpath in localmods:
185 return (n, dottedpath, True)
186 return (n, dottedpath, True)
186 return False
187 return False
187 return fromlocal
188 return fromlocal
188
189
189 def populateextmods(localmods):
190 def populateextmods(localmods):
190 """Populate C extension modules based on pure modules"""
191 """Populate C extension modules based on pure modules"""
191 newlocalmods = set(localmods)
192 newlocalmods = set(localmods)
192 for n in localmods:
193 for n in localmods:
193 if n.startswith('mercurial.pure.'):
194 if n.startswith('mercurial.pure.'):
194 m = n[len('mercurial.pure.'):]
195 m = n[len('mercurial.pure.'):]
195 newlocalmods.add('mercurial.cext.' + m)
196 newlocalmods.add('mercurial.cext.' + m)
196 newlocalmods.add('mercurial.cffi._' + m)
197 newlocalmods.add('mercurial.cffi._' + m)
197 return newlocalmods
198 return newlocalmods
198
199
199 def list_stdlib_modules():
200 def list_stdlib_modules():
200 """List the modules present in the stdlib.
201 """List the modules present in the stdlib.
201
202
202 >>> py3 = sys.version_info[0] >= 3
203 >>> py3 = sys.version_info[0] >= 3
203 >>> mods = set(list_stdlib_modules())
204 >>> mods = set(list_stdlib_modules())
204 >>> 'BaseHTTPServer' in mods or py3
205 >>> 'BaseHTTPServer' in mods or py3
205 True
206 True
206
207
207 os.path isn't really a module, so it's missing:
208 os.path isn't really a module, so it's missing:
208
209
209 >>> 'os.path' in mods
210 >>> 'os.path' in mods
210 False
211 False
211
212
212 sys requires special treatment, because it's baked into the
213 sys requires special treatment, because it's baked into the
213 interpreter, but it should still appear:
214 interpreter, but it should still appear:
214
215
215 >>> 'sys' in mods
216 >>> 'sys' in mods
216 True
217 True
217
218
218 >>> 'collections' in mods
219 >>> 'collections' in mods
219 True
220 True
220
221
221 >>> 'cStringIO' in mods or py3
222 >>> 'cStringIO' in mods or py3
222 True
223 True
223
224
224 >>> 'cffi' in mods
225 >>> 'cffi' in mods
225 True
226 True
226 """
227 """
227 for m in sys.builtin_module_names:
228 for m in sys.builtin_module_names:
228 yield m
229 yield m
229 # These modules only exist on windows, but we should always
230 # These modules only exist on windows, but we should always
230 # consider them stdlib.
231 # consider them stdlib.
231 for m in ['msvcrt', '_winreg']:
232 for m in ['msvcrt', '_winreg']:
232 yield m
233 yield m
233 yield '__builtin__'
234 yield '__builtin__'
234 yield 'builtins' # python3 only
235 yield 'builtins' # python3 only
235 yield 'importlib.abc' # python3 only
236 yield 'importlib.abc' # python3 only
236 yield 'importlib.machinery' # python3 only
237 yield 'importlib.machinery' # python3 only
237 yield 'importlib.util' # python3 only
238 yield 'importlib.util' # python3 only
238 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
239 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
239 yield m
240 yield m
240 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
241 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
241 yield m
242 yield m
242 for m in ['cffi']:
243 for m in ['cffi']:
243 yield m
244 yield m
244 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
245 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
245 # We need to supplement the list of prefixes for the search to work
246 # We need to supplement the list of prefixes for the search to work
246 # when run from within a virtualenv.
247 # when run from within a virtualenv.
247 for mod in (basehttpserver, zlib):
248 for mod in (basehttpserver, zlib):
248 if mod is None:
249 if mod is None:
249 continue
250 continue
250 try:
251 try:
251 # Not all module objects have a __file__ attribute.
252 # Not all module objects have a __file__ attribute.
252 filename = mod.__file__
253 filename = mod.__file__
253 except AttributeError:
254 except AttributeError:
254 continue
255 continue
255 dirname = os.path.dirname(filename)
256 dirname = os.path.dirname(filename)
256 for prefix in stdlib_prefixes:
257 for prefix in stdlib_prefixes:
257 if dirname.startswith(prefix):
258 if dirname.startswith(prefix):
258 # Then this directory is redundant.
259 # Then this directory is redundant.
259 break
260 break
260 else:
261 else:
261 stdlib_prefixes.add(dirname)
262 stdlib_prefixes.add(dirname)
262 sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
263 sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
263 for libpath in sys.path:
264 for libpath in sys.path:
264 # We want to walk everything in sys.path that starts with something in
265 # We want to walk everything in sys.path that starts with something in
265 # stdlib_prefixes, but not directories from the hg sources.
266 # stdlib_prefixes, but not directories from the hg sources.
266 if (os.path.abspath(libpath).startswith(sourceroot)
267 if (os.path.abspath(libpath).startswith(sourceroot)
267 or not any(libpath.startswith(p) for p in stdlib_prefixes)):
268 or not any(libpath.startswith(p) for p in stdlib_prefixes)):
268 continue
269 continue
269 for top, dirs, files in os.walk(libpath):
270 for top, dirs, files in os.walk(libpath):
270 for i, d in reversed(list(enumerate(dirs))):
271 for i, d in reversed(list(enumerate(dirs))):
271 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
272 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
272 or top == libpath and d in ('hgdemandimport', 'hgext',
273 or top == libpath and d in ('hgdemandimport', 'hgext',
273 'mercurial')):
274 'mercurial')):
274 del dirs[i]
275 del dirs[i]
275 for name in files:
276 for name in files:
276 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
277 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
277 continue
278 continue
278 if name.startswith('__init__.py'):
279 if name.startswith('__init__.py'):
279 full_path = top
280 full_path = top
280 else:
281 else:
281 full_path = os.path.join(top, name)
282 full_path = os.path.join(top, name)
282 rel_path = full_path[len(libpath) + 1:]
283 rel_path = full_path[len(libpath) + 1:]
283 mod = dotted_name_of_path(rel_path)
284 mod = dotted_name_of_path(rel_path)
284 yield mod
285 yield mod
285
286
286 stdlib_modules = set(list_stdlib_modules())
287 stdlib_modules = set(list_stdlib_modules())
287
288
288 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
289 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
289 """Given the source of a file as a string, yield the names
290 """Given the source of a file as a string, yield the names
290 imported by that file.
291 imported by that file.
291
292
292 Args:
293 Args:
293 source: The python source to examine as a string.
294 source: The python source to examine as a string.
294 modulename: of specified python source (may have `__init__`)
295 modulename: of specified python source (may have `__init__`)
295 localmods: set of locally defined module names (may have `__init__`)
296 localmods: set of locally defined module names (may have `__init__`)
296 ignore_nested: If true, import statements that do not start in
297 ignore_nested: If true, import statements that do not start in
297 column zero will be ignored.
298 column zero will be ignored.
298
299
299 Returns:
300 Returns:
300 A list of absolute module names imported by the given source.
301 A list of absolute module names imported by the given source.
301
302
302 >>> f = 'foo/xxx.py'
303 >>> f = 'foo/xxx.py'
303 >>> modulename = 'foo.xxx'
304 >>> modulename = 'foo.xxx'
304 >>> localmods = {'foo.__init__': True,
305 >>> localmods = {'foo.__init__': True,
305 ... 'foo.foo1': True, 'foo.foo2': True,
306 ... 'foo.foo1': True, 'foo.foo2': True,
306 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
307 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
307 ... 'baz.__init__': True, 'baz.baz1': True }
308 ... 'baz.__init__': True, 'baz.baz1': True }
308 >>> # standard library (= not locally defined ones)
309 >>> # standard library (= not locally defined ones)
309 >>> sorted(imported_modules(
310 >>> sorted(imported_modules(
310 ... 'from stdlib1 import foo, bar; import stdlib2',
311 ... 'from stdlib1 import foo, bar; import stdlib2',
311 ... modulename, f, localmods))
312 ... modulename, f, localmods))
312 []
313 []
313 >>> # relative importing
314 >>> # relative importing
314 >>> sorted(imported_modules(
315 >>> sorted(imported_modules(
315 ... 'import foo1; from bar import bar1',
316 ... 'import foo1; from bar import bar1',
316 ... modulename, f, localmods))
317 ... modulename, f, localmods))
317 ['foo.bar.bar1', 'foo.foo1']
318 ['foo.bar.bar1', 'foo.foo1']
318 >>> sorted(imported_modules(
319 >>> sorted(imported_modules(
319 ... 'from bar.bar1 import name1, name2, name3',
320 ... 'from bar.bar1 import name1, name2, name3',
320 ... modulename, f, localmods))
321 ... modulename, f, localmods))
321 ['foo.bar.bar1']
322 ['foo.bar.bar1']
322 >>> # absolute importing
323 >>> # absolute importing
323 >>> sorted(imported_modules(
324 >>> sorted(imported_modules(
324 ... 'from baz import baz1, name1',
325 ... 'from baz import baz1, name1',
325 ... modulename, f, localmods))
326 ... modulename, f, localmods))
326 ['baz.__init__', 'baz.baz1']
327 ['baz.__init__', 'baz.baz1']
327 >>> # mixed importing, even though it shouldn't be recommended
328 >>> # mixed importing, even though it shouldn't be recommended
328 >>> sorted(imported_modules(
329 >>> sorted(imported_modules(
329 ... 'import stdlib, foo1, baz',
330 ... 'import stdlib, foo1, baz',
330 ... modulename, f, localmods))
331 ... modulename, f, localmods))
331 ['baz.__init__', 'foo.foo1']
332 ['baz.__init__', 'foo.foo1']
332 >>> # ignore_nested
333 >>> # ignore_nested
333 >>> sorted(imported_modules(
334 >>> sorted(imported_modules(
334 ... '''import foo
335 ... '''import foo
335 ... def wat():
336 ... def wat():
336 ... import bar
337 ... import bar
337 ... ''', modulename, f, localmods))
338 ... ''', modulename, f, localmods))
338 ['foo.__init__', 'foo.bar.__init__']
339 ['foo.__init__', 'foo.bar.__init__']
339 >>> sorted(imported_modules(
340 >>> sorted(imported_modules(
340 ... '''import foo
341 ... '''import foo
341 ... def wat():
342 ... def wat():
342 ... import bar
343 ... import bar
343 ... ''', modulename, f, localmods, ignore_nested=True))
344 ... ''', modulename, f, localmods, ignore_nested=True))
344 ['foo.__init__']
345 ['foo.__init__']
345 """
346 """
346 fromlocal = fromlocalfunc(modulename, localmods)
347 fromlocal = fromlocalfunc(modulename, localmods)
347 for node in ast.walk(ast.parse(source, f)):
348 for node in ast.walk(ast.parse(source, f)):
348 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
349 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
349 continue
350 continue
350 if isinstance(node, ast.Import):
351 if isinstance(node, ast.Import):
351 for n in node.names:
352 for n in node.names:
352 found = fromlocal(n.name)
353 found = fromlocal(n.name)
353 if not found:
354 if not found:
354 # this should import standard library
355 # this should import standard library
355 continue
356 continue
356 yield found[1]
357 yield found[1]
357 elif isinstance(node, ast.ImportFrom):
358 elif isinstance(node, ast.ImportFrom):
358 found = fromlocal(node.module, node.level)
359 found = fromlocal(node.module, node.level)
359 if not found:
360 if not found:
360 # this should import standard library
361 # this should import standard library
361 continue
362 continue
362
363
363 absname, dottedpath, hassubmod = found
364 absname, dottedpath, hassubmod = found
364 if not hassubmod:
365 if not hassubmod:
365 # "dottedpath" is not a package; must be imported
366 # "dottedpath" is not a package; must be imported
366 yield dottedpath
367 yield dottedpath
367 # examination of "node.names" should be redundant
368 # examination of "node.names" should be redundant
368 # e.g.: from mercurial.node import nullid, nullrev
369 # e.g.: from mercurial.node import nullid, nullrev
369 continue
370 continue
370
371
371 modnotfound = False
372 modnotfound = False
372 prefix = absname + '.'
373 prefix = absname + '.'
373 for n in node.names:
374 for n in node.names:
374 found = fromlocal(prefix + n.name)
375 found = fromlocal(prefix + n.name)
375 if not found:
376 if not found:
376 # this should be a function or a property of "node.module"
377 # this should be a function or a property of "node.module"
377 modnotfound = True
378 modnotfound = True
378 continue
379 continue
379 yield found[1]
380 yield found[1]
380 if modnotfound:
381 if modnotfound:
381 # "dottedpath" is a package, but imported because of non-module
382 # "dottedpath" is a package, but imported because of non-module
382 # lookup
383 # lookup
383 yield dottedpath
384 yield dottedpath
384
385
385 def verify_import_convention(module, source, localmods):
386 def verify_import_convention(module, source, localmods):
386 """Verify imports match our established coding convention.
387 """Verify imports match our established coding convention.
387
388
388 We have 2 conventions: legacy and modern. The modern convention is in
389 We have 2 conventions: legacy and modern. The modern convention is in
389 effect when using absolute imports.
390 effect when using absolute imports.
390
391
391 The legacy convention only looks for mixed imports. The modern convention
392 The legacy convention only looks for mixed imports. The modern convention
392 is much more thorough.
393 is much more thorough.
393 """
394 """
394 root = ast.parse(source)
395 root = ast.parse(source)
395 absolute = usingabsolute(root)
396 absolute = usingabsolute(root)
396
397
397 if absolute:
398 if absolute:
398 return verify_modern_convention(module, root, localmods)
399 return verify_modern_convention(module, root, localmods)
399 else:
400 else:
400 return verify_stdlib_on_own_line(root)
401 return verify_stdlib_on_own_line(root)
401
402
402 def verify_modern_convention(module, root, localmods, root_col_offset=0):
403 def verify_modern_convention(module, root, localmods, root_col_offset=0):
403 """Verify a file conforms to the modern import convention rules.
404 """Verify a file conforms to the modern import convention rules.
404
405
405 The rules of the modern convention are:
406 The rules of the modern convention are:
406
407
407 * Ordering is stdlib followed by local imports. Each group is lexically
408 * Ordering is stdlib followed by local imports. Each group is lexically
408 sorted.
409 sorted.
409 * Importing multiple modules via "import X, Y" is not allowed: use
410 * Importing multiple modules via "import X, Y" is not allowed: use
410 separate import statements.
411 separate import statements.
411 * Importing multiple modules via "from X import ..." is allowed if using
412 * Importing multiple modules via "from X import ..." is allowed if using
412 parenthesis and one entry per line.
413 parenthesis and one entry per line.
413 * Only 1 relative import statement per import level ("from .", "from ..")
414 * Only 1 relative import statement per import level ("from .", "from ..")
414 is allowed.
415 is allowed.
415 * Relative imports from higher levels must occur before lower levels. e.g.
416 * Relative imports from higher levels must occur before lower levels. e.g.
416 "from .." must be before "from .".
417 "from .." must be before "from .".
417 * Imports from peer packages should use relative import (e.g. do not
418 * Imports from peer packages should use relative import (e.g. do not
418 "import mercurial.foo" from a "mercurial.*" module).
419 "import mercurial.foo" from a "mercurial.*" module).
419 * Symbols can only be imported from specific modules (see
420 * Symbols can only be imported from specific modules (see
420 `allowsymbolimports`). For other modules, first import the module then
421 `allowsymbolimports`). For other modules, first import the module then
421 assign the symbol to a module-level variable. In addition, these imports
422 assign the symbol to a module-level variable. In addition, these imports
422 must be performed before other local imports. This rule only
423 must be performed before other local imports. This rule only
423 applies to import statements outside of any blocks.
424 applies to import statements outside of any blocks.
424 * Relative imports from the standard library are not allowed, unless that
425 * Relative imports from the standard library are not allowed, unless that
425 library is also a local module.
426 library is also a local module.
426 * Certain modules must be aliased to alternate names to avoid aliasing
427 * Certain modules must be aliased to alternate names to avoid aliasing
427 and readability problems. See `requirealias`.
428 and readability problems. See `requirealias`.
428 """
429 """
429 if not isinstance(module, str):
430 if not isinstance(module, str):
430 module = module.decode('ascii')
431 module = module.decode('ascii')
431 topmodule = module.split('.')[0]
432 topmodule = module.split('.')[0]
432 fromlocal = fromlocalfunc(module, localmods)
433 fromlocal = fromlocalfunc(module, localmods)
433
434
434 # Whether a local/non-stdlib import has been performed.
435 # Whether a local/non-stdlib import has been performed.
435 seenlocal = None
436 seenlocal = None
436 # Whether a local/non-stdlib, non-symbol import has been seen.
437 # Whether a local/non-stdlib, non-symbol import has been seen.
437 seennonsymbollocal = False
438 seennonsymbollocal = False
438 # The last name to be imported (for sorting).
439 # The last name to be imported (for sorting).
439 lastname = None
440 lastname = None
440 laststdlib = None
441 laststdlib = None
441 # Relative import levels encountered so far.
442 # Relative import levels encountered so far.
442 seenlevels = set()
443 seenlevels = set()
443
444
444 for node, newscope in walklocal(root):
445 for node, newscope in walklocal(root):
445 def msg(fmt, *args):
446 def msg(fmt, *args):
446 return (fmt % args, node.lineno)
447 return (fmt % args, node.lineno)
447 if newscope:
448 if newscope:
448 # Check for local imports in function
449 # Check for local imports in function
449 for r in verify_modern_convention(module, node, localmods,
450 for r in verify_modern_convention(module, node, localmods,
450 node.col_offset + 4):
451 node.col_offset + 4):
451 yield r
452 yield r
452 elif isinstance(node, ast.Import):
453 elif isinstance(node, ast.Import):
453 # Disallow "import foo, bar" and require separate imports
454 # Disallow "import foo, bar" and require separate imports
454 # for each module.
455 # for each module.
455 if len(node.names) > 1:
456 if len(node.names) > 1:
456 yield msg('multiple imported names: %s',
457 yield msg('multiple imported names: %s',
457 ', '.join(n.name for n in node.names))
458 ', '.join(n.name for n in node.names))
458
459
459 name = node.names[0].name
460 name = node.names[0].name
460 asname = node.names[0].asname
461 asname = node.names[0].asname
461
462
462 stdlib = name in stdlib_modules
463 stdlib = name in stdlib_modules
463
464
464 # Ignore sorting rules on imports inside blocks.
465 # Ignore sorting rules on imports inside blocks.
465 if node.col_offset == root_col_offset:
466 if node.col_offset == root_col_offset:
466 if lastname and name < lastname and laststdlib == stdlib:
467 if lastname and name < lastname and laststdlib == stdlib:
467 yield msg('imports not lexically sorted: %s < %s',
468 yield msg('imports not lexically sorted: %s < %s',
468 name, lastname)
469 name, lastname)
469
470
470 lastname = name
471 lastname = name
471 laststdlib = stdlib
472 laststdlib = stdlib
472
473
473 # stdlib imports should be before local imports.
474 # stdlib imports should be before local imports.
474 if stdlib and seenlocal and node.col_offset == root_col_offset:
475 if stdlib and seenlocal and node.col_offset == root_col_offset:
475 yield msg('stdlib import "%s" follows local import: %s',
476 yield msg('stdlib import "%s" follows local import: %s',
476 name, seenlocal)
477 name, seenlocal)
477
478
478 if not stdlib:
479 if not stdlib:
479 seenlocal = name
480 seenlocal = name
480
481
481 # Import of sibling modules should use relative imports.
482 # Import of sibling modules should use relative imports.
482 topname = name.split('.')[0]
483 topname = name.split('.')[0]
483 if topname == topmodule:
484 if topname == topmodule:
484 yield msg('import should be relative: %s', name)
485 yield msg('import should be relative: %s', name)
485
486
486 if name in requirealias and asname != requirealias[name]:
487 if name in requirealias and asname != requirealias[name]:
487 yield msg('%s module must be "as" aliased to %s',
488 yield msg('%s module must be "as" aliased to %s',
488 name, requirealias[name])
489 name, requirealias[name])
489
490
490 elif isinstance(node, ast.ImportFrom):
491 elif isinstance(node, ast.ImportFrom):
491 # Resolve the full imported module name.
492 # Resolve the full imported module name.
492 if node.level > 0:
493 if node.level > 0:
493 fullname = '.'.join(module.split('.')[:-node.level])
494 fullname = '.'.join(module.split('.')[:-node.level])
494 if node.module:
495 if node.module:
495 fullname += '.%s' % node.module
496 fullname += '.%s' % node.module
496 else:
497 else:
497 assert node.module
498 assert node.module
498 fullname = node.module
499 fullname = node.module
499
500
500 topname = fullname.split('.')[0]
501 topname = fullname.split('.')[0]
501 if topname == topmodule:
502 if topname == topmodule:
502 yield msg('import should be relative: %s', fullname)
503 yield msg('import should be relative: %s', fullname)
503
504
504 # __future__ is special since it needs to come first and use
505 # __future__ is special since it needs to come first and use
505 # symbol import.
506 # symbol import.
506 if fullname != '__future__':
507 if fullname != '__future__':
507 if not fullname or (
508 if not fullname or (
508 fullname in stdlib_modules
509 fullname in stdlib_modules
509 and fullname not in localmods
510 and fullname not in localmods
510 and fullname + '.__init__' not in localmods):
511 and fullname + '.__init__' not in localmods):
511 yield msg('relative import of stdlib module')
512 yield msg('relative import of stdlib module')
512 else:
513 else:
513 seenlocal = fullname
514 seenlocal = fullname
514
515
515 # Direct symbol import is only allowed from certain modules and
516 # Direct symbol import is only allowed from certain modules and
516 # must occur before non-symbol imports.
517 # must occur before non-symbol imports.
517 found = fromlocal(node.module, node.level)
518 found = fromlocal(node.module, node.level)
518 if found and found[2]: # node.module is a package
519 if found and found[2]: # node.module is a package
519 prefix = found[0] + '.'
520 prefix = found[0] + '.'
520 symbols = (n.name for n in node.names
521 symbols = (n.name for n in node.names
521 if not fromlocal(prefix + n.name))
522 if not fromlocal(prefix + n.name))
522 else:
523 else:
523 symbols = (n.name for n in node.names)
524 symbols = (n.name for n in node.names)
524 symbols = [sym for sym in symbols if sym not in directsymbols]
525 symbols = [sym for sym in symbols if sym not in directsymbols]
525 if node.module and node.col_offset == root_col_offset:
526 if node.module and node.col_offset == root_col_offset:
526 if symbols and fullname not in allowsymbolimports:
527 if symbols and fullname not in allowsymbolimports:
527 yield msg('direct symbol import %s from %s',
528 yield msg('direct symbol import %s from %s',
528 ', '.join(symbols), fullname)
529 ', '.join(symbols), fullname)
529
530
530 if symbols and seennonsymbollocal:
531 if symbols and seennonsymbollocal:
531 yield msg('symbol import follows non-symbol import: %s',
532 yield msg('symbol import follows non-symbol import: %s',
532 fullname)
533 fullname)
533 if not symbols and fullname not in stdlib_modules:
534 if not symbols and fullname not in stdlib_modules:
534 seennonsymbollocal = True
535 seennonsymbollocal = True
535
536
536 if not node.module:
537 if not node.module:
537 assert node.level
538 assert node.level
538
539
539 # Only allow 1 group per level.
540 # Only allow 1 group per level.
540 if (node.level in seenlevels
541 if (node.level in seenlevels
541 and node.col_offset == root_col_offset):
542 and node.col_offset == root_col_offset):
542 yield msg('multiple "from %s import" statements',
543 yield msg('multiple "from %s import" statements',
543 '.' * node.level)
544 '.' * node.level)
544
545
545 # Higher-level groups come before lower-level groups.
546 # Higher-level groups come before lower-level groups.
546 if any(node.level > l for l in seenlevels):
547 if any(node.level > l for l in seenlevels):
547 yield msg('higher-level import should come first: %s',
548 yield msg('higher-level import should come first: %s',
548 fullname)
549 fullname)
549
550
550 seenlevels.add(node.level)
551 seenlevels.add(node.level)
551
552
552 # Entries in "from .X import ( ... )" lists must be lexically
553 # Entries in "from .X import ( ... )" lists must be lexically
553 # sorted.
554 # sorted.
554 lastentryname = None
555 lastentryname = None
555
556
556 for n in node.names:
557 for n in node.names:
557 if lastentryname and n.name < lastentryname:
558 if lastentryname and n.name < lastentryname:
558 yield msg('imports from %s not lexically sorted: %s < %s',
559 yield msg('imports from %s not lexically sorted: %s < %s',
559 fullname, n.name, lastentryname)
560 fullname, n.name, lastentryname)
560
561
561 lastentryname = n.name
562 lastentryname = n.name
562
563
563 if n.name in requirealias and n.asname != requirealias[n.name]:
564 if n.name in requirealias and n.asname != requirealias[n.name]:
564 yield msg('%s from %s must be "as" aliased to %s',
565 yield msg('%s from %s must be "as" aliased to %s',
565 n.name, fullname, requirealias[n.name])
566 n.name, fullname, requirealias[n.name])
566
567
567 def verify_stdlib_on_own_line(root):
568 def verify_stdlib_on_own_line(root):
568 """Given some python source, verify that stdlib imports are done
569 """Given some python source, verify that stdlib imports are done
569 in separate statements from relative local module imports.
570 in separate statements from relative local module imports.
570
571
571 >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
572 >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
572 [('mixed imports\\n stdlib: sys\\n relative: foo', 1)]
573 [('mixed imports\\n stdlib: sys\\n relative: foo', 1)]
573 >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
574 >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
574 []
575 []
575 >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
576 >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
576 []
577 []
577 """
578 """
578 for node in ast.walk(root):
579 for node in ast.walk(root):
579 if isinstance(node, ast.Import):
580 if isinstance(node, ast.Import):
580 from_stdlib = {False: [], True: []}
581 from_stdlib = {False: [], True: []}
581 for n in node.names:
582 for n in node.names:
582 from_stdlib[n.name in stdlib_modules].append(n.name)
583 from_stdlib[n.name in stdlib_modules].append(n.name)
583 if from_stdlib[True] and from_stdlib[False]:
584 if from_stdlib[True] and from_stdlib[False]:
584 yield ('mixed imports\n stdlib: %s\n relative: %s' %
585 yield ('mixed imports\n stdlib: %s\n relative: %s' %
585 (', '.join(sorted(from_stdlib[True])),
586 (', '.join(sorted(from_stdlib[True])),
586 ', '.join(sorted(from_stdlib[False]))), node.lineno)
587 ', '.join(sorted(from_stdlib[False]))), node.lineno)
587
588
588 class CircularImport(Exception):
589 class CircularImport(Exception):
589 pass
590 pass
590
591
591 def checkmod(mod, imports):
592 def checkmod(mod, imports):
592 shortest = {}
593 shortest = {}
593 visit = [[mod]]
594 visit = [[mod]]
594 while visit:
595 while visit:
595 path = visit.pop(0)
596 path = visit.pop(0)
596 for i in sorted(imports.get(path[-1], [])):
597 for i in sorted(imports.get(path[-1], [])):
597 if len(path) < shortest.get(i, 1000):
598 if len(path) < shortest.get(i, 1000):
598 shortest[i] = len(path)
599 shortest[i] = len(path)
599 if i in path:
600 if i in path:
600 if i == path[0]:
601 if i == path[0]:
601 raise CircularImport(path)
602 raise CircularImport(path)
602 continue
603 continue
603 visit.append(path + [i])
604 visit.append(path + [i])
604
605
605 def rotatecycle(cycle):
606 def rotatecycle(cycle):
606 """arrange a cycle so that the lexicographically first module listed first
607 """arrange a cycle so that the lexicographically first module listed first
607
608
608 >>> rotatecycle(['foo', 'bar'])
609 >>> rotatecycle(['foo', 'bar'])
609 ['bar', 'foo', 'bar']
610 ['bar', 'foo', 'bar']
610 """
611 """
611 lowest = min(cycle)
612 lowest = min(cycle)
612 idx = cycle.index(lowest)
613 idx = cycle.index(lowest)
613 return cycle[idx:] + cycle[:idx] + [lowest]
614 return cycle[idx:] + cycle[:idx] + [lowest]
614
615
615 def find_cycles(imports):
616 def find_cycles(imports):
616 """Find cycles in an already-loaded import graph.
617 """Find cycles in an already-loaded import graph.
617
618
618 All module names recorded in `imports` should be absolute one.
619 All module names recorded in `imports` should be absolute one.
619
620
620 >>> from __future__ import print_function
621 >>> from __future__ import print_function
621 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
622 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
622 ... 'top.bar': ['top.baz', 'sys'],
623 ... 'top.bar': ['top.baz', 'sys'],
623 ... 'top.baz': ['top.foo'],
624 ... 'top.baz': ['top.foo'],
624 ... 'top.qux': ['top.foo']}
625 ... 'top.qux': ['top.foo']}
625 >>> print('\\n'.join(sorted(find_cycles(imports))))
626 >>> print('\\n'.join(sorted(find_cycles(imports))))
626 top.bar -> top.baz -> top.foo -> top.bar
627 top.bar -> top.baz -> top.foo -> top.bar
627 top.foo -> top.qux -> top.foo
628 top.foo -> top.qux -> top.foo
628 """
629 """
629 cycles = set()
630 cycles = set()
630 for mod in sorted(imports.keys()):
631 for mod in sorted(imports.keys()):
631 try:
632 try:
632 checkmod(mod, imports)
633 checkmod(mod, imports)
633 except CircularImport as e:
634 except CircularImport as e:
634 cycle = e.args[0]
635 cycle = e.args[0]
635 cycles.add(" -> ".join(rotatecycle(cycle)))
636 cycles.add(" -> ".join(rotatecycle(cycle)))
636 return cycles
637 return cycles
637
638
638 def _cycle_sortkey(c):
639 def _cycle_sortkey(c):
639 return len(c), c
640 return len(c), c
640
641
641 def embedded(f, modname, src):
642 def embedded(f, modname, src):
642 """Extract embedded python code
643 """Extract embedded python code
643
644
644 >>> def _forcestr(thing):
645 >>> def _forcestr(thing):
645 ... if not isinstance(thing, str):
646 ... if not isinstance(thing, str):
646 ... return thing.decode('ascii')
647 ... return thing.decode('ascii')
647 ... return thing
648 ... return thing
648 >>> def test(fn, lines):
649 >>> def test(fn, lines):
649 ... for s, m, f, l in embedded(fn, b"example", lines):
650 ... for s, m, f, l in embedded(fn, b"example", lines):
650 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
651 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
651 ... print(repr(_forcestr(s)))
652 ... print(repr(_forcestr(s)))
652 >>> lines = [
653 >>> lines = [
653 ... 'comment',
654 ... 'comment',
654 ... ' >>> from __future__ import print_function',
655 ... ' >>> from __future__ import print_function',
655 ... " >>> ' multiline",
656 ... " >>> ' multiline",
656 ... " ... string'",
657 ... " ... string'",
657 ... ' ',
658 ... ' ',
658 ... 'comment',
659 ... 'comment',
659 ... ' $ cat > foo.py <<EOF',
660 ... ' $ cat > foo.py <<EOF',
660 ... ' > from __future__ import print_function',
661 ... ' > from __future__ import print_function',
661 ... ' > EOF',
662 ... ' > EOF',
662 ... ]
663 ... ]
663 >>> test(b"example.t", lines)
664 >>> test(b"example.t", lines)
664 example[2] doctest.py 1
665 example[2] doctest.py 1
665 "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
666 "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
666 example[8] foo.py 7
667 example[8] foo.py 7
667 'from __future__ import print_function\\n'
668 'from __future__ import print_function\\n'
668 """
669 """
669 errors = []
670 errors = []
670 for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
671 for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
671 if not name:
672 if not name:
672 # use 'doctest.py', in order to make already existing
673 # use 'doctest.py', in order to make already existing
673 # doctest above pass instantly
674 # doctest above pass instantly
674 name = 'doctest.py'
675 name = 'doctest.py'
675 # "starts" is "line number" (1-origin), but embedded() is
676 # "starts" is "line number" (1-origin), but embedded() is
676 # expected to return "line offset" (0-origin). Therefore, this
677 # expected to return "line offset" (0-origin). Therefore, this
677 # yields "starts - 1".
678 # yields "starts - 1".
678 if not isinstance(modname, str):
679 if not isinstance(modname, str):
679 modname = modname.decode('utf8')
680 modname = modname.decode('utf8')
680 yield code, "%s[%d]" % (modname, starts), name, starts - 1
681 yield code, "%s[%d]" % (modname, starts), name, starts - 1
681
682
682 def sources(f, modname):
683 def sources(f, modname):
683 """Yields possibly multiple sources from a filepath
684 """Yields possibly multiple sources from a filepath
684
685
685 input: filepath, modulename
686 input: filepath, modulename
686 yields: script(string), modulename, filepath, linenumber
687 yields: script(string), modulename, filepath, linenumber
687
688
688 For embedded scripts, the modulename and filepath will be different
689 For embedded scripts, the modulename and filepath will be different
689 from the function arguments. linenumber is an offset relative to
690 from the function arguments. linenumber is an offset relative to
690 the input file.
691 the input file.
691 """
692 """
692 py = False
693 py = False
693 if not f.endswith('.t'):
694 if not f.endswith('.t'):
694 with open(f, 'rb') as src:
695 with open(f, 'rb') as src:
695 yield src.read(), modname, f, 0
696 yield src.read(), modname, f, 0
696 py = True
697 py = True
697 if py or f.endswith('.t'):
698 if py or f.endswith('.t'):
698 with open(f, 'r') as src:
699 with open(f, 'r') as src:
699 for script, modname, t, line in embedded(f, modname, src):
700 for script, modname, t, line in embedded(f, modname, src):
700 yield script, modname.encode('utf8'), t, line
701 yield script, modname.encode('utf8'), t, line
701
702
702 def main(argv):
703 def main(argv):
703 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
704 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
704 print('Usage: %s {-|file [file] [file] ...}')
705 print('Usage: %s {-|file [file] [file] ...}')
705 return 1
706 return 1
706 if argv[1] == '-':
707 if argv[1] == '-':
707 argv = argv[:1]
708 argv = argv[:1]
708 argv.extend(l.rstrip() for l in sys.stdin.readlines())
709 argv.extend(l.rstrip() for l in sys.stdin.readlines())
709 localmodpaths = {}
710 localmodpaths = {}
710 used_imports = {}
711 used_imports = {}
711 any_errors = False
712 any_errors = False
712 for source_path in argv[1:]:
713 for source_path in argv[1:]:
713 modname = dotted_name_of_path(source_path)
714 modname = dotted_name_of_path(source_path)
714 localmodpaths[modname] = source_path
715 localmodpaths[modname] = source_path
715 localmods = populateextmods(localmodpaths)
716 localmods = populateextmods(localmodpaths)
716 for localmodname, source_path in sorted(localmodpaths.items()):
717 for localmodname, source_path in sorted(localmodpaths.items()):
717 if not isinstance(localmodname, bytes):
718 if not isinstance(localmodname, bytes):
718 # This is only safe because all hg's files are ascii
719 # This is only safe because all hg's files are ascii
719 localmodname = localmodname.encode('ascii')
720 localmodname = localmodname.encode('ascii')
720 for src, modname, name, line in sources(source_path, localmodname):
721 for src, modname, name, line in sources(source_path, localmodname):
721 try:
722 try:
722 used_imports[modname] = sorted(
723 used_imports[modname] = sorted(
723 imported_modules(src, modname, name, localmods,
724 imported_modules(src, modname, name, localmods,
724 ignore_nested=True))
725 ignore_nested=True))
725 for error, lineno in verify_import_convention(modname, src,
726 for error, lineno in verify_import_convention(modname, src,
726 localmods):
727 localmods):
727 any_errors = True
728 any_errors = True
728 print('%s:%d: %s' % (source_path, lineno + line, error))
729 print('%s:%d: %s' % (source_path, lineno + line, error))
729 except SyntaxError as e:
730 except SyntaxError as e:
730 print('%s:%d: SyntaxError: %s' %
731 print('%s:%d: SyntaxError: %s' %
731 (source_path, e.lineno + line, e))
732 (source_path, e.lineno + line, e))
732 cycles = find_cycles(used_imports)
733 cycles = find_cycles(used_imports)
733 if cycles:
734 if cycles:
734 firstmods = set()
735 firstmods = set()
735 for c in sorted(cycles, key=_cycle_sortkey):
736 for c in sorted(cycles, key=_cycle_sortkey):
736 first = c.split()[0]
737 first = c.split()[0]
737 # As a rough cut, ignore any cycle that starts with the
738 # As a rough cut, ignore any cycle that starts with the
738 # same module as some other cycle. Otherwise we see lots
739 # same module as some other cycle. Otherwise we see lots
739 # of cycles that are effectively duplicates.
740 # of cycles that are effectively duplicates.
740 if first in firstmods:
741 if first in firstmods:
741 continue
742 continue
742 print('Import cycle:', c)
743 print('Import cycle:', c)
743 firstmods.add(first)
744 firstmods.add(first)
744 any_errors = True
745 any_errors = True
745 return any_errors != 0
746 return any_errors != 0
746
747
747 if __name__ == '__main__':
748 if __name__ == '__main__':
748 sys.exit(int(main(sys.argv)))
749 sys.exit(int(main(sys.argv)))
@@ -1,383 +1,386
1 # lfs - hash-preserving large file support using Git-LFS protocol
1 # lfs - hash-preserving large file support using Git-LFS protocol
2 #
2 #
3 # Copyright 2017 Facebook, Inc.
3 # Copyright 2017 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 """lfs - large file support (EXPERIMENTAL)
8 """lfs - large file support (EXPERIMENTAL)
9
9
10 This extension allows large files to be tracked outside of the normal
10 This extension allows large files to be tracked outside of the normal
11 repository storage and stored on a centralized server, similar to the
11 repository storage and stored on a centralized server, similar to the
12 ``largefiles`` extension. The ``git-lfs`` protocol is used when
12 ``largefiles`` extension. The ``git-lfs`` protocol is used when
13 communicating with the server, so existing git infrastructure can be
13 communicating with the server, so existing git infrastructure can be
14 harnessed. Even though the files are stored outside of the repository,
14 harnessed. Even though the files are stored outside of the repository,
15 they are still integrity checked in the same manner as normal files.
15 they are still integrity checked in the same manner as normal files.
16
16
17 The files stored outside of the repository are downloaded on demand,
17 The files stored outside of the repository are downloaded on demand,
18 which reduces the time to clone, and possibly the local disk usage.
18 which reduces the time to clone, and possibly the local disk usage.
19 This changes fundamental workflows in a DVCS, so careful thought
19 This changes fundamental workflows in a DVCS, so careful thought
20 should be given before deploying it. :hg:`convert` can be used to
20 should be given before deploying it. :hg:`convert` can be used to
21 convert LFS repositories to normal repositories that no longer
21 convert LFS repositories to normal repositories that no longer
22 require this extension, and do so without changing the commit hashes.
22 require this extension, and do so without changing the commit hashes.
23 This allows the extension to be disabled if the centralized workflow
23 This allows the extension to be disabled if the centralized workflow
24 becomes burdensome. However, the pre and post convert clones will
24 becomes burdensome. However, the pre and post convert clones will
25 not be able to communicate with each other unless the extension is
25 not be able to communicate with each other unless the extension is
26 enabled on both.
26 enabled on both.
27
27
28 To start a new repository, or to add LFS files to an existing one, just
28 To start a new repository, or to add LFS files to an existing one, just
29 create an ``.hglfs`` file as described below in the root directory of
29 create an ``.hglfs`` file as described below in the root directory of
30 the repository. Typically, this file should be put under version
30 the repository. Typically, this file should be put under version
31 control, so that the settings will propagate to other repositories with
31 control, so that the settings will propagate to other repositories with
32 push and pull. During any commit, Mercurial will consult this file to
32 push and pull. During any commit, Mercurial will consult this file to
33 determine if an added or modified file should be stored externally. The
33 determine if an added or modified file should be stored externally. The
34 type of storage depends on the characteristics of the file at each
34 type of storage depends on the characteristics of the file at each
35 commit. A file that is near a size threshold may switch back and forth
35 commit. A file that is near a size threshold may switch back and forth
36 between LFS and normal storage, as needed.
36 between LFS and normal storage, as needed.
37
37
38 Alternately, both normal repositories and largefile controlled
38 Alternately, both normal repositories and largefile controlled
39 repositories can be converted to LFS by using :hg:`convert` and the
39 repositories can be converted to LFS by using :hg:`convert` and the
40 ``lfs.track`` config option described below. The ``.hglfs`` file
40 ``lfs.track`` config option described below. The ``.hglfs`` file
41 should then be created and added, to control subsequent LFS selection.
41 should then be created and added, to control subsequent LFS selection.
42 The hashes are also unchanged in this case. The LFS and non-LFS
42 The hashes are also unchanged in this case. The LFS and non-LFS
43 repositories can be distinguished because the LFS repository will
43 repositories can be distinguished because the LFS repository will
44 abort any command if this extension is disabled.
44 abort any command if this extension is disabled.
45
45
46 Committed LFS files are held locally, until the repository is pushed.
46 Committed LFS files are held locally, until the repository is pushed.
47 Prior to pushing the normal repository data, the LFS files that are
47 Prior to pushing the normal repository data, the LFS files that are
48 tracked by the outgoing commits are automatically uploaded to the
48 tracked by the outgoing commits are automatically uploaded to the
49 configured central server. No LFS files are transferred on
49 configured central server. No LFS files are transferred on
50 :hg:`pull` or :hg:`clone`. Instead, the files are downloaded on
50 :hg:`pull` or :hg:`clone`. Instead, the files are downloaded on
51 demand as they need to be read, if a cached copy cannot be found
51 demand as they need to be read, if a cached copy cannot be found
52 locally. Both committing and downloading an LFS file will link the
52 locally. Both committing and downloading an LFS file will link the
53 file to a usercache, to speed up future access. See the `usercache`
53 file to a usercache, to speed up future access. See the `usercache`
54 config setting described below.
54 config setting described below.
55
55
56 .hglfs::
56 .hglfs::
57
57
58 The extension reads its configuration from a versioned ``.hglfs``
58 The extension reads its configuration from a versioned ``.hglfs``
59 configuration file found in the root of the working directory. The
59 configuration file found in the root of the working directory. The
60 ``.hglfs`` file uses the same syntax as all other Mercurial
60 ``.hglfs`` file uses the same syntax as all other Mercurial
61 configuration files. It uses a single section, ``[track]``.
61 configuration files. It uses a single section, ``[track]``.
62
62
63 The ``[track]`` section specifies which files are stored as LFS (or
63 The ``[track]`` section specifies which files are stored as LFS (or
64 not). Each line is keyed by a file pattern, with a predicate value.
64 not). Each line is keyed by a file pattern, with a predicate value.
65 The first file pattern match is used, so put more specific patterns
65 The first file pattern match is used, so put more specific patterns
66 first. The available predicates are ``all()``, ``none()``, and
66 first. The available predicates are ``all()``, ``none()``, and
67 ``size()``. See "hg help filesets.size" for the latter.
67 ``size()``. See "hg help filesets.size" for the latter.
68
68
69 Example versioned ``.hglfs`` file::
69 Example versioned ``.hglfs`` file::
70
70
71 [track]
71 [track]
72 # No Makefile or python file, anywhere, will be LFS
72 # No Makefile or python file, anywhere, will be LFS
73 **Makefile = none()
73 **Makefile = none()
74 **.py = none()
74 **.py = none()
75
75
76 **.zip = all()
76 **.zip = all()
77 **.exe = size(">1MB")
77 **.exe = size(">1MB")
78
78
79 # Catchall for everything not matched above
79 # Catchall for everything not matched above
80 ** = size(">10MB")
80 ** = size(">10MB")
81
81
82 Configs::
82 Configs::
83
83
84 [lfs]
84 [lfs]
85 # Remote endpoint. Multiple protocols are supported:
85 # Remote endpoint. Multiple protocols are supported:
86 # - http(s)://user:pass@example.com/path
86 # - http(s)://user:pass@example.com/path
87 # git-lfs endpoint
87 # git-lfs endpoint
88 # - file:///tmp/path
88 # - file:///tmp/path
89 # local filesystem, usually for testing
89 # local filesystem, usually for testing
90 # if unset, lfs will assume the remote repository also handles blob storage
90 # if unset, lfs will assume the remote repository also handles blob storage
91 # for http(s) URLs. Otherwise, lfs will prompt to set this when it must
91 # for http(s) URLs. Otherwise, lfs will prompt to set this when it must
92 # use this value.
92 # use this value.
93 # (default: unset)
93 # (default: unset)
94 url = https://example.com/repo.git/info/lfs
94 url = https://example.com/repo.git/info/lfs
95
95
96 # Which files to track in LFS. Path tests are "**.extname" for file
96 # Which files to track in LFS. Path tests are "**.extname" for file
97 # extensions, and "path:under/some/directory" for path prefix. Both
97 # extensions, and "path:under/some/directory" for path prefix. Both
98 # are relative to the repository root.
98 # are relative to the repository root.
99 # File size can be tested with the "size()" fileset, and tests can be
99 # File size can be tested with the "size()" fileset, and tests can be
100 # joined with fileset operators. (See "hg help filesets.operators".)
100 # joined with fileset operators. (See "hg help filesets.operators".)
101 #
101 #
102 # Some examples:
102 # Some examples:
103 # - all() # everything
103 # - all() # everything
104 # - none() # nothing
104 # - none() # nothing
105 # - size(">20MB") # larger than 20MB
105 # - size(">20MB") # larger than 20MB
106 # - !**.txt # anything not a *.txt file
106 # - !**.txt # anything not a *.txt file
107 # - **.zip | **.tar.gz | **.7z # some types of compressed files
107 # - **.zip | **.tar.gz | **.7z # some types of compressed files
108 # - path:bin # files under "bin" in the project root
108 # - path:bin # files under "bin" in the project root
109 # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz
109 # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz
110 # | (path:bin & !path:/bin/README) | size(">1GB")
110 # | (path:bin & !path:/bin/README) | size(">1GB")
111 # (default: none())
111 # (default: none())
112 #
112 #
113 # This is ignored if there is a tracked '.hglfs' file, and this setting
113 # This is ignored if there is a tracked '.hglfs' file, and this setting
114 # will eventually be deprecated and removed.
114 # will eventually be deprecated and removed.
115 track = size(">10M")
115 track = size(">10M")
116
116
117 # how many times to retry before giving up on transferring an object
117 # how many times to retry before giving up on transferring an object
118 retry = 5
118 retry = 5
119
119
120 # the local directory to store lfs files for sharing across local clones.
120 # the local directory to store lfs files for sharing across local clones.
121 # If not set, the cache is located in an OS specific cache location.
121 # If not set, the cache is located in an OS specific cache location.
122 usercache = /path/to/global/cache
122 usercache = /path/to/global/cache
123 """
123 """
124
124
125 from __future__ import absolute_import
125 from __future__ import absolute_import
126
126
127 import sys
127 import sys
128
128
129 from mercurial.i18n import _
129 from mercurial.i18n import _
130
130
131 from mercurial import (
131 from mercurial import (
132 config,
132 config,
133 context,
133 context,
134 error,
134 error,
135 exchange,
135 exchange,
136 extensions,
136 extensions,
137 exthelper,
137 exthelper,
138 filelog,
138 filelog,
139 filesetlang,
139 filesetlang,
140 localrepo,
140 localrepo,
141 minifileset,
141 minifileset,
142 node,
142 node,
143 pycompat,
143 pycompat,
144 repository,
145 revlog,
144 revlog,
146 scmutil,
145 scmutil,
147 templateutil,
146 templateutil,
148 util,
147 util,
149 )
148 )
150
149
150 from mercurial.interfaces import (
151 repository,
152 )
153
151 from . import (
154 from . import (
152 blobstore,
155 blobstore,
153 wireprotolfsserver,
156 wireprotolfsserver,
154 wrapper,
157 wrapper,
155 )
158 )
156
159
157 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
160 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
158 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
161 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
159 # be specifying the version(s) of Mercurial they are tested with, or
162 # be specifying the version(s) of Mercurial they are tested with, or
160 # leave the attribute unspecified.
163 # leave the attribute unspecified.
161 testedwith = 'ships-with-hg-core'
164 testedwith = 'ships-with-hg-core'
162
165
163 eh = exthelper.exthelper()
166 eh = exthelper.exthelper()
164 eh.merge(wrapper.eh)
167 eh.merge(wrapper.eh)
165 eh.merge(wireprotolfsserver.eh)
168 eh.merge(wireprotolfsserver.eh)
166
169
167 cmdtable = eh.cmdtable
170 cmdtable = eh.cmdtable
168 configtable = eh.configtable
171 configtable = eh.configtable
169 extsetup = eh.finalextsetup
172 extsetup = eh.finalextsetup
170 uisetup = eh.finaluisetup
173 uisetup = eh.finaluisetup
171 filesetpredicate = eh.filesetpredicate
174 filesetpredicate = eh.filesetpredicate
172 reposetup = eh.finalreposetup
175 reposetup = eh.finalreposetup
173 templatekeyword = eh.templatekeyword
176 templatekeyword = eh.templatekeyword
174
177
175 eh.configitem('experimental', 'lfs.serve',
178 eh.configitem('experimental', 'lfs.serve',
176 default=True,
179 default=True,
177 )
180 )
178 eh.configitem('experimental', 'lfs.user-agent',
181 eh.configitem('experimental', 'lfs.user-agent',
179 default=None,
182 default=None,
180 )
183 )
181 eh.configitem('experimental', 'lfs.disableusercache',
184 eh.configitem('experimental', 'lfs.disableusercache',
182 default=False,
185 default=False,
183 )
186 )
184 eh.configitem('experimental', 'lfs.worker-enable',
187 eh.configitem('experimental', 'lfs.worker-enable',
185 default=False,
188 default=False,
186 )
189 )
187
190
188 eh.configitem('lfs', 'url',
191 eh.configitem('lfs', 'url',
189 default=None,
192 default=None,
190 )
193 )
191 eh.configitem('lfs', 'usercache',
194 eh.configitem('lfs', 'usercache',
192 default=None,
195 default=None,
193 )
196 )
194 # Deprecated
197 # Deprecated
195 eh.configitem('lfs', 'threshold',
198 eh.configitem('lfs', 'threshold',
196 default=None,
199 default=None,
197 )
200 )
198 eh.configitem('lfs', 'track',
201 eh.configitem('lfs', 'track',
199 default='none()',
202 default='none()',
200 )
203 )
201 eh.configitem('lfs', 'retry',
204 eh.configitem('lfs', 'retry',
202 default=5,
205 default=5,
203 )
206 )
204
207
205 lfsprocessor = (
208 lfsprocessor = (
206 wrapper.readfromstore,
209 wrapper.readfromstore,
207 wrapper.writetostore,
210 wrapper.writetostore,
208 wrapper.bypasscheckhash,
211 wrapper.bypasscheckhash,
209 )
212 )
210
213
211 def featuresetup(ui, supported):
214 def featuresetup(ui, supported):
212 # don't die on seeing a repo with the lfs requirement
215 # don't die on seeing a repo with the lfs requirement
213 supported |= {'lfs'}
216 supported |= {'lfs'}
214
217
215 @eh.uisetup
218 @eh.uisetup
216 def _uisetup(ui):
219 def _uisetup(ui):
217 localrepo.featuresetupfuncs.add(featuresetup)
220 localrepo.featuresetupfuncs.add(featuresetup)
218
221
219 @eh.reposetup
222 @eh.reposetup
220 def _reposetup(ui, repo):
223 def _reposetup(ui, repo):
221 # Nothing to do with a remote repo
224 # Nothing to do with a remote repo
222 if not repo.local():
225 if not repo.local():
223 return
226 return
224
227
225 repo.svfs.lfslocalblobstore = blobstore.local(repo)
228 repo.svfs.lfslocalblobstore = blobstore.local(repo)
226 repo.svfs.lfsremoteblobstore = blobstore.remote(repo)
229 repo.svfs.lfsremoteblobstore = blobstore.remote(repo)
227
230
228 class lfsrepo(repo.__class__):
231 class lfsrepo(repo.__class__):
229 @localrepo.unfilteredmethod
232 @localrepo.unfilteredmethod
230 def commitctx(self, ctx, error=False, origctx=None):
233 def commitctx(self, ctx, error=False, origctx=None):
231 repo.svfs.options['lfstrack'] = _trackedmatcher(self)
234 repo.svfs.options['lfstrack'] = _trackedmatcher(self)
232 return super(lfsrepo, self).commitctx(ctx, error, origctx=origctx)
235 return super(lfsrepo, self).commitctx(ctx, error, origctx=origctx)
233
236
234 repo.__class__ = lfsrepo
237 repo.__class__ = lfsrepo
235
238
236 if 'lfs' not in repo.requirements:
239 if 'lfs' not in repo.requirements:
237 def checkrequireslfs(ui, repo, **kwargs):
240 def checkrequireslfs(ui, repo, **kwargs):
238 if 'lfs' in repo.requirements:
241 if 'lfs' in repo.requirements:
239 return 0
242 return 0
240
243
241 last = kwargs.get(r'node_last')
244 last = kwargs.get(r'node_last')
242 _bin = node.bin
245 _bin = node.bin
243 if last:
246 if last:
244 s = repo.set('%n:%n', _bin(kwargs[r'node']), _bin(last))
247 s = repo.set('%n:%n', _bin(kwargs[r'node']), _bin(last))
245 else:
248 else:
246 s = repo.set('%n', _bin(kwargs[r'node']))
249 s = repo.set('%n', _bin(kwargs[r'node']))
247 match = repo._storenarrowmatch
250 match = repo._storenarrowmatch
248 for ctx in s:
251 for ctx in s:
249 # TODO: is there a way to just walk the files in the commit?
252 # TODO: is there a way to just walk the files in the commit?
250 if any(ctx[f].islfs() for f in ctx.files()
253 if any(ctx[f].islfs() for f in ctx.files()
251 if f in ctx and match(f)):
254 if f in ctx and match(f)):
252 repo.requirements.add('lfs')
255 repo.requirements.add('lfs')
253 repo.features.add(repository.REPO_FEATURE_LFS)
256 repo.features.add(repository.REPO_FEATURE_LFS)
254 repo._writerequirements()
257 repo._writerequirements()
255 repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
258 repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
256 break
259 break
257
260
258 ui.setconfig('hooks', 'commit.lfs', checkrequireslfs, 'lfs')
261 ui.setconfig('hooks', 'commit.lfs', checkrequireslfs, 'lfs')
259 ui.setconfig('hooks', 'pretxnchangegroup.lfs', checkrequireslfs, 'lfs')
262 ui.setconfig('hooks', 'pretxnchangegroup.lfs', checkrequireslfs, 'lfs')
260 else:
263 else:
261 repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
264 repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
262
265
263 def _trackedmatcher(repo):
266 def _trackedmatcher(repo):
264 """Return a function (path, size) -> bool indicating whether or not to
267 """Return a function (path, size) -> bool indicating whether or not to
265 track a given file with lfs."""
268 track a given file with lfs."""
266 if not repo.wvfs.exists('.hglfs'):
269 if not repo.wvfs.exists('.hglfs'):
267 # No '.hglfs' in wdir. Fallback to config for now.
270 # No '.hglfs' in wdir. Fallback to config for now.
268 trackspec = repo.ui.config('lfs', 'track')
271 trackspec = repo.ui.config('lfs', 'track')
269
272
270 # deprecated config: lfs.threshold
273 # deprecated config: lfs.threshold
271 threshold = repo.ui.configbytes('lfs', 'threshold')
274 threshold = repo.ui.configbytes('lfs', 'threshold')
272 if threshold:
275 if threshold:
273 filesetlang.parse(trackspec) # make sure syntax errors are confined
276 filesetlang.parse(trackspec) # make sure syntax errors are confined
274 trackspec = "(%s) | size('>%d')" % (trackspec, threshold)
277 trackspec = "(%s) | size('>%d')" % (trackspec, threshold)
275
278
276 return minifileset.compile(trackspec)
279 return minifileset.compile(trackspec)
277
280
278 data = repo.wvfs.tryread('.hglfs')
281 data = repo.wvfs.tryread('.hglfs')
279 if not data:
282 if not data:
280 return lambda p, s: False
283 return lambda p, s: False
281
284
282 # Parse errors here will abort with a message that points to the .hglfs file
285 # Parse errors here will abort with a message that points to the .hglfs file
283 # and line number.
286 # and line number.
284 cfg = config.config()
287 cfg = config.config()
285 cfg.parse('.hglfs', data)
288 cfg.parse('.hglfs', data)
286
289
287 try:
290 try:
288 rules = [(minifileset.compile(pattern), minifileset.compile(rule))
291 rules = [(minifileset.compile(pattern), minifileset.compile(rule))
289 for pattern, rule in cfg.items('track')]
292 for pattern, rule in cfg.items('track')]
290 except error.ParseError as e:
293 except error.ParseError as e:
291 # The original exception gives no indicator that the error is in the
294 # The original exception gives no indicator that the error is in the
292 # .hglfs file, so add that.
295 # .hglfs file, so add that.
293
296
294 # TODO: See if the line number of the file can be made available.
297 # TODO: See if the line number of the file can be made available.
295 raise error.Abort(_('parse error in .hglfs: %s') % e)
298 raise error.Abort(_('parse error in .hglfs: %s') % e)
296
299
297 def _match(path, size):
300 def _match(path, size):
298 for pat, rule in rules:
301 for pat, rule in rules:
299 if pat(path, size):
302 if pat(path, size):
300 return rule(path, size)
303 return rule(path, size)
301
304
302 return False
305 return False
303
306
304 return _match
307 return _match
305
308
306 # Called by remotefilelog
309 # Called by remotefilelog
307 def wrapfilelog(filelog):
310 def wrapfilelog(filelog):
308 wrapfunction = extensions.wrapfunction
311 wrapfunction = extensions.wrapfunction
309
312
310 wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision)
313 wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision)
311 wrapfunction(filelog, 'renamed', wrapper.filelogrenamed)
314 wrapfunction(filelog, 'renamed', wrapper.filelogrenamed)
312 wrapfunction(filelog, 'size', wrapper.filelogsize)
315 wrapfunction(filelog, 'size', wrapper.filelogsize)
313
316
314 @eh.wrapfunction(localrepo, 'resolverevlogstorevfsoptions')
317 @eh.wrapfunction(localrepo, 'resolverevlogstorevfsoptions')
315 def _resolverevlogstorevfsoptions(orig, ui, requirements, features):
318 def _resolverevlogstorevfsoptions(orig, ui, requirements, features):
316 opts = orig(ui, requirements, features)
319 opts = orig(ui, requirements, features)
317 for name, module in extensions.extensions(ui):
320 for name, module in extensions.extensions(ui):
318 if module is sys.modules[__name__]:
321 if module is sys.modules[__name__]:
319 if revlog.REVIDX_EXTSTORED in opts[b'flagprocessors']:
322 if revlog.REVIDX_EXTSTORED in opts[b'flagprocessors']:
320 msg = (_(b"cannot register multiple processors on flag '%#x'.")
323 msg = (_(b"cannot register multiple processors on flag '%#x'.")
321 % revlog.REVIDX_EXTSTORED)
324 % revlog.REVIDX_EXTSTORED)
322 raise error.Abort(msg)
325 raise error.Abort(msg)
323
326
324 opts[b'flagprocessors'][revlog.REVIDX_EXTSTORED] = lfsprocessor
327 opts[b'flagprocessors'][revlog.REVIDX_EXTSTORED] = lfsprocessor
325 break
328 break
326
329
327 return opts
330 return opts
328
331
329 @eh.extsetup
332 @eh.extsetup
330 def _extsetup(ui):
333 def _extsetup(ui):
331 wrapfilelog(filelog.filelog)
334 wrapfilelog(filelog.filelog)
332
335
333 context.basefilectx.islfs = wrapper.filectxislfs
336 context.basefilectx.islfs = wrapper.filectxislfs
334
337
335 scmutil.fileprefetchhooks.add('lfs', wrapper._prefetchfiles)
338 scmutil.fileprefetchhooks.add('lfs', wrapper._prefetchfiles)
336
339
337 # Make bundle choose changegroup3 instead of changegroup2. This affects
340 # Make bundle choose changegroup3 instead of changegroup2. This affects
338 # "hg bundle" command. Note: it does not cover all bundle formats like
341 # "hg bundle" command. Note: it does not cover all bundle formats like
339 # "packed1". Using "packed1" with lfs will likely cause trouble.
342 # "packed1". Using "packed1" with lfs will likely cause trouble.
340 exchange._bundlespeccontentopts["v2"]["cg.version"] = "03"
343 exchange._bundlespeccontentopts["v2"]["cg.version"] = "03"
341
344
342 @eh.filesetpredicate('lfs()')
345 @eh.filesetpredicate('lfs()')
343 def lfsfileset(mctx, x):
346 def lfsfileset(mctx, x):
344 """File that uses LFS storage."""
347 """File that uses LFS storage."""
345 # i18n: "lfs" is a keyword
348 # i18n: "lfs" is a keyword
346 filesetlang.getargs(x, 0, 0, _("lfs takes no arguments"))
349 filesetlang.getargs(x, 0, 0, _("lfs takes no arguments"))
347 ctx = mctx.ctx
350 ctx = mctx.ctx
348 def lfsfilep(f):
351 def lfsfilep(f):
349 return wrapper.pointerfromctx(ctx, f, removed=True) is not None
352 return wrapper.pointerfromctx(ctx, f, removed=True) is not None
350 return mctx.predicate(lfsfilep, predrepr='<lfs>')
353 return mctx.predicate(lfsfilep, predrepr='<lfs>')
351
354
352 @eh.templatekeyword('lfs_files', requires={'ctx'})
355 @eh.templatekeyword('lfs_files', requires={'ctx'})
353 def lfsfiles(context, mapping):
356 def lfsfiles(context, mapping):
354 """List of strings. All files modified, added, or removed by this
357 """List of strings. All files modified, added, or removed by this
355 changeset."""
358 changeset."""
356 ctx = context.resource(mapping, 'ctx')
359 ctx = context.resource(mapping, 'ctx')
357
360
358 pointers = wrapper.pointersfromctx(ctx, removed=True) # {path: pointer}
361 pointers = wrapper.pointersfromctx(ctx, removed=True) # {path: pointer}
359 files = sorted(pointers.keys())
362 files = sorted(pointers.keys())
360
363
361 def pointer(v):
364 def pointer(v):
362 # In the file spec, version is first and the other keys are sorted.
365 # In the file spec, version is first and the other keys are sorted.
363 sortkeyfunc = lambda x: (x[0] != 'version', x)
366 sortkeyfunc = lambda x: (x[0] != 'version', x)
364 items = sorted(pointers[v].iteritems(), key=sortkeyfunc)
367 items = sorted(pointers[v].iteritems(), key=sortkeyfunc)
365 return util.sortdict(items)
368 return util.sortdict(items)
366
369
367 makemap = lambda v: {
370 makemap = lambda v: {
368 'file': v,
371 'file': v,
369 'lfsoid': pointers[v].oid() if pointers[v] else None,
372 'lfsoid': pointers[v].oid() if pointers[v] else None,
370 'lfspointer': templateutil.hybriddict(pointer(v)),
373 'lfspointer': templateutil.hybriddict(pointer(v)),
371 }
374 }
372
375
373 # TODO: make the separator ', '?
376 # TODO: make the separator ', '?
374 f = templateutil._showcompatlist(context, mapping, 'lfs_file', files)
377 f = templateutil._showcompatlist(context, mapping, 'lfs_file', files)
375 return templateutil.hybrid(f, files, makemap, pycompat.identity)
378 return templateutil.hybrid(f, files, makemap, pycompat.identity)
376
379
377 @eh.command('debuglfsupload',
380 @eh.command('debuglfsupload',
378 [('r', 'rev', [], _('upload large files introduced by REV'))])
381 [('r', 'rev', [], _('upload large files introduced by REV'))])
379 def debuglfsupload(ui, repo, **opts):
382 def debuglfsupload(ui, repo, **opts):
380 """upload lfs blobs added by the working copy parent or given revisions"""
383 """upload lfs blobs added by the working copy parent or given revisions"""
381 revs = opts.get(r'rev', [])
384 revs = opts.get(r'rev', [])
382 pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs))
385 pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs))
383 wrapper.uploadblobs(repo, pointers)
386 wrapper.uploadblobs(repo, pointers)
@@ -1,446 +1,449
1 # wrapper.py - methods wrapping core mercurial logic
1 # wrapper.py - methods wrapping core mercurial logic
2 #
2 #
3 # Copyright 2017 Facebook, Inc.
3 # Copyright 2017 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import hashlib
10 import hashlib
11
11
12 from mercurial.i18n import _
12 from mercurial.i18n import _
13 from mercurial.node import bin, hex, nullid, short
13 from mercurial.node import bin, hex, nullid, short
14
14
15 from mercurial import (
15 from mercurial import (
16 bundle2,
16 bundle2,
17 changegroup,
17 changegroup,
18 cmdutil,
18 cmdutil,
19 context,
19 context,
20 error,
20 error,
21 exchange,
21 exchange,
22 exthelper,
22 exthelper,
23 localrepo,
23 localrepo,
24 repository,
25 revlog,
24 revlog,
26 scmutil,
25 scmutil,
27 upgrade,
26 upgrade,
28 util,
27 util,
29 vfs as vfsmod,
28 vfs as vfsmod,
30 wireprotov1server,
29 wireprotov1server,
31 )
30 )
32
31
32 from mercurial.interfaces import (
33 repository,
34 )
35
33 from mercurial.utils import (
36 from mercurial.utils import (
34 storageutil,
37 storageutil,
35 stringutil,
38 stringutil,
36 )
39 )
37
40
38 from ..largefiles import lfutil
41 from ..largefiles import lfutil
39
42
40 from . import (
43 from . import (
41 blobstore,
44 blobstore,
42 pointer,
45 pointer,
43 )
46 )
44
47
45 eh = exthelper.exthelper()
48 eh = exthelper.exthelper()
46
49
47 @eh.wrapfunction(localrepo, 'makefilestorage')
50 @eh.wrapfunction(localrepo, 'makefilestorage')
48 def localrepomakefilestorage(orig, requirements, features, **kwargs):
51 def localrepomakefilestorage(orig, requirements, features, **kwargs):
49 if b'lfs' in requirements:
52 if b'lfs' in requirements:
50 features.add(repository.REPO_FEATURE_LFS)
53 features.add(repository.REPO_FEATURE_LFS)
51
54
52 return orig(requirements=requirements, features=features, **kwargs)
55 return orig(requirements=requirements, features=features, **kwargs)
53
56
54 @eh.wrapfunction(changegroup, 'allsupportedversions')
57 @eh.wrapfunction(changegroup, 'allsupportedversions')
55 def allsupportedversions(orig, ui):
58 def allsupportedversions(orig, ui):
56 versions = orig(ui)
59 versions = orig(ui)
57 versions.add('03')
60 versions.add('03')
58 return versions
61 return versions
59
62
60 @eh.wrapfunction(wireprotov1server, '_capabilities')
63 @eh.wrapfunction(wireprotov1server, '_capabilities')
61 def _capabilities(orig, repo, proto):
64 def _capabilities(orig, repo, proto):
62 '''Wrap server command to announce lfs server capability'''
65 '''Wrap server command to announce lfs server capability'''
63 caps = orig(repo, proto)
66 caps = orig(repo, proto)
64 if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
67 if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
65 # Advertise a slightly different capability when lfs is *required*, so
68 # Advertise a slightly different capability when lfs is *required*, so
66 # that the client knows it MUST load the extension. If lfs is not
69 # that the client knows it MUST load the extension. If lfs is not
67 # required on the server, there's no reason to autoload the extension
70 # required on the server, there's no reason to autoload the extension
68 # on the client.
71 # on the client.
69 if b'lfs' in repo.requirements:
72 if b'lfs' in repo.requirements:
70 caps.append('lfs-serve')
73 caps.append('lfs-serve')
71
74
72 caps.append('lfs')
75 caps.append('lfs')
73 return caps
76 return caps
74
77
75 def bypasscheckhash(self, text):
78 def bypasscheckhash(self, text):
76 return False
79 return False
77
80
78 def readfromstore(self, text):
81 def readfromstore(self, text):
79 """Read filelog content from local blobstore transform for flagprocessor.
82 """Read filelog content from local blobstore transform for flagprocessor.
80
83
81 Default tranform for flagprocessor, returning contents from blobstore.
84 Default tranform for flagprocessor, returning contents from blobstore.
82 Returns a 2-typle (text, validatehash) where validatehash is True as the
85 Returns a 2-typle (text, validatehash) where validatehash is True as the
83 contents of the blobstore should be checked using checkhash.
86 contents of the blobstore should be checked using checkhash.
84 """
87 """
85 p = pointer.deserialize(text)
88 p = pointer.deserialize(text)
86 oid = p.oid()
89 oid = p.oid()
87 store = self.opener.lfslocalblobstore
90 store = self.opener.lfslocalblobstore
88 if not store.has(oid):
91 if not store.has(oid):
89 p.filename = self.filename
92 p.filename = self.filename
90 self.opener.lfsremoteblobstore.readbatch([p], store)
93 self.opener.lfsremoteblobstore.readbatch([p], store)
91
94
92 # The caller will validate the content
95 # The caller will validate the content
93 text = store.read(oid, verify=False)
96 text = store.read(oid, verify=False)
94
97
95 # pack hg filelog metadata
98 # pack hg filelog metadata
96 hgmeta = {}
99 hgmeta = {}
97 for k in p.keys():
100 for k in p.keys():
98 if k.startswith('x-hg-'):
101 if k.startswith('x-hg-'):
99 name = k[len('x-hg-'):]
102 name = k[len('x-hg-'):]
100 hgmeta[name] = p[k]
103 hgmeta[name] = p[k]
101 if hgmeta or text.startswith('\1\n'):
104 if hgmeta or text.startswith('\1\n'):
102 text = storageutil.packmeta(hgmeta, text)
105 text = storageutil.packmeta(hgmeta, text)
103
106
104 return (text, True)
107 return (text, True)
105
108
106 def writetostore(self, text):
109 def writetostore(self, text):
107 # hg filelog metadata (includes rename, etc)
110 # hg filelog metadata (includes rename, etc)
108 hgmeta, offset = storageutil.parsemeta(text)
111 hgmeta, offset = storageutil.parsemeta(text)
109 if offset and offset > 0:
112 if offset and offset > 0:
110 # lfs blob does not contain hg filelog metadata
113 # lfs blob does not contain hg filelog metadata
111 text = text[offset:]
114 text = text[offset:]
112
115
113 # git-lfs only supports sha256
116 # git-lfs only supports sha256
114 oid = hex(hashlib.sha256(text).digest())
117 oid = hex(hashlib.sha256(text).digest())
115 self.opener.lfslocalblobstore.write(oid, text)
118 self.opener.lfslocalblobstore.write(oid, text)
116
119
117 # replace contents with metadata
120 # replace contents with metadata
118 longoid = 'sha256:%s' % oid
121 longoid = 'sha256:%s' % oid
119 metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
122 metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
120
123
121 # by default, we expect the content to be binary. however, LFS could also
124 # by default, we expect the content to be binary. however, LFS could also
122 # be used for non-binary content. add a special entry for non-binary data.
125 # be used for non-binary content. add a special entry for non-binary data.
123 # this will be used by filectx.isbinary().
126 # this will be used by filectx.isbinary().
124 if not stringutil.binary(text):
127 if not stringutil.binary(text):
125 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
128 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
126 metadata['x-is-binary'] = '0'
129 metadata['x-is-binary'] = '0'
127
130
128 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
131 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
129 if hgmeta is not None:
132 if hgmeta is not None:
130 for k, v in hgmeta.iteritems():
133 for k, v in hgmeta.iteritems():
131 metadata['x-hg-%s' % k] = v
134 metadata['x-hg-%s' % k] = v
132
135
133 rawtext = metadata.serialize()
136 rawtext = metadata.serialize()
134 return (rawtext, False)
137 return (rawtext, False)
135
138
136 def _islfs(rlog, node=None, rev=None):
139 def _islfs(rlog, node=None, rev=None):
137 if rev is None:
140 if rev is None:
138 if node is None:
141 if node is None:
139 # both None - likely working copy content where node is not ready
142 # both None - likely working copy content where node is not ready
140 return False
143 return False
141 rev = rlog._revlog.rev(node)
144 rev = rlog._revlog.rev(node)
142 else:
145 else:
143 node = rlog._revlog.node(rev)
146 node = rlog._revlog.node(rev)
144 if node == nullid:
147 if node == nullid:
145 return False
148 return False
146 flags = rlog._revlog.flags(rev)
149 flags = rlog._revlog.flags(rev)
147 return bool(flags & revlog.REVIDX_EXTSTORED)
150 return bool(flags & revlog.REVIDX_EXTSTORED)
148
151
149 # Wrapping may also be applied by remotefilelog
152 # Wrapping may also be applied by remotefilelog
150 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
153 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
151 cachedelta=None, node=None,
154 cachedelta=None, node=None,
152 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
155 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
153 # The matcher isn't available if reposetup() wasn't called.
156 # The matcher isn't available if reposetup() wasn't called.
154 lfstrack = self._revlog.opener.options.get('lfstrack')
157 lfstrack = self._revlog.opener.options.get('lfstrack')
155
158
156 if lfstrack:
159 if lfstrack:
157 textlen = len(text)
160 textlen = len(text)
158 # exclude hg rename meta from file size
161 # exclude hg rename meta from file size
159 meta, offset = storageutil.parsemeta(text)
162 meta, offset = storageutil.parsemeta(text)
160 if offset:
163 if offset:
161 textlen -= offset
164 textlen -= offset
162
165
163 if lfstrack(self._revlog.filename, textlen):
166 if lfstrack(self._revlog.filename, textlen):
164 flags |= revlog.REVIDX_EXTSTORED
167 flags |= revlog.REVIDX_EXTSTORED
165
168
166 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
169 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
167 node=node, flags=flags, **kwds)
170 node=node, flags=flags, **kwds)
168
171
169 # Wrapping may also be applied by remotefilelog
172 # Wrapping may also be applied by remotefilelog
170 def filelogrenamed(orig, self, node):
173 def filelogrenamed(orig, self, node):
171 if _islfs(self, node):
174 if _islfs(self, node):
172 rawtext = self._revlog.rawdata(node)
175 rawtext = self._revlog.rawdata(node)
173 if not rawtext:
176 if not rawtext:
174 return False
177 return False
175 metadata = pointer.deserialize(rawtext)
178 metadata = pointer.deserialize(rawtext)
176 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
179 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
177 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
180 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
178 else:
181 else:
179 return False
182 return False
180 return orig(self, node)
183 return orig(self, node)
181
184
182 # Wrapping may also be applied by remotefilelog
185 # Wrapping may also be applied by remotefilelog
183 def filelogsize(orig, self, rev):
186 def filelogsize(orig, self, rev):
184 if _islfs(self, rev=rev):
187 if _islfs(self, rev=rev):
185 # fast path: use lfs metadata to answer size
188 # fast path: use lfs metadata to answer size
186 rawtext = self._revlog.rawdata(rev)
189 rawtext = self._revlog.rawdata(rev)
187 metadata = pointer.deserialize(rawtext)
190 metadata = pointer.deserialize(rawtext)
188 return int(metadata['size'])
191 return int(metadata['size'])
189 return orig(self, rev)
192 return orig(self, rev)
190
193
191 @eh.wrapfunction(context.basefilectx, 'cmp')
194 @eh.wrapfunction(context.basefilectx, 'cmp')
192 def filectxcmp(orig, self, fctx):
195 def filectxcmp(orig, self, fctx):
193 """returns True if text is different than fctx"""
196 """returns True if text is different than fctx"""
194 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
197 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
195 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
198 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
196 # fast path: check LFS oid
199 # fast path: check LFS oid
197 p1 = pointer.deserialize(self.rawdata())
200 p1 = pointer.deserialize(self.rawdata())
198 p2 = pointer.deserialize(fctx.rawdata())
201 p2 = pointer.deserialize(fctx.rawdata())
199 return p1.oid() != p2.oid()
202 return p1.oid() != p2.oid()
200 return orig(self, fctx)
203 return orig(self, fctx)
201
204
202 @eh.wrapfunction(context.basefilectx, 'isbinary')
205 @eh.wrapfunction(context.basefilectx, 'isbinary')
203 def filectxisbinary(orig, self):
206 def filectxisbinary(orig, self):
204 if self.islfs():
207 if self.islfs():
205 # fast path: use lfs metadata to answer isbinary
208 # fast path: use lfs metadata to answer isbinary
206 metadata = pointer.deserialize(self.rawdata())
209 metadata = pointer.deserialize(self.rawdata())
207 # if lfs metadata says nothing, assume it's binary by default
210 # if lfs metadata says nothing, assume it's binary by default
208 return bool(int(metadata.get('x-is-binary', 1)))
211 return bool(int(metadata.get('x-is-binary', 1)))
209 return orig(self)
212 return orig(self)
210
213
211 def filectxislfs(self):
214 def filectxislfs(self):
212 return _islfs(self.filelog(), self.filenode())
215 return _islfs(self.filelog(), self.filenode())
213
216
214 @eh.wrapfunction(cmdutil, '_updatecatformatter')
217 @eh.wrapfunction(cmdutil, '_updatecatformatter')
215 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
218 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
216 orig(fm, ctx, matcher, path, decode)
219 orig(fm, ctx, matcher, path, decode)
217 fm.data(rawdata=ctx[path].rawdata())
220 fm.data(rawdata=ctx[path].rawdata())
218
221
219 @eh.wrapfunction(scmutil, 'wrapconvertsink')
222 @eh.wrapfunction(scmutil, 'wrapconvertsink')
220 def convertsink(orig, sink):
223 def convertsink(orig, sink):
221 sink = orig(sink)
224 sink = orig(sink)
222 if sink.repotype == 'hg':
225 if sink.repotype == 'hg':
223 class lfssink(sink.__class__):
226 class lfssink(sink.__class__):
224 def putcommit(self, files, copies, parents, commit, source, revmap,
227 def putcommit(self, files, copies, parents, commit, source, revmap,
225 full, cleanp2):
228 full, cleanp2):
226 pc = super(lfssink, self).putcommit
229 pc = super(lfssink, self).putcommit
227 node = pc(files, copies, parents, commit, source, revmap, full,
230 node = pc(files, copies, parents, commit, source, revmap, full,
228 cleanp2)
231 cleanp2)
229
232
230 if 'lfs' not in self.repo.requirements:
233 if 'lfs' not in self.repo.requirements:
231 ctx = self.repo[node]
234 ctx = self.repo[node]
232
235
233 # The file list may contain removed files, so check for
236 # The file list may contain removed files, so check for
234 # membership before assuming it is in the context.
237 # membership before assuming it is in the context.
235 if any(f in ctx and ctx[f].islfs() for f, n in files):
238 if any(f in ctx and ctx[f].islfs() for f, n in files):
236 self.repo.requirements.add('lfs')
239 self.repo.requirements.add('lfs')
237 self.repo._writerequirements()
240 self.repo._writerequirements()
238
241
239 return node
242 return node
240
243
241 sink.__class__ = lfssink
244 sink.__class__ = lfssink
242
245
243 return sink
246 return sink
244
247
245 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
248 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
246 # options and blob stores are passed from othervfs to the new readonlyvfs.
249 # options and blob stores are passed from othervfs to the new readonlyvfs.
247 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
250 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
248 def vfsinit(orig, self, othervfs):
251 def vfsinit(orig, self, othervfs):
249 orig(self, othervfs)
252 orig(self, othervfs)
250 # copy lfs related options
253 # copy lfs related options
251 for k, v in othervfs.options.items():
254 for k, v in othervfs.options.items():
252 if k.startswith('lfs'):
255 if k.startswith('lfs'):
253 self.options[k] = v
256 self.options[k] = v
254 # also copy lfs blobstores. note: this can run before reposetup, so lfs
257 # also copy lfs blobstores. note: this can run before reposetup, so lfs
255 # blobstore attributes are not always ready at this time.
258 # blobstore attributes are not always ready at this time.
256 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
259 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
257 if util.safehasattr(othervfs, name):
260 if util.safehasattr(othervfs, name):
258 setattr(self, name, getattr(othervfs, name))
261 setattr(self, name, getattr(othervfs, name))
259
262
260 def _prefetchfiles(repo, revs, match):
263 def _prefetchfiles(repo, revs, match):
261 """Ensure that required LFS blobs are present, fetching them as a group if
264 """Ensure that required LFS blobs are present, fetching them as a group if
262 needed."""
265 needed."""
263 if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
266 if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
264 return
267 return
265
268
266 pointers = []
269 pointers = []
267 oids = set()
270 oids = set()
268 localstore = repo.svfs.lfslocalblobstore
271 localstore = repo.svfs.lfslocalblobstore
269
272
270 for rev in revs:
273 for rev in revs:
271 ctx = repo[rev]
274 ctx = repo[rev]
272 for f in ctx.walk(match):
275 for f in ctx.walk(match):
273 p = pointerfromctx(ctx, f)
276 p = pointerfromctx(ctx, f)
274 if p and p.oid() not in oids and not localstore.has(p.oid()):
277 if p and p.oid() not in oids and not localstore.has(p.oid()):
275 p.filename = f
278 p.filename = f
276 pointers.append(p)
279 pointers.append(p)
277 oids.add(p.oid())
280 oids.add(p.oid())
278
281
279 if pointers:
282 if pointers:
280 # Recalculating the repo store here allows 'paths.default' that is set
283 # Recalculating the repo store here allows 'paths.default' that is set
281 # on the repo by a clone command to be used for the update.
284 # on the repo by a clone command to be used for the update.
282 blobstore.remote(repo).readbatch(pointers, localstore)
285 blobstore.remote(repo).readbatch(pointers, localstore)
283
286
284 def _canskipupload(repo):
287 def _canskipupload(repo):
285 # Skip if this hasn't been passed to reposetup()
288 # Skip if this hasn't been passed to reposetup()
286 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
289 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
287 return True
290 return True
288
291
289 # if remotestore is a null store, upload is a no-op and can be skipped
292 # if remotestore is a null store, upload is a no-op and can be skipped
290 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
293 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
291
294
292 def candownload(repo):
295 def candownload(repo):
293 # Skip if this hasn't been passed to reposetup()
296 # Skip if this hasn't been passed to reposetup()
294 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
297 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
295 return False
298 return False
296
299
297 # if remotestore is a null store, downloads will lead to nothing
300 # if remotestore is a null store, downloads will lead to nothing
298 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
301 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
299
302
300 def uploadblobsfromrevs(repo, revs):
303 def uploadblobsfromrevs(repo, revs):
301 '''upload lfs blobs introduced by revs
304 '''upload lfs blobs introduced by revs
302
305
303 Note: also used by other extensions e. g. infinitepush. avoid renaming.
306 Note: also used by other extensions e. g. infinitepush. avoid renaming.
304 '''
307 '''
305 if _canskipupload(repo):
308 if _canskipupload(repo):
306 return
309 return
307 pointers = extractpointers(repo, revs)
310 pointers = extractpointers(repo, revs)
308 uploadblobs(repo, pointers)
311 uploadblobs(repo, pointers)
309
312
310 def prepush(pushop):
313 def prepush(pushop):
311 """Prepush hook.
314 """Prepush hook.
312
315
313 Read through the revisions to push, looking for filelog entries that can be
316 Read through the revisions to push, looking for filelog entries that can be
314 deserialized into metadata so that we can block the push on their upload to
317 deserialized into metadata so that we can block the push on their upload to
315 the remote blobstore.
318 the remote blobstore.
316 """
319 """
317 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
320 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
318
321
319 @eh.wrapfunction(exchange, 'push')
322 @eh.wrapfunction(exchange, 'push')
320 def push(orig, repo, remote, *args, **kwargs):
323 def push(orig, repo, remote, *args, **kwargs):
321 """bail on push if the extension isn't enabled on remote when needed, and
324 """bail on push if the extension isn't enabled on remote when needed, and
322 update the remote store based on the destination path."""
325 update the remote store based on the destination path."""
323 if 'lfs' in repo.requirements:
326 if 'lfs' in repo.requirements:
324 # If the remote peer is for a local repo, the requirement tests in the
327 # If the remote peer is for a local repo, the requirement tests in the
325 # base class method enforce lfs support. Otherwise, some revisions in
328 # base class method enforce lfs support. Otherwise, some revisions in
326 # this repo use lfs, and the remote repo needs the extension loaded.
329 # this repo use lfs, and the remote repo needs the extension loaded.
327 if not remote.local() and not remote.capable('lfs'):
330 if not remote.local() and not remote.capable('lfs'):
328 # This is a copy of the message in exchange.push() when requirements
331 # This is a copy of the message in exchange.push() when requirements
329 # are missing between local repos.
332 # are missing between local repos.
330 m = _("required features are not supported in the destination: %s")
333 m = _("required features are not supported in the destination: %s")
331 raise error.Abort(m % 'lfs',
334 raise error.Abort(m % 'lfs',
332 hint=_('enable the lfs extension on the server'))
335 hint=_('enable the lfs extension on the server'))
333
336
334 # Repositories where this extension is disabled won't have the field.
337 # Repositories where this extension is disabled won't have the field.
335 # But if there's a requirement, then the extension must be loaded AND
338 # But if there's a requirement, then the extension must be loaded AND
336 # there may be blobs to push.
339 # there may be blobs to push.
337 remotestore = repo.svfs.lfsremoteblobstore
340 remotestore = repo.svfs.lfsremoteblobstore
338 try:
341 try:
339 repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
342 repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
340 return orig(repo, remote, *args, **kwargs)
343 return orig(repo, remote, *args, **kwargs)
341 finally:
344 finally:
342 repo.svfs.lfsremoteblobstore = remotestore
345 repo.svfs.lfsremoteblobstore = remotestore
343 else:
346 else:
344 return orig(repo, remote, *args, **kwargs)
347 return orig(repo, remote, *args, **kwargs)
345
348
346 # when writing a bundle via "hg bundle" command, upload related LFS blobs
349 # when writing a bundle via "hg bundle" command, upload related LFS blobs
347 @eh.wrapfunction(bundle2, 'writenewbundle')
350 @eh.wrapfunction(bundle2, 'writenewbundle')
348 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
351 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
349 *args, **kwargs):
352 *args, **kwargs):
350 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
353 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
351 uploadblobsfromrevs(repo, outgoing.missing)
354 uploadblobsfromrevs(repo, outgoing.missing)
352 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
355 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
353 **kwargs)
356 **kwargs)
354
357
355 def extractpointers(repo, revs):
358 def extractpointers(repo, revs):
356 """return a list of lfs pointers added by given revs"""
359 """return a list of lfs pointers added by given revs"""
357 repo.ui.debug('lfs: computing set of blobs to upload\n')
360 repo.ui.debug('lfs: computing set of blobs to upload\n')
358 pointers = {}
361 pointers = {}
359
362
360 makeprogress = repo.ui.makeprogress
363 makeprogress = repo.ui.makeprogress
361 with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
364 with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
362 for r in revs:
365 for r in revs:
363 ctx = repo[r]
366 ctx = repo[r]
364 for p in pointersfromctx(ctx).values():
367 for p in pointersfromctx(ctx).values():
365 pointers[p.oid()] = p
368 pointers[p.oid()] = p
366 progress.increment()
369 progress.increment()
367 return sorted(pointers.values(), key=lambda p: p.oid())
370 return sorted(pointers.values(), key=lambda p: p.oid())
368
371
369 def pointerfromctx(ctx, f, removed=False):
372 def pointerfromctx(ctx, f, removed=False):
370 """return a pointer for the named file from the given changectx, or None if
373 """return a pointer for the named file from the given changectx, or None if
371 the file isn't LFS.
374 the file isn't LFS.
372
375
373 Optionally, the pointer for a file deleted from the context can be returned.
376 Optionally, the pointer for a file deleted from the context can be returned.
374 Since no such pointer is actually stored, and to distinguish from a non LFS
377 Since no such pointer is actually stored, and to distinguish from a non LFS
375 file, this pointer is represented by an empty dict.
378 file, this pointer is represented by an empty dict.
376 """
379 """
377 _ctx = ctx
380 _ctx = ctx
378 if f not in ctx:
381 if f not in ctx:
379 if not removed:
382 if not removed:
380 return None
383 return None
381 if f in ctx.p1():
384 if f in ctx.p1():
382 _ctx = ctx.p1()
385 _ctx = ctx.p1()
383 elif f in ctx.p2():
386 elif f in ctx.p2():
384 _ctx = ctx.p2()
387 _ctx = ctx.p2()
385 else:
388 else:
386 return None
389 return None
387 fctx = _ctx[f]
390 fctx = _ctx[f]
388 if not _islfs(fctx.filelog(), fctx.filenode()):
391 if not _islfs(fctx.filelog(), fctx.filenode()):
389 return None
392 return None
390 try:
393 try:
391 p = pointer.deserialize(fctx.rawdata())
394 p = pointer.deserialize(fctx.rawdata())
392 if ctx == _ctx:
395 if ctx == _ctx:
393 return p
396 return p
394 return {}
397 return {}
395 except pointer.InvalidPointer as ex:
398 except pointer.InvalidPointer as ex:
396 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
399 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
397 % (f, short(_ctx.node()), ex))
400 % (f, short(_ctx.node()), ex))
398
401
399 def pointersfromctx(ctx, removed=False):
402 def pointersfromctx(ctx, removed=False):
400 """return a dict {path: pointer} for given single changectx.
403 """return a dict {path: pointer} for given single changectx.
401
404
402 If ``removed`` == True and the LFS file was removed from ``ctx``, the value
405 If ``removed`` == True and the LFS file was removed from ``ctx``, the value
403 stored for the path is an empty dict.
406 stored for the path is an empty dict.
404 """
407 """
405 result = {}
408 result = {}
406 m = ctx.repo().narrowmatch()
409 m = ctx.repo().narrowmatch()
407
410
408 # TODO: consider manifest.fastread() instead
411 # TODO: consider manifest.fastread() instead
409 for f in ctx.files():
412 for f in ctx.files():
410 if not m(f):
413 if not m(f):
411 continue
414 continue
412 p = pointerfromctx(ctx, f, removed=removed)
415 p = pointerfromctx(ctx, f, removed=removed)
413 if p is not None:
416 if p is not None:
414 result[f] = p
417 result[f] = p
415 return result
418 return result
416
419
417 def uploadblobs(repo, pointers):
420 def uploadblobs(repo, pointers):
418 """upload given pointers from local blobstore"""
421 """upload given pointers from local blobstore"""
419 if not pointers:
422 if not pointers:
420 return
423 return
421
424
422 remoteblob = repo.svfs.lfsremoteblobstore
425 remoteblob = repo.svfs.lfsremoteblobstore
423 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
426 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
424
427
425 @eh.wrapfunction(upgrade, '_finishdatamigration')
428 @eh.wrapfunction(upgrade, '_finishdatamigration')
426 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
429 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
427 orig(ui, srcrepo, dstrepo, requirements)
430 orig(ui, srcrepo, dstrepo, requirements)
428
431
429 # Skip if this hasn't been passed to reposetup()
432 # Skip if this hasn't been passed to reposetup()
430 if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
433 if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
431 util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
434 util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
432 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
435 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
433 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
436 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
434
437
435 for dirpath, dirs, files in srclfsvfs.walk():
438 for dirpath, dirs, files in srclfsvfs.walk():
436 for oid in files:
439 for oid in files:
437 ui.write(_('copying lfs blob %s\n') % oid)
440 ui.write(_('copying lfs blob %s\n') % oid)
438 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
441 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
439
442
440 @eh.wrapfunction(upgrade, 'preservedrequirements')
443 @eh.wrapfunction(upgrade, 'preservedrequirements')
441 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
444 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
442 def upgraderequirements(orig, repo):
445 def upgraderequirements(orig, repo):
443 reqs = orig(repo)
446 reqs = orig(repo)
444 if 'lfs' in repo.requirements:
447 if 'lfs' in repo.requirements:
445 reqs.add('lfs')
448 reqs.add('lfs')
446 return reqs
449 return reqs
@@ -1,71 +1,74
1 # __init__.py - narrowhg extension
1 # __init__.py - narrowhg extension
2 #
2 #
3 # Copyright 2017 Google, Inc.
3 # Copyright 2017 Google, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7 '''create clones which fetch history data for subset of files (EXPERIMENTAL)'''
7 '''create clones which fetch history data for subset of files (EXPERIMENTAL)'''
8
8
9 from __future__ import absolute_import
9 from __future__ import absolute_import
10
10
11 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
11 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
12 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
12 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
13 # be specifying the version(s) of Mercurial they are tested with, or
13 # be specifying the version(s) of Mercurial they are tested with, or
14 # leave the attribute unspecified.
14 # leave the attribute unspecified.
15 testedwith = 'ships-with-hg-core'
15 testedwith = 'ships-with-hg-core'
16
16
17 from mercurial import (
17 from mercurial import (
18 localrepo,
18 localrepo,
19 registrar,
19 registrar,
20 )
21
22 from mercurial.interfaces import (
20 repository,
23 repository,
21 )
24 )
22
25
23 from . import (
26 from . import (
24 narrowbundle2,
27 narrowbundle2,
25 narrowcommands,
28 narrowcommands,
26 narrowrepo,
29 narrowrepo,
27 narrowtemplates,
30 narrowtemplates,
28 narrowwirepeer,
31 narrowwirepeer,
29 )
32 )
30
33
31 configtable = {}
34 configtable = {}
32 configitem = registrar.configitem(configtable)
35 configitem = registrar.configitem(configtable)
33 # Narrowhg *has* support for serving ellipsis nodes (which are used at
36 # Narrowhg *has* support for serving ellipsis nodes (which are used at
34 # least by Google's internal server), but that support is pretty
37 # least by Google's internal server), but that support is pretty
35 # fragile and has a lot of problems on real-world repositories that
38 # fragile and has a lot of problems on real-world repositories that
36 # have complex graph topologies. This could probably be corrected, but
39 # have complex graph topologies. This could probably be corrected, but
37 # absent someone needing the full support for ellipsis nodes in
40 # absent someone needing the full support for ellipsis nodes in
38 # repositories with merges, it's unlikely this work will get done. As
41 # repositories with merges, it's unlikely this work will get done. As
39 # of this writining in late 2017, all repositories large enough for
42 # of this writining in late 2017, all repositories large enough for
40 # ellipsis nodes to be a hard requirement also enforce strictly linear
43 # ellipsis nodes to be a hard requirement also enforce strictly linear
41 # history for other scaling reasons.
44 # history for other scaling reasons.
42 configitem('experimental', 'narrowservebrokenellipses',
45 configitem('experimental', 'narrowservebrokenellipses',
43 default=False,
46 default=False,
44 alias=[('narrow', 'serveellipses')],
47 alias=[('narrow', 'serveellipses')],
45 )
48 )
46
49
47 # Export the commands table for Mercurial to see.
50 # Export the commands table for Mercurial to see.
48 cmdtable = narrowcommands.table
51 cmdtable = narrowcommands.table
49
52
50 def featuresetup(ui, features):
53 def featuresetup(ui, features):
51 features.add(repository.NARROW_REQUIREMENT)
54 features.add(repository.NARROW_REQUIREMENT)
52
55
53 def uisetup(ui):
56 def uisetup(ui):
54 """Wraps user-facing mercurial commands with narrow-aware versions."""
57 """Wraps user-facing mercurial commands with narrow-aware versions."""
55 localrepo.featuresetupfuncs.add(featuresetup)
58 localrepo.featuresetupfuncs.add(featuresetup)
56 narrowbundle2.setup()
59 narrowbundle2.setup()
57 narrowcommands.setup()
60 narrowcommands.setup()
58 narrowwirepeer.uisetup()
61 narrowwirepeer.uisetup()
59
62
60 def reposetup(ui, repo):
63 def reposetup(ui, repo):
61 """Wraps local repositories with narrow repo support."""
64 """Wraps local repositories with narrow repo support."""
62 if not repo.local():
65 if not repo.local():
63 return
66 return
64
67
65 repo.ui.setconfig('experimental', 'narrow', True, 'narrow-ext')
68 repo.ui.setconfig('experimental', 'narrow', True, 'narrow-ext')
66 if repository.NARROW_REQUIREMENT in repo.requirements:
69 if repository.NARROW_REQUIREMENT in repo.requirements:
67 narrowrepo.wraprepo(repo)
70 narrowrepo.wraprepo(repo)
68 narrowwirepeer.reposetup(repo)
71 narrowwirepeer.reposetup(repo)
69
72
70 templatekeyword = narrowtemplates.templatekeyword
73 templatekeyword = narrowtemplates.templatekeyword
71 revsetpredicate = narrowtemplates.revsetpredicate
74 revsetpredicate = narrowtemplates.revsetpredicate
@@ -1,301 +1,303
1 # narrowbundle2.py - bundle2 extensions for narrow repository support
1 # narrowbundle2.py - bundle2 extensions for narrow repository support
2 #
2 #
3 # Copyright 2017 Google, Inc.
3 # Copyright 2017 Google, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import errno
10 import errno
11 import struct
11 import struct
12
12
13 from mercurial.i18n import _
13 from mercurial.i18n import _
14 from mercurial.node import (
14 from mercurial.node import (
15 bin,
15 bin,
16 nullid,
16 nullid,
17 )
17 )
18 from mercurial import (
18 from mercurial import (
19 bundle2,
19 bundle2,
20 changegroup,
20 changegroup,
21 error,
21 error,
22 exchange,
22 exchange,
23 localrepo,
23 localrepo,
24 narrowspec,
24 narrowspec,
25 repair,
25 repair,
26 repository,
27 util,
26 util,
28 wireprototypes,
27 wireprototypes,
29 )
28 )
29 from mercurial.interfaces import (
30 repository,
31 )
30 from mercurial.utils import (
32 from mercurial.utils import (
31 stringutil,
33 stringutil,
32 )
34 )
33
35
34 _NARROWACL_SECTION = 'narrowacl'
36 _NARROWACL_SECTION = 'narrowacl'
35 _CHANGESPECPART = 'narrow:changespec'
37 _CHANGESPECPART = 'narrow:changespec'
36 _RESSPECS = 'narrow:responsespec'
38 _RESSPECS = 'narrow:responsespec'
37 _SPECPART = 'narrow:spec'
39 _SPECPART = 'narrow:spec'
38 _SPECPART_INCLUDE = 'include'
40 _SPECPART_INCLUDE = 'include'
39 _SPECPART_EXCLUDE = 'exclude'
41 _SPECPART_EXCLUDE = 'exclude'
40 _KILLNODESIGNAL = 'KILL'
42 _KILLNODESIGNAL = 'KILL'
41 _DONESIGNAL = 'DONE'
43 _DONESIGNAL = 'DONE'
42 _ELIDEDCSHEADER = '>20s20s20sl' # cset id, p1, p2, len(text)
44 _ELIDEDCSHEADER = '>20s20s20sl' # cset id, p1, p2, len(text)
43 _ELIDEDMFHEADER = '>20s20s20s20sl' # manifest id, p1, p2, link id, len(text)
45 _ELIDEDMFHEADER = '>20s20s20s20sl' # manifest id, p1, p2, link id, len(text)
44 _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
46 _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
45 _MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
47 _MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
46
48
47 # Serve a changegroup for a client with a narrow clone.
49 # Serve a changegroup for a client with a narrow clone.
48 def getbundlechangegrouppart_narrow(bundler, repo, source,
50 def getbundlechangegrouppart_narrow(bundler, repo, source,
49 bundlecaps=None, b2caps=None, heads=None,
51 bundlecaps=None, b2caps=None, heads=None,
50 common=None, **kwargs):
52 common=None, **kwargs):
51 assert repo.ui.configbool('experimental', 'narrowservebrokenellipses')
53 assert repo.ui.configbool('experimental', 'narrowservebrokenellipses')
52
54
53 cgversions = b2caps.get('changegroup')
55 cgversions = b2caps.get('changegroup')
54 cgversions = [v for v in cgversions
56 cgversions = [v for v in cgversions
55 if v in changegroup.supportedoutgoingversions(repo)]
57 if v in changegroup.supportedoutgoingversions(repo)]
56 if not cgversions:
58 if not cgversions:
57 raise ValueError(_('no common changegroup version'))
59 raise ValueError(_('no common changegroup version'))
58 version = max(cgversions)
60 version = max(cgversions)
59
61
60 oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
62 oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
61 oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
63 oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
62 newinclude = sorted(filter(bool, kwargs.get(r'includepats', [])))
64 newinclude = sorted(filter(bool, kwargs.get(r'includepats', [])))
63 newexclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
65 newexclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
64 known = {bin(n) for n in kwargs.get(r'known', [])}
66 known = {bin(n) for n in kwargs.get(r'known', [])}
65 generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
67 generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
66 newexclude, version, common, heads, known,
68 newexclude, version, common, heads, known,
67 kwargs.get(r'depth', None))
69 kwargs.get(r'depth', None))
68
70
69 def generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
71 def generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
70 newexclude, version, common, heads, known, depth):
72 newexclude, version, common, heads, known, depth):
71 newmatch = narrowspec.match(repo.root, include=newinclude,
73 newmatch = narrowspec.match(repo.root, include=newinclude,
72 exclude=newexclude)
74 exclude=newexclude)
73 if depth is not None:
75 if depth is not None:
74 depth = int(depth)
76 depth = int(depth)
75 if depth < 1:
77 if depth < 1:
76 raise error.Abort(_('depth must be positive, got %d') % depth)
78 raise error.Abort(_('depth must be positive, got %d') % depth)
77
79
78 heads = set(heads or repo.heads())
80 heads = set(heads or repo.heads())
79 common = set(common or [nullid])
81 common = set(common or [nullid])
80 if known and (oldinclude != newinclude or oldexclude != newexclude):
82 if known and (oldinclude != newinclude or oldexclude != newexclude):
81 # Steps:
83 # Steps:
82 # 1. Send kill for "$known & ::common"
84 # 1. Send kill for "$known & ::common"
83 #
85 #
84 # 2. Send changegroup for ::common
86 # 2. Send changegroup for ::common
85 #
87 #
86 # 3. Proceed.
88 # 3. Proceed.
87 #
89 #
88 # In the future, we can send kills for only the specific
90 # In the future, we can send kills for only the specific
89 # nodes we know should go away or change shape, and then
91 # nodes we know should go away or change shape, and then
90 # send a data stream that tells the client something like this:
92 # send a data stream that tells the client something like this:
91 #
93 #
92 # a) apply this changegroup
94 # a) apply this changegroup
93 # b) apply nodes XXX, YYY, ZZZ that you already have
95 # b) apply nodes XXX, YYY, ZZZ that you already have
94 # c) goto a
96 # c) goto a
95 #
97 #
96 # until they've built up the full new state.
98 # until they've built up the full new state.
97 # Convert to revnums and intersect with "common". The client should
99 # Convert to revnums and intersect with "common". The client should
98 # have made it a subset of "common" already, but let's be safe.
100 # have made it a subset of "common" already, but let's be safe.
99 known = set(repo.revs("%ln & ::%ln", known, common))
101 known = set(repo.revs("%ln & ::%ln", known, common))
100 # TODO: we could send only roots() of this set, and the
102 # TODO: we could send only roots() of this set, and the
101 # list of nodes in common, and the client could work out
103 # list of nodes in common, and the client could work out
102 # what to strip, instead of us explicitly sending every
104 # what to strip, instead of us explicitly sending every
103 # single node.
105 # single node.
104 deadrevs = known
106 deadrevs = known
105 def genkills():
107 def genkills():
106 for r in deadrevs:
108 for r in deadrevs:
107 yield _KILLNODESIGNAL
109 yield _KILLNODESIGNAL
108 yield repo.changelog.node(r)
110 yield repo.changelog.node(r)
109 yield _DONESIGNAL
111 yield _DONESIGNAL
110 bundler.newpart(_CHANGESPECPART, data=genkills())
112 bundler.newpart(_CHANGESPECPART, data=genkills())
111 newvisit, newfull, newellipsis = exchange._computeellipsis(
113 newvisit, newfull, newellipsis = exchange._computeellipsis(
112 repo, set(), common, known, newmatch)
114 repo, set(), common, known, newmatch)
113 if newvisit:
115 if newvisit:
114 packer = changegroup.getbundler(version, repo,
116 packer = changegroup.getbundler(version, repo,
115 matcher=newmatch,
117 matcher=newmatch,
116 ellipses=True,
118 ellipses=True,
117 shallow=depth is not None,
119 shallow=depth is not None,
118 ellipsisroots=newellipsis,
120 ellipsisroots=newellipsis,
119 fullnodes=newfull)
121 fullnodes=newfull)
120 cgdata = packer.generate(common, newvisit, False, 'narrow_widen')
122 cgdata = packer.generate(common, newvisit, False, 'narrow_widen')
121
123
122 part = bundler.newpart('changegroup', data=cgdata)
124 part = bundler.newpart('changegroup', data=cgdata)
123 part.addparam('version', version)
125 part.addparam('version', version)
124 if 'treemanifest' in repo.requirements:
126 if 'treemanifest' in repo.requirements:
125 part.addparam('treemanifest', '1')
127 part.addparam('treemanifest', '1')
126
128
127 visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
129 visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
128 repo, common, heads, set(), newmatch, depth=depth)
130 repo, common, heads, set(), newmatch, depth=depth)
129
131
130 repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
132 repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
131 if visitnodes:
133 if visitnodes:
132 packer = changegroup.getbundler(version, repo,
134 packer = changegroup.getbundler(version, repo,
133 matcher=newmatch,
135 matcher=newmatch,
134 ellipses=True,
136 ellipses=True,
135 shallow=depth is not None,
137 shallow=depth is not None,
136 ellipsisroots=ellipsisroots,
138 ellipsisroots=ellipsisroots,
137 fullnodes=relevant_nodes)
139 fullnodes=relevant_nodes)
138 cgdata = packer.generate(common, visitnodes, False, 'narrow_widen')
140 cgdata = packer.generate(common, visitnodes, False, 'narrow_widen')
139
141
140 part = bundler.newpart('changegroup', data=cgdata)
142 part = bundler.newpart('changegroup', data=cgdata)
141 part.addparam('version', version)
143 part.addparam('version', version)
142 if 'treemanifest' in repo.requirements:
144 if 'treemanifest' in repo.requirements:
143 part.addparam('treemanifest', '1')
145 part.addparam('treemanifest', '1')
144
146
145 @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
147 @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
146 def _handlechangespec_2(op, inpart):
148 def _handlechangespec_2(op, inpart):
147 # XXX: This bundle2 handling is buggy and should be removed after hg5.2 is
149 # XXX: This bundle2 handling is buggy and should be removed after hg5.2 is
148 # released. New servers will send a mandatory bundle2 part named
150 # released. New servers will send a mandatory bundle2 part named
149 # 'Narrowspec' and will send specs as data instead of params.
151 # 'Narrowspec' and will send specs as data instead of params.
150 # Refer to issue5952 and 6019
152 # Refer to issue5952 and 6019
151 includepats = set(inpart.params.get(_SPECPART_INCLUDE, '').splitlines())
153 includepats = set(inpart.params.get(_SPECPART_INCLUDE, '').splitlines())
152 excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, '').splitlines())
154 excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, '').splitlines())
153 narrowspec.validatepatterns(includepats)
155 narrowspec.validatepatterns(includepats)
154 narrowspec.validatepatterns(excludepats)
156 narrowspec.validatepatterns(excludepats)
155
157
156 if not repository.NARROW_REQUIREMENT in op.repo.requirements:
158 if not repository.NARROW_REQUIREMENT in op.repo.requirements:
157 op.repo.requirements.add(repository.NARROW_REQUIREMENT)
159 op.repo.requirements.add(repository.NARROW_REQUIREMENT)
158 op.repo._writerequirements()
160 op.repo._writerequirements()
159 op.repo.setnarrowpats(includepats, excludepats)
161 op.repo.setnarrowpats(includepats, excludepats)
160 narrowspec.copytoworkingcopy(op.repo)
162 narrowspec.copytoworkingcopy(op.repo)
161
163
162 @bundle2.parthandler(_RESSPECS)
164 @bundle2.parthandler(_RESSPECS)
163 def _handlenarrowspecs(op, inpart):
165 def _handlenarrowspecs(op, inpart):
164 data = inpart.read()
166 data = inpart.read()
165 inc, exc = data.split('\0')
167 inc, exc = data.split('\0')
166 includepats = set(inc.splitlines())
168 includepats = set(inc.splitlines())
167 excludepats = set(exc.splitlines())
169 excludepats = set(exc.splitlines())
168 narrowspec.validatepatterns(includepats)
170 narrowspec.validatepatterns(includepats)
169 narrowspec.validatepatterns(excludepats)
171 narrowspec.validatepatterns(excludepats)
170
172
171 if repository.NARROW_REQUIREMENT not in op.repo.requirements:
173 if repository.NARROW_REQUIREMENT not in op.repo.requirements:
172 op.repo.requirements.add(repository.NARROW_REQUIREMENT)
174 op.repo.requirements.add(repository.NARROW_REQUIREMENT)
173 op.repo._writerequirements()
175 op.repo._writerequirements()
174 op.repo.setnarrowpats(includepats, excludepats)
176 op.repo.setnarrowpats(includepats, excludepats)
175 narrowspec.copytoworkingcopy(op.repo)
177 narrowspec.copytoworkingcopy(op.repo)
176
178
177 @bundle2.parthandler(_CHANGESPECPART)
179 @bundle2.parthandler(_CHANGESPECPART)
178 def _handlechangespec(op, inpart):
180 def _handlechangespec(op, inpart):
179 repo = op.repo
181 repo = op.repo
180 cl = repo.changelog
182 cl = repo.changelog
181
183
182 # changesets which need to be stripped entirely. either they're no longer
184 # changesets which need to be stripped entirely. either they're no longer
183 # needed in the new narrow spec, or the server is sending a replacement
185 # needed in the new narrow spec, or the server is sending a replacement
184 # in the changegroup part.
186 # in the changegroup part.
185 clkills = set()
187 clkills = set()
186
188
187 # A changespec part contains all the updates to ellipsis nodes
189 # A changespec part contains all the updates to ellipsis nodes
188 # that will happen as a result of widening or narrowing a
190 # that will happen as a result of widening or narrowing a
189 # repo. All the changes that this block encounters are ellipsis
191 # repo. All the changes that this block encounters are ellipsis
190 # nodes or flags to kill an existing ellipsis.
192 # nodes or flags to kill an existing ellipsis.
191 chunksignal = changegroup.readexactly(inpart, 4)
193 chunksignal = changegroup.readexactly(inpart, 4)
192 while chunksignal != _DONESIGNAL:
194 while chunksignal != _DONESIGNAL:
193 if chunksignal == _KILLNODESIGNAL:
195 if chunksignal == _KILLNODESIGNAL:
194 # a node used to be an ellipsis but isn't anymore
196 # a node used to be an ellipsis but isn't anymore
195 ck = changegroup.readexactly(inpart, 20)
197 ck = changegroup.readexactly(inpart, 20)
196 if cl.hasnode(ck):
198 if cl.hasnode(ck):
197 clkills.add(ck)
199 clkills.add(ck)
198 else:
200 else:
199 raise error.Abort(
201 raise error.Abort(
200 _('unexpected changespec node chunk type: %s') % chunksignal)
202 _('unexpected changespec node chunk type: %s') % chunksignal)
201 chunksignal = changegroup.readexactly(inpart, 4)
203 chunksignal = changegroup.readexactly(inpart, 4)
202
204
203 if clkills:
205 if clkills:
204 # preserve bookmarks that repair.strip() would otherwise strip
206 # preserve bookmarks that repair.strip() would otherwise strip
205 op._bookmarksbackup = repo._bookmarks
207 op._bookmarksbackup = repo._bookmarks
206 class dummybmstore(dict):
208 class dummybmstore(dict):
207 def applychanges(self, repo, tr, changes):
209 def applychanges(self, repo, tr, changes):
208 pass
210 pass
209 localrepo.localrepository._bookmarks.set(repo, dummybmstore())
211 localrepo.localrepository._bookmarks.set(repo, dummybmstore())
210 chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
212 chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
211 topic='widen')
213 topic='widen')
212 if chgrpfile:
214 if chgrpfile:
213 op._widen_uninterr = repo.ui.uninterruptible()
215 op._widen_uninterr = repo.ui.uninterruptible()
214 op._widen_uninterr.__enter__()
216 op._widen_uninterr.__enter__()
215 # presence of _widen_bundle attribute activates widen handler later
217 # presence of _widen_bundle attribute activates widen handler later
216 op._widen_bundle = chgrpfile
218 op._widen_bundle = chgrpfile
217 # Set the new narrowspec if we're widening. The setnewnarrowpats() method
219 # Set the new narrowspec if we're widening. The setnewnarrowpats() method
218 # will currently always be there when using the core+narrowhg server, but
220 # will currently always be there when using the core+narrowhg server, but
219 # other servers may include a changespec part even when not widening (e.g.
221 # other servers may include a changespec part even when not widening (e.g.
220 # because we're deepening a shallow repo).
222 # because we're deepening a shallow repo).
221 if util.safehasattr(repo, 'setnewnarrowpats'):
223 if util.safehasattr(repo, 'setnewnarrowpats'):
222 repo.setnewnarrowpats()
224 repo.setnewnarrowpats()
223
225
224 def handlechangegroup_widen(op, inpart):
226 def handlechangegroup_widen(op, inpart):
225 """Changegroup exchange handler which restores temporarily-stripped nodes"""
227 """Changegroup exchange handler which restores temporarily-stripped nodes"""
226 # We saved a bundle with stripped node data we must now restore.
228 # We saved a bundle with stripped node data we must now restore.
227 # This approach is based on mercurial/repair.py@6ee26a53c111.
229 # This approach is based on mercurial/repair.py@6ee26a53c111.
228 repo = op.repo
230 repo = op.repo
229 ui = op.ui
231 ui = op.ui
230
232
231 chgrpfile = op._widen_bundle
233 chgrpfile = op._widen_bundle
232 del op._widen_bundle
234 del op._widen_bundle
233 vfs = repo.vfs
235 vfs = repo.vfs
234
236
235 ui.note(_("adding branch\n"))
237 ui.note(_("adding branch\n"))
236 f = vfs.open(chgrpfile, "rb")
238 f = vfs.open(chgrpfile, "rb")
237 try:
239 try:
238 gen = exchange.readbundle(ui, f, chgrpfile, vfs)
240 gen = exchange.readbundle(ui, f, chgrpfile, vfs)
239 if not ui.verbose:
241 if not ui.verbose:
240 # silence internal shuffling chatter
242 # silence internal shuffling chatter
241 ui.pushbuffer()
243 ui.pushbuffer()
242 if isinstance(gen, bundle2.unbundle20):
244 if isinstance(gen, bundle2.unbundle20):
243 with repo.transaction('strip') as tr:
245 with repo.transaction('strip') as tr:
244 bundle2.processbundle(repo, gen, lambda: tr)
246 bundle2.processbundle(repo, gen, lambda: tr)
245 else:
247 else:
246 gen.apply(repo, 'strip', 'bundle:' + vfs.join(chgrpfile), True)
248 gen.apply(repo, 'strip', 'bundle:' + vfs.join(chgrpfile), True)
247 if not ui.verbose:
249 if not ui.verbose:
248 ui.popbuffer()
250 ui.popbuffer()
249 finally:
251 finally:
250 f.close()
252 f.close()
251
253
252 # remove undo files
254 # remove undo files
253 for undovfs, undofile in repo.undofiles():
255 for undovfs, undofile in repo.undofiles():
254 try:
256 try:
255 undovfs.unlink(undofile)
257 undovfs.unlink(undofile)
256 except OSError as e:
258 except OSError as e:
257 if e.errno != errno.ENOENT:
259 if e.errno != errno.ENOENT:
258 ui.warn(_('error removing %s: %s\n') %
260 ui.warn(_('error removing %s: %s\n') %
259 (undovfs.join(undofile), stringutil.forcebytestr(e)))
261 (undovfs.join(undofile), stringutil.forcebytestr(e)))
260
262
261 # Remove partial backup only if there were no exceptions
263 # Remove partial backup only if there were no exceptions
262 op._widen_uninterr.__exit__(None, None, None)
264 op._widen_uninterr.__exit__(None, None, None)
263 vfs.unlink(chgrpfile)
265 vfs.unlink(chgrpfile)
264
266
265 def setup():
267 def setup():
266 """Enable narrow repo support in bundle2-related extension points."""
268 """Enable narrow repo support in bundle2-related extension points."""
267 getbundleargs = wireprototypes.GETBUNDLE_ARGUMENTS
269 getbundleargs = wireprototypes.GETBUNDLE_ARGUMENTS
268
270
269 getbundleargs['narrow'] = 'boolean'
271 getbundleargs['narrow'] = 'boolean'
270 getbundleargs['depth'] = 'plain'
272 getbundleargs['depth'] = 'plain'
271 getbundleargs['oldincludepats'] = 'csv'
273 getbundleargs['oldincludepats'] = 'csv'
272 getbundleargs['oldexcludepats'] = 'csv'
274 getbundleargs['oldexcludepats'] = 'csv'
273 getbundleargs['known'] = 'csv'
275 getbundleargs['known'] = 'csv'
274
276
275 # Extend changegroup serving to handle requests from narrow clients.
277 # Extend changegroup serving to handle requests from narrow clients.
276 origcgfn = exchange.getbundle2partsmapping['changegroup']
278 origcgfn = exchange.getbundle2partsmapping['changegroup']
277 def wrappedcgfn(*args, **kwargs):
279 def wrappedcgfn(*args, **kwargs):
278 repo = args[1]
280 repo = args[1]
279 if repo.ui.has_section(_NARROWACL_SECTION):
281 if repo.ui.has_section(_NARROWACL_SECTION):
280 kwargs = exchange.applynarrowacl(repo, kwargs)
282 kwargs = exchange.applynarrowacl(repo, kwargs)
281
283
282 if (kwargs.get(r'narrow', False) and
284 if (kwargs.get(r'narrow', False) and
283 repo.ui.configbool('experimental', 'narrowservebrokenellipses')):
285 repo.ui.configbool('experimental', 'narrowservebrokenellipses')):
284 getbundlechangegrouppart_narrow(*args, **kwargs)
286 getbundlechangegrouppart_narrow(*args, **kwargs)
285 else:
287 else:
286 origcgfn(*args, **kwargs)
288 origcgfn(*args, **kwargs)
287 exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
289 exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
288
290
289 # Extend changegroup receiver so client can fixup after widen requests.
291 # Extend changegroup receiver so client can fixup after widen requests.
290 origcghandler = bundle2.parthandlermapping['changegroup']
292 origcghandler = bundle2.parthandlermapping['changegroup']
291 def wrappedcghandler(op, inpart):
293 def wrappedcghandler(op, inpart):
292 origcghandler(op, inpart)
294 origcghandler(op, inpart)
293 if util.safehasattr(op, '_widen_bundle'):
295 if util.safehasattr(op, '_widen_bundle'):
294 handlechangegroup_widen(op, inpart)
296 handlechangegroup_widen(op, inpart)
295 if util.safehasattr(op, '_bookmarksbackup'):
297 if util.safehasattr(op, '_bookmarksbackup'):
296 localrepo.localrepository._bookmarks.set(op.repo,
298 localrepo.localrepository._bookmarks.set(op.repo,
297 op._bookmarksbackup)
299 op._bookmarksbackup)
298 del op._bookmarksbackup
300 del op._bookmarksbackup
299
301
300 wrappedcghandler.params = origcghandler.params
302 wrappedcghandler.params = origcghandler.params
301 bundle2.parthandlermapping['changegroup'] = wrappedcghandler
303 bundle2.parthandlermapping['changegroup'] = wrappedcghandler
@@ -1,478 +1,480
1 # narrowcommands.py - command modifications for narrowhg extension
1 # narrowcommands.py - command modifications for narrowhg extension
2 #
2 #
3 # Copyright 2017 Google, Inc.
3 # Copyright 2017 Google, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7 from __future__ import absolute_import
7 from __future__ import absolute_import
8
8
9 import itertools
9 import itertools
10 import os
10 import os
11
11
12 from mercurial.i18n import _
12 from mercurial.i18n import _
13 from mercurial import (
13 from mercurial import (
14 bundle2,
14 bundle2,
15 cmdutil,
15 cmdutil,
16 commands,
16 commands,
17 discovery,
17 discovery,
18 encoding,
18 encoding,
19 error,
19 error,
20 exchange,
20 exchange,
21 extensions,
21 extensions,
22 hg,
22 hg,
23 narrowspec,
23 narrowspec,
24 node,
24 node,
25 pycompat,
25 pycompat,
26 registrar,
26 registrar,
27 repair,
27 repair,
28 repository,
29 repoview,
28 repoview,
30 sparse,
29 sparse,
31 util,
30 util,
32 wireprototypes,
31 wireprototypes,
33 )
32 )
33 from mercurial.interfaces import (
34 repository,
35 )
34
36
35 table = {}
37 table = {}
36 command = registrar.command(table)
38 command = registrar.command(table)
37
39
38 def setup():
40 def setup():
39 """Wraps user-facing mercurial commands with narrow-aware versions."""
41 """Wraps user-facing mercurial commands with narrow-aware versions."""
40
42
41 entry = extensions.wrapcommand(commands.table, 'clone', clonenarrowcmd)
43 entry = extensions.wrapcommand(commands.table, 'clone', clonenarrowcmd)
42 entry[1].append(('', 'narrow', None,
44 entry[1].append(('', 'narrow', None,
43 _("create a narrow clone of select files")))
45 _("create a narrow clone of select files")))
44 entry[1].append(('', 'depth', '',
46 entry[1].append(('', 'depth', '',
45 _("limit the history fetched by distance from heads")))
47 _("limit the history fetched by distance from heads")))
46 entry[1].append(('', 'narrowspec', '',
48 entry[1].append(('', 'narrowspec', '',
47 _("read narrowspecs from file")))
49 _("read narrowspecs from file")))
48 # TODO(durin42): unify sparse/narrow --include/--exclude logic a bit
50 # TODO(durin42): unify sparse/narrow --include/--exclude logic a bit
49 if 'sparse' not in extensions.enabled():
51 if 'sparse' not in extensions.enabled():
50 entry[1].append(('', 'include', [],
52 entry[1].append(('', 'include', [],
51 _("specifically fetch this file/directory")))
53 _("specifically fetch this file/directory")))
52 entry[1].append(
54 entry[1].append(
53 ('', 'exclude', [],
55 ('', 'exclude', [],
54 _("do not fetch this file/directory, even if included")))
56 _("do not fetch this file/directory, even if included")))
55
57
56 entry = extensions.wrapcommand(commands.table, 'pull', pullnarrowcmd)
58 entry = extensions.wrapcommand(commands.table, 'pull', pullnarrowcmd)
57 entry[1].append(('', 'depth', '',
59 entry[1].append(('', 'depth', '',
58 _("limit the history fetched by distance from heads")))
60 _("limit the history fetched by distance from heads")))
59
61
60 extensions.wrapcommand(commands.table, 'archive', archivenarrowcmd)
62 extensions.wrapcommand(commands.table, 'archive', archivenarrowcmd)
61
63
62 def clonenarrowcmd(orig, ui, repo, *args, **opts):
64 def clonenarrowcmd(orig, ui, repo, *args, **opts):
63 """Wraps clone command, so 'hg clone' first wraps localrepo.clone()."""
65 """Wraps clone command, so 'hg clone' first wraps localrepo.clone()."""
64 opts = pycompat.byteskwargs(opts)
66 opts = pycompat.byteskwargs(opts)
65 wrappedextraprepare = util.nullcontextmanager()
67 wrappedextraprepare = util.nullcontextmanager()
66 narrowspecfile = opts['narrowspec']
68 narrowspecfile = opts['narrowspec']
67
69
68 if narrowspecfile:
70 if narrowspecfile:
69 filepath = os.path.join(encoding.getcwd(), narrowspecfile)
71 filepath = os.path.join(encoding.getcwd(), narrowspecfile)
70 ui.status(_("reading narrowspec from '%s'\n") % filepath)
72 ui.status(_("reading narrowspec from '%s'\n") % filepath)
71 try:
73 try:
72 fdata = util.readfile(filepath)
74 fdata = util.readfile(filepath)
73 except IOError as inst:
75 except IOError as inst:
74 raise error.Abort(_("cannot read narrowspecs from '%s': %s") %
76 raise error.Abort(_("cannot read narrowspecs from '%s': %s") %
75 (filepath, encoding.strtolocal(inst.strerror)))
77 (filepath, encoding.strtolocal(inst.strerror)))
76
78
77 includes, excludes, profiles = sparse.parseconfig(ui, fdata, 'narrow')
79 includes, excludes, profiles = sparse.parseconfig(ui, fdata, 'narrow')
78 if profiles:
80 if profiles:
79 raise error.Abort(_("cannot specify other files using '%include' in"
81 raise error.Abort(_("cannot specify other files using '%include' in"
80 " narrowspec"))
82 " narrowspec"))
81
83
82 narrowspec.validatepatterns(includes)
84 narrowspec.validatepatterns(includes)
83 narrowspec.validatepatterns(excludes)
85 narrowspec.validatepatterns(excludes)
84
86
85 # narrowspec is passed so we should assume that user wants narrow clone
87 # narrowspec is passed so we should assume that user wants narrow clone
86 opts['narrow'] = True
88 opts['narrow'] = True
87 opts['include'].extend(includes)
89 opts['include'].extend(includes)
88 opts['exclude'].extend(excludes)
90 opts['exclude'].extend(excludes)
89
91
90 if opts['narrow']:
92 if opts['narrow']:
91 def pullbundle2extraprepare_widen(orig, pullop, kwargs):
93 def pullbundle2extraprepare_widen(orig, pullop, kwargs):
92 orig(pullop, kwargs)
94 orig(pullop, kwargs)
93
95
94 if opts.get('depth'):
96 if opts.get('depth'):
95 kwargs['depth'] = opts['depth']
97 kwargs['depth'] = opts['depth']
96 wrappedextraprepare = extensions.wrappedfunction(exchange,
98 wrappedextraprepare = extensions.wrappedfunction(exchange,
97 '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
99 '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
98
100
99 with wrappedextraprepare:
101 with wrappedextraprepare:
100 return orig(ui, repo, *args, **pycompat.strkwargs(opts))
102 return orig(ui, repo, *args, **pycompat.strkwargs(opts))
101
103
102 def pullnarrowcmd(orig, ui, repo, *args, **opts):
104 def pullnarrowcmd(orig, ui, repo, *args, **opts):
103 """Wraps pull command to allow modifying narrow spec."""
105 """Wraps pull command to allow modifying narrow spec."""
104 wrappedextraprepare = util.nullcontextmanager()
106 wrappedextraprepare = util.nullcontextmanager()
105 if repository.NARROW_REQUIREMENT in repo.requirements:
107 if repository.NARROW_REQUIREMENT in repo.requirements:
106
108
107 def pullbundle2extraprepare_widen(orig, pullop, kwargs):
109 def pullbundle2extraprepare_widen(orig, pullop, kwargs):
108 orig(pullop, kwargs)
110 orig(pullop, kwargs)
109 if opts.get(r'depth'):
111 if opts.get(r'depth'):
110 kwargs['depth'] = opts[r'depth']
112 kwargs['depth'] = opts[r'depth']
111 wrappedextraprepare = extensions.wrappedfunction(exchange,
113 wrappedextraprepare = extensions.wrappedfunction(exchange,
112 '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
114 '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
113
115
114 with wrappedextraprepare:
116 with wrappedextraprepare:
115 return orig(ui, repo, *args, **opts)
117 return orig(ui, repo, *args, **opts)
116
118
117 def archivenarrowcmd(orig, ui, repo, *args, **opts):
119 def archivenarrowcmd(orig, ui, repo, *args, **opts):
118 """Wraps archive command to narrow the default includes."""
120 """Wraps archive command to narrow the default includes."""
119 if repository.NARROW_REQUIREMENT in repo.requirements:
121 if repository.NARROW_REQUIREMENT in repo.requirements:
120 repo_includes, repo_excludes = repo.narrowpats
122 repo_includes, repo_excludes = repo.narrowpats
121 includes = set(opts.get(r'include', []))
123 includes = set(opts.get(r'include', []))
122 excludes = set(opts.get(r'exclude', []))
124 excludes = set(opts.get(r'exclude', []))
123 includes, excludes, unused_invalid = narrowspec.restrictpatterns(
125 includes, excludes, unused_invalid = narrowspec.restrictpatterns(
124 includes, excludes, repo_includes, repo_excludes)
126 includes, excludes, repo_includes, repo_excludes)
125 if includes:
127 if includes:
126 opts[r'include'] = includes
128 opts[r'include'] = includes
127 if excludes:
129 if excludes:
128 opts[r'exclude'] = excludes
130 opts[r'exclude'] = excludes
129 return orig(ui, repo, *args, **opts)
131 return orig(ui, repo, *args, **opts)
130
132
131 def pullbundle2extraprepare(orig, pullop, kwargs):
133 def pullbundle2extraprepare(orig, pullop, kwargs):
132 repo = pullop.repo
134 repo = pullop.repo
133 if repository.NARROW_REQUIREMENT not in repo.requirements:
135 if repository.NARROW_REQUIREMENT not in repo.requirements:
134 return orig(pullop, kwargs)
136 return orig(pullop, kwargs)
135
137
136 if wireprototypes.NARROWCAP not in pullop.remote.capabilities():
138 if wireprototypes.NARROWCAP not in pullop.remote.capabilities():
137 raise error.Abort(_("server does not support narrow clones"))
139 raise error.Abort(_("server does not support narrow clones"))
138 orig(pullop, kwargs)
140 orig(pullop, kwargs)
139 kwargs['narrow'] = True
141 kwargs['narrow'] = True
140 include, exclude = repo.narrowpats
142 include, exclude = repo.narrowpats
141 kwargs['oldincludepats'] = include
143 kwargs['oldincludepats'] = include
142 kwargs['oldexcludepats'] = exclude
144 kwargs['oldexcludepats'] = exclude
143 if include:
145 if include:
144 kwargs['includepats'] = include
146 kwargs['includepats'] = include
145 if exclude:
147 if exclude:
146 kwargs['excludepats'] = exclude
148 kwargs['excludepats'] = exclude
147 # calculate known nodes only in ellipses cases because in non-ellipses cases
149 # calculate known nodes only in ellipses cases because in non-ellipses cases
148 # we have all the nodes
150 # we have all the nodes
149 if wireprototypes.ELLIPSESCAP1 in pullop.remote.capabilities():
151 if wireprototypes.ELLIPSESCAP1 in pullop.remote.capabilities():
150 kwargs['known'] = [node.hex(ctx.node()) for ctx in
152 kwargs['known'] = [node.hex(ctx.node()) for ctx in
151 repo.set('::%ln', pullop.common)
153 repo.set('::%ln', pullop.common)
152 if ctx.node() != node.nullid]
154 if ctx.node() != node.nullid]
153 if not kwargs['known']:
155 if not kwargs['known']:
154 # Mercurial serializes an empty list as '' and deserializes it as
156 # Mercurial serializes an empty list as '' and deserializes it as
155 # [''], so delete it instead to avoid handling the empty string on
157 # [''], so delete it instead to avoid handling the empty string on
156 # the server.
158 # the server.
157 del kwargs['known']
159 del kwargs['known']
158
160
159 extensions.wrapfunction(exchange,'_pullbundle2extraprepare',
161 extensions.wrapfunction(exchange,'_pullbundle2extraprepare',
160 pullbundle2extraprepare)
162 pullbundle2extraprepare)
161
163
162 def _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
164 def _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
163 newincludes, newexcludes, force):
165 newincludes, newexcludes, force):
164 oldmatch = narrowspec.match(repo.root, oldincludes, oldexcludes)
166 oldmatch = narrowspec.match(repo.root, oldincludes, oldexcludes)
165 newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
167 newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
166
168
167 # This is essentially doing "hg outgoing" to find all local-only
169 # This is essentially doing "hg outgoing" to find all local-only
168 # commits. We will then check that the local-only commits don't
170 # commits. We will then check that the local-only commits don't
169 # have any changes to files that will be untracked.
171 # have any changes to files that will be untracked.
170 unfi = repo.unfiltered()
172 unfi = repo.unfiltered()
171 outgoing = discovery.findcommonoutgoing(unfi, remote,
173 outgoing = discovery.findcommonoutgoing(unfi, remote,
172 commoninc=commoninc)
174 commoninc=commoninc)
173 ui.status(_('looking for local changes to affected paths\n'))
175 ui.status(_('looking for local changes to affected paths\n'))
174 localnodes = []
176 localnodes = []
175 for n in itertools.chain(outgoing.missing, outgoing.excluded):
177 for n in itertools.chain(outgoing.missing, outgoing.excluded):
176 if any(oldmatch(f) and not newmatch(f) for f in unfi[n].files()):
178 if any(oldmatch(f) and not newmatch(f) for f in unfi[n].files()):
177 localnodes.append(n)
179 localnodes.append(n)
178 revstostrip = unfi.revs('descendants(%ln)', localnodes)
180 revstostrip = unfi.revs('descendants(%ln)', localnodes)
179 hiddenrevs = repoview.filterrevs(repo, 'visible')
181 hiddenrevs = repoview.filterrevs(repo, 'visible')
180 visibletostrip = list(repo.changelog.node(r)
182 visibletostrip = list(repo.changelog.node(r)
181 for r in (revstostrip - hiddenrevs))
183 for r in (revstostrip - hiddenrevs))
182 if visibletostrip:
184 if visibletostrip:
183 ui.status(_('The following changeset(s) or their ancestors have '
185 ui.status(_('The following changeset(s) or their ancestors have '
184 'local changes not on the remote:\n'))
186 'local changes not on the remote:\n'))
185 maxnodes = 10
187 maxnodes = 10
186 if ui.verbose or len(visibletostrip) <= maxnodes:
188 if ui.verbose or len(visibletostrip) <= maxnodes:
187 for n in visibletostrip:
189 for n in visibletostrip:
188 ui.status('%s\n' % node.short(n))
190 ui.status('%s\n' % node.short(n))
189 else:
191 else:
190 for n in visibletostrip[:maxnodes]:
192 for n in visibletostrip[:maxnodes]:
191 ui.status('%s\n' % node.short(n))
193 ui.status('%s\n' % node.short(n))
192 ui.status(_('...and %d more, use --verbose to list all\n') %
194 ui.status(_('...and %d more, use --verbose to list all\n') %
193 (len(visibletostrip) - maxnodes))
195 (len(visibletostrip) - maxnodes))
194 if not force:
196 if not force:
195 raise error.Abort(_('local changes found'),
197 raise error.Abort(_('local changes found'),
196 hint=_('use --force-delete-local-changes to '
198 hint=_('use --force-delete-local-changes to '
197 'ignore'))
199 'ignore'))
198
200
199 with ui.uninterruptible():
201 with ui.uninterruptible():
200 if revstostrip:
202 if revstostrip:
201 tostrip = [unfi.changelog.node(r) for r in revstostrip]
203 tostrip = [unfi.changelog.node(r) for r in revstostrip]
202 if repo['.'].node() in tostrip:
204 if repo['.'].node() in tostrip:
203 # stripping working copy, so move to a different commit first
205 # stripping working copy, so move to a different commit first
204 urev = max(repo.revs('(::%n) - %ln + null',
206 urev = max(repo.revs('(::%n) - %ln + null',
205 repo['.'].node(), visibletostrip))
207 repo['.'].node(), visibletostrip))
206 hg.clean(repo, urev)
208 hg.clean(repo, urev)
207 overrides = {('devel', 'strip-obsmarkers'): False}
209 overrides = {('devel', 'strip-obsmarkers'): False}
208 with ui.configoverride(overrides, 'narrow'):
210 with ui.configoverride(overrides, 'narrow'):
209 repair.strip(ui, unfi, tostrip, topic='narrow')
211 repair.strip(ui, unfi, tostrip, topic='narrow')
210
212
211 todelete = []
213 todelete = []
212 for f, f2, size in repo.store.datafiles():
214 for f, f2, size in repo.store.datafiles():
213 if f.startswith('data/'):
215 if f.startswith('data/'):
214 file = f[5:-2]
216 file = f[5:-2]
215 if not newmatch(file):
217 if not newmatch(file):
216 todelete.append(f)
218 todelete.append(f)
217 elif f.startswith('meta/'):
219 elif f.startswith('meta/'):
218 dir = f[5:-13]
220 dir = f[5:-13]
219 dirs = sorted(util.dirs({dir})) + [dir]
221 dirs = sorted(util.dirs({dir})) + [dir]
220 include = True
222 include = True
221 for d in dirs:
223 for d in dirs:
222 visit = newmatch.visitdir(d)
224 visit = newmatch.visitdir(d)
223 if not visit:
225 if not visit:
224 include = False
226 include = False
225 break
227 break
226 if visit == 'all':
228 if visit == 'all':
227 break
229 break
228 if not include:
230 if not include:
229 todelete.append(f)
231 todelete.append(f)
230
232
231 repo.destroying()
233 repo.destroying()
232
234
233 with repo.transaction('narrowing'):
235 with repo.transaction('narrowing'):
234 # Update narrowspec before removing revlogs, so repo won't be
236 # Update narrowspec before removing revlogs, so repo won't be
235 # corrupt in case of crash
237 # corrupt in case of crash
236 repo.setnarrowpats(newincludes, newexcludes)
238 repo.setnarrowpats(newincludes, newexcludes)
237
239
238 for f in todelete:
240 for f in todelete:
239 ui.status(_('deleting %s\n') % f)
241 ui.status(_('deleting %s\n') % f)
240 util.unlinkpath(repo.svfs.join(f))
242 util.unlinkpath(repo.svfs.join(f))
241 repo.store.markremoved(f)
243 repo.store.markremoved(f)
242
244
243 narrowspec.updateworkingcopy(repo, assumeclean=True)
245 narrowspec.updateworkingcopy(repo, assumeclean=True)
244 narrowspec.copytoworkingcopy(repo)
246 narrowspec.copytoworkingcopy(repo)
245
247
246 repo.destroyed()
248 repo.destroyed()
247
249
248 def _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
250 def _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
249 newincludes, newexcludes):
251 newincludes, newexcludes):
250 # for now we assume that if a server has ellipses enabled, we will be
252 # for now we assume that if a server has ellipses enabled, we will be
251 # exchanging ellipses nodes. In future we should add ellipses as a client
253 # exchanging ellipses nodes. In future we should add ellipses as a client
252 # side requirement (maybe) to distinguish a client is shallow or not and
254 # side requirement (maybe) to distinguish a client is shallow or not and
253 # then send that information to server whether we want ellipses or not.
255 # then send that information to server whether we want ellipses or not.
254 # Theoretically a non-ellipses repo should be able to use narrow
256 # Theoretically a non-ellipses repo should be able to use narrow
255 # functionality from an ellipses enabled server
257 # functionality from an ellipses enabled server
256 remotecap = remote.capabilities()
258 remotecap = remote.capabilities()
257 ellipsesremote = any(cap in remotecap
259 ellipsesremote = any(cap in remotecap
258 for cap in wireprototypes.SUPPORTED_ELLIPSESCAP)
260 for cap in wireprototypes.SUPPORTED_ELLIPSESCAP)
259
261
260 # check whether we are talking to a server which supports old version of
262 # check whether we are talking to a server which supports old version of
261 # ellipses capabilities
263 # ellipses capabilities
262 isoldellipses = (ellipsesremote and wireprototypes.ELLIPSESCAP1 in
264 isoldellipses = (ellipsesremote and wireprototypes.ELLIPSESCAP1 in
263 remotecap and wireprototypes.ELLIPSESCAP not in remotecap)
265 remotecap and wireprototypes.ELLIPSESCAP not in remotecap)
264
266
265 def pullbundle2extraprepare_widen(orig, pullop, kwargs):
267 def pullbundle2extraprepare_widen(orig, pullop, kwargs):
266 orig(pullop, kwargs)
268 orig(pullop, kwargs)
267 # The old{in,ex}cludepats have already been set by orig()
269 # The old{in,ex}cludepats have already been set by orig()
268 kwargs['includepats'] = newincludes
270 kwargs['includepats'] = newincludes
269 kwargs['excludepats'] = newexcludes
271 kwargs['excludepats'] = newexcludes
270 wrappedextraprepare = extensions.wrappedfunction(exchange,
272 wrappedextraprepare = extensions.wrappedfunction(exchange,
271 '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
273 '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
272
274
273 # define a function that narrowbundle2 can call after creating the
275 # define a function that narrowbundle2 can call after creating the
274 # backup bundle, but before applying the bundle from the server
276 # backup bundle, but before applying the bundle from the server
275 def setnewnarrowpats():
277 def setnewnarrowpats():
276 repo.setnarrowpats(newincludes, newexcludes)
278 repo.setnarrowpats(newincludes, newexcludes)
277 repo.setnewnarrowpats = setnewnarrowpats
279 repo.setnewnarrowpats = setnewnarrowpats
278 # silence the devel-warning of applying an empty changegroup
280 # silence the devel-warning of applying an empty changegroup
279 overrides = {('devel', 'all-warnings'): False}
281 overrides = {('devel', 'all-warnings'): False}
280
282
281 common = commoninc[0]
283 common = commoninc[0]
282 with ui.uninterruptible():
284 with ui.uninterruptible():
283 if ellipsesremote:
285 if ellipsesremote:
284 ds = repo.dirstate
286 ds = repo.dirstate
285 p1, p2 = ds.p1(), ds.p2()
287 p1, p2 = ds.p1(), ds.p2()
286 with ds.parentchange():
288 with ds.parentchange():
287 ds.setparents(node.nullid, node.nullid)
289 ds.setparents(node.nullid, node.nullid)
288 if isoldellipses:
290 if isoldellipses:
289 with wrappedextraprepare:
291 with wrappedextraprepare:
290 exchange.pull(repo, remote, heads=common)
292 exchange.pull(repo, remote, heads=common)
291 else:
293 else:
292 known = []
294 known = []
293 if ellipsesremote:
295 if ellipsesremote:
294 known = [node.hex(ctx.node()) for ctx in
296 known = [node.hex(ctx.node()) for ctx in
295 repo.set('::%ln', common)
297 repo.set('::%ln', common)
296 if ctx.node() != node.nullid]
298 if ctx.node() != node.nullid]
297 with remote.commandexecutor() as e:
299 with remote.commandexecutor() as e:
298 bundle = e.callcommand('narrow_widen', {
300 bundle = e.callcommand('narrow_widen', {
299 'oldincludes': oldincludes,
301 'oldincludes': oldincludes,
300 'oldexcludes': oldexcludes,
302 'oldexcludes': oldexcludes,
301 'newincludes': newincludes,
303 'newincludes': newincludes,
302 'newexcludes': newexcludes,
304 'newexcludes': newexcludes,
303 'cgversion': '03',
305 'cgversion': '03',
304 'commonheads': common,
306 'commonheads': common,
305 'known': known,
307 'known': known,
306 'ellipses': ellipsesremote,
308 'ellipses': ellipsesremote,
307 }).result()
309 }).result()
308
310
309 trmanager = exchange.transactionmanager(repo, 'widen', remote.url())
311 trmanager = exchange.transactionmanager(repo, 'widen', remote.url())
310 with trmanager, repo.ui.configoverride(overrides, 'widen'):
312 with trmanager, repo.ui.configoverride(overrides, 'widen'):
311 op = bundle2.bundleoperation(repo, trmanager.transaction,
313 op = bundle2.bundleoperation(repo, trmanager.transaction,
312 source='widen')
314 source='widen')
313 # TODO: we should catch error.Abort here
315 # TODO: we should catch error.Abort here
314 bundle2.processbundle(repo, bundle, op=op)
316 bundle2.processbundle(repo, bundle, op=op)
315
317
316 if ellipsesremote:
318 if ellipsesremote:
317 with ds.parentchange():
319 with ds.parentchange():
318 ds.setparents(p1, p2)
320 ds.setparents(p1, p2)
319
321
320 with repo.transaction('widening'):
322 with repo.transaction('widening'):
321 repo.setnewnarrowpats()
323 repo.setnewnarrowpats()
322 narrowspec.updateworkingcopy(repo)
324 narrowspec.updateworkingcopy(repo)
323 narrowspec.copytoworkingcopy(repo)
325 narrowspec.copytoworkingcopy(repo)
324
326
325 # TODO(rdamazio): Make new matcher format and update description
327 # TODO(rdamazio): Make new matcher format and update description
326 @command('tracked',
328 @command('tracked',
327 [('', 'addinclude', [], _('new paths to include')),
329 [('', 'addinclude', [], _('new paths to include')),
328 ('', 'removeinclude', [], _('old paths to no longer include')),
330 ('', 'removeinclude', [], _('old paths to no longer include')),
329 ('', 'addexclude', [], _('new paths to exclude')),
331 ('', 'addexclude', [], _('new paths to exclude')),
330 ('', 'import-rules', '', _('import narrowspecs from a file')),
332 ('', 'import-rules', '', _('import narrowspecs from a file')),
331 ('', 'removeexclude', [], _('old paths to no longer exclude')),
333 ('', 'removeexclude', [], _('old paths to no longer exclude')),
332 ('', 'clear', False, _('whether to replace the existing narrowspec')),
334 ('', 'clear', False, _('whether to replace the existing narrowspec')),
333 ('', 'force-delete-local-changes', False,
335 ('', 'force-delete-local-changes', False,
334 _('forces deletion of local changes when narrowing')),
336 _('forces deletion of local changes when narrowing')),
335 ('', 'update-working-copy', False,
337 ('', 'update-working-copy', False,
336 _('update working copy when the store has changed')),
338 _('update working copy when the store has changed')),
337 ] + commands.remoteopts,
339 ] + commands.remoteopts,
338 _('[OPTIONS]... [REMOTE]'),
340 _('[OPTIONS]... [REMOTE]'),
339 inferrepo=True)
341 inferrepo=True)
340 def trackedcmd(ui, repo, remotepath=None, *pats, **opts):
342 def trackedcmd(ui, repo, remotepath=None, *pats, **opts):
341 """show or change the current narrowspec
343 """show or change the current narrowspec
342
344
343 With no argument, shows the current narrowspec entries, one per line. Each
345 With no argument, shows the current narrowspec entries, one per line. Each
344 line will be prefixed with 'I' or 'X' for included or excluded patterns,
346 line will be prefixed with 'I' or 'X' for included or excluded patterns,
345 respectively.
347 respectively.
346
348
347 The narrowspec is comprised of expressions to match remote files and/or
349 The narrowspec is comprised of expressions to match remote files and/or
348 directories that should be pulled into your client.
350 directories that should be pulled into your client.
349 The narrowspec has *include* and *exclude* expressions, with excludes always
351 The narrowspec has *include* and *exclude* expressions, with excludes always
350 trumping includes: that is, if a file matches an exclude expression, it will
352 trumping includes: that is, if a file matches an exclude expression, it will
351 be excluded even if it also matches an include expression.
353 be excluded even if it also matches an include expression.
352 Excluding files that were never included has no effect.
354 Excluding files that were never included has no effect.
353
355
354 Each included or excluded entry is in the format described by
356 Each included or excluded entry is in the format described by
355 'hg help patterns'.
357 'hg help patterns'.
356
358
357 The options allow you to add or remove included and excluded expressions.
359 The options allow you to add or remove included and excluded expressions.
358
360
359 If --clear is specified, then all previous includes and excludes are DROPPED
361 If --clear is specified, then all previous includes and excludes are DROPPED
360 and replaced by the new ones specified to --addinclude and --addexclude.
362 and replaced by the new ones specified to --addinclude and --addexclude.
361 If --clear is specified without any further options, the narrowspec will be
363 If --clear is specified without any further options, the narrowspec will be
362 empty and will not match any files.
364 empty and will not match any files.
363
365
364 --import-rules accepts a path to a file containing rules, allowing you to
366 --import-rules accepts a path to a file containing rules, allowing you to
365 add --addinclude, --addexclude rules in bulk. Like the other include and
367 add --addinclude, --addexclude rules in bulk. Like the other include and
366 exclude switches, the changes are applied immediately.
368 exclude switches, the changes are applied immediately.
367 """
369 """
368 opts = pycompat.byteskwargs(opts)
370 opts = pycompat.byteskwargs(opts)
369 if repository.NARROW_REQUIREMENT not in repo.requirements:
371 if repository.NARROW_REQUIREMENT not in repo.requirements:
370 raise error.Abort(_('the tracked command is only supported on '
372 raise error.Abort(_('the tracked command is only supported on '
371 'repositories cloned with --narrow'))
373 'repositories cloned with --narrow'))
372
374
373 # Before supporting, decide whether it "hg tracked --clear" should mean
375 # Before supporting, decide whether it "hg tracked --clear" should mean
374 # tracking no paths or all paths.
376 # tracking no paths or all paths.
375 if opts['clear']:
377 if opts['clear']:
376 raise error.Abort(_('the --clear option is not yet supported'))
378 raise error.Abort(_('the --clear option is not yet supported'))
377
379
378 # import rules from a file
380 # import rules from a file
379 newrules = opts.get('import_rules')
381 newrules = opts.get('import_rules')
380 if newrules:
382 if newrules:
381 try:
383 try:
382 filepath = os.path.join(encoding.getcwd(), newrules)
384 filepath = os.path.join(encoding.getcwd(), newrules)
383 fdata = util.readfile(filepath)
385 fdata = util.readfile(filepath)
384 except IOError as inst:
386 except IOError as inst:
385 raise error.Abort(_("cannot read narrowspecs from '%s': %s") %
387 raise error.Abort(_("cannot read narrowspecs from '%s': %s") %
386 (filepath, encoding.strtolocal(inst.strerror)))
388 (filepath, encoding.strtolocal(inst.strerror)))
387 includepats, excludepats, profiles = sparse.parseconfig(ui, fdata,
389 includepats, excludepats, profiles = sparse.parseconfig(ui, fdata,
388 'narrow')
390 'narrow')
389 if profiles:
391 if profiles:
390 raise error.Abort(_("including other spec files using '%include' "
392 raise error.Abort(_("including other spec files using '%include' "
391 "is not supported in narrowspec"))
393 "is not supported in narrowspec"))
392 opts['addinclude'].extend(includepats)
394 opts['addinclude'].extend(includepats)
393 opts['addexclude'].extend(excludepats)
395 opts['addexclude'].extend(excludepats)
394
396
395 addedincludes = narrowspec.parsepatterns(opts['addinclude'])
397 addedincludes = narrowspec.parsepatterns(opts['addinclude'])
396 removedincludes = narrowspec.parsepatterns(opts['removeinclude'])
398 removedincludes = narrowspec.parsepatterns(opts['removeinclude'])
397 addedexcludes = narrowspec.parsepatterns(opts['addexclude'])
399 addedexcludes = narrowspec.parsepatterns(opts['addexclude'])
398 removedexcludes = narrowspec.parsepatterns(opts['removeexclude'])
400 removedexcludes = narrowspec.parsepatterns(opts['removeexclude'])
399
401
400 update_working_copy = opts['update_working_copy']
402 update_working_copy = opts['update_working_copy']
401 only_show = not (addedincludes or removedincludes or addedexcludes or
403 only_show = not (addedincludes or removedincludes or addedexcludes or
402 removedexcludes or newrules or update_working_copy)
404 removedexcludes or newrules or update_working_copy)
403
405
404 oldincludes, oldexcludes = repo.narrowpats
406 oldincludes, oldexcludes = repo.narrowpats
405
407
406 # filter the user passed additions and deletions into actual additions and
408 # filter the user passed additions and deletions into actual additions and
407 # deletions of excludes and includes
409 # deletions of excludes and includes
408 addedincludes -= oldincludes
410 addedincludes -= oldincludes
409 removedincludes &= oldincludes
411 removedincludes &= oldincludes
410 addedexcludes -= oldexcludes
412 addedexcludes -= oldexcludes
411 removedexcludes &= oldexcludes
413 removedexcludes &= oldexcludes
412
414
413 widening = addedincludes or removedexcludes
415 widening = addedincludes or removedexcludes
414 narrowing = removedincludes or addedexcludes
416 narrowing = removedincludes or addedexcludes
415
417
416 # Only print the current narrowspec.
418 # Only print the current narrowspec.
417 if only_show:
419 if only_show:
418 ui.pager('tracked')
420 ui.pager('tracked')
419 fm = ui.formatter('narrow', opts)
421 fm = ui.formatter('narrow', opts)
420 for i in sorted(oldincludes):
422 for i in sorted(oldincludes):
421 fm.startitem()
423 fm.startitem()
422 fm.write('status', '%s ', 'I', label='narrow.included')
424 fm.write('status', '%s ', 'I', label='narrow.included')
423 fm.write('pat', '%s\n', i, label='narrow.included')
425 fm.write('pat', '%s\n', i, label='narrow.included')
424 for i in sorted(oldexcludes):
426 for i in sorted(oldexcludes):
425 fm.startitem()
427 fm.startitem()
426 fm.write('status', '%s ', 'X', label='narrow.excluded')
428 fm.write('status', '%s ', 'X', label='narrow.excluded')
427 fm.write('pat', '%s\n', i, label='narrow.excluded')
429 fm.write('pat', '%s\n', i, label='narrow.excluded')
428 fm.end()
430 fm.end()
429 return 0
431 return 0
430
432
431 if update_working_copy:
433 if update_working_copy:
432 with repo.wlock(), repo.lock(), repo.transaction('narrow-wc'):
434 with repo.wlock(), repo.lock(), repo.transaction('narrow-wc'):
433 narrowspec.updateworkingcopy(repo)
435 narrowspec.updateworkingcopy(repo)
434 narrowspec.copytoworkingcopy(repo)
436 narrowspec.copytoworkingcopy(repo)
435 return 0
437 return 0
436
438
437 if not widening and not narrowing:
439 if not widening and not narrowing:
438 ui.status(_("nothing to widen or narrow\n"))
440 ui.status(_("nothing to widen or narrow\n"))
439 return 0
441 return 0
440
442
441 with repo.wlock(), repo.lock():
443 with repo.wlock(), repo.lock():
442 cmdutil.bailifchanged(repo)
444 cmdutil.bailifchanged(repo)
443
445
444 # Find the revisions we have in common with the remote. These will
446 # Find the revisions we have in common with the remote. These will
445 # be used for finding local-only changes for narrowing. They will
447 # be used for finding local-only changes for narrowing. They will
446 # also define the set of revisions to update for widening.
448 # also define the set of revisions to update for widening.
447 remotepath = ui.expandpath(remotepath or 'default')
449 remotepath = ui.expandpath(remotepath or 'default')
448 url, branches = hg.parseurl(remotepath)
450 url, branches = hg.parseurl(remotepath)
449 ui.status(_('comparing with %s\n') % util.hidepassword(url))
451 ui.status(_('comparing with %s\n') % util.hidepassword(url))
450 remote = hg.peer(repo, opts, url)
452 remote = hg.peer(repo, opts, url)
451
453
452 # check narrow support before doing anything if widening needs to be
454 # check narrow support before doing anything if widening needs to be
453 # performed. In future we should also abort if client is ellipses and
455 # performed. In future we should also abort if client is ellipses and
454 # server does not support ellipses
456 # server does not support ellipses
455 if widening and wireprototypes.NARROWCAP not in remote.capabilities():
457 if widening and wireprototypes.NARROWCAP not in remote.capabilities():
456 raise error.Abort(_("server does not support narrow clones"))
458 raise error.Abort(_("server does not support narrow clones"))
457
459
458 commoninc = discovery.findcommonincoming(repo, remote)
460 commoninc = discovery.findcommonincoming(repo, remote)
459
461
460 if narrowing:
462 if narrowing:
461 newincludes = oldincludes - removedincludes
463 newincludes = oldincludes - removedincludes
462 newexcludes = oldexcludes | addedexcludes
464 newexcludes = oldexcludes | addedexcludes
463 _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
465 _narrow(ui, repo, remote, commoninc, oldincludes, oldexcludes,
464 newincludes, newexcludes,
466 newincludes, newexcludes,
465 opts['force_delete_local_changes'])
467 opts['force_delete_local_changes'])
466 # _narrow() updated the narrowspec and _widen() below needs to
468 # _narrow() updated the narrowspec and _widen() below needs to
467 # use the updated values as its base (otherwise removed includes
469 # use the updated values as its base (otherwise removed includes
468 # and addedexcludes will be lost in the resulting narrowspec)
470 # and addedexcludes will be lost in the resulting narrowspec)
469 oldincludes = newincludes
471 oldincludes = newincludes
470 oldexcludes = newexcludes
472 oldexcludes = newexcludes
471
473
472 if widening:
474 if widening:
473 newincludes = oldincludes | addedincludes
475 newincludes = oldincludes | addedincludes
474 newexcludes = oldexcludes - removedexcludes
476 newexcludes = oldexcludes - removedexcludes
475 _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
477 _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
476 newincludes, newexcludes)
478 newincludes, newexcludes)
477
479
478 return 0
480 return 0
@@ -1,1174 +1,1176
1 # sqlitestore.py - Storage backend that uses SQLite
1 # sqlitestore.py - Storage backend that uses SQLite
2 #
2 #
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 """store repository data in SQLite (EXPERIMENTAL)
8 """store repository data in SQLite (EXPERIMENTAL)
9
9
10 The sqlitestore extension enables the storage of repository data in SQLite.
10 The sqlitestore extension enables the storage of repository data in SQLite.
11
11
12 This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY
12 This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY
13 GUARANTEES. This means that repositories created with this extension may
13 GUARANTEES. This means that repositories created with this extension may
14 only be usable with the exact version of this extension/Mercurial that was
14 only be usable with the exact version of this extension/Mercurial that was
15 used. The extension attempts to enforce this in order to prevent repository
15 used. The extension attempts to enforce this in order to prevent repository
16 corruption.
16 corruption.
17
17
18 In addition, several features are not yet supported or have known bugs:
18 In addition, several features are not yet supported or have known bugs:
19
19
20 * Only some data is stored in SQLite. Changeset, manifest, and other repository
20 * Only some data is stored in SQLite. Changeset, manifest, and other repository
21 data is not yet stored in SQLite.
21 data is not yet stored in SQLite.
22 * Transactions are not robust. If the process is aborted at the right time
22 * Transactions are not robust. If the process is aborted at the right time
23 during transaction close/rollback, the repository could be in an inconsistent
23 during transaction close/rollback, the repository could be in an inconsistent
24 state. This problem will diminish once all repository data is tracked by
24 state. This problem will diminish once all repository data is tracked by
25 SQLite.
25 SQLite.
26 * Bundle repositories do not work (the ability to use e.g.
26 * Bundle repositories do not work (the ability to use e.g.
27 `hg -R <bundle-file> log` to automatically overlay a bundle on top of the
27 `hg -R <bundle-file> log` to automatically overlay a bundle on top of the
28 existing repository).
28 existing repository).
29 * Various other features don't work.
29 * Various other features don't work.
30
30
31 This extension should work for basic clone/pull, update, and commit workflows.
31 This extension should work for basic clone/pull, update, and commit workflows.
32 Some history rewriting operations may fail due to lack of support for bundle
32 Some history rewriting operations may fail due to lack of support for bundle
33 repositories.
33 repositories.
34
34
35 To use, activate the extension and set the ``storage.new-repo-backend`` config
35 To use, activate the extension and set the ``storage.new-repo-backend`` config
36 option to ``sqlite`` to enable new repositories to use SQLite for storage.
36 option to ``sqlite`` to enable new repositories to use SQLite for storage.
37 """
37 """
38
38
39 # To run the test suite with repos using SQLite by default, execute the
39 # To run the test suite with repos using SQLite by default, execute the
40 # following:
40 # following:
41 #
41 #
42 # HGREPOFEATURES="sqlitestore" run-tests.py \
42 # HGREPOFEATURES="sqlitestore" run-tests.py \
43 # --extra-config-opt extensions.sqlitestore= \
43 # --extra-config-opt extensions.sqlitestore= \
44 # --extra-config-opt storage.new-repo-backend=sqlite
44 # --extra-config-opt storage.new-repo-backend=sqlite
45
45
46 from __future__ import absolute_import
46 from __future__ import absolute_import
47
47
48 import hashlib
48 import hashlib
49 import sqlite3
49 import sqlite3
50 import struct
50 import struct
51 import threading
51 import threading
52 import zlib
52 import zlib
53
53
54 from mercurial.i18n import _
54 from mercurial.i18n import _
55 from mercurial.node import (
55 from mercurial.node import (
56 nullid,
56 nullid,
57 nullrev,
57 nullrev,
58 short,
58 short,
59 )
59 )
60 from mercurial.thirdparty import (
60 from mercurial.thirdparty import (
61 attr,
61 attr,
62 )
62 )
63 from mercurial import (
63 from mercurial import (
64 ancestor,
64 ancestor,
65 dagop,
65 dagop,
66 encoding,
66 encoding,
67 error,
67 error,
68 extensions,
68 extensions,
69 localrepo,
69 localrepo,
70 mdiff,
70 mdiff,
71 pycompat,
71 pycompat,
72 registrar,
72 registrar,
73 repository,
74 util,
73 util,
75 verify,
74 verify,
76 )
75 )
76 from mercurial.interfaces import (
77 repository,
78 )
77 from mercurial.utils import (
79 from mercurial.utils import (
78 interfaceutil,
80 interfaceutil,
79 storageutil,
81 storageutil,
80 )
82 )
81
83
82 try:
84 try:
83 from mercurial import zstd
85 from mercurial import zstd
84 zstd.__version__
86 zstd.__version__
85 except ImportError:
87 except ImportError:
86 zstd = None
88 zstd = None
87
89
88 configtable = {}
90 configtable = {}
89 configitem = registrar.configitem(configtable)
91 configitem = registrar.configitem(configtable)
90
92
91 # experimental config: storage.sqlite.compression
93 # experimental config: storage.sqlite.compression
92 configitem('storage', 'sqlite.compression',
94 configitem('storage', 'sqlite.compression',
93 default='zstd' if zstd else 'zlib',
95 default='zstd' if zstd else 'zlib',
94 experimental=True)
96 experimental=True)
95
97
96 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
98 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
97 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
99 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
98 # be specifying the version(s) of Mercurial they are tested with, or
100 # be specifying the version(s) of Mercurial they are tested with, or
99 # leave the attribute unspecified.
101 # leave the attribute unspecified.
100 testedwith = 'ships-with-hg-core'
102 testedwith = 'ships-with-hg-core'
101
103
102 REQUIREMENT = b'exp-sqlite-001'
104 REQUIREMENT = b'exp-sqlite-001'
103 REQUIREMENT_ZSTD = b'exp-sqlite-comp-001=zstd'
105 REQUIREMENT_ZSTD = b'exp-sqlite-comp-001=zstd'
104 REQUIREMENT_ZLIB = b'exp-sqlite-comp-001=zlib'
106 REQUIREMENT_ZLIB = b'exp-sqlite-comp-001=zlib'
105 REQUIREMENT_NONE = b'exp-sqlite-comp-001=none'
107 REQUIREMENT_NONE = b'exp-sqlite-comp-001=none'
106 REQUIREMENT_SHALLOW_FILES = b'exp-sqlite-shallow-files'
108 REQUIREMENT_SHALLOW_FILES = b'exp-sqlite-shallow-files'
107
109
108 CURRENT_SCHEMA_VERSION = 1
110 CURRENT_SCHEMA_VERSION = 1
109
111
110 COMPRESSION_NONE = 1
112 COMPRESSION_NONE = 1
111 COMPRESSION_ZSTD = 2
113 COMPRESSION_ZSTD = 2
112 COMPRESSION_ZLIB = 3
114 COMPRESSION_ZLIB = 3
113
115
114 FLAG_CENSORED = 1
116 FLAG_CENSORED = 1
115 FLAG_MISSING_P1 = 2
117 FLAG_MISSING_P1 = 2
116 FLAG_MISSING_P2 = 4
118 FLAG_MISSING_P2 = 4
117
119
118 CREATE_SCHEMA = [
120 CREATE_SCHEMA = [
119 # Deltas are stored as content-indexed blobs.
121 # Deltas are stored as content-indexed blobs.
120 # compression column holds COMPRESSION_* constant for how the
122 # compression column holds COMPRESSION_* constant for how the
121 # delta is encoded.
123 # delta is encoded.
122
124
123 r'CREATE TABLE delta ('
125 r'CREATE TABLE delta ('
124 r' id INTEGER PRIMARY KEY, '
126 r' id INTEGER PRIMARY KEY, '
125 r' compression INTEGER NOT NULL, '
127 r' compression INTEGER NOT NULL, '
126 r' hash BLOB UNIQUE ON CONFLICT ABORT, '
128 r' hash BLOB UNIQUE ON CONFLICT ABORT, '
127 r' delta BLOB NOT NULL '
129 r' delta BLOB NOT NULL '
128 r')',
130 r')',
129
131
130 # Tracked paths are denormalized to integers to avoid redundant
132 # Tracked paths are denormalized to integers to avoid redundant
131 # storage of the path name.
133 # storage of the path name.
132 r'CREATE TABLE filepath ('
134 r'CREATE TABLE filepath ('
133 r' id INTEGER PRIMARY KEY, '
135 r' id INTEGER PRIMARY KEY, '
134 r' path BLOB NOT NULL '
136 r' path BLOB NOT NULL '
135 r')',
137 r')',
136
138
137 r'CREATE UNIQUE INDEX filepath_path '
139 r'CREATE UNIQUE INDEX filepath_path '
138 r' ON filepath (path)',
140 r' ON filepath (path)',
139
141
140 # We have a single table for all file revision data.
142 # We have a single table for all file revision data.
141 # Each file revision is uniquely described by a (path, rev) and
143 # Each file revision is uniquely described by a (path, rev) and
142 # (path, node).
144 # (path, node).
143 #
145 #
144 # Revision data is stored as a pointer to the delta producing this
146 # Revision data is stored as a pointer to the delta producing this
145 # revision and the file revision whose delta should be applied before
147 # revision and the file revision whose delta should be applied before
146 # that one. One can reconstruct the delta chain by recursively following
148 # that one. One can reconstruct the delta chain by recursively following
147 # the delta base revision pointers until one encounters NULL.
149 # the delta base revision pointers until one encounters NULL.
148 #
150 #
149 # flags column holds bitwise integer flags controlling storage options.
151 # flags column holds bitwise integer flags controlling storage options.
150 # These flags are defined by the FLAG_* constants.
152 # These flags are defined by the FLAG_* constants.
151 r'CREATE TABLE fileindex ('
153 r'CREATE TABLE fileindex ('
152 r' id INTEGER PRIMARY KEY, '
154 r' id INTEGER PRIMARY KEY, '
153 r' pathid INTEGER REFERENCES filepath(id), '
155 r' pathid INTEGER REFERENCES filepath(id), '
154 r' revnum INTEGER NOT NULL, '
156 r' revnum INTEGER NOT NULL, '
155 r' p1rev INTEGER NOT NULL, '
157 r' p1rev INTEGER NOT NULL, '
156 r' p2rev INTEGER NOT NULL, '
158 r' p2rev INTEGER NOT NULL, '
157 r' linkrev INTEGER NOT NULL, '
159 r' linkrev INTEGER NOT NULL, '
158 r' flags INTEGER NOT NULL, '
160 r' flags INTEGER NOT NULL, '
159 r' deltaid INTEGER REFERENCES delta(id), '
161 r' deltaid INTEGER REFERENCES delta(id), '
160 r' deltabaseid INTEGER REFERENCES fileindex(id), '
162 r' deltabaseid INTEGER REFERENCES fileindex(id), '
161 r' node BLOB NOT NULL '
163 r' node BLOB NOT NULL '
162 r')',
164 r')',
163
165
164 r'CREATE UNIQUE INDEX fileindex_pathrevnum '
166 r'CREATE UNIQUE INDEX fileindex_pathrevnum '
165 r' ON fileindex (pathid, revnum)',
167 r' ON fileindex (pathid, revnum)',
166
168
167 r'CREATE UNIQUE INDEX fileindex_pathnode '
169 r'CREATE UNIQUE INDEX fileindex_pathnode '
168 r' ON fileindex (pathid, node)',
170 r' ON fileindex (pathid, node)',
169
171
170 # Provide a view over all file data for convenience.
172 # Provide a view over all file data for convenience.
171 r'CREATE VIEW filedata AS '
173 r'CREATE VIEW filedata AS '
172 r'SELECT '
174 r'SELECT '
173 r' fileindex.id AS id, '
175 r' fileindex.id AS id, '
174 r' filepath.id AS pathid, '
176 r' filepath.id AS pathid, '
175 r' filepath.path AS path, '
177 r' filepath.path AS path, '
176 r' fileindex.revnum AS revnum, '
178 r' fileindex.revnum AS revnum, '
177 r' fileindex.node AS node, '
179 r' fileindex.node AS node, '
178 r' fileindex.p1rev AS p1rev, '
180 r' fileindex.p1rev AS p1rev, '
179 r' fileindex.p2rev AS p2rev, '
181 r' fileindex.p2rev AS p2rev, '
180 r' fileindex.linkrev AS linkrev, '
182 r' fileindex.linkrev AS linkrev, '
181 r' fileindex.flags AS flags, '
183 r' fileindex.flags AS flags, '
182 r' fileindex.deltaid AS deltaid, '
184 r' fileindex.deltaid AS deltaid, '
183 r' fileindex.deltabaseid AS deltabaseid '
185 r' fileindex.deltabaseid AS deltabaseid '
184 r'FROM filepath, fileindex '
186 r'FROM filepath, fileindex '
185 r'WHERE fileindex.pathid=filepath.id',
187 r'WHERE fileindex.pathid=filepath.id',
186
188
187 r'PRAGMA user_version=%d' % CURRENT_SCHEMA_VERSION,
189 r'PRAGMA user_version=%d' % CURRENT_SCHEMA_VERSION,
188 ]
190 ]
189
191
190 def resolvedeltachain(db, pathid, node, revisioncache,
192 def resolvedeltachain(db, pathid, node, revisioncache,
191 stoprids, zstddctx=None):
193 stoprids, zstddctx=None):
192 """Resolve a delta chain for a file node."""
194 """Resolve a delta chain for a file node."""
193
195
194 # TODO the "not in ({stops})" here is possibly slowing down the query
196 # TODO the "not in ({stops})" here is possibly slowing down the query
195 # because it needs to perform the lookup on every recursive invocation.
197 # because it needs to perform the lookup on every recursive invocation.
196 # This could possibly be faster if we created a temporary query with
198 # This could possibly be faster if we created a temporary query with
197 # baseid "poisoned" to null and limited the recursive filter to
199 # baseid "poisoned" to null and limited the recursive filter to
198 # "is not null".
200 # "is not null".
199 res = db.execute(
201 res = db.execute(
200 r'WITH RECURSIVE '
202 r'WITH RECURSIVE '
201 r' deltachain(deltaid, baseid) AS ('
203 r' deltachain(deltaid, baseid) AS ('
202 r' SELECT deltaid, deltabaseid FROM fileindex '
204 r' SELECT deltaid, deltabaseid FROM fileindex '
203 r' WHERE pathid=? AND node=? '
205 r' WHERE pathid=? AND node=? '
204 r' UNION ALL '
206 r' UNION ALL '
205 r' SELECT fileindex.deltaid, deltabaseid '
207 r' SELECT fileindex.deltaid, deltabaseid '
206 r' FROM fileindex, deltachain '
208 r' FROM fileindex, deltachain '
207 r' WHERE '
209 r' WHERE '
208 r' fileindex.id=deltachain.baseid '
210 r' fileindex.id=deltachain.baseid '
209 r' AND deltachain.baseid IS NOT NULL '
211 r' AND deltachain.baseid IS NOT NULL '
210 r' AND fileindex.id NOT IN ({stops}) '
212 r' AND fileindex.id NOT IN ({stops}) '
211 r' ) '
213 r' ) '
212 r'SELECT deltachain.baseid, compression, delta '
214 r'SELECT deltachain.baseid, compression, delta '
213 r'FROM deltachain, delta '
215 r'FROM deltachain, delta '
214 r'WHERE delta.id=deltachain.deltaid'.format(
216 r'WHERE delta.id=deltachain.deltaid'.format(
215 stops=r','.join([r'?'] * len(stoprids))),
217 stops=r','.join([r'?'] * len(stoprids))),
216 tuple([pathid, node] + list(stoprids.keys())))
218 tuple([pathid, node] + list(stoprids.keys())))
217
219
218 deltas = []
220 deltas = []
219 lastdeltabaseid = None
221 lastdeltabaseid = None
220
222
221 for deltabaseid, compression, delta in res:
223 for deltabaseid, compression, delta in res:
222 lastdeltabaseid = deltabaseid
224 lastdeltabaseid = deltabaseid
223
225
224 if compression == COMPRESSION_ZSTD:
226 if compression == COMPRESSION_ZSTD:
225 delta = zstddctx.decompress(delta)
227 delta = zstddctx.decompress(delta)
226 elif compression == COMPRESSION_NONE:
228 elif compression == COMPRESSION_NONE:
227 delta = delta
229 delta = delta
228 elif compression == COMPRESSION_ZLIB:
230 elif compression == COMPRESSION_ZLIB:
229 delta = zlib.decompress(delta)
231 delta = zlib.decompress(delta)
230 else:
232 else:
231 raise SQLiteStoreError('unhandled compression type: %d' %
233 raise SQLiteStoreError('unhandled compression type: %d' %
232 compression)
234 compression)
233
235
234 deltas.append(delta)
236 deltas.append(delta)
235
237
236 if lastdeltabaseid in stoprids:
238 if lastdeltabaseid in stoprids:
237 basetext = revisioncache[stoprids[lastdeltabaseid]]
239 basetext = revisioncache[stoprids[lastdeltabaseid]]
238 else:
240 else:
239 basetext = deltas.pop()
241 basetext = deltas.pop()
240
242
241 deltas.reverse()
243 deltas.reverse()
242 fulltext = mdiff.patches(basetext, deltas)
244 fulltext = mdiff.patches(basetext, deltas)
243
245
244 # SQLite returns buffer instances for blob columns on Python 2. This
246 # SQLite returns buffer instances for blob columns on Python 2. This
245 # type can propagate through the delta application layer. Because
247 # type can propagate through the delta application layer. Because
246 # downstream callers assume revisions are bytes, cast as needed.
248 # downstream callers assume revisions are bytes, cast as needed.
247 if not isinstance(fulltext, bytes):
249 if not isinstance(fulltext, bytes):
248 fulltext = bytes(delta)
250 fulltext = bytes(delta)
249
251
250 return fulltext
252 return fulltext
251
253
252 def insertdelta(db, compression, hash, delta):
254 def insertdelta(db, compression, hash, delta):
253 try:
255 try:
254 return db.execute(
256 return db.execute(
255 r'INSERT INTO delta (compression, hash, delta) '
257 r'INSERT INTO delta (compression, hash, delta) '
256 r'VALUES (?, ?, ?)',
258 r'VALUES (?, ?, ?)',
257 (compression, hash, delta)).lastrowid
259 (compression, hash, delta)).lastrowid
258 except sqlite3.IntegrityError:
260 except sqlite3.IntegrityError:
259 return db.execute(
261 return db.execute(
260 r'SELECT id FROM delta WHERE hash=?',
262 r'SELECT id FROM delta WHERE hash=?',
261 (hash,)).fetchone()[0]
263 (hash,)).fetchone()[0]
262
264
263 class SQLiteStoreError(error.StorageError):
265 class SQLiteStoreError(error.StorageError):
264 pass
266 pass
265
267
266 @attr.s
268 @attr.s
267 class revisionentry(object):
269 class revisionentry(object):
268 rid = attr.ib()
270 rid = attr.ib()
269 rev = attr.ib()
271 rev = attr.ib()
270 node = attr.ib()
272 node = attr.ib()
271 p1rev = attr.ib()
273 p1rev = attr.ib()
272 p2rev = attr.ib()
274 p2rev = attr.ib()
273 p1node = attr.ib()
275 p1node = attr.ib()
274 p2node = attr.ib()
276 p2node = attr.ib()
275 linkrev = attr.ib()
277 linkrev = attr.ib()
276 flags = attr.ib()
278 flags = attr.ib()
277
279
278 @interfaceutil.implementer(repository.irevisiondelta)
280 @interfaceutil.implementer(repository.irevisiondelta)
279 @attr.s(slots=True)
281 @attr.s(slots=True)
280 class sqliterevisiondelta(object):
282 class sqliterevisiondelta(object):
281 node = attr.ib()
283 node = attr.ib()
282 p1node = attr.ib()
284 p1node = attr.ib()
283 p2node = attr.ib()
285 p2node = attr.ib()
284 basenode = attr.ib()
286 basenode = attr.ib()
285 flags = attr.ib()
287 flags = attr.ib()
286 baserevisionsize = attr.ib()
288 baserevisionsize = attr.ib()
287 revision = attr.ib()
289 revision = attr.ib()
288 delta = attr.ib()
290 delta = attr.ib()
289 linknode = attr.ib(default=None)
291 linknode = attr.ib(default=None)
290
292
291 @interfaceutil.implementer(repository.iverifyproblem)
293 @interfaceutil.implementer(repository.iverifyproblem)
292 @attr.s(frozen=True)
294 @attr.s(frozen=True)
293 class sqliteproblem(object):
295 class sqliteproblem(object):
294 warning = attr.ib(default=None)
296 warning = attr.ib(default=None)
295 error = attr.ib(default=None)
297 error = attr.ib(default=None)
296 node = attr.ib(default=None)
298 node = attr.ib(default=None)
297
299
298 @interfaceutil.implementer(repository.ifilestorage)
300 @interfaceutil.implementer(repository.ifilestorage)
299 class sqlitefilestore(object):
301 class sqlitefilestore(object):
300 """Implements storage for an individual tracked path."""
302 """Implements storage for an individual tracked path."""
301
303
302 def __init__(self, db, path, compression):
304 def __init__(self, db, path, compression):
303 self._db = db
305 self._db = db
304 self._path = path
306 self._path = path
305
307
306 self._pathid = None
308 self._pathid = None
307
309
308 # revnum -> node
310 # revnum -> node
309 self._revtonode = {}
311 self._revtonode = {}
310 # node -> revnum
312 # node -> revnum
311 self._nodetorev = {}
313 self._nodetorev = {}
312 # node -> data structure
314 # node -> data structure
313 self._revisions = {}
315 self._revisions = {}
314
316
315 self._revisioncache = util.lrucachedict(10)
317 self._revisioncache = util.lrucachedict(10)
316
318
317 self._compengine = compression
319 self._compengine = compression
318
320
319 if compression == 'zstd':
321 if compression == 'zstd':
320 self._cctx = zstd.ZstdCompressor(level=3)
322 self._cctx = zstd.ZstdCompressor(level=3)
321 self._dctx = zstd.ZstdDecompressor()
323 self._dctx = zstd.ZstdDecompressor()
322 else:
324 else:
323 self._cctx = None
325 self._cctx = None
324 self._dctx = None
326 self._dctx = None
325
327
326 self._refreshindex()
328 self._refreshindex()
327
329
328 def _refreshindex(self):
330 def _refreshindex(self):
329 self._revtonode = {}
331 self._revtonode = {}
330 self._nodetorev = {}
332 self._nodetorev = {}
331 self._revisions = {}
333 self._revisions = {}
332
334
333 res = list(self._db.execute(
335 res = list(self._db.execute(
334 r'SELECT id FROM filepath WHERE path=?', (self._path,)))
336 r'SELECT id FROM filepath WHERE path=?', (self._path,)))
335
337
336 if not res:
338 if not res:
337 self._pathid = None
339 self._pathid = None
338 return
340 return
339
341
340 self._pathid = res[0][0]
342 self._pathid = res[0][0]
341
343
342 res = self._db.execute(
344 res = self._db.execute(
343 r'SELECT id, revnum, node, p1rev, p2rev, linkrev, flags '
345 r'SELECT id, revnum, node, p1rev, p2rev, linkrev, flags '
344 r'FROM fileindex '
346 r'FROM fileindex '
345 r'WHERE pathid=? '
347 r'WHERE pathid=? '
346 r'ORDER BY revnum ASC',
348 r'ORDER BY revnum ASC',
347 (self._pathid,))
349 (self._pathid,))
348
350
349 for i, row in enumerate(res):
351 for i, row in enumerate(res):
350 rid, rev, node, p1rev, p2rev, linkrev, flags = row
352 rid, rev, node, p1rev, p2rev, linkrev, flags = row
351
353
352 if i != rev:
354 if i != rev:
353 raise SQLiteStoreError(_('sqlite database has inconsistent '
355 raise SQLiteStoreError(_('sqlite database has inconsistent '
354 'revision numbers'))
356 'revision numbers'))
355
357
356 if p1rev == nullrev:
358 if p1rev == nullrev:
357 p1node = nullid
359 p1node = nullid
358 else:
360 else:
359 p1node = self._revtonode[p1rev]
361 p1node = self._revtonode[p1rev]
360
362
361 if p2rev == nullrev:
363 if p2rev == nullrev:
362 p2node = nullid
364 p2node = nullid
363 else:
365 else:
364 p2node = self._revtonode[p2rev]
366 p2node = self._revtonode[p2rev]
365
367
366 entry = revisionentry(
368 entry = revisionentry(
367 rid=rid,
369 rid=rid,
368 rev=rev,
370 rev=rev,
369 node=node,
371 node=node,
370 p1rev=p1rev,
372 p1rev=p1rev,
371 p2rev=p2rev,
373 p2rev=p2rev,
372 p1node=p1node,
374 p1node=p1node,
373 p2node=p2node,
375 p2node=p2node,
374 linkrev=linkrev,
376 linkrev=linkrev,
375 flags=flags)
377 flags=flags)
376
378
377 self._revtonode[rev] = node
379 self._revtonode[rev] = node
378 self._nodetorev[node] = rev
380 self._nodetorev[node] = rev
379 self._revisions[node] = entry
381 self._revisions[node] = entry
380
382
381 # Start of ifileindex interface.
383 # Start of ifileindex interface.
382
384
383 def __len__(self):
385 def __len__(self):
384 return len(self._revisions)
386 return len(self._revisions)
385
387
386 def __iter__(self):
388 def __iter__(self):
387 return iter(pycompat.xrange(len(self._revisions)))
389 return iter(pycompat.xrange(len(self._revisions)))
388
390
389 def hasnode(self, node):
391 def hasnode(self, node):
390 if node == nullid:
392 if node == nullid:
391 return False
393 return False
392
394
393 return node in self._nodetorev
395 return node in self._nodetorev
394
396
395 def revs(self, start=0, stop=None):
397 def revs(self, start=0, stop=None):
396 return storageutil.iterrevs(len(self._revisions), start=start,
398 return storageutil.iterrevs(len(self._revisions), start=start,
397 stop=stop)
399 stop=stop)
398
400
399 def parents(self, node):
401 def parents(self, node):
400 if node == nullid:
402 if node == nullid:
401 return nullid, nullid
403 return nullid, nullid
402
404
403 if node not in self._revisions:
405 if node not in self._revisions:
404 raise error.LookupError(node, self._path, _('no node'))
406 raise error.LookupError(node, self._path, _('no node'))
405
407
406 entry = self._revisions[node]
408 entry = self._revisions[node]
407 return entry.p1node, entry.p2node
409 return entry.p1node, entry.p2node
408
410
409 def parentrevs(self, rev):
411 def parentrevs(self, rev):
410 if rev == nullrev:
412 if rev == nullrev:
411 return nullrev, nullrev
413 return nullrev, nullrev
412
414
413 if rev not in self._revtonode:
415 if rev not in self._revtonode:
414 raise IndexError(rev)
416 raise IndexError(rev)
415
417
416 entry = self._revisions[self._revtonode[rev]]
418 entry = self._revisions[self._revtonode[rev]]
417 return entry.p1rev, entry.p2rev
419 return entry.p1rev, entry.p2rev
418
420
419 def rev(self, node):
421 def rev(self, node):
420 if node == nullid:
422 if node == nullid:
421 return nullrev
423 return nullrev
422
424
423 if node not in self._nodetorev:
425 if node not in self._nodetorev:
424 raise error.LookupError(node, self._path, _('no node'))
426 raise error.LookupError(node, self._path, _('no node'))
425
427
426 return self._nodetorev[node]
428 return self._nodetorev[node]
427
429
428 def node(self, rev):
430 def node(self, rev):
429 if rev == nullrev:
431 if rev == nullrev:
430 return nullid
432 return nullid
431
433
432 if rev not in self._revtonode:
434 if rev not in self._revtonode:
433 raise IndexError(rev)
435 raise IndexError(rev)
434
436
435 return self._revtonode[rev]
437 return self._revtonode[rev]
436
438
437 def lookup(self, node):
439 def lookup(self, node):
438 return storageutil.fileidlookup(self, node, self._path)
440 return storageutil.fileidlookup(self, node, self._path)
439
441
440 def linkrev(self, rev):
442 def linkrev(self, rev):
441 if rev == nullrev:
443 if rev == nullrev:
442 return nullrev
444 return nullrev
443
445
444 if rev not in self._revtonode:
446 if rev not in self._revtonode:
445 raise IndexError(rev)
447 raise IndexError(rev)
446
448
447 entry = self._revisions[self._revtonode[rev]]
449 entry = self._revisions[self._revtonode[rev]]
448 return entry.linkrev
450 return entry.linkrev
449
451
450 def iscensored(self, rev):
452 def iscensored(self, rev):
451 if rev == nullrev:
453 if rev == nullrev:
452 return False
454 return False
453
455
454 if rev not in self._revtonode:
456 if rev not in self._revtonode:
455 raise IndexError(rev)
457 raise IndexError(rev)
456
458
457 return self._revisions[self._revtonode[rev]].flags & FLAG_CENSORED
459 return self._revisions[self._revtonode[rev]].flags & FLAG_CENSORED
458
460
459 def commonancestorsheads(self, node1, node2):
461 def commonancestorsheads(self, node1, node2):
460 rev1 = self.rev(node1)
462 rev1 = self.rev(node1)
461 rev2 = self.rev(node2)
463 rev2 = self.rev(node2)
462
464
463 ancestors = ancestor.commonancestorsheads(self.parentrevs, rev1, rev2)
465 ancestors = ancestor.commonancestorsheads(self.parentrevs, rev1, rev2)
464 return pycompat.maplist(self.node, ancestors)
466 return pycompat.maplist(self.node, ancestors)
465
467
466 def descendants(self, revs):
468 def descendants(self, revs):
467 # TODO we could implement this using a recursive SQL query, which
469 # TODO we could implement this using a recursive SQL query, which
468 # might be faster.
470 # might be faster.
469 return dagop.descendantrevs(revs, self.revs, self.parentrevs)
471 return dagop.descendantrevs(revs, self.revs, self.parentrevs)
470
472
471 def heads(self, start=None, stop=None):
473 def heads(self, start=None, stop=None):
472 if start is None and stop is None:
474 if start is None and stop is None:
473 if not len(self):
475 if not len(self):
474 return [nullid]
476 return [nullid]
475
477
476 startrev = self.rev(start) if start is not None else nullrev
478 startrev = self.rev(start) if start is not None else nullrev
477 stoprevs = {self.rev(n) for n in stop or []}
479 stoprevs = {self.rev(n) for n in stop or []}
478
480
479 revs = dagop.headrevssubset(self.revs, self.parentrevs,
481 revs = dagop.headrevssubset(self.revs, self.parentrevs,
480 startrev=startrev, stoprevs=stoprevs)
482 startrev=startrev, stoprevs=stoprevs)
481
483
482 return [self.node(rev) for rev in revs]
484 return [self.node(rev) for rev in revs]
483
485
484 def children(self, node):
486 def children(self, node):
485 rev = self.rev(node)
487 rev = self.rev(node)
486
488
487 res = self._db.execute(
489 res = self._db.execute(
488 r'SELECT'
490 r'SELECT'
489 r' node '
491 r' node '
490 r' FROM filedata '
492 r' FROM filedata '
491 r' WHERE path=? AND (p1rev=? OR p2rev=?) '
493 r' WHERE path=? AND (p1rev=? OR p2rev=?) '
492 r' ORDER BY revnum ASC',
494 r' ORDER BY revnum ASC',
493 (self._path, rev, rev))
495 (self._path, rev, rev))
494
496
495 return [row[0] for row in res]
497 return [row[0] for row in res]
496
498
497 # End of ifileindex interface.
499 # End of ifileindex interface.
498
500
499 # Start of ifiledata interface.
501 # Start of ifiledata interface.
500
502
501 def size(self, rev):
503 def size(self, rev):
502 if rev == nullrev:
504 if rev == nullrev:
503 return 0
505 return 0
504
506
505 if rev not in self._revtonode:
507 if rev not in self._revtonode:
506 raise IndexError(rev)
508 raise IndexError(rev)
507
509
508 node = self._revtonode[rev]
510 node = self._revtonode[rev]
509
511
510 if self.renamed(node):
512 if self.renamed(node):
511 return len(self.read(node))
513 return len(self.read(node))
512
514
513 return len(self.revision(node))
515 return len(self.revision(node))
514
516
515 def revision(self, node, raw=False, _verifyhash=True):
517 def revision(self, node, raw=False, _verifyhash=True):
516 if node in (nullid, nullrev):
518 if node in (nullid, nullrev):
517 return b''
519 return b''
518
520
519 if isinstance(node, int):
521 if isinstance(node, int):
520 node = self.node(node)
522 node = self.node(node)
521
523
522 if node not in self._nodetorev:
524 if node not in self._nodetorev:
523 raise error.LookupError(node, self._path, _('no node'))
525 raise error.LookupError(node, self._path, _('no node'))
524
526
525 if node in self._revisioncache:
527 if node in self._revisioncache:
526 return self._revisioncache[node]
528 return self._revisioncache[node]
527
529
528 # Because we have a fulltext revision cache, we are able to
530 # Because we have a fulltext revision cache, we are able to
529 # short-circuit delta chain traversal and decompression as soon as
531 # short-circuit delta chain traversal and decompression as soon as
530 # we encounter a revision in the cache.
532 # we encounter a revision in the cache.
531
533
532 stoprids = {self._revisions[n].rid: n
534 stoprids = {self._revisions[n].rid: n
533 for n in self._revisioncache}
535 for n in self._revisioncache}
534
536
535 if not stoprids:
537 if not stoprids:
536 stoprids[-1] = None
538 stoprids[-1] = None
537
539
538 fulltext = resolvedeltachain(self._db, self._pathid, node,
540 fulltext = resolvedeltachain(self._db, self._pathid, node,
539 self._revisioncache, stoprids,
541 self._revisioncache, stoprids,
540 zstddctx=self._dctx)
542 zstddctx=self._dctx)
541
543
542 # Don't verify hashes if parent nodes were rewritten, as the hash
544 # Don't verify hashes if parent nodes were rewritten, as the hash
543 # wouldn't verify.
545 # wouldn't verify.
544 if self._revisions[node].flags & (FLAG_MISSING_P1 | FLAG_MISSING_P2):
546 if self._revisions[node].flags & (FLAG_MISSING_P1 | FLAG_MISSING_P2):
545 _verifyhash = False
547 _verifyhash = False
546
548
547 if _verifyhash:
549 if _verifyhash:
548 self._checkhash(fulltext, node)
550 self._checkhash(fulltext, node)
549 self._revisioncache[node] = fulltext
551 self._revisioncache[node] = fulltext
550
552
551 return fulltext
553 return fulltext
552
554
553 def rawdata(self, *args, **kwargs):
555 def rawdata(self, *args, **kwargs):
554 return self.revision(*args, **kwargs)
556 return self.revision(*args, **kwargs)
555
557
556 def read(self, node):
558 def read(self, node):
557 return storageutil.filtermetadata(self.revision(node))
559 return storageutil.filtermetadata(self.revision(node))
558
560
559 def renamed(self, node):
561 def renamed(self, node):
560 return storageutil.filerevisioncopied(self, node)
562 return storageutil.filerevisioncopied(self, node)
561
563
562 def cmp(self, node, fulltext):
564 def cmp(self, node, fulltext):
563 return not storageutil.filedataequivalent(self, node, fulltext)
565 return not storageutil.filedataequivalent(self, node, fulltext)
564
566
565 def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
567 def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
566 assumehaveparentrevisions=False,
568 assumehaveparentrevisions=False,
567 deltamode=repository.CG_DELTAMODE_STD):
569 deltamode=repository.CG_DELTAMODE_STD):
568 if nodesorder not in ('nodes', 'storage', 'linear', None):
570 if nodesorder not in ('nodes', 'storage', 'linear', None):
569 raise error.ProgrammingError('unhandled value for nodesorder: %s' %
571 raise error.ProgrammingError('unhandled value for nodesorder: %s' %
570 nodesorder)
572 nodesorder)
571
573
572 nodes = [n for n in nodes if n != nullid]
574 nodes = [n for n in nodes if n != nullid]
573
575
574 if not nodes:
576 if not nodes:
575 return
577 return
576
578
577 # TODO perform in a single query.
579 # TODO perform in a single query.
578 res = self._db.execute(
580 res = self._db.execute(
579 r'SELECT revnum, deltaid FROM fileindex '
581 r'SELECT revnum, deltaid FROM fileindex '
580 r'WHERE pathid=? '
582 r'WHERE pathid=? '
581 r' AND node in (%s)' % (r','.join([r'?'] * len(nodes))),
583 r' AND node in (%s)' % (r','.join([r'?'] * len(nodes))),
582 tuple([self._pathid] + nodes))
584 tuple([self._pathid] + nodes))
583
585
584 deltabases = {}
586 deltabases = {}
585
587
586 for rev, deltaid in res:
588 for rev, deltaid in res:
587 res = self._db.execute(
589 res = self._db.execute(
588 r'SELECT revnum from fileindex WHERE pathid=? AND deltaid=?',
590 r'SELECT revnum from fileindex WHERE pathid=? AND deltaid=?',
589 (self._pathid, deltaid))
591 (self._pathid, deltaid))
590 deltabases[rev] = res.fetchone()[0]
592 deltabases[rev] = res.fetchone()[0]
591
593
592 # TODO define revdifffn so we can use delta from storage.
594 # TODO define revdifffn so we can use delta from storage.
593 for delta in storageutil.emitrevisions(
595 for delta in storageutil.emitrevisions(
594 self, nodes, nodesorder, sqliterevisiondelta,
596 self, nodes, nodesorder, sqliterevisiondelta,
595 deltaparentfn=deltabases.__getitem__,
597 deltaparentfn=deltabases.__getitem__,
596 revisiondata=revisiondata,
598 revisiondata=revisiondata,
597 assumehaveparentrevisions=assumehaveparentrevisions,
599 assumehaveparentrevisions=assumehaveparentrevisions,
598 deltamode=deltamode):
600 deltamode=deltamode):
599
601
600 yield delta
602 yield delta
601
603
602 # End of ifiledata interface.
604 # End of ifiledata interface.
603
605
604 # Start of ifilemutation interface.
606 # Start of ifilemutation interface.
605
607
606 def add(self, filedata, meta, transaction, linkrev, p1, p2):
608 def add(self, filedata, meta, transaction, linkrev, p1, p2):
607 if meta or filedata.startswith(b'\x01\n'):
609 if meta or filedata.startswith(b'\x01\n'):
608 filedata = storageutil.packmeta(meta, filedata)
610 filedata = storageutil.packmeta(meta, filedata)
609
611
610 return self.addrevision(filedata, transaction, linkrev, p1, p2)
612 return self.addrevision(filedata, transaction, linkrev, p1, p2)
611
613
612 def addrevision(self, revisiondata, transaction, linkrev, p1, p2, node=None,
614 def addrevision(self, revisiondata, transaction, linkrev, p1, p2, node=None,
613 flags=0, cachedelta=None):
615 flags=0, cachedelta=None):
614 if flags:
616 if flags:
615 raise SQLiteStoreError(_('flags not supported on revisions'))
617 raise SQLiteStoreError(_('flags not supported on revisions'))
616
618
617 validatehash = node is not None
619 validatehash = node is not None
618 node = node or storageutil.hashrevisionsha1(revisiondata, p1, p2)
620 node = node or storageutil.hashrevisionsha1(revisiondata, p1, p2)
619
621
620 if validatehash:
622 if validatehash:
621 self._checkhash(revisiondata, node, p1, p2)
623 self._checkhash(revisiondata, node, p1, p2)
622
624
623 if node in self._nodetorev:
625 if node in self._nodetorev:
624 return node
626 return node
625
627
626 node = self._addrawrevision(node, revisiondata, transaction, linkrev,
628 node = self._addrawrevision(node, revisiondata, transaction, linkrev,
627 p1, p2)
629 p1, p2)
628
630
629 self._revisioncache[node] = revisiondata
631 self._revisioncache[node] = revisiondata
630 return node
632 return node
631
633
632 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None,
634 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None,
633 maybemissingparents=False):
635 maybemissingparents=False):
634 nodes = []
636 nodes = []
635
637
636 for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
638 for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
637 storeflags = 0
639 storeflags = 0
638
640
639 if wireflags & repository.REVISION_FLAG_CENSORED:
641 if wireflags & repository.REVISION_FLAG_CENSORED:
640 storeflags |= FLAG_CENSORED
642 storeflags |= FLAG_CENSORED
641
643
642 if wireflags & ~repository.REVISION_FLAG_CENSORED:
644 if wireflags & ~repository.REVISION_FLAG_CENSORED:
643 raise SQLiteStoreError('unhandled revision flag')
645 raise SQLiteStoreError('unhandled revision flag')
644
646
645 if maybemissingparents:
647 if maybemissingparents:
646 if p1 != nullid and not self.hasnode(p1):
648 if p1 != nullid and not self.hasnode(p1):
647 p1 = nullid
649 p1 = nullid
648 storeflags |= FLAG_MISSING_P1
650 storeflags |= FLAG_MISSING_P1
649
651
650 if p2 != nullid and not self.hasnode(p2):
652 if p2 != nullid and not self.hasnode(p2):
651 p2 = nullid
653 p2 = nullid
652 storeflags |= FLAG_MISSING_P2
654 storeflags |= FLAG_MISSING_P2
653
655
654 baserev = self.rev(deltabase)
656 baserev = self.rev(deltabase)
655
657
656 # If base is censored, delta must be full replacement in a single
658 # If base is censored, delta must be full replacement in a single
657 # patch operation.
659 # patch operation.
658 if baserev != nullrev and self.iscensored(baserev):
660 if baserev != nullrev and self.iscensored(baserev):
659 hlen = struct.calcsize('>lll')
661 hlen = struct.calcsize('>lll')
660 oldlen = len(self.rawdata(deltabase, _verifyhash=False))
662 oldlen = len(self.rawdata(deltabase, _verifyhash=False))
661 newlen = len(delta) - hlen
663 newlen = len(delta) - hlen
662
664
663 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
665 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
664 raise error.CensoredBaseError(self._path,
666 raise error.CensoredBaseError(self._path,
665 deltabase)
667 deltabase)
666
668
667 if (not (storeflags & FLAG_CENSORED)
669 if (not (storeflags & FLAG_CENSORED)
668 and storageutil.deltaiscensored(
670 and storageutil.deltaiscensored(
669 delta, baserev, lambda x: len(self.rawdata(x)))):
671 delta, baserev, lambda x: len(self.rawdata(x)))):
670 storeflags |= FLAG_CENSORED
672 storeflags |= FLAG_CENSORED
671
673
672 linkrev = linkmapper(linknode)
674 linkrev = linkmapper(linknode)
673
675
674 nodes.append(node)
676 nodes.append(node)
675
677
676 if node in self._revisions:
678 if node in self._revisions:
677 # Possibly reset parents to make them proper.
679 # Possibly reset parents to make them proper.
678 entry = self._revisions[node]
680 entry = self._revisions[node]
679
681
680 if entry.flags & FLAG_MISSING_P1 and p1 != nullid:
682 if entry.flags & FLAG_MISSING_P1 and p1 != nullid:
681 entry.p1node = p1
683 entry.p1node = p1
682 entry.p1rev = self._nodetorev[p1]
684 entry.p1rev = self._nodetorev[p1]
683 entry.flags &= ~FLAG_MISSING_P1
685 entry.flags &= ~FLAG_MISSING_P1
684
686
685 self._db.execute(
687 self._db.execute(
686 r'UPDATE fileindex SET p1rev=?, flags=? '
688 r'UPDATE fileindex SET p1rev=?, flags=? '
687 r'WHERE id=?',
689 r'WHERE id=?',
688 (self._nodetorev[p1], entry.flags, entry.rid))
690 (self._nodetorev[p1], entry.flags, entry.rid))
689
691
690 if entry.flags & FLAG_MISSING_P2 and p2 != nullid:
692 if entry.flags & FLAG_MISSING_P2 and p2 != nullid:
691 entry.p2node = p2
693 entry.p2node = p2
692 entry.p2rev = self._nodetorev[p2]
694 entry.p2rev = self._nodetorev[p2]
693 entry.flags &= ~FLAG_MISSING_P2
695 entry.flags &= ~FLAG_MISSING_P2
694
696
695 self._db.execute(
697 self._db.execute(
696 r'UPDATE fileindex SET p2rev=?, flags=? '
698 r'UPDATE fileindex SET p2rev=?, flags=? '
697 r'WHERE id=?',
699 r'WHERE id=?',
698 (self._nodetorev[p1], entry.flags, entry.rid))
700 (self._nodetorev[p1], entry.flags, entry.rid))
699
701
700 continue
702 continue
701
703
702 if deltabase == nullid:
704 if deltabase == nullid:
703 text = mdiff.patch(b'', delta)
705 text = mdiff.patch(b'', delta)
704 storedelta = None
706 storedelta = None
705 else:
707 else:
706 text = None
708 text = None
707 storedelta = (deltabase, delta)
709 storedelta = (deltabase, delta)
708
710
709 self._addrawrevision(node, text, transaction, linkrev, p1, p2,
711 self._addrawrevision(node, text, transaction, linkrev, p1, p2,
710 storedelta=storedelta, flags=storeflags)
712 storedelta=storedelta, flags=storeflags)
711
713
712 if addrevisioncb:
714 if addrevisioncb:
713 addrevisioncb(self, node)
715 addrevisioncb(self, node)
714
716
715 return nodes
717 return nodes
716
718
717 def censorrevision(self, tr, censornode, tombstone=b''):
719 def censorrevision(self, tr, censornode, tombstone=b''):
718 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
720 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
719
721
720 # This restriction is cargo culted from revlogs and makes no sense for
722 # This restriction is cargo culted from revlogs and makes no sense for
721 # SQLite, since columns can be resized at will.
723 # SQLite, since columns can be resized at will.
722 if len(tombstone) > len(self.rawdata(censornode)):
724 if len(tombstone) > len(self.rawdata(censornode)):
723 raise error.Abort(_('censor tombstone must be no longer than '
725 raise error.Abort(_('censor tombstone must be no longer than '
724 'censored data'))
726 'censored data'))
725
727
726 # We need to replace the censored revision's data with the tombstone.
728 # We need to replace the censored revision's data with the tombstone.
727 # But replacing that data will have implications for delta chains that
729 # But replacing that data will have implications for delta chains that
728 # reference it.
730 # reference it.
729 #
731 #
730 # While "better," more complex strategies are possible, we do something
732 # While "better," more complex strategies are possible, we do something
731 # simple: we find delta chain children of the censored revision and we
733 # simple: we find delta chain children of the censored revision and we
732 # replace those incremental deltas with fulltexts of their corresponding
734 # replace those incremental deltas with fulltexts of their corresponding
733 # revision. Then we delete the now-unreferenced delta and original
735 # revision. Then we delete the now-unreferenced delta and original
734 # revision and insert a replacement.
736 # revision and insert a replacement.
735
737
736 # Find the delta to be censored.
738 # Find the delta to be censored.
737 censoreddeltaid = self._db.execute(
739 censoreddeltaid = self._db.execute(
738 r'SELECT deltaid FROM fileindex WHERE id=?',
740 r'SELECT deltaid FROM fileindex WHERE id=?',
739 (self._revisions[censornode].rid,)).fetchone()[0]
741 (self._revisions[censornode].rid,)).fetchone()[0]
740
742
741 # Find all its delta chain children.
743 # Find all its delta chain children.
742 # TODO once we support storing deltas for !files, we'll need to look
744 # TODO once we support storing deltas for !files, we'll need to look
743 # for those delta chains too.
745 # for those delta chains too.
744 rows = list(self._db.execute(
746 rows = list(self._db.execute(
745 r'SELECT id, pathid, node FROM fileindex '
747 r'SELECT id, pathid, node FROM fileindex '
746 r'WHERE deltabaseid=? OR deltaid=?',
748 r'WHERE deltabaseid=? OR deltaid=?',
747 (censoreddeltaid, censoreddeltaid)))
749 (censoreddeltaid, censoreddeltaid)))
748
750
749 for row in rows:
751 for row in rows:
750 rid, pathid, node = row
752 rid, pathid, node = row
751
753
752 fulltext = resolvedeltachain(self._db, pathid, node, {}, {-1: None},
754 fulltext = resolvedeltachain(self._db, pathid, node, {}, {-1: None},
753 zstddctx=self._dctx)
755 zstddctx=self._dctx)
754
756
755 deltahash = hashlib.sha1(fulltext).digest()
757 deltahash = hashlib.sha1(fulltext).digest()
756
758
757 if self._compengine == 'zstd':
759 if self._compengine == 'zstd':
758 deltablob = self._cctx.compress(fulltext)
760 deltablob = self._cctx.compress(fulltext)
759 compression = COMPRESSION_ZSTD
761 compression = COMPRESSION_ZSTD
760 elif self._compengine == 'zlib':
762 elif self._compengine == 'zlib':
761 deltablob = zlib.compress(fulltext)
763 deltablob = zlib.compress(fulltext)
762 compression = COMPRESSION_ZLIB
764 compression = COMPRESSION_ZLIB
763 elif self._compengine == 'none':
765 elif self._compengine == 'none':
764 deltablob = fulltext
766 deltablob = fulltext
765 compression = COMPRESSION_NONE
767 compression = COMPRESSION_NONE
766 else:
768 else:
767 raise error.ProgrammingError('unhandled compression engine: %s'
769 raise error.ProgrammingError('unhandled compression engine: %s'
768 % self._compengine)
770 % self._compengine)
769
771
770 if len(deltablob) >= len(fulltext):
772 if len(deltablob) >= len(fulltext):
771 deltablob = fulltext
773 deltablob = fulltext
772 compression = COMPRESSION_NONE
774 compression = COMPRESSION_NONE
773
775
774 deltaid = insertdelta(self._db, compression, deltahash, deltablob)
776 deltaid = insertdelta(self._db, compression, deltahash, deltablob)
775
777
776 self._db.execute(
778 self._db.execute(
777 r'UPDATE fileindex SET deltaid=?, deltabaseid=NULL '
779 r'UPDATE fileindex SET deltaid=?, deltabaseid=NULL '
778 r'WHERE id=?', (deltaid, rid))
780 r'WHERE id=?', (deltaid, rid))
779
781
780 # Now create the tombstone delta and replace the delta on the censored
782 # Now create the tombstone delta and replace the delta on the censored
781 # node.
783 # node.
782 deltahash = hashlib.sha1(tombstone).digest()
784 deltahash = hashlib.sha1(tombstone).digest()
783 tombstonedeltaid = insertdelta(self._db, COMPRESSION_NONE,
785 tombstonedeltaid = insertdelta(self._db, COMPRESSION_NONE,
784 deltahash, tombstone)
786 deltahash, tombstone)
785
787
786 flags = self._revisions[censornode].flags
788 flags = self._revisions[censornode].flags
787 flags |= FLAG_CENSORED
789 flags |= FLAG_CENSORED
788
790
789 self._db.execute(
791 self._db.execute(
790 r'UPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL '
792 r'UPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL '
791 r'WHERE pathid=? AND node=?',
793 r'WHERE pathid=? AND node=?',
792 (flags, tombstonedeltaid, self._pathid, censornode))
794 (flags, tombstonedeltaid, self._pathid, censornode))
793
795
794 self._db.execute(
796 self._db.execute(
795 r'DELETE FROM delta WHERE id=?', (censoreddeltaid,))
797 r'DELETE FROM delta WHERE id=?', (censoreddeltaid,))
796
798
797 self._refreshindex()
799 self._refreshindex()
798 self._revisioncache.clear()
800 self._revisioncache.clear()
799
801
800 def getstrippoint(self, minlink):
802 def getstrippoint(self, minlink):
801 return storageutil.resolvestripinfo(minlink, len(self) - 1,
803 return storageutil.resolvestripinfo(minlink, len(self) - 1,
802 [self.rev(n) for n in self.heads()],
804 [self.rev(n) for n in self.heads()],
803 self.linkrev,
805 self.linkrev,
804 self.parentrevs)
806 self.parentrevs)
805
807
806 def strip(self, minlink, transaction):
808 def strip(self, minlink, transaction):
807 if not len(self):
809 if not len(self):
808 return
810 return
809
811
810 rev, _ignored = self.getstrippoint(minlink)
812 rev, _ignored = self.getstrippoint(minlink)
811
813
812 if rev == len(self):
814 if rev == len(self):
813 return
815 return
814
816
815 for rev in self.revs(rev):
817 for rev in self.revs(rev):
816 self._db.execute(
818 self._db.execute(
817 r'DELETE FROM fileindex WHERE pathid=? AND node=?',
819 r'DELETE FROM fileindex WHERE pathid=? AND node=?',
818 (self._pathid, self.node(rev)))
820 (self._pathid, self.node(rev)))
819
821
820 # TODO how should we garbage collect data in delta table?
822 # TODO how should we garbage collect data in delta table?
821
823
822 self._refreshindex()
824 self._refreshindex()
823
825
824 # End of ifilemutation interface.
826 # End of ifilemutation interface.
825
827
826 # Start of ifilestorage interface.
828 # Start of ifilestorage interface.
827
829
828 def files(self):
830 def files(self):
829 return []
831 return []
830
832
831 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
833 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
832 revisionscount=False, trackedsize=False,
834 revisionscount=False, trackedsize=False,
833 storedsize=False):
835 storedsize=False):
834 d = {}
836 d = {}
835
837
836 if exclusivefiles:
838 if exclusivefiles:
837 d['exclusivefiles'] = []
839 d['exclusivefiles'] = []
838
840
839 if sharedfiles:
841 if sharedfiles:
840 # TODO list sqlite file(s) here.
842 # TODO list sqlite file(s) here.
841 d['sharedfiles'] = []
843 d['sharedfiles'] = []
842
844
843 if revisionscount:
845 if revisionscount:
844 d['revisionscount'] = len(self)
846 d['revisionscount'] = len(self)
845
847
846 if trackedsize:
848 if trackedsize:
847 d['trackedsize'] = sum(len(self.revision(node))
849 d['trackedsize'] = sum(len(self.revision(node))
848 for node in self._nodetorev)
850 for node in self._nodetorev)
849
851
850 if storedsize:
852 if storedsize:
851 # TODO implement this?
853 # TODO implement this?
852 d['storedsize'] = None
854 d['storedsize'] = None
853
855
854 return d
856 return d
855
857
856 def verifyintegrity(self, state):
858 def verifyintegrity(self, state):
857 state['skipread'] = set()
859 state['skipread'] = set()
858
860
859 for rev in self:
861 for rev in self:
860 node = self.node(rev)
862 node = self.node(rev)
861
863
862 try:
864 try:
863 self.revision(node)
865 self.revision(node)
864 except Exception as e:
866 except Exception as e:
865 yield sqliteproblem(
867 yield sqliteproblem(
866 error=_('unpacking %s: %s') % (short(node), e),
868 error=_('unpacking %s: %s') % (short(node), e),
867 node=node)
869 node=node)
868
870
869 state['skipread'].add(node)
871 state['skipread'].add(node)
870
872
871 # End of ifilestorage interface.
873 # End of ifilestorage interface.
872
874
873 def _checkhash(self, fulltext, node, p1=None, p2=None):
875 def _checkhash(self, fulltext, node, p1=None, p2=None):
874 if p1 is None and p2 is None:
876 if p1 is None and p2 is None:
875 p1, p2 = self.parents(node)
877 p1, p2 = self.parents(node)
876
878
877 if node == storageutil.hashrevisionsha1(fulltext, p1, p2):
879 if node == storageutil.hashrevisionsha1(fulltext, p1, p2):
878 return
880 return
879
881
880 try:
882 try:
881 del self._revisioncache[node]
883 del self._revisioncache[node]
882 except KeyError:
884 except KeyError:
883 pass
885 pass
884
886
885 if storageutil.iscensoredtext(fulltext):
887 if storageutil.iscensoredtext(fulltext):
886 raise error.CensoredNodeError(self._path, node, fulltext)
888 raise error.CensoredNodeError(self._path, node, fulltext)
887
889
888 raise SQLiteStoreError(_('integrity check failed on %s') %
890 raise SQLiteStoreError(_('integrity check failed on %s') %
889 self._path)
891 self._path)
890
892
891 def _addrawrevision(self, node, revisiondata, transaction, linkrev,
893 def _addrawrevision(self, node, revisiondata, transaction, linkrev,
892 p1, p2, storedelta=None, flags=0):
894 p1, p2, storedelta=None, flags=0):
893 if self._pathid is None:
895 if self._pathid is None:
894 res = self._db.execute(
896 res = self._db.execute(
895 r'INSERT INTO filepath (path) VALUES (?)', (self._path,))
897 r'INSERT INTO filepath (path) VALUES (?)', (self._path,))
896 self._pathid = res.lastrowid
898 self._pathid = res.lastrowid
897
899
898 # For simplicity, always store a delta against p1.
900 # For simplicity, always store a delta against p1.
899 # TODO we need a lot more logic here to make behavior reasonable.
901 # TODO we need a lot more logic here to make behavior reasonable.
900
902
901 if storedelta:
903 if storedelta:
902 deltabase, delta = storedelta
904 deltabase, delta = storedelta
903
905
904 if isinstance(deltabase, int):
906 if isinstance(deltabase, int):
905 deltabase = self.node(deltabase)
907 deltabase = self.node(deltabase)
906
908
907 else:
909 else:
908 assert revisiondata is not None
910 assert revisiondata is not None
909 deltabase = p1
911 deltabase = p1
910
912
911 if deltabase == nullid:
913 if deltabase == nullid:
912 delta = revisiondata
914 delta = revisiondata
913 else:
915 else:
914 delta = mdiff.textdiff(self.revision(self.rev(deltabase)),
916 delta = mdiff.textdiff(self.revision(self.rev(deltabase)),
915 revisiondata)
917 revisiondata)
916
918
917 # File index stores a pointer to its delta and the parent delta.
919 # File index stores a pointer to its delta and the parent delta.
918 # The parent delta is stored via a pointer to the fileindex PK.
920 # The parent delta is stored via a pointer to the fileindex PK.
919 if deltabase == nullid:
921 if deltabase == nullid:
920 baseid = None
922 baseid = None
921 else:
923 else:
922 baseid = self._revisions[deltabase].rid
924 baseid = self._revisions[deltabase].rid
923
925
924 # Deltas are stored with a hash of their content. This allows
926 # Deltas are stored with a hash of their content. This allows
925 # us to de-duplicate. The table is configured to ignore conflicts
927 # us to de-duplicate. The table is configured to ignore conflicts
926 # and it is faster to just insert and silently noop than to look
928 # and it is faster to just insert and silently noop than to look
927 # first.
929 # first.
928 deltahash = hashlib.sha1(delta).digest()
930 deltahash = hashlib.sha1(delta).digest()
929
931
930 if self._compengine == 'zstd':
932 if self._compengine == 'zstd':
931 deltablob = self._cctx.compress(delta)
933 deltablob = self._cctx.compress(delta)
932 compression = COMPRESSION_ZSTD
934 compression = COMPRESSION_ZSTD
933 elif self._compengine == 'zlib':
935 elif self._compengine == 'zlib':
934 deltablob = zlib.compress(delta)
936 deltablob = zlib.compress(delta)
935 compression = COMPRESSION_ZLIB
937 compression = COMPRESSION_ZLIB
936 elif self._compengine == 'none':
938 elif self._compengine == 'none':
937 deltablob = delta
939 deltablob = delta
938 compression = COMPRESSION_NONE
940 compression = COMPRESSION_NONE
939 else:
941 else:
940 raise error.ProgrammingError('unhandled compression engine: %s' %
942 raise error.ProgrammingError('unhandled compression engine: %s' %
941 self._compengine)
943 self._compengine)
942
944
943 # Don't store compressed data if it isn't practical.
945 # Don't store compressed data if it isn't practical.
944 if len(deltablob) >= len(delta):
946 if len(deltablob) >= len(delta):
945 deltablob = delta
947 deltablob = delta
946 compression = COMPRESSION_NONE
948 compression = COMPRESSION_NONE
947
949
948 deltaid = insertdelta(self._db, compression, deltahash, deltablob)
950 deltaid = insertdelta(self._db, compression, deltahash, deltablob)
949
951
950 rev = len(self)
952 rev = len(self)
951
953
952 if p1 == nullid:
954 if p1 == nullid:
953 p1rev = nullrev
955 p1rev = nullrev
954 else:
956 else:
955 p1rev = self._nodetorev[p1]
957 p1rev = self._nodetorev[p1]
956
958
957 if p2 == nullid:
959 if p2 == nullid:
958 p2rev = nullrev
960 p2rev = nullrev
959 else:
961 else:
960 p2rev = self._nodetorev[p2]
962 p2rev = self._nodetorev[p2]
961
963
962 rid = self._db.execute(
964 rid = self._db.execute(
963 r'INSERT INTO fileindex ('
965 r'INSERT INTO fileindex ('
964 r' pathid, revnum, node, p1rev, p2rev, linkrev, flags, '
966 r' pathid, revnum, node, p1rev, p2rev, linkrev, flags, '
965 r' deltaid, deltabaseid) '
967 r' deltaid, deltabaseid) '
966 r' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
968 r' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
967 (self._pathid, rev, node, p1rev, p2rev, linkrev, flags,
969 (self._pathid, rev, node, p1rev, p2rev, linkrev, flags,
968 deltaid, baseid)
970 deltaid, baseid)
969 ).lastrowid
971 ).lastrowid
970
972
971 entry = revisionentry(
973 entry = revisionentry(
972 rid=rid,
974 rid=rid,
973 rev=rev,
975 rev=rev,
974 node=node,
976 node=node,
975 p1rev=p1rev,
977 p1rev=p1rev,
976 p2rev=p2rev,
978 p2rev=p2rev,
977 p1node=p1,
979 p1node=p1,
978 p2node=p2,
980 p2node=p2,
979 linkrev=linkrev,
981 linkrev=linkrev,
980 flags=flags)
982 flags=flags)
981
983
982 self._nodetorev[node] = rev
984 self._nodetorev[node] = rev
983 self._revtonode[rev] = node
985 self._revtonode[rev] = node
984 self._revisions[node] = entry
986 self._revisions[node] = entry
985
987
986 return node
988 return node
987
989
988 class sqliterepository(localrepo.localrepository):
990 class sqliterepository(localrepo.localrepository):
989 def cancopy(self):
991 def cancopy(self):
990 return False
992 return False
991
993
992 def transaction(self, *args, **kwargs):
994 def transaction(self, *args, **kwargs):
993 current = self.currenttransaction()
995 current = self.currenttransaction()
994
996
995 tr = super(sqliterepository, self).transaction(*args, **kwargs)
997 tr = super(sqliterepository, self).transaction(*args, **kwargs)
996
998
997 if current:
999 if current:
998 return tr
1000 return tr
999
1001
1000 self._dbconn.execute(r'BEGIN TRANSACTION')
1002 self._dbconn.execute(r'BEGIN TRANSACTION')
1001
1003
1002 def committransaction(_):
1004 def committransaction(_):
1003 self._dbconn.commit()
1005 self._dbconn.commit()
1004
1006
1005 tr.addfinalize('sqlitestore', committransaction)
1007 tr.addfinalize('sqlitestore', committransaction)
1006
1008
1007 return tr
1009 return tr
1008
1010
1009 @property
1011 @property
1010 def _dbconn(self):
1012 def _dbconn(self):
1011 # SQLite connections can only be used on the thread that created
1013 # SQLite connections can only be used on the thread that created
1012 # them. In most cases, this "just works." However, hgweb uses
1014 # them. In most cases, this "just works." However, hgweb uses
1013 # multiple threads.
1015 # multiple threads.
1014 tid = threading.current_thread().ident
1016 tid = threading.current_thread().ident
1015
1017
1016 if self._db:
1018 if self._db:
1017 if self._db[0] == tid:
1019 if self._db[0] == tid:
1018 return self._db[1]
1020 return self._db[1]
1019
1021
1020 db = makedb(self.svfs.join('db.sqlite'))
1022 db = makedb(self.svfs.join('db.sqlite'))
1021 self._db = (tid, db)
1023 self._db = (tid, db)
1022
1024
1023 return db
1025 return db
1024
1026
1025 def makedb(path):
1027 def makedb(path):
1026 """Construct a database handle for a database at path."""
1028 """Construct a database handle for a database at path."""
1027
1029
1028 db = sqlite3.connect(encoding.strfromlocal(path))
1030 db = sqlite3.connect(encoding.strfromlocal(path))
1029 db.text_factory = bytes
1031 db.text_factory = bytes
1030
1032
1031 res = db.execute(r'PRAGMA user_version').fetchone()[0]
1033 res = db.execute(r'PRAGMA user_version').fetchone()[0]
1032
1034
1033 # New database.
1035 # New database.
1034 if res == 0:
1036 if res == 0:
1035 for statement in CREATE_SCHEMA:
1037 for statement in CREATE_SCHEMA:
1036 db.execute(statement)
1038 db.execute(statement)
1037
1039
1038 db.commit()
1040 db.commit()
1039
1041
1040 elif res == CURRENT_SCHEMA_VERSION:
1042 elif res == CURRENT_SCHEMA_VERSION:
1041 pass
1043 pass
1042
1044
1043 else:
1045 else:
1044 raise error.Abort(_('sqlite database has unrecognized version'))
1046 raise error.Abort(_('sqlite database has unrecognized version'))
1045
1047
1046 db.execute(r'PRAGMA journal_mode=WAL')
1048 db.execute(r'PRAGMA journal_mode=WAL')
1047
1049
1048 return db
1050 return db
1049
1051
1050 def featuresetup(ui, supported):
1052 def featuresetup(ui, supported):
1051 supported.add(REQUIREMENT)
1053 supported.add(REQUIREMENT)
1052
1054
1053 if zstd:
1055 if zstd:
1054 supported.add(REQUIREMENT_ZSTD)
1056 supported.add(REQUIREMENT_ZSTD)
1055
1057
1056 supported.add(REQUIREMENT_ZLIB)
1058 supported.add(REQUIREMENT_ZLIB)
1057 supported.add(REQUIREMENT_NONE)
1059 supported.add(REQUIREMENT_NONE)
1058 supported.add(REQUIREMENT_SHALLOW_FILES)
1060 supported.add(REQUIREMENT_SHALLOW_FILES)
1059 supported.add(repository.NARROW_REQUIREMENT)
1061 supported.add(repository.NARROW_REQUIREMENT)
1060
1062
1061 def newreporequirements(orig, ui, createopts):
1063 def newreporequirements(orig, ui, createopts):
1062 if createopts['backend'] != 'sqlite':
1064 if createopts['backend'] != 'sqlite':
1063 return orig(ui, createopts)
1065 return orig(ui, createopts)
1064
1066
1065 # This restriction can be lifted once we have more confidence.
1067 # This restriction can be lifted once we have more confidence.
1066 if 'sharedrepo' in createopts:
1068 if 'sharedrepo' in createopts:
1067 raise error.Abort(_('shared repositories not supported with SQLite '
1069 raise error.Abort(_('shared repositories not supported with SQLite '
1068 'store'))
1070 'store'))
1069
1071
1070 # This filtering is out of an abundance of caution: we want to ensure
1072 # This filtering is out of an abundance of caution: we want to ensure
1071 # we honor creation options and we do that by annotating exactly the
1073 # we honor creation options and we do that by annotating exactly the
1072 # creation options we recognize.
1074 # creation options we recognize.
1073 known = {
1075 known = {
1074 'narrowfiles',
1076 'narrowfiles',
1075 'backend',
1077 'backend',
1076 'shallowfilestore',
1078 'shallowfilestore',
1077 }
1079 }
1078
1080
1079 unsupported = set(createopts) - known
1081 unsupported = set(createopts) - known
1080 if unsupported:
1082 if unsupported:
1081 raise error.Abort(_('SQLite store does not support repo creation '
1083 raise error.Abort(_('SQLite store does not support repo creation '
1082 'option: %s') % ', '.join(sorted(unsupported)))
1084 'option: %s') % ', '.join(sorted(unsupported)))
1083
1085
1084 # Since we're a hybrid store that still relies on revlogs, we fall back
1086 # Since we're a hybrid store that still relies on revlogs, we fall back
1085 # to using the revlogv1 backend's storage requirements then adding our
1087 # to using the revlogv1 backend's storage requirements then adding our
1086 # own requirement.
1088 # own requirement.
1087 createopts['backend'] = 'revlogv1'
1089 createopts['backend'] = 'revlogv1'
1088 requirements = orig(ui, createopts)
1090 requirements = orig(ui, createopts)
1089 requirements.add(REQUIREMENT)
1091 requirements.add(REQUIREMENT)
1090
1092
1091 compression = ui.config('storage', 'sqlite.compression')
1093 compression = ui.config('storage', 'sqlite.compression')
1092
1094
1093 if compression == 'zstd' and not zstd:
1095 if compression == 'zstd' and not zstd:
1094 raise error.Abort(_('storage.sqlite.compression set to "zstd" but '
1096 raise error.Abort(_('storage.sqlite.compression set to "zstd" but '
1095 'zstandard compression not available to this '
1097 'zstandard compression not available to this '
1096 'Mercurial install'))
1098 'Mercurial install'))
1097
1099
1098 if compression == 'zstd':
1100 if compression == 'zstd':
1099 requirements.add(REQUIREMENT_ZSTD)
1101 requirements.add(REQUIREMENT_ZSTD)
1100 elif compression == 'zlib':
1102 elif compression == 'zlib':
1101 requirements.add(REQUIREMENT_ZLIB)
1103 requirements.add(REQUIREMENT_ZLIB)
1102 elif compression == 'none':
1104 elif compression == 'none':
1103 requirements.add(REQUIREMENT_NONE)
1105 requirements.add(REQUIREMENT_NONE)
1104 else:
1106 else:
1105 raise error.Abort(_('unknown compression engine defined in '
1107 raise error.Abort(_('unknown compression engine defined in '
1106 'storage.sqlite.compression: %s') % compression)
1108 'storage.sqlite.compression: %s') % compression)
1107
1109
1108 if createopts.get('shallowfilestore'):
1110 if createopts.get('shallowfilestore'):
1109 requirements.add(REQUIREMENT_SHALLOW_FILES)
1111 requirements.add(REQUIREMENT_SHALLOW_FILES)
1110
1112
1111 return requirements
1113 return requirements
1112
1114
1113 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
1115 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
1114 class sqlitefilestorage(object):
1116 class sqlitefilestorage(object):
1115 """Repository file storage backed by SQLite."""
1117 """Repository file storage backed by SQLite."""
1116 def file(self, path):
1118 def file(self, path):
1117 if path[0] == b'/':
1119 if path[0] == b'/':
1118 path = path[1:]
1120 path = path[1:]
1119
1121
1120 if REQUIREMENT_ZSTD in self.requirements:
1122 if REQUIREMENT_ZSTD in self.requirements:
1121 compression = 'zstd'
1123 compression = 'zstd'
1122 elif REQUIREMENT_ZLIB in self.requirements:
1124 elif REQUIREMENT_ZLIB in self.requirements:
1123 compression = 'zlib'
1125 compression = 'zlib'
1124 elif REQUIREMENT_NONE in self.requirements:
1126 elif REQUIREMENT_NONE in self.requirements:
1125 compression = 'none'
1127 compression = 'none'
1126 else:
1128 else:
1127 raise error.Abort(_('unable to determine what compression engine '
1129 raise error.Abort(_('unable to determine what compression engine '
1128 'to use for SQLite storage'))
1130 'to use for SQLite storage'))
1129
1131
1130 return sqlitefilestore(self._dbconn, path, compression)
1132 return sqlitefilestore(self._dbconn, path, compression)
1131
1133
1132 def makefilestorage(orig, requirements, features, **kwargs):
1134 def makefilestorage(orig, requirements, features, **kwargs):
1133 """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
1135 """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
1134 if REQUIREMENT in requirements:
1136 if REQUIREMENT in requirements:
1135 if REQUIREMENT_SHALLOW_FILES in requirements:
1137 if REQUIREMENT_SHALLOW_FILES in requirements:
1136 features.add(repository.REPO_FEATURE_SHALLOW_FILE_STORAGE)
1138 features.add(repository.REPO_FEATURE_SHALLOW_FILE_STORAGE)
1137
1139
1138 return sqlitefilestorage
1140 return sqlitefilestorage
1139 else:
1141 else:
1140 return orig(requirements=requirements, features=features, **kwargs)
1142 return orig(requirements=requirements, features=features, **kwargs)
1141
1143
1142 def makemain(orig, ui, requirements, **kwargs):
1144 def makemain(orig, ui, requirements, **kwargs):
1143 if REQUIREMENT in requirements:
1145 if REQUIREMENT in requirements:
1144 if REQUIREMENT_ZSTD in requirements and not zstd:
1146 if REQUIREMENT_ZSTD in requirements and not zstd:
1145 raise error.Abort(_('repository uses zstandard compression, which '
1147 raise error.Abort(_('repository uses zstandard compression, which '
1146 'is not available to this Mercurial install'))
1148 'is not available to this Mercurial install'))
1147
1149
1148 return sqliterepository
1150 return sqliterepository
1149
1151
1150 return orig(requirements=requirements, **kwargs)
1152 return orig(requirements=requirements, **kwargs)
1151
1153
1152 def verifierinit(orig, self, *args, **kwargs):
1154 def verifierinit(orig, self, *args, **kwargs):
1153 orig(self, *args, **kwargs)
1155 orig(self, *args, **kwargs)
1154
1156
1155 # We don't care that files in the store don't align with what is
1157 # We don't care that files in the store don't align with what is
1156 # advertised. So suppress these warnings.
1158 # advertised. So suppress these warnings.
1157 self.warnorphanstorefiles = False
1159 self.warnorphanstorefiles = False
1158
1160
1159 def extsetup(ui):
1161 def extsetup(ui):
1160 localrepo.featuresetupfuncs.add(featuresetup)
1162 localrepo.featuresetupfuncs.add(featuresetup)
1161 extensions.wrapfunction(localrepo, 'newreporequirements',
1163 extensions.wrapfunction(localrepo, 'newreporequirements',
1162 newreporequirements)
1164 newreporequirements)
1163 extensions.wrapfunction(localrepo, 'makefilestorage',
1165 extensions.wrapfunction(localrepo, 'makefilestorage',
1164 makefilestorage)
1166 makefilestorage)
1165 extensions.wrapfunction(localrepo, 'makemain',
1167 extensions.wrapfunction(localrepo, 'makemain',
1166 makemain)
1168 makemain)
1167 extensions.wrapfunction(verify.verifier, '__init__',
1169 extensions.wrapfunction(verify.verifier, '__init__',
1168 verifierinit)
1170 verifierinit)
1169
1171
1170 def reposetup(ui, repo):
1172 def reposetup(ui, repo):
1171 if isinstance(repo, sqliterepository):
1173 if isinstance(repo, sqliterepository):
1172 repo._db = None
1174 repo._db = None
1173
1175
1174 # TODO check for bundlerepository?
1176 # TODO check for bundlerepository?
@@ -1,1423 +1,1426
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import weakref
12 import weakref
13
13
14 from .i18n import _
14 from .i18n import _
15 from .node import (
15 from .node import (
16 hex,
16 hex,
17 nullid,
17 nullid,
18 nullrev,
18 nullrev,
19 short,
19 short,
20 )
20 )
21
21
22 from . import (
22 from . import (
23 error,
23 error,
24 match as matchmod,
24 match as matchmod,
25 mdiff,
25 mdiff,
26 phases,
26 phases,
27 pycompat,
27 pycompat,
28 util,
29 )
30
31 from .interfaces import (
28 repository,
32 repository,
29 util,
30 )
33 )
31
34
32 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
35 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
33 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
36 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
34 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
37 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
35
38
36 LFS_REQUIREMENT = 'lfs'
39 LFS_REQUIREMENT = 'lfs'
37
40
38 readexactly = util.readexactly
41 readexactly = util.readexactly
39
42
40 def getchunk(stream):
43 def getchunk(stream):
41 """return the next chunk from stream as a string"""
44 """return the next chunk from stream as a string"""
42 d = readexactly(stream, 4)
45 d = readexactly(stream, 4)
43 l = struct.unpack(">l", d)[0]
46 l = struct.unpack(">l", d)[0]
44 if l <= 4:
47 if l <= 4:
45 if l:
48 if l:
46 raise error.Abort(_("invalid chunk length %d") % l)
49 raise error.Abort(_("invalid chunk length %d") % l)
47 return ""
50 return ""
48 return readexactly(stream, l - 4)
51 return readexactly(stream, l - 4)
49
52
50 def chunkheader(length):
53 def chunkheader(length):
51 """return a changegroup chunk header (string)"""
54 """return a changegroup chunk header (string)"""
52 return struct.pack(">l", length + 4)
55 return struct.pack(">l", length + 4)
53
56
54 def closechunk():
57 def closechunk():
55 """return a changegroup chunk header (string) for a zero-length chunk"""
58 """return a changegroup chunk header (string) for a zero-length chunk"""
56 return struct.pack(">l", 0)
59 return struct.pack(">l", 0)
57
60
58 def _fileheader(path):
61 def _fileheader(path):
59 """Obtain a changegroup chunk header for a named path."""
62 """Obtain a changegroup chunk header for a named path."""
60 return chunkheader(len(path)) + path
63 return chunkheader(len(path)) + path
61
64
62 def writechunks(ui, chunks, filename, vfs=None):
65 def writechunks(ui, chunks, filename, vfs=None):
63 """Write chunks to a file and return its filename.
66 """Write chunks to a file and return its filename.
64
67
65 The stream is assumed to be a bundle file.
68 The stream is assumed to be a bundle file.
66 Existing files will not be overwritten.
69 Existing files will not be overwritten.
67 If no filename is specified, a temporary file is created.
70 If no filename is specified, a temporary file is created.
68 """
71 """
69 fh = None
72 fh = None
70 cleanup = None
73 cleanup = None
71 try:
74 try:
72 if filename:
75 if filename:
73 if vfs:
76 if vfs:
74 fh = vfs.open(filename, "wb")
77 fh = vfs.open(filename, "wb")
75 else:
78 else:
76 # Increase default buffer size because default is usually
79 # Increase default buffer size because default is usually
77 # small (4k is common on Linux).
80 # small (4k is common on Linux).
78 fh = open(filename, "wb", 131072)
81 fh = open(filename, "wb", 131072)
79 else:
82 else:
80 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
83 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
81 fh = os.fdopen(fd, r"wb")
84 fh = os.fdopen(fd, r"wb")
82 cleanup = filename
85 cleanup = filename
83 for c in chunks:
86 for c in chunks:
84 fh.write(c)
87 fh.write(c)
85 cleanup = None
88 cleanup = None
86 return filename
89 return filename
87 finally:
90 finally:
88 if fh is not None:
91 if fh is not None:
89 fh.close()
92 fh.close()
90 if cleanup is not None:
93 if cleanup is not None:
91 if filename and vfs:
94 if filename and vfs:
92 vfs.unlink(cleanup)
95 vfs.unlink(cleanup)
93 else:
96 else:
94 os.unlink(cleanup)
97 os.unlink(cleanup)
95
98
96 class cg1unpacker(object):
99 class cg1unpacker(object):
97 """Unpacker for cg1 changegroup streams.
100 """Unpacker for cg1 changegroup streams.
98
101
99 A changegroup unpacker handles the framing of the revision data in
102 A changegroup unpacker handles the framing of the revision data in
100 the wire format. Most consumers will want to use the apply()
103 the wire format. Most consumers will want to use the apply()
101 method to add the changes from the changegroup to a repository.
104 method to add the changes from the changegroup to a repository.
102
105
103 If you're forwarding a changegroup unmodified to another consumer,
106 If you're forwarding a changegroup unmodified to another consumer,
104 use getchunks(), which returns an iterator of changegroup
107 use getchunks(), which returns an iterator of changegroup
105 chunks. This is mostly useful for cases where you need to know the
108 chunks. This is mostly useful for cases where you need to know the
106 data stream has ended by observing the end of the changegroup.
109 data stream has ended by observing the end of the changegroup.
107
110
108 deltachunk() is useful only if you're applying delta data. Most
111 deltachunk() is useful only if you're applying delta data. Most
109 consumers should prefer apply() instead.
112 consumers should prefer apply() instead.
110
113
111 A few other public methods exist. Those are used only for
114 A few other public methods exist. Those are used only for
112 bundlerepo and some debug commands - their use is discouraged.
115 bundlerepo and some debug commands - their use is discouraged.
113 """
116 """
114 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
117 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
115 deltaheadersize = deltaheader.size
118 deltaheadersize = deltaheader.size
116 version = '01'
119 version = '01'
117 _grouplistcount = 1 # One list of files after the manifests
120 _grouplistcount = 1 # One list of files after the manifests
118
121
119 def __init__(self, fh, alg, extras=None):
122 def __init__(self, fh, alg, extras=None):
120 if alg is None:
123 if alg is None:
121 alg = 'UN'
124 alg = 'UN'
122 if alg not in util.compengines.supportedbundletypes:
125 if alg not in util.compengines.supportedbundletypes:
123 raise error.Abort(_('unknown stream compression type: %s')
126 raise error.Abort(_('unknown stream compression type: %s')
124 % alg)
127 % alg)
125 if alg == 'BZ':
128 if alg == 'BZ':
126 alg = '_truncatedBZ'
129 alg = '_truncatedBZ'
127
130
128 compengine = util.compengines.forbundletype(alg)
131 compengine = util.compengines.forbundletype(alg)
129 self._stream = compengine.decompressorreader(fh)
132 self._stream = compengine.decompressorreader(fh)
130 self._type = alg
133 self._type = alg
131 self.extras = extras or {}
134 self.extras = extras or {}
132 self.callback = None
135 self.callback = None
133
136
134 # These methods (compressed, read, seek, tell) all appear to only
137 # These methods (compressed, read, seek, tell) all appear to only
135 # be used by bundlerepo, but it's a little hard to tell.
138 # be used by bundlerepo, but it's a little hard to tell.
136 def compressed(self):
139 def compressed(self):
137 return self._type is not None and self._type != 'UN'
140 return self._type is not None and self._type != 'UN'
138 def read(self, l):
141 def read(self, l):
139 return self._stream.read(l)
142 return self._stream.read(l)
140 def seek(self, pos):
143 def seek(self, pos):
141 return self._stream.seek(pos)
144 return self._stream.seek(pos)
142 def tell(self):
145 def tell(self):
143 return self._stream.tell()
146 return self._stream.tell()
144 def close(self):
147 def close(self):
145 return self._stream.close()
148 return self._stream.close()
146
149
147 def _chunklength(self):
150 def _chunklength(self):
148 d = readexactly(self._stream, 4)
151 d = readexactly(self._stream, 4)
149 l = struct.unpack(">l", d)[0]
152 l = struct.unpack(">l", d)[0]
150 if l <= 4:
153 if l <= 4:
151 if l:
154 if l:
152 raise error.Abort(_("invalid chunk length %d") % l)
155 raise error.Abort(_("invalid chunk length %d") % l)
153 return 0
156 return 0
154 if self.callback:
157 if self.callback:
155 self.callback()
158 self.callback()
156 return l - 4
159 return l - 4
157
160
158 def changelogheader(self):
161 def changelogheader(self):
159 """v10 does not have a changelog header chunk"""
162 """v10 does not have a changelog header chunk"""
160 return {}
163 return {}
161
164
162 def manifestheader(self):
165 def manifestheader(self):
163 """v10 does not have a manifest header chunk"""
166 """v10 does not have a manifest header chunk"""
164 return {}
167 return {}
165
168
166 def filelogheader(self):
169 def filelogheader(self):
167 """return the header of the filelogs chunk, v10 only has the filename"""
170 """return the header of the filelogs chunk, v10 only has the filename"""
168 l = self._chunklength()
171 l = self._chunklength()
169 if not l:
172 if not l:
170 return {}
173 return {}
171 fname = readexactly(self._stream, l)
174 fname = readexactly(self._stream, l)
172 return {'filename': fname}
175 return {'filename': fname}
173
176
174 def _deltaheader(self, headertuple, prevnode):
177 def _deltaheader(self, headertuple, prevnode):
175 node, p1, p2, cs = headertuple
178 node, p1, p2, cs = headertuple
176 if prevnode is None:
179 if prevnode is None:
177 deltabase = p1
180 deltabase = p1
178 else:
181 else:
179 deltabase = prevnode
182 deltabase = prevnode
180 flags = 0
183 flags = 0
181 return node, p1, p2, deltabase, cs, flags
184 return node, p1, p2, deltabase, cs, flags
182
185
183 def deltachunk(self, prevnode):
186 def deltachunk(self, prevnode):
184 l = self._chunklength()
187 l = self._chunklength()
185 if not l:
188 if not l:
186 return {}
189 return {}
187 headerdata = readexactly(self._stream, self.deltaheadersize)
190 headerdata = readexactly(self._stream, self.deltaheadersize)
188 header = self.deltaheader.unpack(headerdata)
191 header = self.deltaheader.unpack(headerdata)
189 delta = readexactly(self._stream, l - self.deltaheadersize)
192 delta = readexactly(self._stream, l - self.deltaheadersize)
190 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
193 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
191 return (node, p1, p2, cs, deltabase, delta, flags)
194 return (node, p1, p2, cs, deltabase, delta, flags)
192
195
193 def getchunks(self):
196 def getchunks(self):
194 """returns all the chunks contains in the bundle
197 """returns all the chunks contains in the bundle
195
198
196 Used when you need to forward the binary stream to a file or another
199 Used when you need to forward the binary stream to a file or another
197 network API. To do so, it parse the changegroup data, otherwise it will
200 network API. To do so, it parse the changegroup data, otherwise it will
198 block in case of sshrepo because it don't know the end of the stream.
201 block in case of sshrepo because it don't know the end of the stream.
199 """
202 """
200 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
203 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
201 # and a list of filelogs. For changegroup 3, we expect 4 parts:
204 # and a list of filelogs. For changegroup 3, we expect 4 parts:
202 # changelog, manifestlog, a list of tree manifestlogs, and a list of
205 # changelog, manifestlog, a list of tree manifestlogs, and a list of
203 # filelogs.
206 # filelogs.
204 #
207 #
205 # Changelog and manifestlog parts are terminated with empty chunks. The
208 # Changelog and manifestlog parts are terminated with empty chunks. The
206 # tree and file parts are a list of entry sections. Each entry section
209 # tree and file parts are a list of entry sections. Each entry section
207 # is a series of chunks terminating in an empty chunk. The list of these
210 # is a series of chunks terminating in an empty chunk. The list of these
208 # entry sections is terminated in yet another empty chunk, so we know
211 # entry sections is terminated in yet another empty chunk, so we know
209 # we've reached the end of the tree/file list when we reach an empty
212 # we've reached the end of the tree/file list when we reach an empty
210 # chunk that was proceeded by no non-empty chunks.
213 # chunk that was proceeded by no non-empty chunks.
211
214
212 parts = 0
215 parts = 0
213 while parts < 2 + self._grouplistcount:
216 while parts < 2 + self._grouplistcount:
214 noentries = True
217 noentries = True
215 while True:
218 while True:
216 chunk = getchunk(self)
219 chunk = getchunk(self)
217 if not chunk:
220 if not chunk:
218 # The first two empty chunks represent the end of the
221 # The first two empty chunks represent the end of the
219 # changelog and the manifestlog portions. The remaining
222 # changelog and the manifestlog portions. The remaining
220 # empty chunks represent either A) the end of individual
223 # empty chunks represent either A) the end of individual
221 # tree or file entries in the file list, or B) the end of
224 # tree or file entries in the file list, or B) the end of
222 # the entire list. It's the end of the entire list if there
225 # the entire list. It's the end of the entire list if there
223 # were no entries (i.e. noentries is True).
226 # were no entries (i.e. noentries is True).
224 if parts < 2:
227 if parts < 2:
225 parts += 1
228 parts += 1
226 elif noentries:
229 elif noentries:
227 parts += 1
230 parts += 1
228 break
231 break
229 noentries = False
232 noentries = False
230 yield chunkheader(len(chunk))
233 yield chunkheader(len(chunk))
231 pos = 0
234 pos = 0
232 while pos < len(chunk):
235 while pos < len(chunk):
233 next = pos + 2**20
236 next = pos + 2**20
234 yield chunk[pos:next]
237 yield chunk[pos:next]
235 pos = next
238 pos = next
236 yield closechunk()
239 yield closechunk()
237
240
238 def _unpackmanifests(self, repo, revmap, trp, prog):
241 def _unpackmanifests(self, repo, revmap, trp, prog):
239 self.callback = prog.increment
242 self.callback = prog.increment
240 # no need to check for empty manifest group here:
243 # no need to check for empty manifest group here:
241 # if the result of the merge of 1 and 2 is the same in 3 and 4,
244 # if the result of the merge of 1 and 2 is the same in 3 and 4,
242 # no new manifest will be created and the manifest group will
245 # no new manifest will be created and the manifest group will
243 # be empty during the pull
246 # be empty during the pull
244 self.manifestheader()
247 self.manifestheader()
245 deltas = self.deltaiter()
248 deltas = self.deltaiter()
246 repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp)
249 repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp)
247 prog.complete()
250 prog.complete()
248 self.callback = None
251 self.callback = None
249
252
250 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
253 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
251 expectedtotal=None):
254 expectedtotal=None):
252 """Add the changegroup returned by source.read() to this repo.
255 """Add the changegroup returned by source.read() to this repo.
253 srctype is a string like 'push', 'pull', or 'unbundle'. url is
256 srctype is a string like 'push', 'pull', or 'unbundle'. url is
254 the URL of the repo where this changegroup is coming from.
257 the URL of the repo where this changegroup is coming from.
255
258
256 Return an integer summarizing the change to this repo:
259 Return an integer summarizing the change to this repo:
257 - nothing changed or no source: 0
260 - nothing changed or no source: 0
258 - more heads than before: 1+added heads (2..n)
261 - more heads than before: 1+added heads (2..n)
259 - fewer heads than before: -1-removed heads (-2..-n)
262 - fewer heads than before: -1-removed heads (-2..-n)
260 - number of heads stays the same: 1
263 - number of heads stays the same: 1
261 """
264 """
262 repo = repo.unfiltered()
265 repo = repo.unfiltered()
263 def csmap(x):
266 def csmap(x):
264 repo.ui.debug("add changeset %s\n" % short(x))
267 repo.ui.debug("add changeset %s\n" % short(x))
265 return len(cl)
268 return len(cl)
266
269
267 def revmap(x):
270 def revmap(x):
268 return cl.rev(x)
271 return cl.rev(x)
269
272
270 changesets = files = revisions = 0
273 changesets = files = revisions = 0
271
274
272 try:
275 try:
273 # The transaction may already carry source information. In this
276 # The transaction may already carry source information. In this
274 # case we use the top level data. We overwrite the argument
277 # case we use the top level data. We overwrite the argument
275 # because we need to use the top level value (if they exist)
278 # because we need to use the top level value (if they exist)
276 # in this function.
279 # in this function.
277 srctype = tr.hookargs.setdefault('source', srctype)
280 srctype = tr.hookargs.setdefault('source', srctype)
278 tr.hookargs.setdefault('url', url)
281 tr.hookargs.setdefault('url', url)
279 repo.hook('prechangegroup',
282 repo.hook('prechangegroup',
280 throw=True, **pycompat.strkwargs(tr.hookargs))
283 throw=True, **pycompat.strkwargs(tr.hookargs))
281
284
282 # write changelog data to temp files so concurrent readers
285 # write changelog data to temp files so concurrent readers
283 # will not see an inconsistent view
286 # will not see an inconsistent view
284 cl = repo.changelog
287 cl = repo.changelog
285 cl.delayupdate(tr)
288 cl.delayupdate(tr)
286 oldheads = set(cl.heads())
289 oldheads = set(cl.heads())
287
290
288 trp = weakref.proxy(tr)
291 trp = weakref.proxy(tr)
289 # pull off the changeset group
292 # pull off the changeset group
290 repo.ui.status(_("adding changesets\n"))
293 repo.ui.status(_("adding changesets\n"))
291 clstart = len(cl)
294 clstart = len(cl)
292 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
295 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
293 total=expectedtotal)
296 total=expectedtotal)
294 self.callback = progress.increment
297 self.callback = progress.increment
295
298
296 efiles = set()
299 efiles = set()
297 def onchangelog(cl, node):
300 def onchangelog(cl, node):
298 efiles.update(cl.readfiles(node))
301 efiles.update(cl.readfiles(node))
299
302
300 self.changelogheader()
303 self.changelogheader()
301 deltas = self.deltaiter()
304 deltas = self.deltaiter()
302 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
305 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
303 efiles = len(efiles)
306 efiles = len(efiles)
304
307
305 if not cgnodes:
308 if not cgnodes:
306 repo.ui.develwarn('applied empty changelog from changegroup',
309 repo.ui.develwarn('applied empty changelog from changegroup',
307 config='warn-empty-changegroup')
310 config='warn-empty-changegroup')
308 clend = len(cl)
311 clend = len(cl)
309 changesets = clend - clstart
312 changesets = clend - clstart
310 progress.complete()
313 progress.complete()
311 self.callback = None
314 self.callback = None
312
315
313 # pull off the manifest group
316 # pull off the manifest group
314 repo.ui.status(_("adding manifests\n"))
317 repo.ui.status(_("adding manifests\n"))
315 # We know that we'll never have more manifests than we had
318 # We know that we'll never have more manifests than we had
316 # changesets.
319 # changesets.
317 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
320 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
318 total=changesets)
321 total=changesets)
319 self._unpackmanifests(repo, revmap, trp, progress)
322 self._unpackmanifests(repo, revmap, trp, progress)
320
323
321 needfiles = {}
324 needfiles = {}
322 if repo.ui.configbool('server', 'validate'):
325 if repo.ui.configbool('server', 'validate'):
323 cl = repo.changelog
326 cl = repo.changelog
324 ml = repo.manifestlog
327 ml = repo.manifestlog
325 # validate incoming csets have their manifests
328 # validate incoming csets have their manifests
326 for cset in pycompat.xrange(clstart, clend):
329 for cset in pycompat.xrange(clstart, clend):
327 mfnode = cl.changelogrevision(cset).manifest
330 mfnode = cl.changelogrevision(cset).manifest
328 mfest = ml[mfnode].readdelta()
331 mfest = ml[mfnode].readdelta()
329 # store file cgnodes we must see
332 # store file cgnodes we must see
330 for f, n in mfest.iteritems():
333 for f, n in mfest.iteritems():
331 needfiles.setdefault(f, set()).add(n)
334 needfiles.setdefault(f, set()).add(n)
332
335
333 # process the files
336 # process the files
334 repo.ui.status(_("adding file changes\n"))
337 repo.ui.status(_("adding file changes\n"))
335 newrevs, newfiles = _addchangegroupfiles(
338 newrevs, newfiles = _addchangegroupfiles(
336 repo, self, revmap, trp, efiles, needfiles)
339 repo, self, revmap, trp, efiles, needfiles)
337 revisions += newrevs
340 revisions += newrevs
338 files += newfiles
341 files += newfiles
339
342
340 deltaheads = 0
343 deltaheads = 0
341 if oldheads:
344 if oldheads:
342 heads = cl.heads()
345 heads = cl.heads()
343 deltaheads = len(heads) - len(oldheads)
346 deltaheads = len(heads) - len(oldheads)
344 for h in heads:
347 for h in heads:
345 if h not in oldheads and repo[h].closesbranch():
348 if h not in oldheads and repo[h].closesbranch():
346 deltaheads -= 1
349 deltaheads -= 1
347 htext = ""
350 htext = ""
348 if deltaheads:
351 if deltaheads:
349 htext = _(" (%+d heads)") % deltaheads
352 htext = _(" (%+d heads)") % deltaheads
350
353
351 repo.ui.status(_("added %d changesets"
354 repo.ui.status(_("added %d changesets"
352 " with %d changes to %d files%s\n")
355 " with %d changes to %d files%s\n")
353 % (changesets, revisions, files, htext))
356 % (changesets, revisions, files, htext))
354 repo.invalidatevolatilesets()
357 repo.invalidatevolatilesets()
355
358
356 if changesets > 0:
359 if changesets > 0:
357 if 'node' not in tr.hookargs:
360 if 'node' not in tr.hookargs:
358 tr.hookargs['node'] = hex(cl.node(clstart))
361 tr.hookargs['node'] = hex(cl.node(clstart))
359 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
362 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
360 hookargs = dict(tr.hookargs)
363 hookargs = dict(tr.hookargs)
361 else:
364 else:
362 hookargs = dict(tr.hookargs)
365 hookargs = dict(tr.hookargs)
363 hookargs['node'] = hex(cl.node(clstart))
366 hookargs['node'] = hex(cl.node(clstart))
364 hookargs['node_last'] = hex(cl.node(clend - 1))
367 hookargs['node_last'] = hex(cl.node(clend - 1))
365 repo.hook('pretxnchangegroup',
368 repo.hook('pretxnchangegroup',
366 throw=True, **pycompat.strkwargs(hookargs))
369 throw=True, **pycompat.strkwargs(hookargs))
367
370
368 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
371 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
369 phaseall = None
372 phaseall = None
370 if srctype in ('push', 'serve'):
373 if srctype in ('push', 'serve'):
371 # Old servers can not push the boundary themselves.
374 # Old servers can not push the boundary themselves.
372 # New servers won't push the boundary if changeset already
375 # New servers won't push the boundary if changeset already
373 # exists locally as secret
376 # exists locally as secret
374 #
377 #
375 # We should not use added here but the list of all change in
378 # We should not use added here but the list of all change in
376 # the bundle
379 # the bundle
377 if repo.publishing():
380 if repo.publishing():
378 targetphase = phaseall = phases.public
381 targetphase = phaseall = phases.public
379 else:
382 else:
380 # closer target phase computation
383 # closer target phase computation
381
384
382 # Those changesets have been pushed from the
385 # Those changesets have been pushed from the
383 # outside, their phases are going to be pushed
386 # outside, their phases are going to be pushed
384 # alongside. Therefor `targetphase` is
387 # alongside. Therefor `targetphase` is
385 # ignored.
388 # ignored.
386 targetphase = phaseall = phases.draft
389 targetphase = phaseall = phases.draft
387 if added:
390 if added:
388 phases.registernew(repo, tr, targetphase, added)
391 phases.registernew(repo, tr, targetphase, added)
389 if phaseall is not None:
392 if phaseall is not None:
390 phases.advanceboundary(repo, tr, phaseall, cgnodes)
393 phases.advanceboundary(repo, tr, phaseall, cgnodes)
391
394
392 if changesets > 0:
395 if changesets > 0:
393
396
394 def runhooks():
397 def runhooks():
395 # These hooks run when the lock releases, not when the
398 # These hooks run when the lock releases, not when the
396 # transaction closes. So it's possible for the changelog
399 # transaction closes. So it's possible for the changelog
397 # to have changed since we last saw it.
400 # to have changed since we last saw it.
398 if clstart >= len(repo):
401 if clstart >= len(repo):
399 return
402 return
400
403
401 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
404 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
402
405
403 for n in added:
406 for n in added:
404 args = hookargs.copy()
407 args = hookargs.copy()
405 args['node'] = hex(n)
408 args['node'] = hex(n)
406 del args['node_last']
409 del args['node_last']
407 repo.hook("incoming", **pycompat.strkwargs(args))
410 repo.hook("incoming", **pycompat.strkwargs(args))
408
411
409 newheads = [h for h in repo.heads()
412 newheads = [h for h in repo.heads()
410 if h not in oldheads]
413 if h not in oldheads]
411 repo.ui.log("incoming",
414 repo.ui.log("incoming",
412 "%d incoming changes - new heads: %s\n",
415 "%d incoming changes - new heads: %s\n",
413 len(added),
416 len(added),
414 ', '.join([hex(c[:6]) for c in newheads]))
417 ', '.join([hex(c[:6]) for c in newheads]))
415
418
416 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
419 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
417 lambda tr: repo._afterlock(runhooks))
420 lambda tr: repo._afterlock(runhooks))
418 finally:
421 finally:
419 repo.ui.flush()
422 repo.ui.flush()
420 # never return 0 here:
423 # never return 0 here:
421 if deltaheads < 0:
424 if deltaheads < 0:
422 ret = deltaheads - 1
425 ret = deltaheads - 1
423 else:
426 else:
424 ret = deltaheads + 1
427 ret = deltaheads + 1
425 return ret
428 return ret
426
429
427 def deltaiter(self):
430 def deltaiter(self):
428 """
431 """
429 returns an iterator of the deltas in this changegroup
432 returns an iterator of the deltas in this changegroup
430
433
431 Useful for passing to the underlying storage system to be stored.
434 Useful for passing to the underlying storage system to be stored.
432 """
435 """
433 chain = None
436 chain = None
434 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
437 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
435 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
438 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
436 yield chunkdata
439 yield chunkdata
437 chain = chunkdata[0]
440 chain = chunkdata[0]
438
441
439 class cg2unpacker(cg1unpacker):
442 class cg2unpacker(cg1unpacker):
440 """Unpacker for cg2 streams.
443 """Unpacker for cg2 streams.
441
444
442 cg2 streams add support for generaldelta, so the delta header
445 cg2 streams add support for generaldelta, so the delta header
443 format is slightly different. All other features about the data
446 format is slightly different. All other features about the data
444 remain the same.
447 remain the same.
445 """
448 """
446 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
449 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
447 deltaheadersize = deltaheader.size
450 deltaheadersize = deltaheader.size
448 version = '02'
451 version = '02'
449
452
450 def _deltaheader(self, headertuple, prevnode):
453 def _deltaheader(self, headertuple, prevnode):
451 node, p1, p2, deltabase, cs = headertuple
454 node, p1, p2, deltabase, cs = headertuple
452 flags = 0
455 flags = 0
453 return node, p1, p2, deltabase, cs, flags
456 return node, p1, p2, deltabase, cs, flags
454
457
455 class cg3unpacker(cg2unpacker):
458 class cg3unpacker(cg2unpacker):
456 """Unpacker for cg3 streams.
459 """Unpacker for cg3 streams.
457
460
458 cg3 streams add support for exchanging treemanifests and revlog
461 cg3 streams add support for exchanging treemanifests and revlog
459 flags. It adds the revlog flags to the delta header and an empty chunk
462 flags. It adds the revlog flags to the delta header and an empty chunk
460 separating manifests and files.
463 separating manifests and files.
461 """
464 """
462 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
465 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
463 deltaheadersize = deltaheader.size
466 deltaheadersize = deltaheader.size
464 version = '03'
467 version = '03'
465 _grouplistcount = 2 # One list of manifests and one list of files
468 _grouplistcount = 2 # One list of manifests and one list of files
466
469
467 def _deltaheader(self, headertuple, prevnode):
470 def _deltaheader(self, headertuple, prevnode):
468 node, p1, p2, deltabase, cs, flags = headertuple
471 node, p1, p2, deltabase, cs, flags = headertuple
469 return node, p1, p2, deltabase, cs, flags
472 return node, p1, p2, deltabase, cs, flags
470
473
471 def _unpackmanifests(self, repo, revmap, trp, prog):
474 def _unpackmanifests(self, repo, revmap, trp, prog):
472 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
475 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
473 for chunkdata in iter(self.filelogheader, {}):
476 for chunkdata in iter(self.filelogheader, {}):
474 # If we get here, there are directory manifests in the changegroup
477 # If we get here, there are directory manifests in the changegroup
475 d = chunkdata["filename"]
478 d = chunkdata["filename"]
476 repo.ui.debug("adding %s revisions\n" % d)
479 repo.ui.debug("adding %s revisions\n" % d)
477 deltas = self.deltaiter()
480 deltas = self.deltaiter()
478 if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
481 if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
479 raise error.Abort(_("received dir revlog group is empty"))
482 raise error.Abort(_("received dir revlog group is empty"))
480
483
481 class headerlessfixup(object):
484 class headerlessfixup(object):
482 def __init__(self, fh, h):
485 def __init__(self, fh, h):
483 self._h = h
486 self._h = h
484 self._fh = fh
487 self._fh = fh
485 def read(self, n):
488 def read(self, n):
486 if self._h:
489 if self._h:
487 d, self._h = self._h[:n], self._h[n:]
490 d, self._h = self._h[:n], self._h[n:]
488 if len(d) < n:
491 if len(d) < n:
489 d += readexactly(self._fh, n - len(d))
492 d += readexactly(self._fh, n - len(d))
490 return d
493 return d
491 return readexactly(self._fh, n)
494 return readexactly(self._fh, n)
492
495
493 def _revisiondeltatochunks(delta, headerfn):
496 def _revisiondeltatochunks(delta, headerfn):
494 """Serialize a revisiondelta to changegroup chunks."""
497 """Serialize a revisiondelta to changegroup chunks."""
495
498
496 # The captured revision delta may be encoded as a delta against
499 # The captured revision delta may be encoded as a delta against
497 # a base revision or as a full revision. The changegroup format
500 # a base revision or as a full revision. The changegroup format
498 # requires that everything on the wire be deltas. So for full
501 # requires that everything on the wire be deltas. So for full
499 # revisions, we need to invent a header that says to rewrite
502 # revisions, we need to invent a header that says to rewrite
500 # data.
503 # data.
501
504
502 if delta.delta is not None:
505 if delta.delta is not None:
503 prefix, data = b'', delta.delta
506 prefix, data = b'', delta.delta
504 elif delta.basenode == nullid:
507 elif delta.basenode == nullid:
505 data = delta.revision
508 data = delta.revision
506 prefix = mdiff.trivialdiffheader(len(data))
509 prefix = mdiff.trivialdiffheader(len(data))
507 else:
510 else:
508 data = delta.revision
511 data = delta.revision
509 prefix = mdiff.replacediffheader(delta.baserevisionsize,
512 prefix = mdiff.replacediffheader(delta.baserevisionsize,
510 len(data))
513 len(data))
511
514
512 meta = headerfn(delta)
515 meta = headerfn(delta)
513
516
514 yield chunkheader(len(meta) + len(prefix) + len(data))
517 yield chunkheader(len(meta) + len(prefix) + len(data))
515 yield meta
518 yield meta
516 if prefix:
519 if prefix:
517 yield prefix
520 yield prefix
518 yield data
521 yield data
519
522
520 def _sortnodesellipsis(store, nodes, cl, lookup):
523 def _sortnodesellipsis(store, nodes, cl, lookup):
521 """Sort nodes for changegroup generation."""
524 """Sort nodes for changegroup generation."""
522 # Ellipses serving mode.
525 # Ellipses serving mode.
523 #
526 #
524 # In a perfect world, we'd generate better ellipsis-ified graphs
527 # In a perfect world, we'd generate better ellipsis-ified graphs
525 # for non-changelog revlogs. In practice, we haven't started doing
528 # for non-changelog revlogs. In practice, we haven't started doing
526 # that yet, so the resulting DAGs for the manifestlog and filelogs
529 # that yet, so the resulting DAGs for the manifestlog and filelogs
527 # are actually full of bogus parentage on all the ellipsis
530 # are actually full of bogus parentage on all the ellipsis
528 # nodes. This has the side effect that, while the contents are
531 # nodes. This has the side effect that, while the contents are
529 # correct, the individual DAGs might be completely out of whack in
532 # correct, the individual DAGs might be completely out of whack in
530 # a case like 882681bc3166 and its ancestors (back about 10
533 # a case like 882681bc3166 and its ancestors (back about 10
531 # revisions or so) in the main hg repo.
534 # revisions or so) in the main hg repo.
532 #
535 #
533 # The one invariant we *know* holds is that the new (potentially
536 # The one invariant we *know* holds is that the new (potentially
534 # bogus) DAG shape will be valid if we order the nodes in the
537 # bogus) DAG shape will be valid if we order the nodes in the
535 # order that they're introduced in dramatis personae by the
538 # order that they're introduced in dramatis personae by the
536 # changelog, so what we do is we sort the non-changelog histories
539 # changelog, so what we do is we sort the non-changelog histories
537 # by the order in which they are used by the changelog.
540 # by the order in which they are used by the changelog.
538 key = lambda n: cl.rev(lookup(n))
541 key = lambda n: cl.rev(lookup(n))
539 return sorted(nodes, key=key)
542 return sorted(nodes, key=key)
540
543
541 def _resolvenarrowrevisioninfo(cl, store, ischangelog, rev, linkrev,
544 def _resolvenarrowrevisioninfo(cl, store, ischangelog, rev, linkrev,
542 linknode, clrevtolocalrev, fullclnodes,
545 linknode, clrevtolocalrev, fullclnodes,
543 precomputedellipsis):
546 precomputedellipsis):
544 linkparents = precomputedellipsis[linkrev]
547 linkparents = precomputedellipsis[linkrev]
545 def local(clrev):
548 def local(clrev):
546 """Turn a changelog revnum into a local revnum.
549 """Turn a changelog revnum into a local revnum.
547
550
548 The ellipsis dag is stored as revnums on the changelog,
551 The ellipsis dag is stored as revnums on the changelog,
549 but when we're producing ellipsis entries for
552 but when we're producing ellipsis entries for
550 non-changelog revlogs, we need to turn those numbers into
553 non-changelog revlogs, we need to turn those numbers into
551 something local. This does that for us, and during the
554 something local. This does that for us, and during the
552 changelog sending phase will also expand the stored
555 changelog sending phase will also expand the stored
553 mappings as needed.
556 mappings as needed.
554 """
557 """
555 if clrev == nullrev:
558 if clrev == nullrev:
556 return nullrev
559 return nullrev
557
560
558 if ischangelog:
561 if ischangelog:
559 return clrev
562 return clrev
560
563
561 # Walk the ellipsis-ized changelog breadth-first looking for a
564 # Walk the ellipsis-ized changelog breadth-first looking for a
562 # change that has been linked from the current revlog.
565 # change that has been linked from the current revlog.
563 #
566 #
564 # For a flat manifest revlog only a single step should be necessary
567 # For a flat manifest revlog only a single step should be necessary
565 # as all relevant changelog entries are relevant to the flat
568 # as all relevant changelog entries are relevant to the flat
566 # manifest.
569 # manifest.
567 #
570 #
568 # For a filelog or tree manifest dirlog however not every changelog
571 # For a filelog or tree manifest dirlog however not every changelog
569 # entry will have been relevant, so we need to skip some changelog
572 # entry will have been relevant, so we need to skip some changelog
570 # nodes even after ellipsis-izing.
573 # nodes even after ellipsis-izing.
571 walk = [clrev]
574 walk = [clrev]
572 while walk:
575 while walk:
573 p = walk[0]
576 p = walk[0]
574 walk = walk[1:]
577 walk = walk[1:]
575 if p in clrevtolocalrev:
578 if p in clrevtolocalrev:
576 return clrevtolocalrev[p]
579 return clrevtolocalrev[p]
577 elif p in fullclnodes:
580 elif p in fullclnodes:
578 walk.extend([pp for pp in cl.parentrevs(p)
581 walk.extend([pp for pp in cl.parentrevs(p)
579 if pp != nullrev])
582 if pp != nullrev])
580 elif p in precomputedellipsis:
583 elif p in precomputedellipsis:
581 walk.extend([pp for pp in precomputedellipsis[p]
584 walk.extend([pp for pp in precomputedellipsis[p]
582 if pp != nullrev])
585 if pp != nullrev])
583 else:
586 else:
584 # In this case, we've got an ellipsis with parents
587 # In this case, we've got an ellipsis with parents
585 # outside the current bundle (likely an
588 # outside the current bundle (likely an
586 # incremental pull). We "know" that we can use the
589 # incremental pull). We "know" that we can use the
587 # value of this same revlog at whatever revision
590 # value of this same revlog at whatever revision
588 # is pointed to by linknode. "Know" is in scare
591 # is pointed to by linknode. "Know" is in scare
589 # quotes because I haven't done enough examination
592 # quotes because I haven't done enough examination
590 # of edge cases to convince myself this is really
593 # of edge cases to convince myself this is really
591 # a fact - it works for all the (admittedly
594 # a fact - it works for all the (admittedly
592 # thorough) cases in our testsuite, but I would be
595 # thorough) cases in our testsuite, but I would be
593 # somewhat unsurprised to find a case in the wild
596 # somewhat unsurprised to find a case in the wild
594 # where this breaks down a bit. That said, I don't
597 # where this breaks down a bit. That said, I don't
595 # know if it would hurt anything.
598 # know if it would hurt anything.
596 for i in pycompat.xrange(rev, 0, -1):
599 for i in pycompat.xrange(rev, 0, -1):
597 if store.linkrev(i) == clrev:
600 if store.linkrev(i) == clrev:
598 return i
601 return i
599 # We failed to resolve a parent for this node, so
602 # We failed to resolve a parent for this node, so
600 # we crash the changegroup construction.
603 # we crash the changegroup construction.
601 raise error.Abort(
604 raise error.Abort(
602 'unable to resolve parent while packing %r %r'
605 'unable to resolve parent while packing %r %r'
603 ' for changeset %r' % (store.indexfile, rev, clrev))
606 ' for changeset %r' % (store.indexfile, rev, clrev))
604
607
605 return nullrev
608 return nullrev
606
609
607 if not linkparents or (
610 if not linkparents or (
608 store.parentrevs(rev) == (nullrev, nullrev)):
611 store.parentrevs(rev) == (nullrev, nullrev)):
609 p1, p2 = nullrev, nullrev
612 p1, p2 = nullrev, nullrev
610 elif len(linkparents) == 1:
613 elif len(linkparents) == 1:
611 p1, = sorted(local(p) for p in linkparents)
614 p1, = sorted(local(p) for p in linkparents)
612 p2 = nullrev
615 p2 = nullrev
613 else:
616 else:
614 p1, p2 = sorted(local(p) for p in linkparents)
617 p1, p2 = sorted(local(p) for p in linkparents)
615
618
616 p1node, p2node = store.node(p1), store.node(p2)
619 p1node, p2node = store.node(p1), store.node(p2)
617
620
618 return p1node, p2node, linknode
621 return p1node, p2node, linknode
619
622
620 def deltagroup(repo, store, nodes, ischangelog, lookup, forcedeltaparentprev,
623 def deltagroup(repo, store, nodes, ischangelog, lookup, forcedeltaparentprev,
621 topic=None,
624 topic=None,
622 ellipses=False, clrevtolocalrev=None, fullclnodes=None,
625 ellipses=False, clrevtolocalrev=None, fullclnodes=None,
623 precomputedellipsis=None):
626 precomputedellipsis=None):
624 """Calculate deltas for a set of revisions.
627 """Calculate deltas for a set of revisions.
625
628
626 Is a generator of ``revisiondelta`` instances.
629 Is a generator of ``revisiondelta`` instances.
627
630
628 If topic is not None, progress detail will be generated using this
631 If topic is not None, progress detail will be generated using this
629 topic name (e.g. changesets, manifests, etc).
632 topic name (e.g. changesets, manifests, etc).
630 """
633 """
631 if not nodes:
634 if not nodes:
632 return
635 return
633
636
634 cl = repo.changelog
637 cl = repo.changelog
635
638
636 if ischangelog:
639 if ischangelog:
637 # `hg log` shows changesets in storage order. To preserve order
640 # `hg log` shows changesets in storage order. To preserve order
638 # across clones, send out changesets in storage order.
641 # across clones, send out changesets in storage order.
639 nodesorder = 'storage'
642 nodesorder = 'storage'
640 elif ellipses:
643 elif ellipses:
641 nodes = _sortnodesellipsis(store, nodes, cl, lookup)
644 nodes = _sortnodesellipsis(store, nodes, cl, lookup)
642 nodesorder = 'nodes'
645 nodesorder = 'nodes'
643 else:
646 else:
644 nodesorder = None
647 nodesorder = None
645
648
646 # Perform ellipses filtering and revision massaging. We do this before
649 # Perform ellipses filtering and revision massaging. We do this before
647 # emitrevisions() because a) filtering out revisions creates less work
650 # emitrevisions() because a) filtering out revisions creates less work
648 # for emitrevisions() b) dropping revisions would break emitrevisions()'s
651 # for emitrevisions() b) dropping revisions would break emitrevisions()'s
649 # assumptions about delta choices and we would possibly send a delta
652 # assumptions about delta choices and we would possibly send a delta
650 # referencing a missing base revision.
653 # referencing a missing base revision.
651 #
654 #
652 # Also, calling lookup() has side-effects with regards to populating
655 # Also, calling lookup() has side-effects with regards to populating
653 # data structures. If we don't call lookup() for each node or if we call
656 # data structures. If we don't call lookup() for each node or if we call
654 # lookup() after the first pass through each node, things can break -
657 # lookup() after the first pass through each node, things can break -
655 # possibly intermittently depending on the python hash seed! For that
658 # possibly intermittently depending on the python hash seed! For that
656 # reason, we store a mapping of all linknodes during the initial node
659 # reason, we store a mapping of all linknodes during the initial node
657 # pass rather than use lookup() on the output side.
660 # pass rather than use lookup() on the output side.
658 if ellipses:
661 if ellipses:
659 filtered = []
662 filtered = []
660 adjustedparents = {}
663 adjustedparents = {}
661 linknodes = {}
664 linknodes = {}
662
665
663 for node in nodes:
666 for node in nodes:
664 rev = store.rev(node)
667 rev = store.rev(node)
665 linknode = lookup(node)
668 linknode = lookup(node)
666 linkrev = cl.rev(linknode)
669 linkrev = cl.rev(linknode)
667 clrevtolocalrev[linkrev] = rev
670 clrevtolocalrev[linkrev] = rev
668
671
669 # If linknode is in fullclnodes, it means the corresponding
672 # If linknode is in fullclnodes, it means the corresponding
670 # changeset was a full changeset and is being sent unaltered.
673 # changeset was a full changeset and is being sent unaltered.
671 if linknode in fullclnodes:
674 if linknode in fullclnodes:
672 linknodes[node] = linknode
675 linknodes[node] = linknode
673
676
674 # If the corresponding changeset wasn't in the set computed
677 # If the corresponding changeset wasn't in the set computed
675 # as relevant to us, it should be dropped outright.
678 # as relevant to us, it should be dropped outright.
676 elif linkrev not in precomputedellipsis:
679 elif linkrev not in precomputedellipsis:
677 continue
680 continue
678
681
679 else:
682 else:
680 # We could probably do this later and avoid the dict
683 # We could probably do this later and avoid the dict
681 # holding state. But it likely doesn't matter.
684 # holding state. But it likely doesn't matter.
682 p1node, p2node, linknode = _resolvenarrowrevisioninfo(
685 p1node, p2node, linknode = _resolvenarrowrevisioninfo(
683 cl, store, ischangelog, rev, linkrev, linknode,
686 cl, store, ischangelog, rev, linkrev, linknode,
684 clrevtolocalrev, fullclnodes, precomputedellipsis)
687 clrevtolocalrev, fullclnodes, precomputedellipsis)
685
688
686 adjustedparents[node] = (p1node, p2node)
689 adjustedparents[node] = (p1node, p2node)
687 linknodes[node] = linknode
690 linknodes[node] = linknode
688
691
689 filtered.append(node)
692 filtered.append(node)
690
693
691 nodes = filtered
694 nodes = filtered
692
695
693 # We expect the first pass to be fast, so we only engage the progress
696 # We expect the first pass to be fast, so we only engage the progress
694 # meter for constructing the revision deltas.
697 # meter for constructing the revision deltas.
695 progress = None
698 progress = None
696 if topic is not None:
699 if topic is not None:
697 progress = repo.ui.makeprogress(topic, unit=_('chunks'),
700 progress = repo.ui.makeprogress(topic, unit=_('chunks'),
698 total=len(nodes))
701 total=len(nodes))
699
702
700 configtarget = repo.ui.config('devel', 'bundle.delta')
703 configtarget = repo.ui.config('devel', 'bundle.delta')
701 if configtarget not in ('', 'p1', 'full'):
704 if configtarget not in ('', 'p1', 'full'):
702 msg = _("""config "devel.bundle.delta" as unknown value: %s""")
705 msg = _("""config "devel.bundle.delta" as unknown value: %s""")
703 repo.ui.warn(msg % configtarget)
706 repo.ui.warn(msg % configtarget)
704
707
705 deltamode = repository.CG_DELTAMODE_STD
708 deltamode = repository.CG_DELTAMODE_STD
706 if forcedeltaparentprev:
709 if forcedeltaparentprev:
707 deltamode = repository.CG_DELTAMODE_PREV
710 deltamode = repository.CG_DELTAMODE_PREV
708 elif configtarget == 'p1':
711 elif configtarget == 'p1':
709 deltamode = repository.CG_DELTAMODE_P1
712 deltamode = repository.CG_DELTAMODE_P1
710 elif configtarget == 'full':
713 elif configtarget == 'full':
711 deltamode = repository.CG_DELTAMODE_FULL
714 deltamode = repository.CG_DELTAMODE_FULL
712
715
713 revisions = store.emitrevisions(
716 revisions = store.emitrevisions(
714 nodes,
717 nodes,
715 nodesorder=nodesorder,
718 nodesorder=nodesorder,
716 revisiondata=True,
719 revisiondata=True,
717 assumehaveparentrevisions=not ellipses,
720 assumehaveparentrevisions=not ellipses,
718 deltamode=deltamode)
721 deltamode=deltamode)
719
722
720 for i, revision in enumerate(revisions):
723 for i, revision in enumerate(revisions):
721 if progress:
724 if progress:
722 progress.update(i + 1)
725 progress.update(i + 1)
723
726
724 if ellipses:
727 if ellipses:
725 linknode = linknodes[revision.node]
728 linknode = linknodes[revision.node]
726
729
727 if revision.node in adjustedparents:
730 if revision.node in adjustedparents:
728 p1node, p2node = adjustedparents[revision.node]
731 p1node, p2node = adjustedparents[revision.node]
729 revision.p1node = p1node
732 revision.p1node = p1node
730 revision.p2node = p2node
733 revision.p2node = p2node
731 revision.flags |= repository.REVISION_FLAG_ELLIPSIS
734 revision.flags |= repository.REVISION_FLAG_ELLIPSIS
732
735
733 else:
736 else:
734 linknode = lookup(revision.node)
737 linknode = lookup(revision.node)
735
738
736 revision.linknode = linknode
739 revision.linknode = linknode
737 yield revision
740 yield revision
738
741
739 if progress:
742 if progress:
740 progress.complete()
743 progress.complete()
741
744
742 class cgpacker(object):
745 class cgpacker(object):
743 def __init__(self, repo, oldmatcher, matcher, version,
746 def __init__(self, repo, oldmatcher, matcher, version,
744 builddeltaheader, manifestsend,
747 builddeltaheader, manifestsend,
745 forcedeltaparentprev=False,
748 forcedeltaparentprev=False,
746 bundlecaps=None, ellipses=False,
749 bundlecaps=None, ellipses=False,
747 shallow=False, ellipsisroots=None, fullnodes=None):
750 shallow=False, ellipsisroots=None, fullnodes=None):
748 """Given a source repo, construct a bundler.
751 """Given a source repo, construct a bundler.
749
752
750 oldmatcher is a matcher that matches on files the client already has.
753 oldmatcher is a matcher that matches on files the client already has.
751 These will not be included in the changegroup.
754 These will not be included in the changegroup.
752
755
753 matcher is a matcher that matches on files to include in the
756 matcher is a matcher that matches on files to include in the
754 changegroup. Used to facilitate sparse changegroups.
757 changegroup. Used to facilitate sparse changegroups.
755
758
756 forcedeltaparentprev indicates whether delta parents must be against
759 forcedeltaparentprev indicates whether delta parents must be against
757 the previous revision in a delta group. This should only be used for
760 the previous revision in a delta group. This should only be used for
758 compatibility with changegroup version 1.
761 compatibility with changegroup version 1.
759
762
760 builddeltaheader is a callable that constructs the header for a group
763 builddeltaheader is a callable that constructs the header for a group
761 delta.
764 delta.
762
765
763 manifestsend is a chunk to send after manifests have been fully emitted.
766 manifestsend is a chunk to send after manifests have been fully emitted.
764
767
765 ellipses indicates whether ellipsis serving mode is enabled.
768 ellipses indicates whether ellipsis serving mode is enabled.
766
769
767 bundlecaps is optional and can be used to specify the set of
770 bundlecaps is optional and can be used to specify the set of
768 capabilities which can be used to build the bundle. While bundlecaps is
771 capabilities which can be used to build the bundle. While bundlecaps is
769 unused in core Mercurial, extensions rely on this feature to communicate
772 unused in core Mercurial, extensions rely on this feature to communicate
770 capabilities to customize the changegroup packer.
773 capabilities to customize the changegroup packer.
771
774
772 shallow indicates whether shallow data might be sent. The packer may
775 shallow indicates whether shallow data might be sent. The packer may
773 need to pack file contents not introduced by the changes being packed.
776 need to pack file contents not introduced by the changes being packed.
774
777
775 fullnodes is the set of changelog nodes which should not be ellipsis
778 fullnodes is the set of changelog nodes which should not be ellipsis
776 nodes. We store this rather than the set of nodes that should be
779 nodes. We store this rather than the set of nodes that should be
777 ellipsis because for very large histories we expect this to be
780 ellipsis because for very large histories we expect this to be
778 significantly smaller.
781 significantly smaller.
779 """
782 """
780 assert oldmatcher
783 assert oldmatcher
781 assert matcher
784 assert matcher
782 self._oldmatcher = oldmatcher
785 self._oldmatcher = oldmatcher
783 self._matcher = matcher
786 self._matcher = matcher
784
787
785 self.version = version
788 self.version = version
786 self._forcedeltaparentprev = forcedeltaparentprev
789 self._forcedeltaparentprev = forcedeltaparentprev
787 self._builddeltaheader = builddeltaheader
790 self._builddeltaheader = builddeltaheader
788 self._manifestsend = manifestsend
791 self._manifestsend = manifestsend
789 self._ellipses = ellipses
792 self._ellipses = ellipses
790
793
791 # Set of capabilities we can use to build the bundle.
794 # Set of capabilities we can use to build the bundle.
792 if bundlecaps is None:
795 if bundlecaps is None:
793 bundlecaps = set()
796 bundlecaps = set()
794 self._bundlecaps = bundlecaps
797 self._bundlecaps = bundlecaps
795 self._isshallow = shallow
798 self._isshallow = shallow
796 self._fullclnodes = fullnodes
799 self._fullclnodes = fullnodes
797
800
798 # Maps ellipsis revs to their roots at the changelog level.
801 # Maps ellipsis revs to their roots at the changelog level.
799 self._precomputedellipsis = ellipsisroots
802 self._precomputedellipsis = ellipsisroots
800
803
801 self._repo = repo
804 self._repo = repo
802
805
803 if self._repo.ui.verbose and not self._repo.ui.debugflag:
806 if self._repo.ui.verbose and not self._repo.ui.debugflag:
804 self._verbosenote = self._repo.ui.note
807 self._verbosenote = self._repo.ui.note
805 else:
808 else:
806 self._verbosenote = lambda s: None
809 self._verbosenote = lambda s: None
807
810
808 def generate(self, commonrevs, clnodes, fastpathlinkrev, source,
811 def generate(self, commonrevs, clnodes, fastpathlinkrev, source,
809 changelog=True):
812 changelog=True):
810 """Yield a sequence of changegroup byte chunks.
813 """Yield a sequence of changegroup byte chunks.
811 If changelog is False, changelog data won't be added to changegroup
814 If changelog is False, changelog data won't be added to changegroup
812 """
815 """
813
816
814 repo = self._repo
817 repo = self._repo
815 cl = repo.changelog
818 cl = repo.changelog
816
819
817 self._verbosenote(_('uncompressed size of bundle content:\n'))
820 self._verbosenote(_('uncompressed size of bundle content:\n'))
818 size = 0
821 size = 0
819
822
820 clstate, deltas = self._generatechangelog(cl, clnodes,
823 clstate, deltas = self._generatechangelog(cl, clnodes,
821 generate=changelog)
824 generate=changelog)
822 for delta in deltas:
825 for delta in deltas:
823 for chunk in _revisiondeltatochunks(delta,
826 for chunk in _revisiondeltatochunks(delta,
824 self._builddeltaheader):
827 self._builddeltaheader):
825 size += len(chunk)
828 size += len(chunk)
826 yield chunk
829 yield chunk
827
830
828 close = closechunk()
831 close = closechunk()
829 size += len(close)
832 size += len(close)
830 yield closechunk()
833 yield closechunk()
831
834
832 self._verbosenote(_('%8.i (changelog)\n') % size)
835 self._verbosenote(_('%8.i (changelog)\n') % size)
833
836
834 clrevorder = clstate['clrevorder']
837 clrevorder = clstate['clrevorder']
835 manifests = clstate['manifests']
838 manifests = clstate['manifests']
836 changedfiles = clstate['changedfiles']
839 changedfiles = clstate['changedfiles']
837
840
838 # We need to make sure that the linkrev in the changegroup refers to
841 # We need to make sure that the linkrev in the changegroup refers to
839 # the first changeset that introduced the manifest or file revision.
842 # the first changeset that introduced the manifest or file revision.
840 # The fastpath is usually safer than the slowpath, because the filelogs
843 # The fastpath is usually safer than the slowpath, because the filelogs
841 # are walked in revlog order.
844 # are walked in revlog order.
842 #
845 #
843 # When taking the slowpath when the manifest revlog uses generaldelta,
846 # When taking the slowpath when the manifest revlog uses generaldelta,
844 # the manifest may be walked in the "wrong" order. Without 'clrevorder',
847 # the manifest may be walked in the "wrong" order. Without 'clrevorder',
845 # we would get an incorrect linkrev (see fix in cc0ff93d0c0c).
848 # we would get an incorrect linkrev (see fix in cc0ff93d0c0c).
846 #
849 #
847 # When taking the fastpath, we are only vulnerable to reordering
850 # When taking the fastpath, we are only vulnerable to reordering
848 # of the changelog itself. The changelog never uses generaldelta and is
851 # of the changelog itself. The changelog never uses generaldelta and is
849 # never reordered. To handle this case, we simply take the slowpath,
852 # never reordered. To handle this case, we simply take the slowpath,
850 # which already has the 'clrevorder' logic. This was also fixed in
853 # which already has the 'clrevorder' logic. This was also fixed in
851 # cc0ff93d0c0c.
854 # cc0ff93d0c0c.
852
855
853 # Treemanifests don't work correctly with fastpathlinkrev
856 # Treemanifests don't work correctly with fastpathlinkrev
854 # either, because we don't discover which directory nodes to
857 # either, because we don't discover which directory nodes to
855 # send along with files. This could probably be fixed.
858 # send along with files. This could probably be fixed.
856 fastpathlinkrev = fastpathlinkrev and (
859 fastpathlinkrev = fastpathlinkrev and (
857 'treemanifest' not in repo.requirements)
860 'treemanifest' not in repo.requirements)
858
861
859 fnodes = {} # needed file nodes
862 fnodes = {} # needed file nodes
860
863
861 size = 0
864 size = 0
862 it = self.generatemanifests(
865 it = self.generatemanifests(
863 commonrevs, clrevorder, fastpathlinkrev, manifests, fnodes, source,
866 commonrevs, clrevorder, fastpathlinkrev, manifests, fnodes, source,
864 clstate['clrevtomanifestrev'])
867 clstate['clrevtomanifestrev'])
865
868
866 for tree, deltas in it:
869 for tree, deltas in it:
867 if tree:
870 if tree:
868 assert self.version == b'03'
871 assert self.version == b'03'
869 chunk = _fileheader(tree)
872 chunk = _fileheader(tree)
870 size += len(chunk)
873 size += len(chunk)
871 yield chunk
874 yield chunk
872
875
873 for delta in deltas:
876 for delta in deltas:
874 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
877 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
875 for chunk in chunks:
878 for chunk in chunks:
876 size += len(chunk)
879 size += len(chunk)
877 yield chunk
880 yield chunk
878
881
879 close = closechunk()
882 close = closechunk()
880 size += len(close)
883 size += len(close)
881 yield close
884 yield close
882
885
883 self._verbosenote(_('%8.i (manifests)\n') % size)
886 self._verbosenote(_('%8.i (manifests)\n') % size)
884 yield self._manifestsend
887 yield self._manifestsend
885
888
886 mfdicts = None
889 mfdicts = None
887 if self._ellipses and self._isshallow:
890 if self._ellipses and self._isshallow:
888 mfdicts = [(self._repo.manifestlog[n].read(), lr)
891 mfdicts = [(self._repo.manifestlog[n].read(), lr)
889 for (n, lr) in manifests.iteritems()]
892 for (n, lr) in manifests.iteritems()]
890
893
891 manifests.clear()
894 manifests.clear()
892 clrevs = set(cl.rev(x) for x in clnodes)
895 clrevs = set(cl.rev(x) for x in clnodes)
893
896
894 it = self.generatefiles(changedfiles, commonrevs,
897 it = self.generatefiles(changedfiles, commonrevs,
895 source, mfdicts, fastpathlinkrev,
898 source, mfdicts, fastpathlinkrev,
896 fnodes, clrevs)
899 fnodes, clrevs)
897
900
898 for path, deltas in it:
901 for path, deltas in it:
899 h = _fileheader(path)
902 h = _fileheader(path)
900 size = len(h)
903 size = len(h)
901 yield h
904 yield h
902
905
903 for delta in deltas:
906 for delta in deltas:
904 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
907 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
905 for chunk in chunks:
908 for chunk in chunks:
906 size += len(chunk)
909 size += len(chunk)
907 yield chunk
910 yield chunk
908
911
909 close = closechunk()
912 close = closechunk()
910 size += len(close)
913 size += len(close)
911 yield close
914 yield close
912
915
913 self._verbosenote(_('%8.i %s\n') % (size, path))
916 self._verbosenote(_('%8.i %s\n') % (size, path))
914
917
915 yield closechunk()
918 yield closechunk()
916
919
917 if clnodes:
920 if clnodes:
918 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
921 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
919
922
920 def _generatechangelog(self, cl, nodes, generate=True):
923 def _generatechangelog(self, cl, nodes, generate=True):
921 """Generate data for changelog chunks.
924 """Generate data for changelog chunks.
922
925
923 Returns a 2-tuple of a dict containing state and an iterable of
926 Returns a 2-tuple of a dict containing state and an iterable of
924 byte chunks. The state will not be fully populated until the
927 byte chunks. The state will not be fully populated until the
925 chunk stream has been fully consumed.
928 chunk stream has been fully consumed.
926
929
927 if generate is False, the state will be fully populated and no chunk
930 if generate is False, the state will be fully populated and no chunk
928 stream will be yielded
931 stream will be yielded
929 """
932 """
930 clrevorder = {}
933 clrevorder = {}
931 manifests = {}
934 manifests = {}
932 mfl = self._repo.manifestlog
935 mfl = self._repo.manifestlog
933 changedfiles = set()
936 changedfiles = set()
934 clrevtomanifestrev = {}
937 clrevtomanifestrev = {}
935
938
936 state = {
939 state = {
937 'clrevorder': clrevorder,
940 'clrevorder': clrevorder,
938 'manifests': manifests,
941 'manifests': manifests,
939 'changedfiles': changedfiles,
942 'changedfiles': changedfiles,
940 'clrevtomanifestrev': clrevtomanifestrev,
943 'clrevtomanifestrev': clrevtomanifestrev,
941 }
944 }
942
945
943 if not (generate or self._ellipses):
946 if not (generate or self._ellipses):
944 # sort the nodes in storage order
947 # sort the nodes in storage order
945 nodes = sorted(nodes, key=cl.rev)
948 nodes = sorted(nodes, key=cl.rev)
946 for node in nodes:
949 for node in nodes:
947 c = cl.changelogrevision(node)
950 c = cl.changelogrevision(node)
948 clrevorder[node] = len(clrevorder)
951 clrevorder[node] = len(clrevorder)
949 # record the first changeset introducing this manifest version
952 # record the first changeset introducing this manifest version
950 manifests.setdefault(c.manifest, node)
953 manifests.setdefault(c.manifest, node)
951 # Record a complete list of potentially-changed files in
954 # Record a complete list of potentially-changed files in
952 # this manifest.
955 # this manifest.
953 changedfiles.update(c.files)
956 changedfiles.update(c.files)
954
957
955 return state, ()
958 return state, ()
956
959
957 # Callback for the changelog, used to collect changed files and
960 # Callback for the changelog, used to collect changed files and
958 # manifest nodes.
961 # manifest nodes.
959 # Returns the linkrev node (identity in the changelog case).
962 # Returns the linkrev node (identity in the changelog case).
960 def lookupcl(x):
963 def lookupcl(x):
961 c = cl.changelogrevision(x)
964 c = cl.changelogrevision(x)
962 clrevorder[x] = len(clrevorder)
965 clrevorder[x] = len(clrevorder)
963
966
964 if self._ellipses:
967 if self._ellipses:
965 # Only update manifests if x is going to be sent. Otherwise we
968 # Only update manifests if x is going to be sent. Otherwise we
966 # end up with bogus linkrevs specified for manifests and
969 # end up with bogus linkrevs specified for manifests and
967 # we skip some manifest nodes that we should otherwise
970 # we skip some manifest nodes that we should otherwise
968 # have sent.
971 # have sent.
969 if (x in self._fullclnodes
972 if (x in self._fullclnodes
970 or cl.rev(x) in self._precomputedellipsis):
973 or cl.rev(x) in self._precomputedellipsis):
971
974
972 manifestnode = c.manifest
975 manifestnode = c.manifest
973 # Record the first changeset introducing this manifest
976 # Record the first changeset introducing this manifest
974 # version.
977 # version.
975 manifests.setdefault(manifestnode, x)
978 manifests.setdefault(manifestnode, x)
976 # Set this narrow-specific dict so we have the lowest
979 # Set this narrow-specific dict so we have the lowest
977 # manifest revnum to look up for this cl revnum. (Part of
980 # manifest revnum to look up for this cl revnum. (Part of
978 # mapping changelog ellipsis parents to manifest ellipsis
981 # mapping changelog ellipsis parents to manifest ellipsis
979 # parents)
982 # parents)
980 clrevtomanifestrev.setdefault(
983 clrevtomanifestrev.setdefault(
981 cl.rev(x), mfl.rev(manifestnode))
984 cl.rev(x), mfl.rev(manifestnode))
982 # We can't trust the changed files list in the changeset if the
985 # We can't trust the changed files list in the changeset if the
983 # client requested a shallow clone.
986 # client requested a shallow clone.
984 if self._isshallow:
987 if self._isshallow:
985 changedfiles.update(mfl[c.manifest].read().keys())
988 changedfiles.update(mfl[c.manifest].read().keys())
986 else:
989 else:
987 changedfiles.update(c.files)
990 changedfiles.update(c.files)
988 else:
991 else:
989 # record the first changeset introducing this manifest version
992 # record the first changeset introducing this manifest version
990 manifests.setdefault(c.manifest, x)
993 manifests.setdefault(c.manifest, x)
991 # Record a complete list of potentially-changed files in
994 # Record a complete list of potentially-changed files in
992 # this manifest.
995 # this manifest.
993 changedfiles.update(c.files)
996 changedfiles.update(c.files)
994
997
995 return x
998 return x
996
999
997 gen = deltagroup(
1000 gen = deltagroup(
998 self._repo, cl, nodes, True, lookupcl,
1001 self._repo, cl, nodes, True, lookupcl,
999 self._forcedeltaparentprev,
1002 self._forcedeltaparentprev,
1000 ellipses=self._ellipses,
1003 ellipses=self._ellipses,
1001 topic=_('changesets'),
1004 topic=_('changesets'),
1002 clrevtolocalrev={},
1005 clrevtolocalrev={},
1003 fullclnodes=self._fullclnodes,
1006 fullclnodes=self._fullclnodes,
1004 precomputedellipsis=self._precomputedellipsis)
1007 precomputedellipsis=self._precomputedellipsis)
1005
1008
1006 return state, gen
1009 return state, gen
1007
1010
1008 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev,
1011 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev,
1009 manifests, fnodes, source, clrevtolocalrev):
1012 manifests, fnodes, source, clrevtolocalrev):
1010 """Returns an iterator of changegroup chunks containing manifests.
1013 """Returns an iterator of changegroup chunks containing manifests.
1011
1014
1012 `source` is unused here, but is used by extensions like remotefilelog to
1015 `source` is unused here, but is used by extensions like remotefilelog to
1013 change what is sent based in pulls vs pushes, etc.
1016 change what is sent based in pulls vs pushes, etc.
1014 """
1017 """
1015 repo = self._repo
1018 repo = self._repo
1016 mfl = repo.manifestlog
1019 mfl = repo.manifestlog
1017 tmfnodes = {'': manifests}
1020 tmfnodes = {'': manifests}
1018
1021
1019 # Callback for the manifest, used to collect linkrevs for filelog
1022 # Callback for the manifest, used to collect linkrevs for filelog
1020 # revisions.
1023 # revisions.
1021 # Returns the linkrev node (collected in lookupcl).
1024 # Returns the linkrev node (collected in lookupcl).
1022 def makelookupmflinknode(tree, nodes):
1025 def makelookupmflinknode(tree, nodes):
1023 if fastpathlinkrev:
1026 if fastpathlinkrev:
1024 assert not tree
1027 assert not tree
1025 return manifests.__getitem__
1028 return manifests.__getitem__
1026
1029
1027 def lookupmflinknode(x):
1030 def lookupmflinknode(x):
1028 """Callback for looking up the linknode for manifests.
1031 """Callback for looking up the linknode for manifests.
1029
1032
1030 Returns the linkrev node for the specified manifest.
1033 Returns the linkrev node for the specified manifest.
1031
1034
1032 SIDE EFFECT:
1035 SIDE EFFECT:
1033
1036
1034 1) fclnodes gets populated with the list of relevant
1037 1) fclnodes gets populated with the list of relevant
1035 file nodes if we're not using fastpathlinkrev
1038 file nodes if we're not using fastpathlinkrev
1036 2) When treemanifests are in use, collects treemanifest nodes
1039 2) When treemanifests are in use, collects treemanifest nodes
1037 to send
1040 to send
1038
1041
1039 Note that this means manifests must be completely sent to
1042 Note that this means manifests must be completely sent to
1040 the client before you can trust the list of files and
1043 the client before you can trust the list of files and
1041 treemanifests to send.
1044 treemanifests to send.
1042 """
1045 """
1043 clnode = nodes[x]
1046 clnode = nodes[x]
1044 mdata = mfl.get(tree, x).readfast(shallow=True)
1047 mdata = mfl.get(tree, x).readfast(shallow=True)
1045 for p, n, fl in mdata.iterentries():
1048 for p, n, fl in mdata.iterentries():
1046 if fl == 't': # subdirectory manifest
1049 if fl == 't': # subdirectory manifest
1047 subtree = tree + p + '/'
1050 subtree = tree + p + '/'
1048 tmfclnodes = tmfnodes.setdefault(subtree, {})
1051 tmfclnodes = tmfnodes.setdefault(subtree, {})
1049 tmfclnode = tmfclnodes.setdefault(n, clnode)
1052 tmfclnode = tmfclnodes.setdefault(n, clnode)
1050 if clrevorder[clnode] < clrevorder[tmfclnode]:
1053 if clrevorder[clnode] < clrevorder[tmfclnode]:
1051 tmfclnodes[n] = clnode
1054 tmfclnodes[n] = clnode
1052 else:
1055 else:
1053 f = tree + p
1056 f = tree + p
1054 fclnodes = fnodes.setdefault(f, {})
1057 fclnodes = fnodes.setdefault(f, {})
1055 fclnode = fclnodes.setdefault(n, clnode)
1058 fclnode = fclnodes.setdefault(n, clnode)
1056 if clrevorder[clnode] < clrevorder[fclnode]:
1059 if clrevorder[clnode] < clrevorder[fclnode]:
1057 fclnodes[n] = clnode
1060 fclnodes[n] = clnode
1058 return clnode
1061 return clnode
1059 return lookupmflinknode
1062 return lookupmflinknode
1060
1063
1061 while tmfnodes:
1064 while tmfnodes:
1062 tree, nodes = tmfnodes.popitem()
1065 tree, nodes = tmfnodes.popitem()
1063
1066
1064 should_visit = self._matcher.visitdir(tree[:-1])
1067 should_visit = self._matcher.visitdir(tree[:-1])
1065 if tree and not should_visit:
1068 if tree and not should_visit:
1066 continue
1069 continue
1067
1070
1068 store = mfl.getstorage(tree)
1071 store = mfl.getstorage(tree)
1069
1072
1070 if not should_visit:
1073 if not should_visit:
1071 # No nodes to send because this directory is out of
1074 # No nodes to send because this directory is out of
1072 # the client's view of the repository (probably
1075 # the client's view of the repository (probably
1073 # because of narrow clones). Do this even for the root
1076 # because of narrow clones). Do this even for the root
1074 # directory (tree=='')
1077 # directory (tree=='')
1075 prunednodes = []
1078 prunednodes = []
1076 else:
1079 else:
1077 # Avoid sending any manifest nodes we can prove the
1080 # Avoid sending any manifest nodes we can prove the
1078 # client already has by checking linkrevs. See the
1081 # client already has by checking linkrevs. See the
1079 # related comment in generatefiles().
1082 # related comment in generatefiles().
1080 prunednodes = self._prunemanifests(store, nodes, commonrevs)
1083 prunednodes = self._prunemanifests(store, nodes, commonrevs)
1081
1084
1082 if tree and not prunednodes:
1085 if tree and not prunednodes:
1083 continue
1086 continue
1084
1087
1085 lookupfn = makelookupmflinknode(tree, nodes)
1088 lookupfn = makelookupmflinknode(tree, nodes)
1086
1089
1087 deltas = deltagroup(
1090 deltas = deltagroup(
1088 self._repo, store, prunednodes, False, lookupfn,
1091 self._repo, store, prunednodes, False, lookupfn,
1089 self._forcedeltaparentprev,
1092 self._forcedeltaparentprev,
1090 ellipses=self._ellipses,
1093 ellipses=self._ellipses,
1091 topic=_('manifests'),
1094 topic=_('manifests'),
1092 clrevtolocalrev=clrevtolocalrev,
1095 clrevtolocalrev=clrevtolocalrev,
1093 fullclnodes=self._fullclnodes,
1096 fullclnodes=self._fullclnodes,
1094 precomputedellipsis=self._precomputedellipsis)
1097 precomputedellipsis=self._precomputedellipsis)
1095
1098
1096 if not self._oldmatcher.visitdir(store.tree[:-1]):
1099 if not self._oldmatcher.visitdir(store.tree[:-1]):
1097 yield tree, deltas
1100 yield tree, deltas
1098 else:
1101 else:
1099 # 'deltas' is a generator and we need to consume it even if
1102 # 'deltas' is a generator and we need to consume it even if
1100 # we are not going to send it because a side-effect is that
1103 # we are not going to send it because a side-effect is that
1101 # it updates tmdnodes (via lookupfn)
1104 # it updates tmdnodes (via lookupfn)
1102 for d in deltas:
1105 for d in deltas:
1103 pass
1106 pass
1104 if not tree:
1107 if not tree:
1105 yield tree, []
1108 yield tree, []
1106
1109
1107 def _prunemanifests(self, store, nodes, commonrevs):
1110 def _prunemanifests(self, store, nodes, commonrevs):
1108 if not self._ellipses:
1111 if not self._ellipses:
1109 # In non-ellipses case and large repositories, it is better to
1112 # In non-ellipses case and large repositories, it is better to
1110 # prevent calling of store.rev and store.linkrev on a lot of
1113 # prevent calling of store.rev and store.linkrev on a lot of
1111 # nodes as compared to sending some extra data
1114 # nodes as compared to sending some extra data
1112 return nodes.copy()
1115 return nodes.copy()
1113 # This is split out as a separate method to allow filtering
1116 # This is split out as a separate method to allow filtering
1114 # commonrevs in extension code.
1117 # commonrevs in extension code.
1115 #
1118 #
1116 # TODO(augie): this shouldn't be required, instead we should
1119 # TODO(augie): this shouldn't be required, instead we should
1117 # make filtering of revisions to send delegated to the store
1120 # make filtering of revisions to send delegated to the store
1118 # layer.
1121 # layer.
1119 frev, flr = store.rev, store.linkrev
1122 frev, flr = store.rev, store.linkrev
1120 return [n for n in nodes if flr(frev(n)) not in commonrevs]
1123 return [n for n in nodes if flr(frev(n)) not in commonrevs]
1121
1124
1122 # The 'source' parameter is useful for extensions
1125 # The 'source' parameter is useful for extensions
1123 def generatefiles(self, changedfiles, commonrevs, source,
1126 def generatefiles(self, changedfiles, commonrevs, source,
1124 mfdicts, fastpathlinkrev, fnodes, clrevs):
1127 mfdicts, fastpathlinkrev, fnodes, clrevs):
1125 changedfiles = [f for f in changedfiles
1128 changedfiles = [f for f in changedfiles
1126 if self._matcher(f) and not self._oldmatcher(f)]
1129 if self._matcher(f) and not self._oldmatcher(f)]
1127
1130
1128 if not fastpathlinkrev:
1131 if not fastpathlinkrev:
1129 def normallinknodes(unused, fname):
1132 def normallinknodes(unused, fname):
1130 return fnodes.get(fname, {})
1133 return fnodes.get(fname, {})
1131 else:
1134 else:
1132 cln = self._repo.changelog.node
1135 cln = self._repo.changelog.node
1133
1136
1134 def normallinknodes(store, fname):
1137 def normallinknodes(store, fname):
1135 flinkrev = store.linkrev
1138 flinkrev = store.linkrev
1136 fnode = store.node
1139 fnode = store.node
1137 revs = ((r, flinkrev(r)) for r in store)
1140 revs = ((r, flinkrev(r)) for r in store)
1138 return dict((fnode(r), cln(lr))
1141 return dict((fnode(r), cln(lr))
1139 for r, lr in revs if lr in clrevs)
1142 for r, lr in revs if lr in clrevs)
1140
1143
1141 clrevtolocalrev = {}
1144 clrevtolocalrev = {}
1142
1145
1143 if self._isshallow:
1146 if self._isshallow:
1144 # In a shallow clone, the linknodes callback needs to also include
1147 # In a shallow clone, the linknodes callback needs to also include
1145 # those file nodes that are in the manifests we sent but weren't
1148 # those file nodes that are in the manifests we sent but weren't
1146 # introduced by those manifests.
1149 # introduced by those manifests.
1147 commonctxs = [self._repo[c] for c in commonrevs]
1150 commonctxs = [self._repo[c] for c in commonrevs]
1148 clrev = self._repo.changelog.rev
1151 clrev = self._repo.changelog.rev
1149
1152
1150 def linknodes(flog, fname):
1153 def linknodes(flog, fname):
1151 for c in commonctxs:
1154 for c in commonctxs:
1152 try:
1155 try:
1153 fnode = c.filenode(fname)
1156 fnode = c.filenode(fname)
1154 clrevtolocalrev[c.rev()] = flog.rev(fnode)
1157 clrevtolocalrev[c.rev()] = flog.rev(fnode)
1155 except error.ManifestLookupError:
1158 except error.ManifestLookupError:
1156 pass
1159 pass
1157 links = normallinknodes(flog, fname)
1160 links = normallinknodes(flog, fname)
1158 if len(links) != len(mfdicts):
1161 if len(links) != len(mfdicts):
1159 for mf, lr in mfdicts:
1162 for mf, lr in mfdicts:
1160 fnode = mf.get(fname, None)
1163 fnode = mf.get(fname, None)
1161 if fnode in links:
1164 if fnode in links:
1162 links[fnode] = min(links[fnode], lr, key=clrev)
1165 links[fnode] = min(links[fnode], lr, key=clrev)
1163 elif fnode:
1166 elif fnode:
1164 links[fnode] = lr
1167 links[fnode] = lr
1165 return links
1168 return links
1166 else:
1169 else:
1167 linknodes = normallinknodes
1170 linknodes = normallinknodes
1168
1171
1169 repo = self._repo
1172 repo = self._repo
1170 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1173 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1171 total=len(changedfiles))
1174 total=len(changedfiles))
1172 for i, fname in enumerate(sorted(changedfiles)):
1175 for i, fname in enumerate(sorted(changedfiles)):
1173 filerevlog = repo.file(fname)
1176 filerevlog = repo.file(fname)
1174 if not filerevlog:
1177 if not filerevlog:
1175 raise error.Abort(_("empty or missing file data for %s") %
1178 raise error.Abort(_("empty or missing file data for %s") %
1176 fname)
1179 fname)
1177
1180
1178 clrevtolocalrev.clear()
1181 clrevtolocalrev.clear()
1179
1182
1180 linkrevnodes = linknodes(filerevlog, fname)
1183 linkrevnodes = linknodes(filerevlog, fname)
1181 # Lookup for filenodes, we collected the linkrev nodes above in the
1184 # Lookup for filenodes, we collected the linkrev nodes above in the
1182 # fastpath case and with lookupmf in the slowpath case.
1185 # fastpath case and with lookupmf in the slowpath case.
1183 def lookupfilelog(x):
1186 def lookupfilelog(x):
1184 return linkrevnodes[x]
1187 return linkrevnodes[x]
1185
1188
1186 frev, flr = filerevlog.rev, filerevlog.linkrev
1189 frev, flr = filerevlog.rev, filerevlog.linkrev
1187 # Skip sending any filenode we know the client already
1190 # Skip sending any filenode we know the client already
1188 # has. This avoids over-sending files relatively
1191 # has. This avoids over-sending files relatively
1189 # inexpensively, so it's not a problem if we under-filter
1192 # inexpensively, so it's not a problem if we under-filter
1190 # here.
1193 # here.
1191 filenodes = [n for n in linkrevnodes
1194 filenodes = [n for n in linkrevnodes
1192 if flr(frev(n)) not in commonrevs]
1195 if flr(frev(n)) not in commonrevs]
1193
1196
1194 if not filenodes:
1197 if not filenodes:
1195 continue
1198 continue
1196
1199
1197 progress.update(i + 1, item=fname)
1200 progress.update(i + 1, item=fname)
1198
1201
1199 deltas = deltagroup(
1202 deltas = deltagroup(
1200 self._repo, filerevlog, filenodes, False, lookupfilelog,
1203 self._repo, filerevlog, filenodes, False, lookupfilelog,
1201 self._forcedeltaparentprev,
1204 self._forcedeltaparentprev,
1202 ellipses=self._ellipses,
1205 ellipses=self._ellipses,
1203 clrevtolocalrev=clrevtolocalrev,
1206 clrevtolocalrev=clrevtolocalrev,
1204 fullclnodes=self._fullclnodes,
1207 fullclnodes=self._fullclnodes,
1205 precomputedellipsis=self._precomputedellipsis)
1208 precomputedellipsis=self._precomputedellipsis)
1206
1209
1207 yield fname, deltas
1210 yield fname, deltas
1208
1211
1209 progress.complete()
1212 progress.complete()
1210
1213
1211 def _makecg1packer(repo, oldmatcher, matcher, bundlecaps,
1214 def _makecg1packer(repo, oldmatcher, matcher, bundlecaps,
1212 ellipses=False, shallow=False, ellipsisroots=None,
1215 ellipses=False, shallow=False, ellipsisroots=None,
1213 fullnodes=None):
1216 fullnodes=None):
1214 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1217 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1215 d.node, d.p1node, d.p2node, d.linknode)
1218 d.node, d.p1node, d.p2node, d.linknode)
1216
1219
1217 return cgpacker(repo, oldmatcher, matcher, b'01',
1220 return cgpacker(repo, oldmatcher, matcher, b'01',
1218 builddeltaheader=builddeltaheader,
1221 builddeltaheader=builddeltaheader,
1219 manifestsend=b'',
1222 manifestsend=b'',
1220 forcedeltaparentprev=True,
1223 forcedeltaparentprev=True,
1221 bundlecaps=bundlecaps,
1224 bundlecaps=bundlecaps,
1222 ellipses=ellipses,
1225 ellipses=ellipses,
1223 shallow=shallow,
1226 shallow=shallow,
1224 ellipsisroots=ellipsisroots,
1227 ellipsisroots=ellipsisroots,
1225 fullnodes=fullnodes)
1228 fullnodes=fullnodes)
1226
1229
1227 def _makecg2packer(repo, oldmatcher, matcher, bundlecaps,
1230 def _makecg2packer(repo, oldmatcher, matcher, bundlecaps,
1228 ellipses=False, shallow=False, ellipsisroots=None,
1231 ellipses=False, shallow=False, ellipsisroots=None,
1229 fullnodes=None):
1232 fullnodes=None):
1230 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1233 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1231 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1234 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1232
1235
1233 return cgpacker(repo, oldmatcher, matcher, b'02',
1236 return cgpacker(repo, oldmatcher, matcher, b'02',
1234 builddeltaheader=builddeltaheader,
1237 builddeltaheader=builddeltaheader,
1235 manifestsend=b'',
1238 manifestsend=b'',
1236 bundlecaps=bundlecaps,
1239 bundlecaps=bundlecaps,
1237 ellipses=ellipses,
1240 ellipses=ellipses,
1238 shallow=shallow,
1241 shallow=shallow,
1239 ellipsisroots=ellipsisroots,
1242 ellipsisroots=ellipsisroots,
1240 fullnodes=fullnodes)
1243 fullnodes=fullnodes)
1241
1244
1242 def _makecg3packer(repo, oldmatcher, matcher, bundlecaps,
1245 def _makecg3packer(repo, oldmatcher, matcher, bundlecaps,
1243 ellipses=False, shallow=False, ellipsisroots=None,
1246 ellipses=False, shallow=False, ellipsisroots=None,
1244 fullnodes=None):
1247 fullnodes=None):
1245 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1248 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1246 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1249 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1247
1250
1248 return cgpacker(repo, oldmatcher, matcher, b'03',
1251 return cgpacker(repo, oldmatcher, matcher, b'03',
1249 builddeltaheader=builddeltaheader,
1252 builddeltaheader=builddeltaheader,
1250 manifestsend=closechunk(),
1253 manifestsend=closechunk(),
1251 bundlecaps=bundlecaps,
1254 bundlecaps=bundlecaps,
1252 ellipses=ellipses,
1255 ellipses=ellipses,
1253 shallow=shallow,
1256 shallow=shallow,
1254 ellipsisroots=ellipsisroots,
1257 ellipsisroots=ellipsisroots,
1255 fullnodes=fullnodes)
1258 fullnodes=fullnodes)
1256
1259
1257 _packermap = {'01': (_makecg1packer, cg1unpacker),
1260 _packermap = {'01': (_makecg1packer, cg1unpacker),
1258 # cg2 adds support for exchanging generaldelta
1261 # cg2 adds support for exchanging generaldelta
1259 '02': (_makecg2packer, cg2unpacker),
1262 '02': (_makecg2packer, cg2unpacker),
1260 # cg3 adds support for exchanging revlog flags and treemanifests
1263 # cg3 adds support for exchanging revlog flags and treemanifests
1261 '03': (_makecg3packer, cg3unpacker),
1264 '03': (_makecg3packer, cg3unpacker),
1262 }
1265 }
1263
1266
1264 def allsupportedversions(repo):
1267 def allsupportedversions(repo):
1265 versions = set(_packermap.keys())
1268 versions = set(_packermap.keys())
1266 if not (repo.ui.configbool('experimental', 'changegroup3') or
1269 if not (repo.ui.configbool('experimental', 'changegroup3') or
1267 repo.ui.configbool('experimental', 'treemanifest') or
1270 repo.ui.configbool('experimental', 'treemanifest') or
1268 'treemanifest' in repo.requirements):
1271 'treemanifest' in repo.requirements):
1269 versions.discard('03')
1272 versions.discard('03')
1270 return versions
1273 return versions
1271
1274
1272 # Changegroup versions that can be applied to the repo
1275 # Changegroup versions that can be applied to the repo
1273 def supportedincomingversions(repo):
1276 def supportedincomingversions(repo):
1274 return allsupportedversions(repo)
1277 return allsupportedversions(repo)
1275
1278
1276 # Changegroup versions that can be created from the repo
1279 # Changegroup versions that can be created from the repo
1277 def supportedoutgoingversions(repo):
1280 def supportedoutgoingversions(repo):
1278 versions = allsupportedversions(repo)
1281 versions = allsupportedversions(repo)
1279 if 'treemanifest' in repo.requirements:
1282 if 'treemanifest' in repo.requirements:
1280 # Versions 01 and 02 support only flat manifests and it's just too
1283 # Versions 01 and 02 support only flat manifests and it's just too
1281 # expensive to convert between the flat manifest and tree manifest on
1284 # expensive to convert between the flat manifest and tree manifest on
1282 # the fly. Since tree manifests are hashed differently, all of history
1285 # the fly. Since tree manifests are hashed differently, all of history
1283 # would have to be converted. Instead, we simply don't even pretend to
1286 # would have to be converted. Instead, we simply don't even pretend to
1284 # support versions 01 and 02.
1287 # support versions 01 and 02.
1285 versions.discard('01')
1288 versions.discard('01')
1286 versions.discard('02')
1289 versions.discard('02')
1287 if repository.NARROW_REQUIREMENT in repo.requirements:
1290 if repository.NARROW_REQUIREMENT in repo.requirements:
1288 # Versions 01 and 02 don't support revlog flags, and we need to
1291 # Versions 01 and 02 don't support revlog flags, and we need to
1289 # support that for stripping and unbundling to work.
1292 # support that for stripping and unbundling to work.
1290 versions.discard('01')
1293 versions.discard('01')
1291 versions.discard('02')
1294 versions.discard('02')
1292 if LFS_REQUIREMENT in repo.requirements:
1295 if LFS_REQUIREMENT in repo.requirements:
1293 # Versions 01 and 02 don't support revlog flags, and we need to
1296 # Versions 01 and 02 don't support revlog flags, and we need to
1294 # mark LFS entries with REVIDX_EXTSTORED.
1297 # mark LFS entries with REVIDX_EXTSTORED.
1295 versions.discard('01')
1298 versions.discard('01')
1296 versions.discard('02')
1299 versions.discard('02')
1297
1300
1298 return versions
1301 return versions
1299
1302
1300 def localversion(repo):
1303 def localversion(repo):
1301 # Finds the best version to use for bundles that are meant to be used
1304 # Finds the best version to use for bundles that are meant to be used
1302 # locally, such as those from strip and shelve, and temporary bundles.
1305 # locally, such as those from strip and shelve, and temporary bundles.
1303 return max(supportedoutgoingversions(repo))
1306 return max(supportedoutgoingversions(repo))
1304
1307
1305 def safeversion(repo):
1308 def safeversion(repo):
1306 # Finds the smallest version that it's safe to assume clients of the repo
1309 # Finds the smallest version that it's safe to assume clients of the repo
1307 # will support. For example, all hg versions that support generaldelta also
1310 # will support. For example, all hg versions that support generaldelta also
1308 # support changegroup 02.
1311 # support changegroup 02.
1309 versions = supportedoutgoingversions(repo)
1312 versions = supportedoutgoingversions(repo)
1310 if 'generaldelta' in repo.requirements:
1313 if 'generaldelta' in repo.requirements:
1311 versions.discard('01')
1314 versions.discard('01')
1312 assert versions
1315 assert versions
1313 return min(versions)
1316 return min(versions)
1314
1317
1315 def getbundler(version, repo, bundlecaps=None, oldmatcher=None,
1318 def getbundler(version, repo, bundlecaps=None, oldmatcher=None,
1316 matcher=None, ellipses=False, shallow=False,
1319 matcher=None, ellipses=False, shallow=False,
1317 ellipsisroots=None, fullnodes=None):
1320 ellipsisroots=None, fullnodes=None):
1318 assert version in supportedoutgoingversions(repo)
1321 assert version in supportedoutgoingversions(repo)
1319
1322
1320 if matcher is None:
1323 if matcher is None:
1321 matcher = matchmod.always()
1324 matcher = matchmod.always()
1322 if oldmatcher is None:
1325 if oldmatcher is None:
1323 oldmatcher = matchmod.never()
1326 oldmatcher = matchmod.never()
1324
1327
1325 if version == '01' and not matcher.always():
1328 if version == '01' and not matcher.always():
1326 raise error.ProgrammingError('version 01 changegroups do not support '
1329 raise error.ProgrammingError('version 01 changegroups do not support '
1327 'sparse file matchers')
1330 'sparse file matchers')
1328
1331
1329 if ellipses and version in (b'01', b'02'):
1332 if ellipses and version in (b'01', b'02'):
1330 raise error.Abort(
1333 raise error.Abort(
1331 _('ellipsis nodes require at least cg3 on client and server, '
1334 _('ellipsis nodes require at least cg3 on client and server, '
1332 'but negotiated version %s') % version)
1335 'but negotiated version %s') % version)
1333
1336
1334 # Requested files could include files not in the local store. So
1337 # Requested files could include files not in the local store. So
1335 # filter those out.
1338 # filter those out.
1336 matcher = repo.narrowmatch(matcher)
1339 matcher = repo.narrowmatch(matcher)
1337
1340
1338 fn = _packermap[version][0]
1341 fn = _packermap[version][0]
1339 return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses,
1342 return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses,
1340 shallow=shallow, ellipsisroots=ellipsisroots,
1343 shallow=shallow, ellipsisroots=ellipsisroots,
1341 fullnodes=fullnodes)
1344 fullnodes=fullnodes)
1342
1345
1343 def getunbundler(version, fh, alg, extras=None):
1346 def getunbundler(version, fh, alg, extras=None):
1344 return _packermap[version][1](fh, alg, extras=extras)
1347 return _packermap[version][1](fh, alg, extras=extras)
1345
1348
1346 def _changegroupinfo(repo, nodes, source):
1349 def _changegroupinfo(repo, nodes, source):
1347 if repo.ui.verbose or source == 'bundle':
1350 if repo.ui.verbose or source == 'bundle':
1348 repo.ui.status(_("%d changesets found\n") % len(nodes))
1351 repo.ui.status(_("%d changesets found\n") % len(nodes))
1349 if repo.ui.debugflag:
1352 if repo.ui.debugflag:
1350 repo.ui.debug("list of changesets:\n")
1353 repo.ui.debug("list of changesets:\n")
1351 for node in nodes:
1354 for node in nodes:
1352 repo.ui.debug("%s\n" % hex(node))
1355 repo.ui.debug("%s\n" % hex(node))
1353
1356
1354 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1357 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1355 bundlecaps=None):
1358 bundlecaps=None):
1356 cgstream = makestream(repo, outgoing, version, source,
1359 cgstream = makestream(repo, outgoing, version, source,
1357 fastpath=fastpath, bundlecaps=bundlecaps)
1360 fastpath=fastpath, bundlecaps=bundlecaps)
1358 return getunbundler(version, util.chunkbuffer(cgstream), None,
1361 return getunbundler(version, util.chunkbuffer(cgstream), None,
1359 {'clcount': len(outgoing.missing) })
1362 {'clcount': len(outgoing.missing) })
1360
1363
1361 def makestream(repo, outgoing, version, source, fastpath=False,
1364 def makestream(repo, outgoing, version, source, fastpath=False,
1362 bundlecaps=None, matcher=None):
1365 bundlecaps=None, matcher=None):
1363 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1366 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1364 matcher=matcher)
1367 matcher=matcher)
1365
1368
1366 repo = repo.unfiltered()
1369 repo = repo.unfiltered()
1367 commonrevs = outgoing.common
1370 commonrevs = outgoing.common
1368 csets = outgoing.missing
1371 csets = outgoing.missing
1369 heads = outgoing.missingheads
1372 heads = outgoing.missingheads
1370 # We go through the fast path if we get told to, or if all (unfiltered
1373 # We go through the fast path if we get told to, or if all (unfiltered
1371 # heads have been requested (since we then know there all linkrevs will
1374 # heads have been requested (since we then know there all linkrevs will
1372 # be pulled by the client).
1375 # be pulled by the client).
1373 heads.sort()
1376 heads.sort()
1374 fastpathlinkrev = fastpath or (
1377 fastpathlinkrev = fastpath or (
1375 repo.filtername is None and heads == sorted(repo.heads()))
1378 repo.filtername is None and heads == sorted(repo.heads()))
1376
1379
1377 repo.hook('preoutgoing', throw=True, source=source)
1380 repo.hook('preoutgoing', throw=True, source=source)
1378 _changegroupinfo(repo, csets, source)
1381 _changegroupinfo(repo, csets, source)
1379 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1382 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1380
1383
1381 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1384 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1382 revisions = 0
1385 revisions = 0
1383 files = 0
1386 files = 0
1384 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1387 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1385 total=expectedfiles)
1388 total=expectedfiles)
1386 for chunkdata in iter(source.filelogheader, {}):
1389 for chunkdata in iter(source.filelogheader, {}):
1387 files += 1
1390 files += 1
1388 f = chunkdata["filename"]
1391 f = chunkdata["filename"]
1389 repo.ui.debug("adding %s revisions\n" % f)
1392 repo.ui.debug("adding %s revisions\n" % f)
1390 progress.increment()
1393 progress.increment()
1391 fl = repo.file(f)
1394 fl = repo.file(f)
1392 o = len(fl)
1395 o = len(fl)
1393 try:
1396 try:
1394 deltas = source.deltaiter()
1397 deltas = source.deltaiter()
1395 if not fl.addgroup(deltas, revmap, trp):
1398 if not fl.addgroup(deltas, revmap, trp):
1396 raise error.Abort(_("received file revlog group is empty"))
1399 raise error.Abort(_("received file revlog group is empty"))
1397 except error.CensoredBaseError as e:
1400 except error.CensoredBaseError as e:
1398 raise error.Abort(_("received delta base is censored: %s") % e)
1401 raise error.Abort(_("received delta base is censored: %s") % e)
1399 revisions += len(fl) - o
1402 revisions += len(fl) - o
1400 if f in needfiles:
1403 if f in needfiles:
1401 needs = needfiles[f]
1404 needs = needfiles[f]
1402 for new in pycompat.xrange(o, len(fl)):
1405 for new in pycompat.xrange(o, len(fl)):
1403 n = fl.node(new)
1406 n = fl.node(new)
1404 if n in needs:
1407 if n in needs:
1405 needs.remove(n)
1408 needs.remove(n)
1406 else:
1409 else:
1407 raise error.Abort(
1410 raise error.Abort(
1408 _("received spurious file revlog entry"))
1411 _("received spurious file revlog entry"))
1409 if not needs:
1412 if not needs:
1410 del needfiles[f]
1413 del needfiles[f]
1411 progress.complete()
1414 progress.complete()
1412
1415
1413 for f, needs in needfiles.iteritems():
1416 for f, needs in needfiles.iteritems():
1414 fl = repo.file(f)
1417 fl = repo.file(f)
1415 for n in needs:
1418 for n in needs:
1416 try:
1419 try:
1417 fl.rev(n)
1420 fl.rev(n)
1418 except error.LookupError:
1421 except error.LookupError:
1419 raise error.Abort(
1422 raise error.Abort(
1420 _('missing file data for %s:%s - run hg verify') %
1423 _('missing file data for %s:%s - run hg verify') %
1421 (f, hex(n)))
1424 (f, hex(n)))
1422
1425
1423 return revisions, files
1426 return revisions, files
@@ -1,2701 +1,2703
1 # exchange.py - utility to exchange data between repos.
1 # exchange.py - utility to exchange data between repos.
2 #
2 #
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import collections
10 import collections
11 import hashlib
11 import hashlib
12
12
13 from .i18n import _
13 from .i18n import _
14 from .node import (
14 from .node import (
15 bin,
15 bin,
16 hex,
16 hex,
17 nullid,
17 nullid,
18 nullrev,
18 nullrev,
19 )
19 )
20 from .thirdparty import (
20 from .thirdparty import (
21 attr,
21 attr,
22 )
22 )
23 from . import (
23 from . import (
24 bookmarks as bookmod,
24 bookmarks as bookmod,
25 bundle2,
25 bundle2,
26 changegroup,
26 changegroup,
27 discovery,
27 discovery,
28 error,
28 error,
29 exchangev2,
29 exchangev2,
30 lock as lockmod,
30 lock as lockmod,
31 logexchange,
31 logexchange,
32 narrowspec,
32 narrowspec,
33 obsolete,
33 obsolete,
34 phases,
34 phases,
35 pushkey,
35 pushkey,
36 pycompat,
36 pycompat,
37 repository,
38 scmutil,
37 scmutil,
39 sslutil,
38 sslutil,
40 streamclone,
39 streamclone,
41 url as urlmod,
40 url as urlmod,
42 util,
41 util,
43 wireprototypes,
42 wireprototypes,
44 )
43 )
44 from .interfaces import (
45 repository,
46 )
45 from .utils import (
47 from .utils import (
46 stringutil,
48 stringutil,
47 )
49 )
48
50
49 urlerr = util.urlerr
51 urlerr = util.urlerr
50 urlreq = util.urlreq
52 urlreq = util.urlreq
51
53
52 _NARROWACL_SECTION = 'narrowacl'
54 _NARROWACL_SECTION = 'narrowacl'
53
55
54 # Maps bundle version human names to changegroup versions.
56 # Maps bundle version human names to changegroup versions.
55 _bundlespeccgversions = {'v1': '01',
57 _bundlespeccgversions = {'v1': '01',
56 'v2': '02',
58 'v2': '02',
57 'packed1': 's1',
59 'packed1': 's1',
58 'bundle2': '02', #legacy
60 'bundle2': '02', #legacy
59 }
61 }
60
62
61 # Maps bundle version with content opts to choose which part to bundle
63 # Maps bundle version with content opts to choose which part to bundle
62 _bundlespeccontentopts = {
64 _bundlespeccontentopts = {
63 'v1': {
65 'v1': {
64 'changegroup': True,
66 'changegroup': True,
65 'cg.version': '01',
67 'cg.version': '01',
66 'obsolescence': False,
68 'obsolescence': False,
67 'phases': False,
69 'phases': False,
68 'tagsfnodescache': False,
70 'tagsfnodescache': False,
69 'revbranchcache': False
71 'revbranchcache': False
70 },
72 },
71 'v2': {
73 'v2': {
72 'changegroup': True,
74 'changegroup': True,
73 'cg.version': '02',
75 'cg.version': '02',
74 'obsolescence': False,
76 'obsolescence': False,
75 'phases': False,
77 'phases': False,
76 'tagsfnodescache': True,
78 'tagsfnodescache': True,
77 'revbranchcache': True
79 'revbranchcache': True
78 },
80 },
79 'packed1' : {
81 'packed1' : {
80 'cg.version': 's1'
82 'cg.version': 's1'
81 }
83 }
82 }
84 }
83 _bundlespeccontentopts['bundle2'] = _bundlespeccontentopts['v2']
85 _bundlespeccontentopts['bundle2'] = _bundlespeccontentopts['v2']
84
86
85 _bundlespecvariants = {"streamv2": {"changegroup": False, "streamv2": True,
87 _bundlespecvariants = {"streamv2": {"changegroup": False, "streamv2": True,
86 "tagsfnodescache": False,
88 "tagsfnodescache": False,
87 "revbranchcache": False}}
89 "revbranchcache": False}}
88
90
89 # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
91 # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
90 _bundlespecv1compengines = {'gzip', 'bzip2', 'none'}
92 _bundlespecv1compengines = {'gzip', 'bzip2', 'none'}
91
93
92 @attr.s
94 @attr.s
93 class bundlespec(object):
95 class bundlespec(object):
94 compression = attr.ib()
96 compression = attr.ib()
95 wirecompression = attr.ib()
97 wirecompression = attr.ib()
96 version = attr.ib()
98 version = attr.ib()
97 wireversion = attr.ib()
99 wireversion = attr.ib()
98 params = attr.ib()
100 params = attr.ib()
99 contentopts = attr.ib()
101 contentopts = attr.ib()
100
102
101 def parsebundlespec(repo, spec, strict=True):
103 def parsebundlespec(repo, spec, strict=True):
102 """Parse a bundle string specification into parts.
104 """Parse a bundle string specification into parts.
103
105
104 Bundle specifications denote a well-defined bundle/exchange format.
106 Bundle specifications denote a well-defined bundle/exchange format.
105 The content of a given specification should not change over time in
107 The content of a given specification should not change over time in
106 order to ensure that bundles produced by a newer version of Mercurial are
108 order to ensure that bundles produced by a newer version of Mercurial are
107 readable from an older version.
109 readable from an older version.
108
110
109 The string currently has the form:
111 The string currently has the form:
110
112
111 <compression>-<type>[;<parameter0>[;<parameter1>]]
113 <compression>-<type>[;<parameter0>[;<parameter1>]]
112
114
113 Where <compression> is one of the supported compression formats
115 Where <compression> is one of the supported compression formats
114 and <type> is (currently) a version string. A ";" can follow the type and
116 and <type> is (currently) a version string. A ";" can follow the type and
115 all text afterwards is interpreted as URI encoded, ";" delimited key=value
117 all text afterwards is interpreted as URI encoded, ";" delimited key=value
116 pairs.
118 pairs.
117
119
118 If ``strict`` is True (the default) <compression> is required. Otherwise,
120 If ``strict`` is True (the default) <compression> is required. Otherwise,
119 it is optional.
121 it is optional.
120
122
121 Returns a bundlespec object of (compression, version, parameters).
123 Returns a bundlespec object of (compression, version, parameters).
122 Compression will be ``None`` if not in strict mode and a compression isn't
124 Compression will be ``None`` if not in strict mode and a compression isn't
123 defined.
125 defined.
124
126
125 An ``InvalidBundleSpecification`` is raised when the specification is
127 An ``InvalidBundleSpecification`` is raised when the specification is
126 not syntactically well formed.
128 not syntactically well formed.
127
129
128 An ``UnsupportedBundleSpecification`` is raised when the compression or
130 An ``UnsupportedBundleSpecification`` is raised when the compression or
129 bundle type/version is not recognized.
131 bundle type/version is not recognized.
130
132
131 Note: this function will likely eventually return a more complex data
133 Note: this function will likely eventually return a more complex data
132 structure, including bundle2 part information.
134 structure, including bundle2 part information.
133 """
135 """
134 def parseparams(s):
136 def parseparams(s):
135 if ';' not in s:
137 if ';' not in s:
136 return s, {}
138 return s, {}
137
139
138 params = {}
140 params = {}
139 version, paramstr = s.split(';', 1)
141 version, paramstr = s.split(';', 1)
140
142
141 for p in paramstr.split(';'):
143 for p in paramstr.split(';'):
142 if '=' not in p:
144 if '=' not in p:
143 raise error.InvalidBundleSpecification(
145 raise error.InvalidBundleSpecification(
144 _('invalid bundle specification: '
146 _('invalid bundle specification: '
145 'missing "=" in parameter: %s') % p)
147 'missing "=" in parameter: %s') % p)
146
148
147 key, value = p.split('=', 1)
149 key, value = p.split('=', 1)
148 key = urlreq.unquote(key)
150 key = urlreq.unquote(key)
149 value = urlreq.unquote(value)
151 value = urlreq.unquote(value)
150 params[key] = value
152 params[key] = value
151
153
152 return version, params
154 return version, params
153
155
154
156
155 if strict and '-' not in spec:
157 if strict and '-' not in spec:
156 raise error.InvalidBundleSpecification(
158 raise error.InvalidBundleSpecification(
157 _('invalid bundle specification; '
159 _('invalid bundle specification; '
158 'must be prefixed with compression: %s') % spec)
160 'must be prefixed with compression: %s') % spec)
159
161
160 if '-' in spec:
162 if '-' in spec:
161 compression, version = spec.split('-', 1)
163 compression, version = spec.split('-', 1)
162
164
163 if compression not in util.compengines.supportedbundlenames:
165 if compression not in util.compengines.supportedbundlenames:
164 raise error.UnsupportedBundleSpecification(
166 raise error.UnsupportedBundleSpecification(
165 _('%s compression is not supported') % compression)
167 _('%s compression is not supported') % compression)
166
168
167 version, params = parseparams(version)
169 version, params = parseparams(version)
168
170
169 if version not in _bundlespeccgversions:
171 if version not in _bundlespeccgversions:
170 raise error.UnsupportedBundleSpecification(
172 raise error.UnsupportedBundleSpecification(
171 _('%s is not a recognized bundle version') % version)
173 _('%s is not a recognized bundle version') % version)
172 else:
174 else:
173 # Value could be just the compression or just the version, in which
175 # Value could be just the compression or just the version, in which
174 # case some defaults are assumed (but only when not in strict mode).
176 # case some defaults are assumed (but only when not in strict mode).
175 assert not strict
177 assert not strict
176
178
177 spec, params = parseparams(spec)
179 spec, params = parseparams(spec)
178
180
179 if spec in util.compengines.supportedbundlenames:
181 if spec in util.compengines.supportedbundlenames:
180 compression = spec
182 compression = spec
181 version = 'v1'
183 version = 'v1'
182 # Generaldelta repos require v2.
184 # Generaldelta repos require v2.
183 if 'generaldelta' in repo.requirements:
185 if 'generaldelta' in repo.requirements:
184 version = 'v2'
186 version = 'v2'
185 # Modern compression engines require v2.
187 # Modern compression engines require v2.
186 if compression not in _bundlespecv1compengines:
188 if compression not in _bundlespecv1compengines:
187 version = 'v2'
189 version = 'v2'
188 elif spec in _bundlespeccgversions:
190 elif spec in _bundlespeccgversions:
189 if spec == 'packed1':
191 if spec == 'packed1':
190 compression = 'none'
192 compression = 'none'
191 else:
193 else:
192 compression = 'bzip2'
194 compression = 'bzip2'
193 version = spec
195 version = spec
194 else:
196 else:
195 raise error.UnsupportedBundleSpecification(
197 raise error.UnsupportedBundleSpecification(
196 _('%s is not a recognized bundle specification') % spec)
198 _('%s is not a recognized bundle specification') % spec)
197
199
198 # Bundle version 1 only supports a known set of compression engines.
200 # Bundle version 1 only supports a known set of compression engines.
199 if version == 'v1' and compression not in _bundlespecv1compengines:
201 if version == 'v1' and compression not in _bundlespecv1compengines:
200 raise error.UnsupportedBundleSpecification(
202 raise error.UnsupportedBundleSpecification(
201 _('compression engine %s is not supported on v1 bundles') %
203 _('compression engine %s is not supported on v1 bundles') %
202 compression)
204 compression)
203
205
204 # The specification for packed1 can optionally declare the data formats
206 # The specification for packed1 can optionally declare the data formats
205 # required to apply it. If we see this metadata, compare against what the
207 # required to apply it. If we see this metadata, compare against what the
206 # repo supports and error if the bundle isn't compatible.
208 # repo supports and error if the bundle isn't compatible.
207 if version == 'packed1' and 'requirements' in params:
209 if version == 'packed1' and 'requirements' in params:
208 requirements = set(params['requirements'].split(','))
210 requirements = set(params['requirements'].split(','))
209 missingreqs = requirements - repo.supportedformats
211 missingreqs = requirements - repo.supportedformats
210 if missingreqs:
212 if missingreqs:
211 raise error.UnsupportedBundleSpecification(
213 raise error.UnsupportedBundleSpecification(
212 _('missing support for repository features: %s') %
214 _('missing support for repository features: %s') %
213 ', '.join(sorted(missingreqs)))
215 ', '.join(sorted(missingreqs)))
214
216
215 # Compute contentopts based on the version
217 # Compute contentopts based on the version
216 contentopts = _bundlespeccontentopts.get(version, {}).copy()
218 contentopts = _bundlespeccontentopts.get(version, {}).copy()
217
219
218 # Process the variants
220 # Process the variants
219 if "stream" in params and params["stream"] == "v2":
221 if "stream" in params and params["stream"] == "v2":
220 variant = _bundlespecvariants["streamv2"]
222 variant = _bundlespecvariants["streamv2"]
221 contentopts.update(variant)
223 contentopts.update(variant)
222
224
223 engine = util.compengines.forbundlename(compression)
225 engine = util.compengines.forbundlename(compression)
224 compression, wirecompression = engine.bundletype()
226 compression, wirecompression = engine.bundletype()
225 wireversion = _bundlespeccgversions[version]
227 wireversion = _bundlespeccgversions[version]
226
228
227 return bundlespec(compression, wirecompression, version, wireversion,
229 return bundlespec(compression, wirecompression, version, wireversion,
228 params, contentopts)
230 params, contentopts)
229
231
230 def readbundle(ui, fh, fname, vfs=None):
232 def readbundle(ui, fh, fname, vfs=None):
231 header = changegroup.readexactly(fh, 4)
233 header = changegroup.readexactly(fh, 4)
232
234
233 alg = None
235 alg = None
234 if not fname:
236 if not fname:
235 fname = "stream"
237 fname = "stream"
236 if not header.startswith('HG') and header.startswith('\0'):
238 if not header.startswith('HG') and header.startswith('\0'):
237 fh = changegroup.headerlessfixup(fh, header)
239 fh = changegroup.headerlessfixup(fh, header)
238 header = "HG10"
240 header = "HG10"
239 alg = 'UN'
241 alg = 'UN'
240 elif vfs:
242 elif vfs:
241 fname = vfs.join(fname)
243 fname = vfs.join(fname)
242
244
243 magic, version = header[0:2], header[2:4]
245 magic, version = header[0:2], header[2:4]
244
246
245 if magic != 'HG':
247 if magic != 'HG':
246 raise error.Abort(_('%s: not a Mercurial bundle') % fname)
248 raise error.Abort(_('%s: not a Mercurial bundle') % fname)
247 if version == '10':
249 if version == '10':
248 if alg is None:
250 if alg is None:
249 alg = changegroup.readexactly(fh, 2)
251 alg = changegroup.readexactly(fh, 2)
250 return changegroup.cg1unpacker(fh, alg)
252 return changegroup.cg1unpacker(fh, alg)
251 elif version.startswith('2'):
253 elif version.startswith('2'):
252 return bundle2.getunbundler(ui, fh, magicstring=magic + version)
254 return bundle2.getunbundler(ui, fh, magicstring=magic + version)
253 elif version == 'S1':
255 elif version == 'S1':
254 return streamclone.streamcloneapplier(fh)
256 return streamclone.streamcloneapplier(fh)
255 else:
257 else:
256 raise error.Abort(_('%s: unknown bundle version %s') % (fname, version))
258 raise error.Abort(_('%s: unknown bundle version %s') % (fname, version))
257
259
258 def getbundlespec(ui, fh):
260 def getbundlespec(ui, fh):
259 """Infer the bundlespec from a bundle file handle.
261 """Infer the bundlespec from a bundle file handle.
260
262
261 The input file handle is seeked and the original seek position is not
263 The input file handle is seeked and the original seek position is not
262 restored.
264 restored.
263 """
265 """
264 def speccompression(alg):
266 def speccompression(alg):
265 try:
267 try:
266 return util.compengines.forbundletype(alg).bundletype()[0]
268 return util.compengines.forbundletype(alg).bundletype()[0]
267 except KeyError:
269 except KeyError:
268 return None
270 return None
269
271
270 b = readbundle(ui, fh, None)
272 b = readbundle(ui, fh, None)
271 if isinstance(b, changegroup.cg1unpacker):
273 if isinstance(b, changegroup.cg1unpacker):
272 alg = b._type
274 alg = b._type
273 if alg == '_truncatedBZ':
275 if alg == '_truncatedBZ':
274 alg = 'BZ'
276 alg = 'BZ'
275 comp = speccompression(alg)
277 comp = speccompression(alg)
276 if not comp:
278 if not comp:
277 raise error.Abort(_('unknown compression algorithm: %s') % alg)
279 raise error.Abort(_('unknown compression algorithm: %s') % alg)
278 return '%s-v1' % comp
280 return '%s-v1' % comp
279 elif isinstance(b, bundle2.unbundle20):
281 elif isinstance(b, bundle2.unbundle20):
280 if 'Compression' in b.params:
282 if 'Compression' in b.params:
281 comp = speccompression(b.params['Compression'])
283 comp = speccompression(b.params['Compression'])
282 if not comp:
284 if not comp:
283 raise error.Abort(_('unknown compression algorithm: %s') % comp)
285 raise error.Abort(_('unknown compression algorithm: %s') % comp)
284 else:
286 else:
285 comp = 'none'
287 comp = 'none'
286
288
287 version = None
289 version = None
288 for part in b.iterparts():
290 for part in b.iterparts():
289 if part.type == 'changegroup':
291 if part.type == 'changegroup':
290 version = part.params['version']
292 version = part.params['version']
291 if version in ('01', '02'):
293 if version in ('01', '02'):
292 version = 'v2'
294 version = 'v2'
293 else:
295 else:
294 raise error.Abort(_('changegroup version %s does not have '
296 raise error.Abort(_('changegroup version %s does not have '
295 'a known bundlespec') % version,
297 'a known bundlespec') % version,
296 hint=_('try upgrading your Mercurial '
298 hint=_('try upgrading your Mercurial '
297 'client'))
299 'client'))
298 elif part.type == 'stream2' and version is None:
300 elif part.type == 'stream2' and version is None:
299 # A stream2 part requires to be part of a v2 bundle
301 # A stream2 part requires to be part of a v2 bundle
300 requirements = urlreq.unquote(part.params['requirements'])
302 requirements = urlreq.unquote(part.params['requirements'])
301 splitted = requirements.split()
303 splitted = requirements.split()
302 params = bundle2._formatrequirementsparams(splitted)
304 params = bundle2._formatrequirementsparams(splitted)
303 return 'none-v2;stream=v2;%s' % params
305 return 'none-v2;stream=v2;%s' % params
304
306
305 if not version:
307 if not version:
306 raise error.Abort(_('could not identify changegroup version in '
308 raise error.Abort(_('could not identify changegroup version in '
307 'bundle'))
309 'bundle'))
308
310
309 return '%s-%s' % (comp, version)
311 return '%s-%s' % (comp, version)
310 elif isinstance(b, streamclone.streamcloneapplier):
312 elif isinstance(b, streamclone.streamcloneapplier):
311 requirements = streamclone.readbundle1header(fh)[2]
313 requirements = streamclone.readbundle1header(fh)[2]
312 formatted = bundle2._formatrequirementsparams(requirements)
314 formatted = bundle2._formatrequirementsparams(requirements)
313 return 'none-packed1;%s' % formatted
315 return 'none-packed1;%s' % formatted
314 else:
316 else:
315 raise error.Abort(_('unknown bundle type: %s') % b)
317 raise error.Abort(_('unknown bundle type: %s') % b)
316
318
317 def _computeoutgoing(repo, heads, common):
319 def _computeoutgoing(repo, heads, common):
318 """Computes which revs are outgoing given a set of common
320 """Computes which revs are outgoing given a set of common
319 and a set of heads.
321 and a set of heads.
320
322
321 This is a separate function so extensions can have access to
323 This is a separate function so extensions can have access to
322 the logic.
324 the logic.
323
325
324 Returns a discovery.outgoing object.
326 Returns a discovery.outgoing object.
325 """
327 """
326 cl = repo.changelog
328 cl = repo.changelog
327 if common:
329 if common:
328 hasnode = cl.hasnode
330 hasnode = cl.hasnode
329 common = [n for n in common if hasnode(n)]
331 common = [n for n in common if hasnode(n)]
330 else:
332 else:
331 common = [nullid]
333 common = [nullid]
332 if not heads:
334 if not heads:
333 heads = cl.heads()
335 heads = cl.heads()
334 return discovery.outgoing(repo, common, heads)
336 return discovery.outgoing(repo, common, heads)
335
337
336 def _checkpublish(pushop):
338 def _checkpublish(pushop):
337 repo = pushop.repo
339 repo = pushop.repo
338 ui = repo.ui
340 ui = repo.ui
339 behavior = ui.config('experimental', 'auto-publish')
341 behavior = ui.config('experimental', 'auto-publish')
340 if pushop.publish or behavior not in ('warn', 'confirm', 'abort'):
342 if pushop.publish or behavior not in ('warn', 'confirm', 'abort'):
341 return
343 return
342 remotephases = listkeys(pushop.remote, 'phases')
344 remotephases = listkeys(pushop.remote, 'phases')
343 if not remotephases.get('publishing', False):
345 if not remotephases.get('publishing', False):
344 return
346 return
345
347
346 if pushop.revs is None:
348 if pushop.revs is None:
347 published = repo.filtered('served').revs('not public()')
349 published = repo.filtered('served').revs('not public()')
348 else:
350 else:
349 published = repo.revs('::%ln - public()', pushop.revs)
351 published = repo.revs('::%ln - public()', pushop.revs)
350 if published:
352 if published:
351 if behavior == 'warn':
353 if behavior == 'warn':
352 ui.warn(_('%i changesets about to be published\n')
354 ui.warn(_('%i changesets about to be published\n')
353 % len(published))
355 % len(published))
354 elif behavior == 'confirm':
356 elif behavior == 'confirm':
355 if ui.promptchoice(_('push and publish %i changesets (yn)?'
357 if ui.promptchoice(_('push and publish %i changesets (yn)?'
356 '$$ &Yes $$ &No') % len(published)):
358 '$$ &Yes $$ &No') % len(published)):
357 raise error.Abort(_('user quit'))
359 raise error.Abort(_('user quit'))
358 elif behavior == 'abort':
360 elif behavior == 'abort':
359 msg = _('push would publish %i changesets') % len(published)
361 msg = _('push would publish %i changesets') % len(published)
360 hint = _("use --publish or adjust 'experimental.auto-publish'"
362 hint = _("use --publish or adjust 'experimental.auto-publish'"
361 " config")
363 " config")
362 raise error.Abort(msg, hint=hint)
364 raise error.Abort(msg, hint=hint)
363
365
364 def _forcebundle1(op):
366 def _forcebundle1(op):
365 """return true if a pull/push must use bundle1
367 """return true if a pull/push must use bundle1
366
368
367 This function is used to allow testing of the older bundle version"""
369 This function is used to allow testing of the older bundle version"""
368 ui = op.repo.ui
370 ui = op.repo.ui
369 # The goal is this config is to allow developer to choose the bundle
371 # The goal is this config is to allow developer to choose the bundle
370 # version used during exchanged. This is especially handy during test.
372 # version used during exchanged. This is especially handy during test.
371 # Value is a list of bundle version to be picked from, highest version
373 # Value is a list of bundle version to be picked from, highest version
372 # should be used.
374 # should be used.
373 #
375 #
374 # developer config: devel.legacy.exchange
376 # developer config: devel.legacy.exchange
375 exchange = ui.configlist('devel', 'legacy.exchange')
377 exchange = ui.configlist('devel', 'legacy.exchange')
376 forcebundle1 = 'bundle2' not in exchange and 'bundle1' in exchange
378 forcebundle1 = 'bundle2' not in exchange and 'bundle1' in exchange
377 return forcebundle1 or not op.remote.capable('bundle2')
379 return forcebundle1 or not op.remote.capable('bundle2')
378
380
379 class pushoperation(object):
381 class pushoperation(object):
380 """A object that represent a single push operation
382 """A object that represent a single push operation
381
383
382 Its purpose is to carry push related state and very common operations.
384 Its purpose is to carry push related state and very common operations.
383
385
384 A new pushoperation should be created at the beginning of each push and
386 A new pushoperation should be created at the beginning of each push and
385 discarded afterward.
387 discarded afterward.
386 """
388 """
387
389
388 def __init__(self, repo, remote, force=False, revs=None, newbranch=False,
390 def __init__(self, repo, remote, force=False, revs=None, newbranch=False,
389 bookmarks=(), publish=False, pushvars=None):
391 bookmarks=(), publish=False, pushvars=None):
390 # repo we push from
392 # repo we push from
391 self.repo = repo
393 self.repo = repo
392 self.ui = repo.ui
394 self.ui = repo.ui
393 # repo we push to
395 # repo we push to
394 self.remote = remote
396 self.remote = remote
395 # force option provided
397 # force option provided
396 self.force = force
398 self.force = force
397 # revs to be pushed (None is "all")
399 # revs to be pushed (None is "all")
398 self.revs = revs
400 self.revs = revs
399 # bookmark explicitly pushed
401 # bookmark explicitly pushed
400 self.bookmarks = bookmarks
402 self.bookmarks = bookmarks
401 # allow push of new branch
403 # allow push of new branch
402 self.newbranch = newbranch
404 self.newbranch = newbranch
403 # step already performed
405 # step already performed
404 # (used to check what steps have been already performed through bundle2)
406 # (used to check what steps have been already performed through bundle2)
405 self.stepsdone = set()
407 self.stepsdone = set()
406 # Integer version of the changegroup push result
408 # Integer version of the changegroup push result
407 # - None means nothing to push
409 # - None means nothing to push
408 # - 0 means HTTP error
410 # - 0 means HTTP error
409 # - 1 means we pushed and remote head count is unchanged *or*
411 # - 1 means we pushed and remote head count is unchanged *or*
410 # we have outgoing changesets but refused to push
412 # we have outgoing changesets but refused to push
411 # - other values as described by addchangegroup()
413 # - other values as described by addchangegroup()
412 self.cgresult = None
414 self.cgresult = None
413 # Boolean value for the bookmark push
415 # Boolean value for the bookmark push
414 self.bkresult = None
416 self.bkresult = None
415 # discover.outgoing object (contains common and outgoing data)
417 # discover.outgoing object (contains common and outgoing data)
416 self.outgoing = None
418 self.outgoing = None
417 # all remote topological heads before the push
419 # all remote topological heads before the push
418 self.remoteheads = None
420 self.remoteheads = None
419 # Details of the remote branch pre and post push
421 # Details of the remote branch pre and post push
420 #
422 #
421 # mapping: {'branch': ([remoteheads],
423 # mapping: {'branch': ([remoteheads],
422 # [newheads],
424 # [newheads],
423 # [unsyncedheads],
425 # [unsyncedheads],
424 # [discardedheads])}
426 # [discardedheads])}
425 # - branch: the branch name
427 # - branch: the branch name
426 # - remoteheads: the list of remote heads known locally
428 # - remoteheads: the list of remote heads known locally
427 # None if the branch is new
429 # None if the branch is new
428 # - newheads: the new remote heads (known locally) with outgoing pushed
430 # - newheads: the new remote heads (known locally) with outgoing pushed
429 # - unsyncedheads: the list of remote heads unknown locally.
431 # - unsyncedheads: the list of remote heads unknown locally.
430 # - discardedheads: the list of remote heads made obsolete by the push
432 # - discardedheads: the list of remote heads made obsolete by the push
431 self.pushbranchmap = None
433 self.pushbranchmap = None
432 # testable as a boolean indicating if any nodes are missing locally.
434 # testable as a boolean indicating if any nodes are missing locally.
433 self.incoming = None
435 self.incoming = None
434 # summary of the remote phase situation
436 # summary of the remote phase situation
435 self.remotephases = None
437 self.remotephases = None
436 # phases changes that must be pushed along side the changesets
438 # phases changes that must be pushed along side the changesets
437 self.outdatedphases = None
439 self.outdatedphases = None
438 # phases changes that must be pushed if changeset push fails
440 # phases changes that must be pushed if changeset push fails
439 self.fallbackoutdatedphases = None
441 self.fallbackoutdatedphases = None
440 # outgoing obsmarkers
442 # outgoing obsmarkers
441 self.outobsmarkers = set()
443 self.outobsmarkers = set()
442 # outgoing bookmarks
444 # outgoing bookmarks
443 self.outbookmarks = []
445 self.outbookmarks = []
444 # transaction manager
446 # transaction manager
445 self.trmanager = None
447 self.trmanager = None
446 # map { pushkey partid -> callback handling failure}
448 # map { pushkey partid -> callback handling failure}
447 # used to handle exception from mandatory pushkey part failure
449 # used to handle exception from mandatory pushkey part failure
448 self.pkfailcb = {}
450 self.pkfailcb = {}
449 # an iterable of pushvars or None
451 # an iterable of pushvars or None
450 self.pushvars = pushvars
452 self.pushvars = pushvars
451 # publish pushed changesets
453 # publish pushed changesets
452 self.publish = publish
454 self.publish = publish
453
455
454 @util.propertycache
456 @util.propertycache
455 def futureheads(self):
457 def futureheads(self):
456 """future remote heads if the changeset push succeeds"""
458 """future remote heads if the changeset push succeeds"""
457 return self.outgoing.missingheads
459 return self.outgoing.missingheads
458
460
459 @util.propertycache
461 @util.propertycache
460 def fallbackheads(self):
462 def fallbackheads(self):
461 """future remote heads if the changeset push fails"""
463 """future remote heads if the changeset push fails"""
462 if self.revs is None:
464 if self.revs is None:
463 # not target to push, all common are relevant
465 # not target to push, all common are relevant
464 return self.outgoing.commonheads
466 return self.outgoing.commonheads
465 unfi = self.repo.unfiltered()
467 unfi = self.repo.unfiltered()
466 # I want cheads = heads(::missingheads and ::commonheads)
468 # I want cheads = heads(::missingheads and ::commonheads)
467 # (missingheads is revs with secret changeset filtered out)
469 # (missingheads is revs with secret changeset filtered out)
468 #
470 #
469 # This can be expressed as:
471 # This can be expressed as:
470 # cheads = ( (missingheads and ::commonheads)
472 # cheads = ( (missingheads and ::commonheads)
471 # + (commonheads and ::missingheads))"
473 # + (commonheads and ::missingheads))"
472 # )
474 # )
473 #
475 #
474 # while trying to push we already computed the following:
476 # while trying to push we already computed the following:
475 # common = (::commonheads)
477 # common = (::commonheads)
476 # missing = ((commonheads::missingheads) - commonheads)
478 # missing = ((commonheads::missingheads) - commonheads)
477 #
479 #
478 # We can pick:
480 # We can pick:
479 # * missingheads part of common (::commonheads)
481 # * missingheads part of common (::commonheads)
480 common = self.outgoing.common
482 common = self.outgoing.common
481 nm = self.repo.changelog.nodemap
483 nm = self.repo.changelog.nodemap
482 cheads = [node for node in self.revs if nm[node] in common]
484 cheads = [node for node in self.revs if nm[node] in common]
483 # and
485 # and
484 # * commonheads parents on missing
486 # * commonheads parents on missing
485 revset = unfi.set('%ln and parents(roots(%ln))',
487 revset = unfi.set('%ln and parents(roots(%ln))',
486 self.outgoing.commonheads,
488 self.outgoing.commonheads,
487 self.outgoing.missing)
489 self.outgoing.missing)
488 cheads.extend(c.node() for c in revset)
490 cheads.extend(c.node() for c in revset)
489 return cheads
491 return cheads
490
492
491 @property
493 @property
492 def commonheads(self):
494 def commonheads(self):
493 """set of all common heads after changeset bundle push"""
495 """set of all common heads after changeset bundle push"""
494 if self.cgresult:
496 if self.cgresult:
495 return self.futureheads
497 return self.futureheads
496 else:
498 else:
497 return self.fallbackheads
499 return self.fallbackheads
498
500
499 # mapping of message used when pushing bookmark
501 # mapping of message used when pushing bookmark
500 bookmsgmap = {'update': (_("updating bookmark %s\n"),
502 bookmsgmap = {'update': (_("updating bookmark %s\n"),
501 _('updating bookmark %s failed!\n')),
503 _('updating bookmark %s failed!\n')),
502 'export': (_("exporting bookmark %s\n"),
504 'export': (_("exporting bookmark %s\n"),
503 _('exporting bookmark %s failed!\n')),
505 _('exporting bookmark %s failed!\n')),
504 'delete': (_("deleting remote bookmark %s\n"),
506 'delete': (_("deleting remote bookmark %s\n"),
505 _('deleting remote bookmark %s failed!\n')),
507 _('deleting remote bookmark %s failed!\n')),
506 }
508 }
507
509
508
510
509 def push(repo, remote, force=False, revs=None, newbranch=False, bookmarks=(),
511 def push(repo, remote, force=False, revs=None, newbranch=False, bookmarks=(),
510 publish=False, opargs=None):
512 publish=False, opargs=None):
511 '''Push outgoing changesets (limited by revs) from a local
513 '''Push outgoing changesets (limited by revs) from a local
512 repository to remote. Return an integer:
514 repository to remote. Return an integer:
513 - None means nothing to push
515 - None means nothing to push
514 - 0 means HTTP error
516 - 0 means HTTP error
515 - 1 means we pushed and remote head count is unchanged *or*
517 - 1 means we pushed and remote head count is unchanged *or*
516 we have outgoing changesets but refused to push
518 we have outgoing changesets but refused to push
517 - other values as described by addchangegroup()
519 - other values as described by addchangegroup()
518 '''
520 '''
519 if opargs is None:
521 if opargs is None:
520 opargs = {}
522 opargs = {}
521 pushop = pushoperation(repo, remote, force, revs, newbranch, bookmarks,
523 pushop = pushoperation(repo, remote, force, revs, newbranch, bookmarks,
522 publish, **pycompat.strkwargs(opargs))
524 publish, **pycompat.strkwargs(opargs))
523 if pushop.remote.local():
525 if pushop.remote.local():
524 missing = (set(pushop.repo.requirements)
526 missing = (set(pushop.repo.requirements)
525 - pushop.remote.local().supported)
527 - pushop.remote.local().supported)
526 if missing:
528 if missing:
527 msg = _("required features are not"
529 msg = _("required features are not"
528 " supported in the destination:"
530 " supported in the destination:"
529 " %s") % (', '.join(sorted(missing)))
531 " %s") % (', '.join(sorted(missing)))
530 raise error.Abort(msg)
532 raise error.Abort(msg)
531
533
532 if not pushop.remote.canpush():
534 if not pushop.remote.canpush():
533 raise error.Abort(_("destination does not support push"))
535 raise error.Abort(_("destination does not support push"))
534
536
535 if not pushop.remote.capable('unbundle'):
537 if not pushop.remote.capable('unbundle'):
536 raise error.Abort(_('cannot push: destination does not support the '
538 raise error.Abort(_('cannot push: destination does not support the '
537 'unbundle wire protocol command'))
539 'unbundle wire protocol command'))
538
540
539 # get lock as we might write phase data
541 # get lock as we might write phase data
540 wlock = lock = None
542 wlock = lock = None
541 try:
543 try:
542 # bundle2 push may receive a reply bundle touching bookmarks
544 # bundle2 push may receive a reply bundle touching bookmarks
543 # requiring the wlock. Take it now to ensure proper ordering.
545 # requiring the wlock. Take it now to ensure proper ordering.
544 maypushback = pushop.ui.configbool('experimental', 'bundle2.pushback')
546 maypushback = pushop.ui.configbool('experimental', 'bundle2.pushback')
545 if ((not _forcebundle1(pushop)) and
547 if ((not _forcebundle1(pushop)) and
546 maypushback and
548 maypushback and
547 not bookmod.bookmarksinstore(repo)):
549 not bookmod.bookmarksinstore(repo)):
548 wlock = pushop.repo.wlock()
550 wlock = pushop.repo.wlock()
549 lock = pushop.repo.lock()
551 lock = pushop.repo.lock()
550 pushop.trmanager = transactionmanager(pushop.repo,
552 pushop.trmanager = transactionmanager(pushop.repo,
551 'push-response',
553 'push-response',
552 pushop.remote.url())
554 pushop.remote.url())
553 except error.LockUnavailable as err:
555 except error.LockUnavailable as err:
554 # source repo cannot be locked.
556 # source repo cannot be locked.
555 # We do not abort the push, but just disable the local phase
557 # We do not abort the push, but just disable the local phase
556 # synchronisation.
558 # synchronisation.
557 msg = ('cannot lock source repository: %s\n'
559 msg = ('cannot lock source repository: %s\n'
558 % stringutil.forcebytestr(err))
560 % stringutil.forcebytestr(err))
559 pushop.ui.debug(msg)
561 pushop.ui.debug(msg)
560
562
561 with wlock or util.nullcontextmanager():
563 with wlock or util.nullcontextmanager():
562 with lock or util.nullcontextmanager():
564 with lock or util.nullcontextmanager():
563 with pushop.trmanager or util.nullcontextmanager():
565 with pushop.trmanager or util.nullcontextmanager():
564 pushop.repo.checkpush(pushop)
566 pushop.repo.checkpush(pushop)
565 _checkpublish(pushop)
567 _checkpublish(pushop)
566 _pushdiscovery(pushop)
568 _pushdiscovery(pushop)
567 if not _forcebundle1(pushop):
569 if not _forcebundle1(pushop):
568 _pushbundle2(pushop)
570 _pushbundle2(pushop)
569 _pushchangeset(pushop)
571 _pushchangeset(pushop)
570 _pushsyncphase(pushop)
572 _pushsyncphase(pushop)
571 _pushobsolete(pushop)
573 _pushobsolete(pushop)
572 _pushbookmark(pushop)
574 _pushbookmark(pushop)
573
575
574 if repo.ui.configbool('experimental', 'remotenames'):
576 if repo.ui.configbool('experimental', 'remotenames'):
575 logexchange.pullremotenames(repo, remote)
577 logexchange.pullremotenames(repo, remote)
576
578
577 return pushop
579 return pushop
578
580
579 # list of steps to perform discovery before push
581 # list of steps to perform discovery before push
580 pushdiscoveryorder = []
582 pushdiscoveryorder = []
581
583
582 # Mapping between step name and function
584 # Mapping between step name and function
583 #
585 #
584 # This exists to help extensions wrap steps if necessary
586 # This exists to help extensions wrap steps if necessary
585 pushdiscoverymapping = {}
587 pushdiscoverymapping = {}
586
588
587 def pushdiscovery(stepname):
589 def pushdiscovery(stepname):
588 """decorator for function performing discovery before push
590 """decorator for function performing discovery before push
589
591
590 The function is added to the step -> function mapping and appended to the
592 The function is added to the step -> function mapping and appended to the
591 list of steps. Beware that decorated function will be added in order (this
593 list of steps. Beware that decorated function will be added in order (this
592 may matter).
594 may matter).
593
595
594 You can only use this decorator for a new step, if you want to wrap a step
596 You can only use this decorator for a new step, if you want to wrap a step
595 from an extension, change the pushdiscovery dictionary directly."""
597 from an extension, change the pushdiscovery dictionary directly."""
596 def dec(func):
598 def dec(func):
597 assert stepname not in pushdiscoverymapping
599 assert stepname not in pushdiscoverymapping
598 pushdiscoverymapping[stepname] = func
600 pushdiscoverymapping[stepname] = func
599 pushdiscoveryorder.append(stepname)
601 pushdiscoveryorder.append(stepname)
600 return func
602 return func
601 return dec
603 return dec
602
604
603 def _pushdiscovery(pushop):
605 def _pushdiscovery(pushop):
604 """Run all discovery steps"""
606 """Run all discovery steps"""
605 for stepname in pushdiscoveryorder:
607 for stepname in pushdiscoveryorder:
606 step = pushdiscoverymapping[stepname]
608 step = pushdiscoverymapping[stepname]
607 step(pushop)
609 step(pushop)
608
610
609 @pushdiscovery('changeset')
611 @pushdiscovery('changeset')
610 def _pushdiscoverychangeset(pushop):
612 def _pushdiscoverychangeset(pushop):
611 """discover the changeset that need to be pushed"""
613 """discover the changeset that need to be pushed"""
612 fci = discovery.findcommonincoming
614 fci = discovery.findcommonincoming
613 if pushop.revs:
615 if pushop.revs:
614 commoninc = fci(pushop.repo, pushop.remote, force=pushop.force,
616 commoninc = fci(pushop.repo, pushop.remote, force=pushop.force,
615 ancestorsof=pushop.revs)
617 ancestorsof=pushop.revs)
616 else:
618 else:
617 commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
619 commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
618 common, inc, remoteheads = commoninc
620 common, inc, remoteheads = commoninc
619 fco = discovery.findcommonoutgoing
621 fco = discovery.findcommonoutgoing
620 outgoing = fco(pushop.repo, pushop.remote, onlyheads=pushop.revs,
622 outgoing = fco(pushop.repo, pushop.remote, onlyheads=pushop.revs,
621 commoninc=commoninc, force=pushop.force)
623 commoninc=commoninc, force=pushop.force)
622 pushop.outgoing = outgoing
624 pushop.outgoing = outgoing
623 pushop.remoteheads = remoteheads
625 pushop.remoteheads = remoteheads
624 pushop.incoming = inc
626 pushop.incoming = inc
625
627
626 @pushdiscovery('phase')
628 @pushdiscovery('phase')
627 def _pushdiscoveryphase(pushop):
629 def _pushdiscoveryphase(pushop):
628 """discover the phase that needs to be pushed
630 """discover the phase that needs to be pushed
629
631
630 (computed for both success and failure case for changesets push)"""
632 (computed for both success and failure case for changesets push)"""
631 outgoing = pushop.outgoing
633 outgoing = pushop.outgoing
632 unfi = pushop.repo.unfiltered()
634 unfi = pushop.repo.unfiltered()
633 remotephases = listkeys(pushop.remote, 'phases')
635 remotephases = listkeys(pushop.remote, 'phases')
634
636
635 if (pushop.ui.configbool('ui', '_usedassubrepo')
637 if (pushop.ui.configbool('ui', '_usedassubrepo')
636 and remotephases # server supports phases
638 and remotephases # server supports phases
637 and not pushop.outgoing.missing # no changesets to be pushed
639 and not pushop.outgoing.missing # no changesets to be pushed
638 and remotephases.get('publishing', False)):
640 and remotephases.get('publishing', False)):
639 # When:
641 # When:
640 # - this is a subrepo push
642 # - this is a subrepo push
641 # - and remote support phase
643 # - and remote support phase
642 # - and no changeset are to be pushed
644 # - and no changeset are to be pushed
643 # - and remote is publishing
645 # - and remote is publishing
644 # We may be in issue 3781 case!
646 # We may be in issue 3781 case!
645 # We drop the possible phase synchronisation done by
647 # We drop the possible phase synchronisation done by
646 # courtesy to publish changesets possibly locally draft
648 # courtesy to publish changesets possibly locally draft
647 # on the remote.
649 # on the remote.
648 pushop.outdatedphases = []
650 pushop.outdatedphases = []
649 pushop.fallbackoutdatedphases = []
651 pushop.fallbackoutdatedphases = []
650 return
652 return
651
653
652 pushop.remotephases = phases.remotephasessummary(pushop.repo,
654 pushop.remotephases = phases.remotephasessummary(pushop.repo,
653 pushop.fallbackheads,
655 pushop.fallbackheads,
654 remotephases)
656 remotephases)
655 droots = pushop.remotephases.draftroots
657 droots = pushop.remotephases.draftroots
656
658
657 extracond = ''
659 extracond = ''
658 if not pushop.remotephases.publishing:
660 if not pushop.remotephases.publishing:
659 extracond = ' and public()'
661 extracond = ' and public()'
660 revset = 'heads((%%ln::%%ln) %s)' % extracond
662 revset = 'heads((%%ln::%%ln) %s)' % extracond
661 # Get the list of all revs draft on remote by public here.
663 # Get the list of all revs draft on remote by public here.
662 # XXX Beware that revset break if droots is not strictly
664 # XXX Beware that revset break if droots is not strictly
663 # XXX root we may want to ensure it is but it is costly
665 # XXX root we may want to ensure it is but it is costly
664 fallback = list(unfi.set(revset, droots, pushop.fallbackheads))
666 fallback = list(unfi.set(revset, droots, pushop.fallbackheads))
665 if not pushop.remotephases.publishing and pushop.publish:
667 if not pushop.remotephases.publishing and pushop.publish:
666 future = list(unfi.set('%ln and (not public() or %ln::)',
668 future = list(unfi.set('%ln and (not public() or %ln::)',
667 pushop.futureheads, droots))
669 pushop.futureheads, droots))
668 elif not outgoing.missing:
670 elif not outgoing.missing:
669 future = fallback
671 future = fallback
670 else:
672 else:
671 # adds changeset we are going to push as draft
673 # adds changeset we are going to push as draft
672 #
674 #
673 # should not be necessary for publishing server, but because of an
675 # should not be necessary for publishing server, but because of an
674 # issue fixed in xxxxx we have to do it anyway.
676 # issue fixed in xxxxx we have to do it anyway.
675 fdroots = list(unfi.set('roots(%ln + %ln::)',
677 fdroots = list(unfi.set('roots(%ln + %ln::)',
676 outgoing.missing, droots))
678 outgoing.missing, droots))
677 fdroots = [f.node() for f in fdroots]
679 fdroots = [f.node() for f in fdroots]
678 future = list(unfi.set(revset, fdroots, pushop.futureheads))
680 future = list(unfi.set(revset, fdroots, pushop.futureheads))
679 pushop.outdatedphases = future
681 pushop.outdatedphases = future
680 pushop.fallbackoutdatedphases = fallback
682 pushop.fallbackoutdatedphases = fallback
681
683
682 @pushdiscovery('obsmarker')
684 @pushdiscovery('obsmarker')
683 def _pushdiscoveryobsmarkers(pushop):
685 def _pushdiscoveryobsmarkers(pushop):
684 if not obsolete.isenabled(pushop.repo, obsolete.exchangeopt):
686 if not obsolete.isenabled(pushop.repo, obsolete.exchangeopt):
685 return
687 return
686
688
687 if not pushop.repo.obsstore:
689 if not pushop.repo.obsstore:
688 return
690 return
689
691
690 if 'obsolete' not in listkeys(pushop.remote, 'namespaces'):
692 if 'obsolete' not in listkeys(pushop.remote, 'namespaces'):
691 return
693 return
692
694
693 repo = pushop.repo
695 repo = pushop.repo
694 # very naive computation, that can be quite expensive on big repo.
696 # very naive computation, that can be quite expensive on big repo.
695 # However: evolution is currently slow on them anyway.
697 # However: evolution is currently slow on them anyway.
696 nodes = (c.node() for c in repo.set('::%ln', pushop.futureheads))
698 nodes = (c.node() for c in repo.set('::%ln', pushop.futureheads))
697 pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
699 pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
698
700
699 @pushdiscovery('bookmarks')
701 @pushdiscovery('bookmarks')
700 def _pushdiscoverybookmarks(pushop):
702 def _pushdiscoverybookmarks(pushop):
701 ui = pushop.ui
703 ui = pushop.ui
702 repo = pushop.repo.unfiltered()
704 repo = pushop.repo.unfiltered()
703 remote = pushop.remote
705 remote = pushop.remote
704 ui.debug("checking for updated bookmarks\n")
706 ui.debug("checking for updated bookmarks\n")
705 ancestors = ()
707 ancestors = ()
706 if pushop.revs:
708 if pushop.revs:
707 revnums = pycompat.maplist(repo.changelog.rev, pushop.revs)
709 revnums = pycompat.maplist(repo.changelog.rev, pushop.revs)
708 ancestors = repo.changelog.ancestors(revnums, inclusive=True)
710 ancestors = repo.changelog.ancestors(revnums, inclusive=True)
709
711
710 remotebookmark = listkeys(remote, 'bookmarks')
712 remotebookmark = listkeys(remote, 'bookmarks')
711
713
712 explicit = {repo._bookmarks.expandname(bookmark)
714 explicit = {repo._bookmarks.expandname(bookmark)
713 for bookmark in pushop.bookmarks}
715 for bookmark in pushop.bookmarks}
714
716
715 remotebookmark = bookmod.unhexlifybookmarks(remotebookmark)
717 remotebookmark = bookmod.unhexlifybookmarks(remotebookmark)
716 comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
718 comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
717
719
718 def safehex(x):
720 def safehex(x):
719 if x is None:
721 if x is None:
720 return x
722 return x
721 return hex(x)
723 return hex(x)
722
724
723 def hexifycompbookmarks(bookmarks):
725 def hexifycompbookmarks(bookmarks):
724 return [(b, safehex(scid), safehex(dcid))
726 return [(b, safehex(scid), safehex(dcid))
725 for (b, scid, dcid) in bookmarks]
727 for (b, scid, dcid) in bookmarks]
726
728
727 comp = [hexifycompbookmarks(marks) for marks in comp]
729 comp = [hexifycompbookmarks(marks) for marks in comp]
728 return _processcompared(pushop, ancestors, explicit, remotebookmark, comp)
730 return _processcompared(pushop, ancestors, explicit, remotebookmark, comp)
729
731
730 def _processcompared(pushop, pushed, explicit, remotebms, comp):
732 def _processcompared(pushop, pushed, explicit, remotebms, comp):
731 """take decision on bookmark to pull from the remote bookmark
733 """take decision on bookmark to pull from the remote bookmark
732
734
733 Exist to help extensions who want to alter this behavior.
735 Exist to help extensions who want to alter this behavior.
734 """
736 """
735 addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
737 addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
736
738
737 repo = pushop.repo
739 repo = pushop.repo
738
740
739 for b, scid, dcid in advsrc:
741 for b, scid, dcid in advsrc:
740 if b in explicit:
742 if b in explicit:
741 explicit.remove(b)
743 explicit.remove(b)
742 if not pushed or repo[scid].rev() in pushed:
744 if not pushed or repo[scid].rev() in pushed:
743 pushop.outbookmarks.append((b, dcid, scid))
745 pushop.outbookmarks.append((b, dcid, scid))
744 # search added bookmark
746 # search added bookmark
745 for b, scid, dcid in addsrc:
747 for b, scid, dcid in addsrc:
746 if b in explicit:
748 if b in explicit:
747 explicit.remove(b)
749 explicit.remove(b)
748 pushop.outbookmarks.append((b, '', scid))
750 pushop.outbookmarks.append((b, '', scid))
749 # search for overwritten bookmark
751 # search for overwritten bookmark
750 for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
752 for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
751 if b in explicit:
753 if b in explicit:
752 explicit.remove(b)
754 explicit.remove(b)
753 pushop.outbookmarks.append((b, dcid, scid))
755 pushop.outbookmarks.append((b, dcid, scid))
754 # search for bookmark to delete
756 # search for bookmark to delete
755 for b, scid, dcid in adddst:
757 for b, scid, dcid in adddst:
756 if b in explicit:
758 if b in explicit:
757 explicit.remove(b)
759 explicit.remove(b)
758 # treat as "deleted locally"
760 # treat as "deleted locally"
759 pushop.outbookmarks.append((b, dcid, ''))
761 pushop.outbookmarks.append((b, dcid, ''))
760 # identical bookmarks shouldn't get reported
762 # identical bookmarks shouldn't get reported
761 for b, scid, dcid in same:
763 for b, scid, dcid in same:
762 if b in explicit:
764 if b in explicit:
763 explicit.remove(b)
765 explicit.remove(b)
764
766
765 if explicit:
767 if explicit:
766 explicit = sorted(explicit)
768 explicit = sorted(explicit)
767 # we should probably list all of them
769 # we should probably list all of them
768 pushop.ui.warn(_('bookmark %s does not exist on the local '
770 pushop.ui.warn(_('bookmark %s does not exist on the local '
769 'or remote repository!\n') % explicit[0])
771 'or remote repository!\n') % explicit[0])
770 pushop.bkresult = 2
772 pushop.bkresult = 2
771
773
772 pushop.outbookmarks.sort()
774 pushop.outbookmarks.sort()
773
775
774 def _pushcheckoutgoing(pushop):
776 def _pushcheckoutgoing(pushop):
775 outgoing = pushop.outgoing
777 outgoing = pushop.outgoing
776 unfi = pushop.repo.unfiltered()
778 unfi = pushop.repo.unfiltered()
777 if not outgoing.missing:
779 if not outgoing.missing:
778 # nothing to push
780 # nothing to push
779 scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
781 scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
780 return False
782 return False
781 # something to push
783 # something to push
782 if not pushop.force:
784 if not pushop.force:
783 # if repo.obsstore == False --> no obsolete
785 # if repo.obsstore == False --> no obsolete
784 # then, save the iteration
786 # then, save the iteration
785 if unfi.obsstore:
787 if unfi.obsstore:
786 # this message are here for 80 char limit reason
788 # this message are here for 80 char limit reason
787 mso = _("push includes obsolete changeset: %s!")
789 mso = _("push includes obsolete changeset: %s!")
788 mspd = _("push includes phase-divergent changeset: %s!")
790 mspd = _("push includes phase-divergent changeset: %s!")
789 mscd = _("push includes content-divergent changeset: %s!")
791 mscd = _("push includes content-divergent changeset: %s!")
790 mst = {"orphan": _("push includes orphan changeset: %s!"),
792 mst = {"orphan": _("push includes orphan changeset: %s!"),
791 "phase-divergent": mspd,
793 "phase-divergent": mspd,
792 "content-divergent": mscd}
794 "content-divergent": mscd}
793 # If we are to push if there is at least one
795 # If we are to push if there is at least one
794 # obsolete or unstable changeset in missing, at
796 # obsolete or unstable changeset in missing, at
795 # least one of the missinghead will be obsolete or
797 # least one of the missinghead will be obsolete or
796 # unstable. So checking heads only is ok
798 # unstable. So checking heads only is ok
797 for node in outgoing.missingheads:
799 for node in outgoing.missingheads:
798 ctx = unfi[node]
800 ctx = unfi[node]
799 if ctx.obsolete():
801 if ctx.obsolete():
800 raise error.Abort(mso % ctx)
802 raise error.Abort(mso % ctx)
801 elif ctx.isunstable():
803 elif ctx.isunstable():
802 # TODO print more than one instability in the abort
804 # TODO print more than one instability in the abort
803 # message
805 # message
804 raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
806 raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
805
807
806 discovery.checkheads(pushop)
808 discovery.checkheads(pushop)
807 return True
809 return True
808
810
809 # List of names of steps to perform for an outgoing bundle2, order matters.
811 # List of names of steps to perform for an outgoing bundle2, order matters.
810 b2partsgenorder = []
812 b2partsgenorder = []
811
813
812 # Mapping between step name and function
814 # Mapping between step name and function
813 #
815 #
814 # This exists to help extensions wrap steps if necessary
816 # This exists to help extensions wrap steps if necessary
815 b2partsgenmapping = {}
817 b2partsgenmapping = {}
816
818
817 def b2partsgenerator(stepname, idx=None):
819 def b2partsgenerator(stepname, idx=None):
818 """decorator for function generating bundle2 part
820 """decorator for function generating bundle2 part
819
821
820 The function is added to the step -> function mapping and appended to the
822 The function is added to the step -> function mapping and appended to the
821 list of steps. Beware that decorated functions will be added in order
823 list of steps. Beware that decorated functions will be added in order
822 (this may matter).
824 (this may matter).
823
825
824 You can only use this decorator for new steps, if you want to wrap a step
826 You can only use this decorator for new steps, if you want to wrap a step
825 from an extension, attack the b2partsgenmapping dictionary directly."""
827 from an extension, attack the b2partsgenmapping dictionary directly."""
826 def dec(func):
828 def dec(func):
827 assert stepname not in b2partsgenmapping
829 assert stepname not in b2partsgenmapping
828 b2partsgenmapping[stepname] = func
830 b2partsgenmapping[stepname] = func
829 if idx is None:
831 if idx is None:
830 b2partsgenorder.append(stepname)
832 b2partsgenorder.append(stepname)
831 else:
833 else:
832 b2partsgenorder.insert(idx, stepname)
834 b2partsgenorder.insert(idx, stepname)
833 return func
835 return func
834 return dec
836 return dec
835
837
836 def _pushb2ctxcheckheads(pushop, bundler):
838 def _pushb2ctxcheckheads(pushop, bundler):
837 """Generate race condition checking parts
839 """Generate race condition checking parts
838
840
839 Exists as an independent function to aid extensions
841 Exists as an independent function to aid extensions
840 """
842 """
841 # * 'force' do not check for push race,
843 # * 'force' do not check for push race,
842 # * if we don't push anything, there are nothing to check.
844 # * if we don't push anything, there are nothing to check.
843 if not pushop.force and pushop.outgoing.missingheads:
845 if not pushop.force and pushop.outgoing.missingheads:
844 allowunrelated = 'related' in bundler.capabilities.get('checkheads', ())
846 allowunrelated = 'related' in bundler.capabilities.get('checkheads', ())
845 emptyremote = pushop.pushbranchmap is None
847 emptyremote = pushop.pushbranchmap is None
846 if not allowunrelated or emptyremote:
848 if not allowunrelated or emptyremote:
847 bundler.newpart('check:heads', data=iter(pushop.remoteheads))
849 bundler.newpart('check:heads', data=iter(pushop.remoteheads))
848 else:
850 else:
849 affected = set()
851 affected = set()
850 for branch, heads in pushop.pushbranchmap.iteritems():
852 for branch, heads in pushop.pushbranchmap.iteritems():
851 remoteheads, newheads, unsyncedheads, discardedheads = heads
853 remoteheads, newheads, unsyncedheads, discardedheads = heads
852 if remoteheads is not None:
854 if remoteheads is not None:
853 remote = set(remoteheads)
855 remote = set(remoteheads)
854 affected |= set(discardedheads) & remote
856 affected |= set(discardedheads) & remote
855 affected |= remote - set(newheads)
857 affected |= remote - set(newheads)
856 if affected:
858 if affected:
857 data = iter(sorted(affected))
859 data = iter(sorted(affected))
858 bundler.newpart('check:updated-heads', data=data)
860 bundler.newpart('check:updated-heads', data=data)
859
861
860 def _pushing(pushop):
862 def _pushing(pushop):
861 """return True if we are pushing anything"""
863 """return True if we are pushing anything"""
862 return bool(pushop.outgoing.missing
864 return bool(pushop.outgoing.missing
863 or pushop.outdatedphases
865 or pushop.outdatedphases
864 or pushop.outobsmarkers
866 or pushop.outobsmarkers
865 or pushop.outbookmarks)
867 or pushop.outbookmarks)
866
868
867 @b2partsgenerator('check-bookmarks')
869 @b2partsgenerator('check-bookmarks')
868 def _pushb2checkbookmarks(pushop, bundler):
870 def _pushb2checkbookmarks(pushop, bundler):
869 """insert bookmark move checking"""
871 """insert bookmark move checking"""
870 if not _pushing(pushop) or pushop.force:
872 if not _pushing(pushop) or pushop.force:
871 return
873 return
872 b2caps = bundle2.bundle2caps(pushop.remote)
874 b2caps = bundle2.bundle2caps(pushop.remote)
873 hasbookmarkcheck = 'bookmarks' in b2caps
875 hasbookmarkcheck = 'bookmarks' in b2caps
874 if not (pushop.outbookmarks and hasbookmarkcheck):
876 if not (pushop.outbookmarks and hasbookmarkcheck):
875 return
877 return
876 data = []
878 data = []
877 for book, old, new in pushop.outbookmarks:
879 for book, old, new in pushop.outbookmarks:
878 old = bin(old)
880 old = bin(old)
879 data.append((book, old))
881 data.append((book, old))
880 checkdata = bookmod.binaryencode(data)
882 checkdata = bookmod.binaryencode(data)
881 bundler.newpart('check:bookmarks', data=checkdata)
883 bundler.newpart('check:bookmarks', data=checkdata)
882
884
883 @b2partsgenerator('check-phases')
885 @b2partsgenerator('check-phases')
884 def _pushb2checkphases(pushop, bundler):
886 def _pushb2checkphases(pushop, bundler):
885 """insert phase move checking"""
887 """insert phase move checking"""
886 if not _pushing(pushop) or pushop.force:
888 if not _pushing(pushop) or pushop.force:
887 return
889 return
888 b2caps = bundle2.bundle2caps(pushop.remote)
890 b2caps = bundle2.bundle2caps(pushop.remote)
889 hasphaseheads = 'heads' in b2caps.get('phases', ())
891 hasphaseheads = 'heads' in b2caps.get('phases', ())
890 if pushop.remotephases is not None and hasphaseheads:
892 if pushop.remotephases is not None and hasphaseheads:
891 # check that the remote phase has not changed
893 # check that the remote phase has not changed
892 checks = [[] for p in phases.allphases]
894 checks = [[] for p in phases.allphases]
893 checks[phases.public].extend(pushop.remotephases.publicheads)
895 checks[phases.public].extend(pushop.remotephases.publicheads)
894 checks[phases.draft].extend(pushop.remotephases.draftroots)
896 checks[phases.draft].extend(pushop.remotephases.draftroots)
895 if any(checks):
897 if any(checks):
896 for nodes in checks:
898 for nodes in checks:
897 nodes.sort()
899 nodes.sort()
898 checkdata = phases.binaryencode(checks)
900 checkdata = phases.binaryencode(checks)
899 bundler.newpart('check:phases', data=checkdata)
901 bundler.newpart('check:phases', data=checkdata)
900
902
901 @b2partsgenerator('changeset')
903 @b2partsgenerator('changeset')
902 def _pushb2ctx(pushop, bundler):
904 def _pushb2ctx(pushop, bundler):
903 """handle changegroup push through bundle2
905 """handle changegroup push through bundle2
904
906
905 addchangegroup result is stored in the ``pushop.cgresult`` attribute.
907 addchangegroup result is stored in the ``pushop.cgresult`` attribute.
906 """
908 """
907 if 'changesets' in pushop.stepsdone:
909 if 'changesets' in pushop.stepsdone:
908 return
910 return
909 pushop.stepsdone.add('changesets')
911 pushop.stepsdone.add('changesets')
910 # Send known heads to the server for race detection.
912 # Send known heads to the server for race detection.
911 if not _pushcheckoutgoing(pushop):
913 if not _pushcheckoutgoing(pushop):
912 return
914 return
913 pushop.repo.prepushoutgoinghooks(pushop)
915 pushop.repo.prepushoutgoinghooks(pushop)
914
916
915 _pushb2ctxcheckheads(pushop, bundler)
917 _pushb2ctxcheckheads(pushop, bundler)
916
918
917 b2caps = bundle2.bundle2caps(pushop.remote)
919 b2caps = bundle2.bundle2caps(pushop.remote)
918 version = '01'
920 version = '01'
919 cgversions = b2caps.get('changegroup')
921 cgversions = b2caps.get('changegroup')
920 if cgversions: # 3.1 and 3.2 ship with an empty value
922 if cgversions: # 3.1 and 3.2 ship with an empty value
921 cgversions = [v for v in cgversions
923 cgversions = [v for v in cgversions
922 if v in changegroup.supportedoutgoingversions(
924 if v in changegroup.supportedoutgoingversions(
923 pushop.repo)]
925 pushop.repo)]
924 if not cgversions:
926 if not cgversions:
925 raise error.Abort(_('no common changegroup version'))
927 raise error.Abort(_('no common changegroup version'))
926 version = max(cgversions)
928 version = max(cgversions)
927 cgstream = changegroup.makestream(pushop.repo, pushop.outgoing, version,
929 cgstream = changegroup.makestream(pushop.repo, pushop.outgoing, version,
928 'push')
930 'push')
929 cgpart = bundler.newpart('changegroup', data=cgstream)
931 cgpart = bundler.newpart('changegroup', data=cgstream)
930 if cgversions:
932 if cgversions:
931 cgpart.addparam('version', version)
933 cgpart.addparam('version', version)
932 if 'treemanifest' in pushop.repo.requirements:
934 if 'treemanifest' in pushop.repo.requirements:
933 cgpart.addparam('treemanifest', '1')
935 cgpart.addparam('treemanifest', '1')
934 def handlereply(op):
936 def handlereply(op):
935 """extract addchangegroup returns from server reply"""
937 """extract addchangegroup returns from server reply"""
936 cgreplies = op.records.getreplies(cgpart.id)
938 cgreplies = op.records.getreplies(cgpart.id)
937 assert len(cgreplies['changegroup']) == 1
939 assert len(cgreplies['changegroup']) == 1
938 pushop.cgresult = cgreplies['changegroup'][0]['return']
940 pushop.cgresult = cgreplies['changegroup'][0]['return']
939 return handlereply
941 return handlereply
940
942
941 @b2partsgenerator('phase')
943 @b2partsgenerator('phase')
942 def _pushb2phases(pushop, bundler):
944 def _pushb2phases(pushop, bundler):
943 """handle phase push through bundle2"""
945 """handle phase push through bundle2"""
944 if 'phases' in pushop.stepsdone:
946 if 'phases' in pushop.stepsdone:
945 return
947 return
946 b2caps = bundle2.bundle2caps(pushop.remote)
948 b2caps = bundle2.bundle2caps(pushop.remote)
947 ui = pushop.repo.ui
949 ui = pushop.repo.ui
948
950
949 legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
951 legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
950 haspushkey = 'pushkey' in b2caps
952 haspushkey = 'pushkey' in b2caps
951 hasphaseheads = 'heads' in b2caps.get('phases', ())
953 hasphaseheads = 'heads' in b2caps.get('phases', ())
952
954
953 if hasphaseheads and not legacyphase:
955 if hasphaseheads and not legacyphase:
954 return _pushb2phaseheads(pushop, bundler)
956 return _pushb2phaseheads(pushop, bundler)
955 elif haspushkey:
957 elif haspushkey:
956 return _pushb2phasespushkey(pushop, bundler)
958 return _pushb2phasespushkey(pushop, bundler)
957
959
958 def _pushb2phaseheads(pushop, bundler):
960 def _pushb2phaseheads(pushop, bundler):
959 """push phase information through a bundle2 - binary part"""
961 """push phase information through a bundle2 - binary part"""
960 pushop.stepsdone.add('phases')
962 pushop.stepsdone.add('phases')
961 if pushop.outdatedphases:
963 if pushop.outdatedphases:
962 updates = [[] for p in phases.allphases]
964 updates = [[] for p in phases.allphases]
963 updates[0].extend(h.node() for h in pushop.outdatedphases)
965 updates[0].extend(h.node() for h in pushop.outdatedphases)
964 phasedata = phases.binaryencode(updates)
966 phasedata = phases.binaryencode(updates)
965 bundler.newpart('phase-heads', data=phasedata)
967 bundler.newpart('phase-heads', data=phasedata)
966
968
967 def _pushb2phasespushkey(pushop, bundler):
969 def _pushb2phasespushkey(pushop, bundler):
968 """push phase information through a bundle2 - pushkey part"""
970 """push phase information through a bundle2 - pushkey part"""
969 pushop.stepsdone.add('phases')
971 pushop.stepsdone.add('phases')
970 part2node = []
972 part2node = []
971
973
972 def handlefailure(pushop, exc):
974 def handlefailure(pushop, exc):
973 targetid = int(exc.partid)
975 targetid = int(exc.partid)
974 for partid, node in part2node:
976 for partid, node in part2node:
975 if partid == targetid:
977 if partid == targetid:
976 raise error.Abort(_('updating %s to public failed') % node)
978 raise error.Abort(_('updating %s to public failed') % node)
977
979
978 enc = pushkey.encode
980 enc = pushkey.encode
979 for newremotehead in pushop.outdatedphases:
981 for newremotehead in pushop.outdatedphases:
980 part = bundler.newpart('pushkey')
982 part = bundler.newpart('pushkey')
981 part.addparam('namespace', enc('phases'))
983 part.addparam('namespace', enc('phases'))
982 part.addparam('key', enc(newremotehead.hex()))
984 part.addparam('key', enc(newremotehead.hex()))
983 part.addparam('old', enc('%d' % phases.draft))
985 part.addparam('old', enc('%d' % phases.draft))
984 part.addparam('new', enc('%d' % phases.public))
986 part.addparam('new', enc('%d' % phases.public))
985 part2node.append((part.id, newremotehead))
987 part2node.append((part.id, newremotehead))
986 pushop.pkfailcb[part.id] = handlefailure
988 pushop.pkfailcb[part.id] = handlefailure
987
989
988 def handlereply(op):
990 def handlereply(op):
989 for partid, node in part2node:
991 for partid, node in part2node:
990 partrep = op.records.getreplies(partid)
992 partrep = op.records.getreplies(partid)
991 results = partrep['pushkey']
993 results = partrep['pushkey']
992 assert len(results) <= 1
994 assert len(results) <= 1
993 msg = None
995 msg = None
994 if not results:
996 if not results:
995 msg = _('server ignored update of %s to public!\n') % node
997 msg = _('server ignored update of %s to public!\n') % node
996 elif not int(results[0]['return']):
998 elif not int(results[0]['return']):
997 msg = _('updating %s to public failed!\n') % node
999 msg = _('updating %s to public failed!\n') % node
998 if msg is not None:
1000 if msg is not None:
999 pushop.ui.warn(msg)
1001 pushop.ui.warn(msg)
1000 return handlereply
1002 return handlereply
1001
1003
1002 @b2partsgenerator('obsmarkers')
1004 @b2partsgenerator('obsmarkers')
1003 def _pushb2obsmarkers(pushop, bundler):
1005 def _pushb2obsmarkers(pushop, bundler):
1004 if 'obsmarkers' in pushop.stepsdone:
1006 if 'obsmarkers' in pushop.stepsdone:
1005 return
1007 return
1006 remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
1008 remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
1007 if obsolete.commonversion(remoteversions) is None:
1009 if obsolete.commonversion(remoteversions) is None:
1008 return
1010 return
1009 pushop.stepsdone.add('obsmarkers')
1011 pushop.stepsdone.add('obsmarkers')
1010 if pushop.outobsmarkers:
1012 if pushop.outobsmarkers:
1011 markers = sorted(pushop.outobsmarkers)
1013 markers = sorted(pushop.outobsmarkers)
1012 bundle2.buildobsmarkerspart(bundler, markers)
1014 bundle2.buildobsmarkerspart(bundler, markers)
1013
1015
1014 @b2partsgenerator('bookmarks')
1016 @b2partsgenerator('bookmarks')
1015 def _pushb2bookmarks(pushop, bundler):
1017 def _pushb2bookmarks(pushop, bundler):
1016 """handle bookmark push through bundle2"""
1018 """handle bookmark push through bundle2"""
1017 if 'bookmarks' in pushop.stepsdone:
1019 if 'bookmarks' in pushop.stepsdone:
1018 return
1020 return
1019 b2caps = bundle2.bundle2caps(pushop.remote)
1021 b2caps = bundle2.bundle2caps(pushop.remote)
1020
1022
1021 legacy = pushop.repo.ui.configlist('devel', 'legacy.exchange')
1023 legacy = pushop.repo.ui.configlist('devel', 'legacy.exchange')
1022 legacybooks = 'bookmarks' in legacy
1024 legacybooks = 'bookmarks' in legacy
1023
1025
1024 if not legacybooks and 'bookmarks' in b2caps:
1026 if not legacybooks and 'bookmarks' in b2caps:
1025 return _pushb2bookmarkspart(pushop, bundler)
1027 return _pushb2bookmarkspart(pushop, bundler)
1026 elif 'pushkey' in b2caps:
1028 elif 'pushkey' in b2caps:
1027 return _pushb2bookmarkspushkey(pushop, bundler)
1029 return _pushb2bookmarkspushkey(pushop, bundler)
1028
1030
1029 def _bmaction(old, new):
1031 def _bmaction(old, new):
1030 """small utility for bookmark pushing"""
1032 """small utility for bookmark pushing"""
1031 if not old:
1033 if not old:
1032 return 'export'
1034 return 'export'
1033 elif not new:
1035 elif not new:
1034 return 'delete'
1036 return 'delete'
1035 return 'update'
1037 return 'update'
1036
1038
1037 def _pushb2bookmarkspart(pushop, bundler):
1039 def _pushb2bookmarkspart(pushop, bundler):
1038 pushop.stepsdone.add('bookmarks')
1040 pushop.stepsdone.add('bookmarks')
1039 if not pushop.outbookmarks:
1041 if not pushop.outbookmarks:
1040 return
1042 return
1041
1043
1042 allactions = []
1044 allactions = []
1043 data = []
1045 data = []
1044 for book, old, new in pushop.outbookmarks:
1046 for book, old, new in pushop.outbookmarks:
1045 new = bin(new)
1047 new = bin(new)
1046 data.append((book, new))
1048 data.append((book, new))
1047 allactions.append((book, _bmaction(old, new)))
1049 allactions.append((book, _bmaction(old, new)))
1048 checkdata = bookmod.binaryencode(data)
1050 checkdata = bookmod.binaryencode(data)
1049 bundler.newpart('bookmarks', data=checkdata)
1051 bundler.newpart('bookmarks', data=checkdata)
1050
1052
1051 def handlereply(op):
1053 def handlereply(op):
1052 ui = pushop.ui
1054 ui = pushop.ui
1053 # if success
1055 # if success
1054 for book, action in allactions:
1056 for book, action in allactions:
1055 ui.status(bookmsgmap[action][0] % book)
1057 ui.status(bookmsgmap[action][0] % book)
1056
1058
1057 return handlereply
1059 return handlereply
1058
1060
1059 def _pushb2bookmarkspushkey(pushop, bundler):
1061 def _pushb2bookmarkspushkey(pushop, bundler):
1060 pushop.stepsdone.add('bookmarks')
1062 pushop.stepsdone.add('bookmarks')
1061 part2book = []
1063 part2book = []
1062 enc = pushkey.encode
1064 enc = pushkey.encode
1063
1065
1064 def handlefailure(pushop, exc):
1066 def handlefailure(pushop, exc):
1065 targetid = int(exc.partid)
1067 targetid = int(exc.partid)
1066 for partid, book, action in part2book:
1068 for partid, book, action in part2book:
1067 if partid == targetid:
1069 if partid == targetid:
1068 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
1070 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
1069 # we should not be called for part we did not generated
1071 # we should not be called for part we did not generated
1070 assert False
1072 assert False
1071
1073
1072 for book, old, new in pushop.outbookmarks:
1074 for book, old, new in pushop.outbookmarks:
1073 part = bundler.newpart('pushkey')
1075 part = bundler.newpart('pushkey')
1074 part.addparam('namespace', enc('bookmarks'))
1076 part.addparam('namespace', enc('bookmarks'))
1075 part.addparam('key', enc(book))
1077 part.addparam('key', enc(book))
1076 part.addparam('old', enc(old))
1078 part.addparam('old', enc(old))
1077 part.addparam('new', enc(new))
1079 part.addparam('new', enc(new))
1078 action = 'update'
1080 action = 'update'
1079 if not old:
1081 if not old:
1080 action = 'export'
1082 action = 'export'
1081 elif not new:
1083 elif not new:
1082 action = 'delete'
1084 action = 'delete'
1083 part2book.append((part.id, book, action))
1085 part2book.append((part.id, book, action))
1084 pushop.pkfailcb[part.id] = handlefailure
1086 pushop.pkfailcb[part.id] = handlefailure
1085
1087
1086 def handlereply(op):
1088 def handlereply(op):
1087 ui = pushop.ui
1089 ui = pushop.ui
1088 for partid, book, action in part2book:
1090 for partid, book, action in part2book:
1089 partrep = op.records.getreplies(partid)
1091 partrep = op.records.getreplies(partid)
1090 results = partrep['pushkey']
1092 results = partrep['pushkey']
1091 assert len(results) <= 1
1093 assert len(results) <= 1
1092 if not results:
1094 if not results:
1093 pushop.ui.warn(_('server ignored bookmark %s update\n') % book)
1095 pushop.ui.warn(_('server ignored bookmark %s update\n') % book)
1094 else:
1096 else:
1095 ret = int(results[0]['return'])
1097 ret = int(results[0]['return'])
1096 if ret:
1098 if ret:
1097 ui.status(bookmsgmap[action][0] % book)
1099 ui.status(bookmsgmap[action][0] % book)
1098 else:
1100 else:
1099 ui.warn(bookmsgmap[action][1] % book)
1101 ui.warn(bookmsgmap[action][1] % book)
1100 if pushop.bkresult is not None:
1102 if pushop.bkresult is not None:
1101 pushop.bkresult = 1
1103 pushop.bkresult = 1
1102 return handlereply
1104 return handlereply
1103
1105
1104 @b2partsgenerator('pushvars', idx=0)
1106 @b2partsgenerator('pushvars', idx=0)
1105 def _getbundlesendvars(pushop, bundler):
1107 def _getbundlesendvars(pushop, bundler):
1106 '''send shellvars via bundle2'''
1108 '''send shellvars via bundle2'''
1107 pushvars = pushop.pushvars
1109 pushvars = pushop.pushvars
1108 if pushvars:
1110 if pushvars:
1109 shellvars = {}
1111 shellvars = {}
1110 for raw in pushvars:
1112 for raw in pushvars:
1111 if '=' not in raw:
1113 if '=' not in raw:
1112 msg = ("unable to parse variable '%s', should follow "
1114 msg = ("unable to parse variable '%s', should follow "
1113 "'KEY=VALUE' or 'KEY=' format")
1115 "'KEY=VALUE' or 'KEY=' format")
1114 raise error.Abort(msg % raw)
1116 raise error.Abort(msg % raw)
1115 k, v = raw.split('=', 1)
1117 k, v = raw.split('=', 1)
1116 shellvars[k] = v
1118 shellvars[k] = v
1117
1119
1118 part = bundler.newpart('pushvars')
1120 part = bundler.newpart('pushvars')
1119
1121
1120 for key, value in shellvars.iteritems():
1122 for key, value in shellvars.iteritems():
1121 part.addparam(key, value, mandatory=False)
1123 part.addparam(key, value, mandatory=False)
1122
1124
1123 def _pushbundle2(pushop):
1125 def _pushbundle2(pushop):
1124 """push data to the remote using bundle2
1126 """push data to the remote using bundle2
1125
1127
1126 The only currently supported type of data is changegroup but this will
1128 The only currently supported type of data is changegroup but this will
1127 evolve in the future."""
1129 evolve in the future."""
1128 bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
1130 bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
1129 pushback = (pushop.trmanager
1131 pushback = (pushop.trmanager
1130 and pushop.ui.configbool('experimental', 'bundle2.pushback'))
1132 and pushop.ui.configbool('experimental', 'bundle2.pushback'))
1131
1133
1132 # create reply capability
1134 # create reply capability
1133 capsblob = bundle2.encodecaps(bundle2.getrepocaps(pushop.repo,
1135 capsblob = bundle2.encodecaps(bundle2.getrepocaps(pushop.repo,
1134 allowpushback=pushback,
1136 allowpushback=pushback,
1135 role='client'))
1137 role='client'))
1136 bundler.newpart('replycaps', data=capsblob)
1138 bundler.newpart('replycaps', data=capsblob)
1137 replyhandlers = []
1139 replyhandlers = []
1138 for partgenname in b2partsgenorder:
1140 for partgenname in b2partsgenorder:
1139 partgen = b2partsgenmapping[partgenname]
1141 partgen = b2partsgenmapping[partgenname]
1140 ret = partgen(pushop, bundler)
1142 ret = partgen(pushop, bundler)
1141 if callable(ret):
1143 if callable(ret):
1142 replyhandlers.append(ret)
1144 replyhandlers.append(ret)
1143 # do not push if nothing to push
1145 # do not push if nothing to push
1144 if bundler.nbparts <= 1:
1146 if bundler.nbparts <= 1:
1145 return
1147 return
1146 stream = util.chunkbuffer(bundler.getchunks())
1148 stream = util.chunkbuffer(bundler.getchunks())
1147 try:
1149 try:
1148 try:
1150 try:
1149 with pushop.remote.commandexecutor() as e:
1151 with pushop.remote.commandexecutor() as e:
1150 reply = e.callcommand('unbundle', {
1152 reply = e.callcommand('unbundle', {
1151 'bundle': stream,
1153 'bundle': stream,
1152 'heads': ['force'],
1154 'heads': ['force'],
1153 'url': pushop.remote.url(),
1155 'url': pushop.remote.url(),
1154 }).result()
1156 }).result()
1155 except error.BundleValueError as exc:
1157 except error.BundleValueError as exc:
1156 raise error.Abort(_('missing support for %s') % exc)
1158 raise error.Abort(_('missing support for %s') % exc)
1157 try:
1159 try:
1158 trgetter = None
1160 trgetter = None
1159 if pushback:
1161 if pushback:
1160 trgetter = pushop.trmanager.transaction
1162 trgetter = pushop.trmanager.transaction
1161 op = bundle2.processbundle(pushop.repo, reply, trgetter)
1163 op = bundle2.processbundle(pushop.repo, reply, trgetter)
1162 except error.BundleValueError as exc:
1164 except error.BundleValueError as exc:
1163 raise error.Abort(_('missing support for %s') % exc)
1165 raise error.Abort(_('missing support for %s') % exc)
1164 except bundle2.AbortFromPart as exc:
1166 except bundle2.AbortFromPart as exc:
1165 pushop.ui.status(_('remote: %s\n') % exc)
1167 pushop.ui.status(_('remote: %s\n') % exc)
1166 if exc.hint is not None:
1168 if exc.hint is not None:
1167 pushop.ui.status(_('remote: %s\n') % ('(%s)' % exc.hint))
1169 pushop.ui.status(_('remote: %s\n') % ('(%s)' % exc.hint))
1168 raise error.Abort(_('push failed on remote'))
1170 raise error.Abort(_('push failed on remote'))
1169 except error.PushkeyFailed as exc:
1171 except error.PushkeyFailed as exc:
1170 partid = int(exc.partid)
1172 partid = int(exc.partid)
1171 if partid not in pushop.pkfailcb:
1173 if partid not in pushop.pkfailcb:
1172 raise
1174 raise
1173 pushop.pkfailcb[partid](pushop, exc)
1175 pushop.pkfailcb[partid](pushop, exc)
1174 for rephand in replyhandlers:
1176 for rephand in replyhandlers:
1175 rephand(op)
1177 rephand(op)
1176
1178
1177 def _pushchangeset(pushop):
1179 def _pushchangeset(pushop):
1178 """Make the actual push of changeset bundle to remote repo"""
1180 """Make the actual push of changeset bundle to remote repo"""
1179 if 'changesets' in pushop.stepsdone:
1181 if 'changesets' in pushop.stepsdone:
1180 return
1182 return
1181 pushop.stepsdone.add('changesets')
1183 pushop.stepsdone.add('changesets')
1182 if not _pushcheckoutgoing(pushop):
1184 if not _pushcheckoutgoing(pushop):
1183 return
1185 return
1184
1186
1185 # Should have verified this in push().
1187 # Should have verified this in push().
1186 assert pushop.remote.capable('unbundle')
1188 assert pushop.remote.capable('unbundle')
1187
1189
1188 pushop.repo.prepushoutgoinghooks(pushop)
1190 pushop.repo.prepushoutgoinghooks(pushop)
1189 outgoing = pushop.outgoing
1191 outgoing = pushop.outgoing
1190 # TODO: get bundlecaps from remote
1192 # TODO: get bundlecaps from remote
1191 bundlecaps = None
1193 bundlecaps = None
1192 # create a changegroup from local
1194 # create a changegroup from local
1193 if pushop.revs is None and not (outgoing.excluded
1195 if pushop.revs is None and not (outgoing.excluded
1194 or pushop.repo.changelog.filteredrevs):
1196 or pushop.repo.changelog.filteredrevs):
1195 # push everything,
1197 # push everything,
1196 # use the fast path, no race possible on push
1198 # use the fast path, no race possible on push
1197 cg = changegroup.makechangegroup(pushop.repo, outgoing, '01', 'push',
1199 cg = changegroup.makechangegroup(pushop.repo, outgoing, '01', 'push',
1198 fastpath=True, bundlecaps=bundlecaps)
1200 fastpath=True, bundlecaps=bundlecaps)
1199 else:
1201 else:
1200 cg = changegroup.makechangegroup(pushop.repo, outgoing, '01',
1202 cg = changegroup.makechangegroup(pushop.repo, outgoing, '01',
1201 'push', bundlecaps=bundlecaps)
1203 'push', bundlecaps=bundlecaps)
1202
1204
1203 # apply changegroup to remote
1205 # apply changegroup to remote
1204 # local repo finds heads on server, finds out what
1206 # local repo finds heads on server, finds out what
1205 # revs it must push. once revs transferred, if server
1207 # revs it must push. once revs transferred, if server
1206 # finds it has different heads (someone else won
1208 # finds it has different heads (someone else won
1207 # commit/push race), server aborts.
1209 # commit/push race), server aborts.
1208 if pushop.force:
1210 if pushop.force:
1209 remoteheads = ['force']
1211 remoteheads = ['force']
1210 else:
1212 else:
1211 remoteheads = pushop.remoteheads
1213 remoteheads = pushop.remoteheads
1212 # ssh: return remote's addchangegroup()
1214 # ssh: return remote's addchangegroup()
1213 # http: return remote's addchangegroup() or 0 for error
1215 # http: return remote's addchangegroup() or 0 for error
1214 pushop.cgresult = pushop.remote.unbundle(cg, remoteheads,
1216 pushop.cgresult = pushop.remote.unbundle(cg, remoteheads,
1215 pushop.repo.url())
1217 pushop.repo.url())
1216
1218
1217 def _pushsyncphase(pushop):
1219 def _pushsyncphase(pushop):
1218 """synchronise phase information locally and remotely"""
1220 """synchronise phase information locally and remotely"""
1219 cheads = pushop.commonheads
1221 cheads = pushop.commonheads
1220 # even when we don't push, exchanging phase data is useful
1222 # even when we don't push, exchanging phase data is useful
1221 remotephases = listkeys(pushop.remote, 'phases')
1223 remotephases = listkeys(pushop.remote, 'phases')
1222 if (pushop.ui.configbool('ui', '_usedassubrepo')
1224 if (pushop.ui.configbool('ui', '_usedassubrepo')
1223 and remotephases # server supports phases
1225 and remotephases # server supports phases
1224 and pushop.cgresult is None # nothing was pushed
1226 and pushop.cgresult is None # nothing was pushed
1225 and remotephases.get('publishing', False)):
1227 and remotephases.get('publishing', False)):
1226 # When:
1228 # When:
1227 # - this is a subrepo push
1229 # - this is a subrepo push
1228 # - and remote support phase
1230 # - and remote support phase
1229 # - and no changeset was pushed
1231 # - and no changeset was pushed
1230 # - and remote is publishing
1232 # - and remote is publishing
1231 # We may be in issue 3871 case!
1233 # We may be in issue 3871 case!
1232 # We drop the possible phase synchronisation done by
1234 # We drop the possible phase synchronisation done by
1233 # courtesy to publish changesets possibly locally draft
1235 # courtesy to publish changesets possibly locally draft
1234 # on the remote.
1236 # on the remote.
1235 remotephases = {'publishing': 'True'}
1237 remotephases = {'publishing': 'True'}
1236 if not remotephases: # old server or public only reply from non-publishing
1238 if not remotephases: # old server or public only reply from non-publishing
1237 _localphasemove(pushop, cheads)
1239 _localphasemove(pushop, cheads)
1238 # don't push any phase data as there is nothing to push
1240 # don't push any phase data as there is nothing to push
1239 else:
1241 else:
1240 ana = phases.analyzeremotephases(pushop.repo, cheads,
1242 ana = phases.analyzeremotephases(pushop.repo, cheads,
1241 remotephases)
1243 remotephases)
1242 pheads, droots = ana
1244 pheads, droots = ana
1243 ### Apply remote phase on local
1245 ### Apply remote phase on local
1244 if remotephases.get('publishing', False):
1246 if remotephases.get('publishing', False):
1245 _localphasemove(pushop, cheads)
1247 _localphasemove(pushop, cheads)
1246 else: # publish = False
1248 else: # publish = False
1247 _localphasemove(pushop, pheads)
1249 _localphasemove(pushop, pheads)
1248 _localphasemove(pushop, cheads, phases.draft)
1250 _localphasemove(pushop, cheads, phases.draft)
1249 ### Apply local phase on remote
1251 ### Apply local phase on remote
1250
1252
1251 if pushop.cgresult:
1253 if pushop.cgresult:
1252 if 'phases' in pushop.stepsdone:
1254 if 'phases' in pushop.stepsdone:
1253 # phases already pushed though bundle2
1255 # phases already pushed though bundle2
1254 return
1256 return
1255 outdated = pushop.outdatedphases
1257 outdated = pushop.outdatedphases
1256 else:
1258 else:
1257 outdated = pushop.fallbackoutdatedphases
1259 outdated = pushop.fallbackoutdatedphases
1258
1260
1259 pushop.stepsdone.add('phases')
1261 pushop.stepsdone.add('phases')
1260
1262
1261 # filter heads already turned public by the push
1263 # filter heads already turned public by the push
1262 outdated = [c for c in outdated if c.node() not in pheads]
1264 outdated = [c for c in outdated if c.node() not in pheads]
1263 # fallback to independent pushkey command
1265 # fallback to independent pushkey command
1264 for newremotehead in outdated:
1266 for newremotehead in outdated:
1265 with pushop.remote.commandexecutor() as e:
1267 with pushop.remote.commandexecutor() as e:
1266 r = e.callcommand('pushkey', {
1268 r = e.callcommand('pushkey', {
1267 'namespace': 'phases',
1269 'namespace': 'phases',
1268 'key': newremotehead.hex(),
1270 'key': newremotehead.hex(),
1269 'old': '%d' % phases.draft,
1271 'old': '%d' % phases.draft,
1270 'new': '%d' % phases.public
1272 'new': '%d' % phases.public
1271 }).result()
1273 }).result()
1272
1274
1273 if not r:
1275 if not r:
1274 pushop.ui.warn(_('updating %s to public failed!\n')
1276 pushop.ui.warn(_('updating %s to public failed!\n')
1275 % newremotehead)
1277 % newremotehead)
1276
1278
1277 def _localphasemove(pushop, nodes, phase=phases.public):
1279 def _localphasemove(pushop, nodes, phase=phases.public):
1278 """move <nodes> to <phase> in the local source repo"""
1280 """move <nodes> to <phase> in the local source repo"""
1279 if pushop.trmanager:
1281 if pushop.trmanager:
1280 phases.advanceboundary(pushop.repo,
1282 phases.advanceboundary(pushop.repo,
1281 pushop.trmanager.transaction(),
1283 pushop.trmanager.transaction(),
1282 phase,
1284 phase,
1283 nodes)
1285 nodes)
1284 else:
1286 else:
1285 # repo is not locked, do not change any phases!
1287 # repo is not locked, do not change any phases!
1286 # Informs the user that phases should have been moved when
1288 # Informs the user that phases should have been moved when
1287 # applicable.
1289 # applicable.
1288 actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
1290 actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
1289 phasestr = phases.phasenames[phase]
1291 phasestr = phases.phasenames[phase]
1290 if actualmoves:
1292 if actualmoves:
1291 pushop.ui.status(_('cannot lock source repo, skipping '
1293 pushop.ui.status(_('cannot lock source repo, skipping '
1292 'local %s phase update\n') % phasestr)
1294 'local %s phase update\n') % phasestr)
1293
1295
1294 def _pushobsolete(pushop):
1296 def _pushobsolete(pushop):
1295 """utility function to push obsolete markers to a remote"""
1297 """utility function to push obsolete markers to a remote"""
1296 if 'obsmarkers' in pushop.stepsdone:
1298 if 'obsmarkers' in pushop.stepsdone:
1297 return
1299 return
1298 repo = pushop.repo
1300 repo = pushop.repo
1299 remote = pushop.remote
1301 remote = pushop.remote
1300 pushop.stepsdone.add('obsmarkers')
1302 pushop.stepsdone.add('obsmarkers')
1301 if pushop.outobsmarkers:
1303 if pushop.outobsmarkers:
1302 pushop.ui.debug('try to push obsolete markers to remote\n')
1304 pushop.ui.debug('try to push obsolete markers to remote\n')
1303 rslts = []
1305 rslts = []
1304 remotedata = obsolete._pushkeyescape(sorted(pushop.outobsmarkers))
1306 remotedata = obsolete._pushkeyescape(sorted(pushop.outobsmarkers))
1305 for key in sorted(remotedata, reverse=True):
1307 for key in sorted(remotedata, reverse=True):
1306 # reverse sort to ensure we end with dump0
1308 # reverse sort to ensure we end with dump0
1307 data = remotedata[key]
1309 data = remotedata[key]
1308 rslts.append(remote.pushkey('obsolete', key, '', data))
1310 rslts.append(remote.pushkey('obsolete', key, '', data))
1309 if [r for r in rslts if not r]:
1311 if [r for r in rslts if not r]:
1310 msg = _('failed to push some obsolete markers!\n')
1312 msg = _('failed to push some obsolete markers!\n')
1311 repo.ui.warn(msg)
1313 repo.ui.warn(msg)
1312
1314
1313 def _pushbookmark(pushop):
1315 def _pushbookmark(pushop):
1314 """Update bookmark position on remote"""
1316 """Update bookmark position on remote"""
1315 if pushop.cgresult == 0 or 'bookmarks' in pushop.stepsdone:
1317 if pushop.cgresult == 0 or 'bookmarks' in pushop.stepsdone:
1316 return
1318 return
1317 pushop.stepsdone.add('bookmarks')
1319 pushop.stepsdone.add('bookmarks')
1318 ui = pushop.ui
1320 ui = pushop.ui
1319 remote = pushop.remote
1321 remote = pushop.remote
1320
1322
1321 for b, old, new in pushop.outbookmarks:
1323 for b, old, new in pushop.outbookmarks:
1322 action = 'update'
1324 action = 'update'
1323 if not old:
1325 if not old:
1324 action = 'export'
1326 action = 'export'
1325 elif not new:
1327 elif not new:
1326 action = 'delete'
1328 action = 'delete'
1327
1329
1328 with remote.commandexecutor() as e:
1330 with remote.commandexecutor() as e:
1329 r = e.callcommand('pushkey', {
1331 r = e.callcommand('pushkey', {
1330 'namespace': 'bookmarks',
1332 'namespace': 'bookmarks',
1331 'key': b,
1333 'key': b,
1332 'old': old,
1334 'old': old,
1333 'new': new,
1335 'new': new,
1334 }).result()
1336 }).result()
1335
1337
1336 if r:
1338 if r:
1337 ui.status(bookmsgmap[action][0] % b)
1339 ui.status(bookmsgmap[action][0] % b)
1338 else:
1340 else:
1339 ui.warn(bookmsgmap[action][1] % b)
1341 ui.warn(bookmsgmap[action][1] % b)
1340 # discovery can have set the value form invalid entry
1342 # discovery can have set the value form invalid entry
1341 if pushop.bkresult is not None:
1343 if pushop.bkresult is not None:
1342 pushop.bkresult = 1
1344 pushop.bkresult = 1
1343
1345
1344 class pulloperation(object):
1346 class pulloperation(object):
1345 """A object that represent a single pull operation
1347 """A object that represent a single pull operation
1346
1348
1347 It purpose is to carry pull related state and very common operation.
1349 It purpose is to carry pull related state and very common operation.
1348
1350
1349 A new should be created at the beginning of each pull and discarded
1351 A new should be created at the beginning of each pull and discarded
1350 afterward.
1352 afterward.
1351 """
1353 """
1352
1354
1353 def __init__(self, repo, remote, heads=None, force=False, bookmarks=(),
1355 def __init__(self, repo, remote, heads=None, force=False, bookmarks=(),
1354 remotebookmarks=None, streamclonerequested=None,
1356 remotebookmarks=None, streamclonerequested=None,
1355 includepats=None, excludepats=None, depth=None):
1357 includepats=None, excludepats=None, depth=None):
1356 # repo we pull into
1358 # repo we pull into
1357 self.repo = repo
1359 self.repo = repo
1358 # repo we pull from
1360 # repo we pull from
1359 self.remote = remote
1361 self.remote = remote
1360 # revision we try to pull (None is "all")
1362 # revision we try to pull (None is "all")
1361 self.heads = heads
1363 self.heads = heads
1362 # bookmark pulled explicitly
1364 # bookmark pulled explicitly
1363 self.explicitbookmarks = [repo._bookmarks.expandname(bookmark)
1365 self.explicitbookmarks = [repo._bookmarks.expandname(bookmark)
1364 for bookmark in bookmarks]
1366 for bookmark in bookmarks]
1365 # do we force pull?
1367 # do we force pull?
1366 self.force = force
1368 self.force = force
1367 # whether a streaming clone was requested
1369 # whether a streaming clone was requested
1368 self.streamclonerequested = streamclonerequested
1370 self.streamclonerequested = streamclonerequested
1369 # transaction manager
1371 # transaction manager
1370 self.trmanager = None
1372 self.trmanager = None
1371 # set of common changeset between local and remote before pull
1373 # set of common changeset between local and remote before pull
1372 self.common = None
1374 self.common = None
1373 # set of pulled head
1375 # set of pulled head
1374 self.rheads = None
1376 self.rheads = None
1375 # list of missing changeset to fetch remotely
1377 # list of missing changeset to fetch remotely
1376 self.fetch = None
1378 self.fetch = None
1377 # remote bookmarks data
1379 # remote bookmarks data
1378 self.remotebookmarks = remotebookmarks
1380 self.remotebookmarks = remotebookmarks
1379 # result of changegroup pulling (used as return code by pull)
1381 # result of changegroup pulling (used as return code by pull)
1380 self.cgresult = None
1382 self.cgresult = None
1381 # list of step already done
1383 # list of step already done
1382 self.stepsdone = set()
1384 self.stepsdone = set()
1383 # Whether we attempted a clone from pre-generated bundles.
1385 # Whether we attempted a clone from pre-generated bundles.
1384 self.clonebundleattempted = False
1386 self.clonebundleattempted = False
1385 # Set of file patterns to include.
1387 # Set of file patterns to include.
1386 self.includepats = includepats
1388 self.includepats = includepats
1387 # Set of file patterns to exclude.
1389 # Set of file patterns to exclude.
1388 self.excludepats = excludepats
1390 self.excludepats = excludepats
1389 # Number of ancestor changesets to pull from each pulled head.
1391 # Number of ancestor changesets to pull from each pulled head.
1390 self.depth = depth
1392 self.depth = depth
1391
1393
1392 @util.propertycache
1394 @util.propertycache
1393 def pulledsubset(self):
1395 def pulledsubset(self):
1394 """heads of the set of changeset target by the pull"""
1396 """heads of the set of changeset target by the pull"""
1395 # compute target subset
1397 # compute target subset
1396 if self.heads is None:
1398 if self.heads is None:
1397 # We pulled every thing possible
1399 # We pulled every thing possible
1398 # sync on everything common
1400 # sync on everything common
1399 c = set(self.common)
1401 c = set(self.common)
1400 ret = list(self.common)
1402 ret = list(self.common)
1401 for n in self.rheads:
1403 for n in self.rheads:
1402 if n not in c:
1404 if n not in c:
1403 ret.append(n)
1405 ret.append(n)
1404 return ret
1406 return ret
1405 else:
1407 else:
1406 # We pulled a specific subset
1408 # We pulled a specific subset
1407 # sync on this subset
1409 # sync on this subset
1408 return self.heads
1410 return self.heads
1409
1411
1410 @util.propertycache
1412 @util.propertycache
1411 def canusebundle2(self):
1413 def canusebundle2(self):
1412 return not _forcebundle1(self)
1414 return not _forcebundle1(self)
1413
1415
1414 @util.propertycache
1416 @util.propertycache
1415 def remotebundle2caps(self):
1417 def remotebundle2caps(self):
1416 return bundle2.bundle2caps(self.remote)
1418 return bundle2.bundle2caps(self.remote)
1417
1419
1418 def gettransaction(self):
1420 def gettransaction(self):
1419 # deprecated; talk to trmanager directly
1421 # deprecated; talk to trmanager directly
1420 return self.trmanager.transaction()
1422 return self.trmanager.transaction()
1421
1423
1422 class transactionmanager(util.transactional):
1424 class transactionmanager(util.transactional):
1423 """An object to manage the life cycle of a transaction
1425 """An object to manage the life cycle of a transaction
1424
1426
1425 It creates the transaction on demand and calls the appropriate hooks when
1427 It creates the transaction on demand and calls the appropriate hooks when
1426 closing the transaction."""
1428 closing the transaction."""
1427 def __init__(self, repo, source, url):
1429 def __init__(self, repo, source, url):
1428 self.repo = repo
1430 self.repo = repo
1429 self.source = source
1431 self.source = source
1430 self.url = url
1432 self.url = url
1431 self._tr = None
1433 self._tr = None
1432
1434
1433 def transaction(self):
1435 def transaction(self):
1434 """Return an open transaction object, constructing if necessary"""
1436 """Return an open transaction object, constructing if necessary"""
1435 if not self._tr:
1437 if not self._tr:
1436 trname = '%s\n%s' % (self.source, util.hidepassword(self.url))
1438 trname = '%s\n%s' % (self.source, util.hidepassword(self.url))
1437 self._tr = self.repo.transaction(trname)
1439 self._tr = self.repo.transaction(trname)
1438 self._tr.hookargs['source'] = self.source
1440 self._tr.hookargs['source'] = self.source
1439 self._tr.hookargs['url'] = self.url
1441 self._tr.hookargs['url'] = self.url
1440 return self._tr
1442 return self._tr
1441
1443
1442 def close(self):
1444 def close(self):
1443 """close transaction if created"""
1445 """close transaction if created"""
1444 if self._tr is not None:
1446 if self._tr is not None:
1445 self._tr.close()
1447 self._tr.close()
1446
1448
1447 def release(self):
1449 def release(self):
1448 """release transaction if created"""
1450 """release transaction if created"""
1449 if self._tr is not None:
1451 if self._tr is not None:
1450 self._tr.release()
1452 self._tr.release()
1451
1453
1452 def listkeys(remote, namespace):
1454 def listkeys(remote, namespace):
1453 with remote.commandexecutor() as e:
1455 with remote.commandexecutor() as e:
1454 return e.callcommand('listkeys', {'namespace': namespace}).result()
1456 return e.callcommand('listkeys', {'namespace': namespace}).result()
1455
1457
1456 def _fullpullbundle2(repo, pullop):
1458 def _fullpullbundle2(repo, pullop):
1457 # The server may send a partial reply, i.e. when inlining
1459 # The server may send a partial reply, i.e. when inlining
1458 # pre-computed bundles. In that case, update the common
1460 # pre-computed bundles. In that case, update the common
1459 # set based on the results and pull another bundle.
1461 # set based on the results and pull another bundle.
1460 #
1462 #
1461 # There are two indicators that the process is finished:
1463 # There are two indicators that the process is finished:
1462 # - no changeset has been added, or
1464 # - no changeset has been added, or
1463 # - all remote heads are known locally.
1465 # - all remote heads are known locally.
1464 # The head check must use the unfiltered view as obsoletion
1466 # The head check must use the unfiltered view as obsoletion
1465 # markers can hide heads.
1467 # markers can hide heads.
1466 unfi = repo.unfiltered()
1468 unfi = repo.unfiltered()
1467 unficl = unfi.changelog
1469 unficl = unfi.changelog
1468 def headsofdiff(h1, h2):
1470 def headsofdiff(h1, h2):
1469 """Returns heads(h1 % h2)"""
1471 """Returns heads(h1 % h2)"""
1470 res = unfi.set('heads(%ln %% %ln)', h1, h2)
1472 res = unfi.set('heads(%ln %% %ln)', h1, h2)
1471 return set(ctx.node() for ctx in res)
1473 return set(ctx.node() for ctx in res)
1472 def headsofunion(h1, h2):
1474 def headsofunion(h1, h2):
1473 """Returns heads((h1 + h2) - null)"""
1475 """Returns heads((h1 + h2) - null)"""
1474 res = unfi.set('heads((%ln + %ln - null))', h1, h2)
1476 res = unfi.set('heads((%ln + %ln - null))', h1, h2)
1475 return set(ctx.node() for ctx in res)
1477 return set(ctx.node() for ctx in res)
1476 while True:
1478 while True:
1477 old_heads = unficl.heads()
1479 old_heads = unficl.heads()
1478 clstart = len(unficl)
1480 clstart = len(unficl)
1479 _pullbundle2(pullop)
1481 _pullbundle2(pullop)
1480 if repository.NARROW_REQUIREMENT in repo.requirements:
1482 if repository.NARROW_REQUIREMENT in repo.requirements:
1481 # XXX narrow clones filter the heads on the server side during
1483 # XXX narrow clones filter the heads on the server side during
1482 # XXX getbundle and result in partial replies as well.
1484 # XXX getbundle and result in partial replies as well.
1483 # XXX Disable pull bundles in this case as band aid to avoid
1485 # XXX Disable pull bundles in this case as band aid to avoid
1484 # XXX extra round trips.
1486 # XXX extra round trips.
1485 break
1487 break
1486 if clstart == len(unficl):
1488 if clstart == len(unficl):
1487 break
1489 break
1488 if all(unficl.hasnode(n) for n in pullop.rheads):
1490 if all(unficl.hasnode(n) for n in pullop.rheads):
1489 break
1491 break
1490 new_heads = headsofdiff(unficl.heads(), old_heads)
1492 new_heads = headsofdiff(unficl.heads(), old_heads)
1491 pullop.common = headsofunion(new_heads, pullop.common)
1493 pullop.common = headsofunion(new_heads, pullop.common)
1492 pullop.rheads = set(pullop.rheads) - pullop.common
1494 pullop.rheads = set(pullop.rheads) - pullop.common
1493
1495
1494 def pull(repo, remote, heads=None, force=False, bookmarks=(), opargs=None,
1496 def pull(repo, remote, heads=None, force=False, bookmarks=(), opargs=None,
1495 streamclonerequested=None, includepats=None, excludepats=None,
1497 streamclonerequested=None, includepats=None, excludepats=None,
1496 depth=None):
1498 depth=None):
1497 """Fetch repository data from a remote.
1499 """Fetch repository data from a remote.
1498
1500
1499 This is the main function used to retrieve data from a remote repository.
1501 This is the main function used to retrieve data from a remote repository.
1500
1502
1501 ``repo`` is the local repository to clone into.
1503 ``repo`` is the local repository to clone into.
1502 ``remote`` is a peer instance.
1504 ``remote`` is a peer instance.
1503 ``heads`` is an iterable of revisions we want to pull. ``None`` (the
1505 ``heads`` is an iterable of revisions we want to pull. ``None`` (the
1504 default) means to pull everything from the remote.
1506 default) means to pull everything from the remote.
1505 ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
1507 ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
1506 default, all remote bookmarks are pulled.
1508 default, all remote bookmarks are pulled.
1507 ``opargs`` are additional keyword arguments to pass to ``pulloperation``
1509 ``opargs`` are additional keyword arguments to pass to ``pulloperation``
1508 initialization.
1510 initialization.
1509 ``streamclonerequested`` is a boolean indicating whether a "streaming
1511 ``streamclonerequested`` is a boolean indicating whether a "streaming
1510 clone" is requested. A "streaming clone" is essentially a raw file copy
1512 clone" is requested. A "streaming clone" is essentially a raw file copy
1511 of revlogs from the server. This only works when the local repository is
1513 of revlogs from the server. This only works when the local repository is
1512 empty. The default value of ``None`` means to respect the server
1514 empty. The default value of ``None`` means to respect the server
1513 configuration for preferring stream clones.
1515 configuration for preferring stream clones.
1514 ``includepats`` and ``excludepats`` define explicit file patterns to
1516 ``includepats`` and ``excludepats`` define explicit file patterns to
1515 include and exclude in storage, respectively. If not defined, narrow
1517 include and exclude in storage, respectively. If not defined, narrow
1516 patterns from the repo instance are used, if available.
1518 patterns from the repo instance are used, if available.
1517 ``depth`` is an integer indicating the DAG depth of history we're
1519 ``depth`` is an integer indicating the DAG depth of history we're
1518 interested in. If defined, for each revision specified in ``heads``, we
1520 interested in. If defined, for each revision specified in ``heads``, we
1519 will fetch up to this many of its ancestors and data associated with them.
1521 will fetch up to this many of its ancestors and data associated with them.
1520
1522
1521 Returns the ``pulloperation`` created for this pull.
1523 Returns the ``pulloperation`` created for this pull.
1522 """
1524 """
1523 if opargs is None:
1525 if opargs is None:
1524 opargs = {}
1526 opargs = {}
1525
1527
1526 # We allow the narrow patterns to be passed in explicitly to provide more
1528 # We allow the narrow patterns to be passed in explicitly to provide more
1527 # flexibility for API consumers.
1529 # flexibility for API consumers.
1528 if includepats or excludepats:
1530 if includepats or excludepats:
1529 includepats = includepats or set()
1531 includepats = includepats or set()
1530 excludepats = excludepats or set()
1532 excludepats = excludepats or set()
1531 else:
1533 else:
1532 includepats, excludepats = repo.narrowpats
1534 includepats, excludepats = repo.narrowpats
1533
1535
1534 narrowspec.validatepatterns(includepats)
1536 narrowspec.validatepatterns(includepats)
1535 narrowspec.validatepatterns(excludepats)
1537 narrowspec.validatepatterns(excludepats)
1536
1538
1537 pullop = pulloperation(repo, remote, heads, force, bookmarks=bookmarks,
1539 pullop = pulloperation(repo, remote, heads, force, bookmarks=bookmarks,
1538 streamclonerequested=streamclonerequested,
1540 streamclonerequested=streamclonerequested,
1539 includepats=includepats, excludepats=excludepats,
1541 includepats=includepats, excludepats=excludepats,
1540 depth=depth,
1542 depth=depth,
1541 **pycompat.strkwargs(opargs))
1543 **pycompat.strkwargs(opargs))
1542
1544
1543 peerlocal = pullop.remote.local()
1545 peerlocal = pullop.remote.local()
1544 if peerlocal:
1546 if peerlocal:
1545 missing = set(peerlocal.requirements) - pullop.repo.supported
1547 missing = set(peerlocal.requirements) - pullop.repo.supported
1546 if missing:
1548 if missing:
1547 msg = _("required features are not"
1549 msg = _("required features are not"
1548 " supported in the destination:"
1550 " supported in the destination:"
1549 " %s") % (', '.join(sorted(missing)))
1551 " %s") % (', '.join(sorted(missing)))
1550 raise error.Abort(msg)
1552 raise error.Abort(msg)
1551
1553
1552 pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
1554 pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
1553 wlock = util.nullcontextmanager()
1555 wlock = util.nullcontextmanager()
1554 if not bookmod.bookmarksinstore(repo):
1556 if not bookmod.bookmarksinstore(repo):
1555 wlock = repo.wlock()
1557 wlock = repo.wlock()
1556 with wlock, repo.lock(), pullop.trmanager:
1558 with wlock, repo.lock(), pullop.trmanager:
1557 # Use the modern wire protocol, if available.
1559 # Use the modern wire protocol, if available.
1558 if remote.capable('command-changesetdata'):
1560 if remote.capable('command-changesetdata'):
1559 exchangev2.pull(pullop)
1561 exchangev2.pull(pullop)
1560 else:
1562 else:
1561 # This should ideally be in _pullbundle2(). However, it needs to run
1563 # This should ideally be in _pullbundle2(). However, it needs to run
1562 # before discovery to avoid extra work.
1564 # before discovery to avoid extra work.
1563 _maybeapplyclonebundle(pullop)
1565 _maybeapplyclonebundle(pullop)
1564 streamclone.maybeperformlegacystreamclone(pullop)
1566 streamclone.maybeperformlegacystreamclone(pullop)
1565 _pulldiscovery(pullop)
1567 _pulldiscovery(pullop)
1566 if pullop.canusebundle2:
1568 if pullop.canusebundle2:
1567 _fullpullbundle2(repo, pullop)
1569 _fullpullbundle2(repo, pullop)
1568 _pullchangeset(pullop)
1570 _pullchangeset(pullop)
1569 _pullphase(pullop)
1571 _pullphase(pullop)
1570 _pullbookmarks(pullop)
1572 _pullbookmarks(pullop)
1571 _pullobsolete(pullop)
1573 _pullobsolete(pullop)
1572
1574
1573 # storing remotenames
1575 # storing remotenames
1574 if repo.ui.configbool('experimental', 'remotenames'):
1576 if repo.ui.configbool('experimental', 'remotenames'):
1575 logexchange.pullremotenames(repo, remote)
1577 logexchange.pullremotenames(repo, remote)
1576
1578
1577 return pullop
1579 return pullop
1578
1580
1579 # list of steps to perform discovery before pull
1581 # list of steps to perform discovery before pull
1580 pulldiscoveryorder = []
1582 pulldiscoveryorder = []
1581
1583
1582 # Mapping between step name and function
1584 # Mapping between step name and function
1583 #
1585 #
1584 # This exists to help extensions wrap steps if necessary
1586 # This exists to help extensions wrap steps if necessary
1585 pulldiscoverymapping = {}
1587 pulldiscoverymapping = {}
1586
1588
1587 def pulldiscovery(stepname):
1589 def pulldiscovery(stepname):
1588 """decorator for function performing discovery before pull
1590 """decorator for function performing discovery before pull
1589
1591
1590 The function is added to the step -> function mapping and appended to the
1592 The function is added to the step -> function mapping and appended to the
1591 list of steps. Beware that decorated function will be added in order (this
1593 list of steps. Beware that decorated function will be added in order (this
1592 may matter).
1594 may matter).
1593
1595
1594 You can only use this decorator for a new step, if you want to wrap a step
1596 You can only use this decorator for a new step, if you want to wrap a step
1595 from an extension, change the pulldiscovery dictionary directly."""
1597 from an extension, change the pulldiscovery dictionary directly."""
1596 def dec(func):
1598 def dec(func):
1597 assert stepname not in pulldiscoverymapping
1599 assert stepname not in pulldiscoverymapping
1598 pulldiscoverymapping[stepname] = func
1600 pulldiscoverymapping[stepname] = func
1599 pulldiscoveryorder.append(stepname)
1601 pulldiscoveryorder.append(stepname)
1600 return func
1602 return func
1601 return dec
1603 return dec
1602
1604
1603 def _pulldiscovery(pullop):
1605 def _pulldiscovery(pullop):
1604 """Run all discovery steps"""
1606 """Run all discovery steps"""
1605 for stepname in pulldiscoveryorder:
1607 for stepname in pulldiscoveryorder:
1606 step = pulldiscoverymapping[stepname]
1608 step = pulldiscoverymapping[stepname]
1607 step(pullop)
1609 step(pullop)
1608
1610
1609 @pulldiscovery('b1:bookmarks')
1611 @pulldiscovery('b1:bookmarks')
1610 def _pullbookmarkbundle1(pullop):
1612 def _pullbookmarkbundle1(pullop):
1611 """fetch bookmark data in bundle1 case
1613 """fetch bookmark data in bundle1 case
1612
1614
1613 If not using bundle2, we have to fetch bookmarks before changeset
1615 If not using bundle2, we have to fetch bookmarks before changeset
1614 discovery to reduce the chance and impact of race conditions."""
1616 discovery to reduce the chance and impact of race conditions."""
1615 if pullop.remotebookmarks is not None:
1617 if pullop.remotebookmarks is not None:
1616 return
1618 return
1617 if pullop.canusebundle2 and 'listkeys' in pullop.remotebundle2caps:
1619 if pullop.canusebundle2 and 'listkeys' in pullop.remotebundle2caps:
1618 # all known bundle2 servers now support listkeys, but lets be nice with
1620 # all known bundle2 servers now support listkeys, but lets be nice with
1619 # new implementation.
1621 # new implementation.
1620 return
1622 return
1621 books = listkeys(pullop.remote, 'bookmarks')
1623 books = listkeys(pullop.remote, 'bookmarks')
1622 pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
1624 pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
1623
1625
1624
1626
1625 @pulldiscovery('changegroup')
1627 @pulldiscovery('changegroup')
1626 def _pulldiscoverychangegroup(pullop):
1628 def _pulldiscoverychangegroup(pullop):
1627 """discovery phase for the pull
1629 """discovery phase for the pull
1628
1630
1629 Current handle changeset discovery only, will change handle all discovery
1631 Current handle changeset discovery only, will change handle all discovery
1630 at some point."""
1632 at some point."""
1631 tmp = discovery.findcommonincoming(pullop.repo,
1633 tmp = discovery.findcommonincoming(pullop.repo,
1632 pullop.remote,
1634 pullop.remote,
1633 heads=pullop.heads,
1635 heads=pullop.heads,
1634 force=pullop.force)
1636 force=pullop.force)
1635 common, fetch, rheads = tmp
1637 common, fetch, rheads = tmp
1636 nm = pullop.repo.unfiltered().changelog.nodemap
1638 nm = pullop.repo.unfiltered().changelog.nodemap
1637 if fetch and rheads:
1639 if fetch and rheads:
1638 # If a remote heads is filtered locally, put in back in common.
1640 # If a remote heads is filtered locally, put in back in common.
1639 #
1641 #
1640 # This is a hackish solution to catch most of "common but locally
1642 # This is a hackish solution to catch most of "common but locally
1641 # hidden situation". We do not performs discovery on unfiltered
1643 # hidden situation". We do not performs discovery on unfiltered
1642 # repository because it end up doing a pathological amount of round
1644 # repository because it end up doing a pathological amount of round
1643 # trip for w huge amount of changeset we do not care about.
1645 # trip for w huge amount of changeset we do not care about.
1644 #
1646 #
1645 # If a set of such "common but filtered" changeset exist on the server
1647 # If a set of such "common but filtered" changeset exist on the server
1646 # but are not including a remote heads, we'll not be able to detect it,
1648 # but are not including a remote heads, we'll not be able to detect it,
1647 scommon = set(common)
1649 scommon = set(common)
1648 for n in rheads:
1650 for n in rheads:
1649 if n in nm:
1651 if n in nm:
1650 if n not in scommon:
1652 if n not in scommon:
1651 common.append(n)
1653 common.append(n)
1652 if set(rheads).issubset(set(common)):
1654 if set(rheads).issubset(set(common)):
1653 fetch = []
1655 fetch = []
1654 pullop.common = common
1656 pullop.common = common
1655 pullop.fetch = fetch
1657 pullop.fetch = fetch
1656 pullop.rheads = rheads
1658 pullop.rheads = rheads
1657
1659
1658 def _pullbundle2(pullop):
1660 def _pullbundle2(pullop):
1659 """pull data using bundle2
1661 """pull data using bundle2
1660
1662
1661 For now, the only supported data are changegroup."""
1663 For now, the only supported data are changegroup."""
1662 kwargs = {'bundlecaps': caps20to10(pullop.repo, role='client')}
1664 kwargs = {'bundlecaps': caps20to10(pullop.repo, role='client')}
1663
1665
1664 # make ui easier to access
1666 # make ui easier to access
1665 ui = pullop.repo.ui
1667 ui = pullop.repo.ui
1666
1668
1667 # At the moment we don't do stream clones over bundle2. If that is
1669 # At the moment we don't do stream clones over bundle2. If that is
1668 # implemented then here's where the check for that will go.
1670 # implemented then here's where the check for that will go.
1669 streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
1671 streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
1670
1672
1671 # declare pull perimeters
1673 # declare pull perimeters
1672 kwargs['common'] = pullop.common
1674 kwargs['common'] = pullop.common
1673 kwargs['heads'] = pullop.heads or pullop.rheads
1675 kwargs['heads'] = pullop.heads or pullop.rheads
1674
1676
1675 # check server supports narrow and then adding includepats and excludepats
1677 # check server supports narrow and then adding includepats and excludepats
1676 servernarrow = pullop.remote.capable(wireprototypes.NARROWCAP)
1678 servernarrow = pullop.remote.capable(wireprototypes.NARROWCAP)
1677 if servernarrow and pullop.includepats:
1679 if servernarrow and pullop.includepats:
1678 kwargs['includepats'] = pullop.includepats
1680 kwargs['includepats'] = pullop.includepats
1679 if servernarrow and pullop.excludepats:
1681 if servernarrow and pullop.excludepats:
1680 kwargs['excludepats'] = pullop.excludepats
1682 kwargs['excludepats'] = pullop.excludepats
1681
1683
1682 if streaming:
1684 if streaming:
1683 kwargs['cg'] = False
1685 kwargs['cg'] = False
1684 kwargs['stream'] = True
1686 kwargs['stream'] = True
1685 pullop.stepsdone.add('changegroup')
1687 pullop.stepsdone.add('changegroup')
1686 pullop.stepsdone.add('phases')
1688 pullop.stepsdone.add('phases')
1687
1689
1688 else:
1690 else:
1689 # pulling changegroup
1691 # pulling changegroup
1690 pullop.stepsdone.add('changegroup')
1692 pullop.stepsdone.add('changegroup')
1691
1693
1692 kwargs['cg'] = pullop.fetch
1694 kwargs['cg'] = pullop.fetch
1693
1695
1694 legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
1696 legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
1695 hasbinaryphase = 'heads' in pullop.remotebundle2caps.get('phases', ())
1697 hasbinaryphase = 'heads' in pullop.remotebundle2caps.get('phases', ())
1696 if (not legacyphase and hasbinaryphase):
1698 if (not legacyphase and hasbinaryphase):
1697 kwargs['phases'] = True
1699 kwargs['phases'] = True
1698 pullop.stepsdone.add('phases')
1700 pullop.stepsdone.add('phases')
1699
1701
1700 if 'listkeys' in pullop.remotebundle2caps:
1702 if 'listkeys' in pullop.remotebundle2caps:
1701 if 'phases' not in pullop.stepsdone:
1703 if 'phases' not in pullop.stepsdone:
1702 kwargs['listkeys'] = ['phases']
1704 kwargs['listkeys'] = ['phases']
1703
1705
1704 bookmarksrequested = False
1706 bookmarksrequested = False
1705 legacybookmark = 'bookmarks' in ui.configlist('devel', 'legacy.exchange')
1707 legacybookmark = 'bookmarks' in ui.configlist('devel', 'legacy.exchange')
1706 hasbinarybook = 'bookmarks' in pullop.remotebundle2caps
1708 hasbinarybook = 'bookmarks' in pullop.remotebundle2caps
1707
1709
1708 if pullop.remotebookmarks is not None:
1710 if pullop.remotebookmarks is not None:
1709 pullop.stepsdone.add('request-bookmarks')
1711 pullop.stepsdone.add('request-bookmarks')
1710
1712
1711 if ('request-bookmarks' not in pullop.stepsdone
1713 if ('request-bookmarks' not in pullop.stepsdone
1712 and pullop.remotebookmarks is None
1714 and pullop.remotebookmarks is None
1713 and not legacybookmark and hasbinarybook):
1715 and not legacybookmark and hasbinarybook):
1714 kwargs['bookmarks'] = True
1716 kwargs['bookmarks'] = True
1715 bookmarksrequested = True
1717 bookmarksrequested = True
1716
1718
1717 if 'listkeys' in pullop.remotebundle2caps:
1719 if 'listkeys' in pullop.remotebundle2caps:
1718 if 'request-bookmarks' not in pullop.stepsdone:
1720 if 'request-bookmarks' not in pullop.stepsdone:
1719 # make sure to always includes bookmark data when migrating
1721 # make sure to always includes bookmark data when migrating
1720 # `hg incoming --bundle` to using this function.
1722 # `hg incoming --bundle` to using this function.
1721 pullop.stepsdone.add('request-bookmarks')
1723 pullop.stepsdone.add('request-bookmarks')
1722 kwargs.setdefault('listkeys', []).append('bookmarks')
1724 kwargs.setdefault('listkeys', []).append('bookmarks')
1723
1725
1724 # If this is a full pull / clone and the server supports the clone bundles
1726 # If this is a full pull / clone and the server supports the clone bundles
1725 # feature, tell the server whether we attempted a clone bundle. The
1727 # feature, tell the server whether we attempted a clone bundle. The
1726 # presence of this flag indicates the client supports clone bundles. This
1728 # presence of this flag indicates the client supports clone bundles. This
1727 # will enable the server to treat clients that support clone bundles
1729 # will enable the server to treat clients that support clone bundles
1728 # differently from those that don't.
1730 # differently from those that don't.
1729 if (pullop.remote.capable('clonebundles')
1731 if (pullop.remote.capable('clonebundles')
1730 and pullop.heads is None and list(pullop.common) == [nullid]):
1732 and pullop.heads is None and list(pullop.common) == [nullid]):
1731 kwargs['cbattempted'] = pullop.clonebundleattempted
1733 kwargs['cbattempted'] = pullop.clonebundleattempted
1732
1734
1733 if streaming:
1735 if streaming:
1734 pullop.repo.ui.status(_('streaming all changes\n'))
1736 pullop.repo.ui.status(_('streaming all changes\n'))
1735 elif not pullop.fetch:
1737 elif not pullop.fetch:
1736 pullop.repo.ui.status(_("no changes found\n"))
1738 pullop.repo.ui.status(_("no changes found\n"))
1737 pullop.cgresult = 0
1739 pullop.cgresult = 0
1738 else:
1740 else:
1739 if pullop.heads is None and list(pullop.common) == [nullid]:
1741 if pullop.heads is None and list(pullop.common) == [nullid]:
1740 pullop.repo.ui.status(_("requesting all changes\n"))
1742 pullop.repo.ui.status(_("requesting all changes\n"))
1741 if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
1743 if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
1742 remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
1744 remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
1743 if obsolete.commonversion(remoteversions) is not None:
1745 if obsolete.commonversion(remoteversions) is not None:
1744 kwargs['obsmarkers'] = True
1746 kwargs['obsmarkers'] = True
1745 pullop.stepsdone.add('obsmarkers')
1747 pullop.stepsdone.add('obsmarkers')
1746 _pullbundle2extraprepare(pullop, kwargs)
1748 _pullbundle2extraprepare(pullop, kwargs)
1747
1749
1748 with pullop.remote.commandexecutor() as e:
1750 with pullop.remote.commandexecutor() as e:
1749 args = dict(kwargs)
1751 args = dict(kwargs)
1750 args['source'] = 'pull'
1752 args['source'] = 'pull'
1751 bundle = e.callcommand('getbundle', args).result()
1753 bundle = e.callcommand('getbundle', args).result()
1752
1754
1753 try:
1755 try:
1754 op = bundle2.bundleoperation(pullop.repo, pullop.gettransaction,
1756 op = bundle2.bundleoperation(pullop.repo, pullop.gettransaction,
1755 source='pull')
1757 source='pull')
1756 op.modes['bookmarks'] = 'records'
1758 op.modes['bookmarks'] = 'records'
1757 bundle2.processbundle(pullop.repo, bundle, op=op)
1759 bundle2.processbundle(pullop.repo, bundle, op=op)
1758 except bundle2.AbortFromPart as exc:
1760 except bundle2.AbortFromPart as exc:
1759 pullop.repo.ui.status(_('remote: abort: %s\n') % exc)
1761 pullop.repo.ui.status(_('remote: abort: %s\n') % exc)
1760 raise error.Abort(_('pull failed on remote'), hint=exc.hint)
1762 raise error.Abort(_('pull failed on remote'), hint=exc.hint)
1761 except error.BundleValueError as exc:
1763 except error.BundleValueError as exc:
1762 raise error.Abort(_('missing support for %s') % exc)
1764 raise error.Abort(_('missing support for %s') % exc)
1763
1765
1764 if pullop.fetch:
1766 if pullop.fetch:
1765 pullop.cgresult = bundle2.combinechangegroupresults(op)
1767 pullop.cgresult = bundle2.combinechangegroupresults(op)
1766
1768
1767 # processing phases change
1769 # processing phases change
1768 for namespace, value in op.records['listkeys']:
1770 for namespace, value in op.records['listkeys']:
1769 if namespace == 'phases':
1771 if namespace == 'phases':
1770 _pullapplyphases(pullop, value)
1772 _pullapplyphases(pullop, value)
1771
1773
1772 # processing bookmark update
1774 # processing bookmark update
1773 if bookmarksrequested:
1775 if bookmarksrequested:
1774 books = {}
1776 books = {}
1775 for record in op.records['bookmarks']:
1777 for record in op.records['bookmarks']:
1776 books[record['bookmark']] = record["node"]
1778 books[record['bookmark']] = record["node"]
1777 pullop.remotebookmarks = books
1779 pullop.remotebookmarks = books
1778 else:
1780 else:
1779 for namespace, value in op.records['listkeys']:
1781 for namespace, value in op.records['listkeys']:
1780 if namespace == 'bookmarks':
1782 if namespace == 'bookmarks':
1781 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
1783 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
1782
1784
1783 # bookmark data were either already there or pulled in the bundle
1785 # bookmark data were either already there or pulled in the bundle
1784 if pullop.remotebookmarks is not None:
1786 if pullop.remotebookmarks is not None:
1785 _pullbookmarks(pullop)
1787 _pullbookmarks(pullop)
1786
1788
1787 def _pullbundle2extraprepare(pullop, kwargs):
1789 def _pullbundle2extraprepare(pullop, kwargs):
1788 """hook function so that extensions can extend the getbundle call"""
1790 """hook function so that extensions can extend the getbundle call"""
1789
1791
1790 def _pullchangeset(pullop):
1792 def _pullchangeset(pullop):
1791 """pull changeset from unbundle into the local repo"""
1793 """pull changeset from unbundle into the local repo"""
1792 # We delay the open of the transaction as late as possible so we
1794 # We delay the open of the transaction as late as possible so we
1793 # don't open transaction for nothing or you break future useful
1795 # don't open transaction for nothing or you break future useful
1794 # rollback call
1796 # rollback call
1795 if 'changegroup' in pullop.stepsdone:
1797 if 'changegroup' in pullop.stepsdone:
1796 return
1798 return
1797 pullop.stepsdone.add('changegroup')
1799 pullop.stepsdone.add('changegroup')
1798 if not pullop.fetch:
1800 if not pullop.fetch:
1799 pullop.repo.ui.status(_("no changes found\n"))
1801 pullop.repo.ui.status(_("no changes found\n"))
1800 pullop.cgresult = 0
1802 pullop.cgresult = 0
1801 return
1803 return
1802 tr = pullop.gettransaction()
1804 tr = pullop.gettransaction()
1803 if pullop.heads is None and list(pullop.common) == [nullid]:
1805 if pullop.heads is None and list(pullop.common) == [nullid]:
1804 pullop.repo.ui.status(_("requesting all changes\n"))
1806 pullop.repo.ui.status(_("requesting all changes\n"))
1805 elif pullop.heads is None and pullop.remote.capable('changegroupsubset'):
1807 elif pullop.heads is None and pullop.remote.capable('changegroupsubset'):
1806 # issue1320, avoid a race if remote changed after discovery
1808 # issue1320, avoid a race if remote changed after discovery
1807 pullop.heads = pullop.rheads
1809 pullop.heads = pullop.rheads
1808
1810
1809 if pullop.remote.capable('getbundle'):
1811 if pullop.remote.capable('getbundle'):
1810 # TODO: get bundlecaps from remote
1812 # TODO: get bundlecaps from remote
1811 cg = pullop.remote.getbundle('pull', common=pullop.common,
1813 cg = pullop.remote.getbundle('pull', common=pullop.common,
1812 heads=pullop.heads or pullop.rheads)
1814 heads=pullop.heads or pullop.rheads)
1813 elif pullop.heads is None:
1815 elif pullop.heads is None:
1814 with pullop.remote.commandexecutor() as e:
1816 with pullop.remote.commandexecutor() as e:
1815 cg = e.callcommand('changegroup', {
1817 cg = e.callcommand('changegroup', {
1816 'nodes': pullop.fetch,
1818 'nodes': pullop.fetch,
1817 'source': 'pull',
1819 'source': 'pull',
1818 }).result()
1820 }).result()
1819
1821
1820 elif not pullop.remote.capable('changegroupsubset'):
1822 elif not pullop.remote.capable('changegroupsubset'):
1821 raise error.Abort(_("partial pull cannot be done because "
1823 raise error.Abort(_("partial pull cannot be done because "
1822 "other repository doesn't support "
1824 "other repository doesn't support "
1823 "changegroupsubset."))
1825 "changegroupsubset."))
1824 else:
1826 else:
1825 with pullop.remote.commandexecutor() as e:
1827 with pullop.remote.commandexecutor() as e:
1826 cg = e.callcommand('changegroupsubset', {
1828 cg = e.callcommand('changegroupsubset', {
1827 'bases': pullop.fetch,
1829 'bases': pullop.fetch,
1828 'heads': pullop.heads,
1830 'heads': pullop.heads,
1829 'source': 'pull',
1831 'source': 'pull',
1830 }).result()
1832 }).result()
1831
1833
1832 bundleop = bundle2.applybundle(pullop.repo, cg, tr, 'pull',
1834 bundleop = bundle2.applybundle(pullop.repo, cg, tr, 'pull',
1833 pullop.remote.url())
1835 pullop.remote.url())
1834 pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
1836 pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
1835
1837
1836 def _pullphase(pullop):
1838 def _pullphase(pullop):
1837 # Get remote phases data from remote
1839 # Get remote phases data from remote
1838 if 'phases' in pullop.stepsdone:
1840 if 'phases' in pullop.stepsdone:
1839 return
1841 return
1840 remotephases = listkeys(pullop.remote, 'phases')
1842 remotephases = listkeys(pullop.remote, 'phases')
1841 _pullapplyphases(pullop, remotephases)
1843 _pullapplyphases(pullop, remotephases)
1842
1844
1843 def _pullapplyphases(pullop, remotephases):
1845 def _pullapplyphases(pullop, remotephases):
1844 """apply phase movement from observed remote state"""
1846 """apply phase movement from observed remote state"""
1845 if 'phases' in pullop.stepsdone:
1847 if 'phases' in pullop.stepsdone:
1846 return
1848 return
1847 pullop.stepsdone.add('phases')
1849 pullop.stepsdone.add('phases')
1848 publishing = bool(remotephases.get('publishing', False))
1850 publishing = bool(remotephases.get('publishing', False))
1849 if remotephases and not publishing:
1851 if remotephases and not publishing:
1850 # remote is new and non-publishing
1852 # remote is new and non-publishing
1851 pheads, _dr = phases.analyzeremotephases(pullop.repo,
1853 pheads, _dr = phases.analyzeremotephases(pullop.repo,
1852 pullop.pulledsubset,
1854 pullop.pulledsubset,
1853 remotephases)
1855 remotephases)
1854 dheads = pullop.pulledsubset
1856 dheads = pullop.pulledsubset
1855 else:
1857 else:
1856 # Remote is old or publishing all common changesets
1858 # Remote is old or publishing all common changesets
1857 # should be seen as public
1859 # should be seen as public
1858 pheads = pullop.pulledsubset
1860 pheads = pullop.pulledsubset
1859 dheads = []
1861 dheads = []
1860 unfi = pullop.repo.unfiltered()
1862 unfi = pullop.repo.unfiltered()
1861 phase = unfi._phasecache.phase
1863 phase = unfi._phasecache.phase
1862 rev = unfi.changelog.nodemap.get
1864 rev = unfi.changelog.nodemap.get
1863 public = phases.public
1865 public = phases.public
1864 draft = phases.draft
1866 draft = phases.draft
1865
1867
1866 # exclude changesets already public locally and update the others
1868 # exclude changesets already public locally and update the others
1867 pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
1869 pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
1868 if pheads:
1870 if pheads:
1869 tr = pullop.gettransaction()
1871 tr = pullop.gettransaction()
1870 phases.advanceboundary(pullop.repo, tr, public, pheads)
1872 phases.advanceboundary(pullop.repo, tr, public, pheads)
1871
1873
1872 # exclude changesets already draft locally and update the others
1874 # exclude changesets already draft locally and update the others
1873 dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
1875 dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
1874 if dheads:
1876 if dheads:
1875 tr = pullop.gettransaction()
1877 tr = pullop.gettransaction()
1876 phases.advanceboundary(pullop.repo, tr, draft, dheads)
1878 phases.advanceboundary(pullop.repo, tr, draft, dheads)
1877
1879
1878 def _pullbookmarks(pullop):
1880 def _pullbookmarks(pullop):
1879 """process the remote bookmark information to update the local one"""
1881 """process the remote bookmark information to update the local one"""
1880 if 'bookmarks' in pullop.stepsdone:
1882 if 'bookmarks' in pullop.stepsdone:
1881 return
1883 return
1882 pullop.stepsdone.add('bookmarks')
1884 pullop.stepsdone.add('bookmarks')
1883 repo = pullop.repo
1885 repo = pullop.repo
1884 remotebookmarks = pullop.remotebookmarks
1886 remotebookmarks = pullop.remotebookmarks
1885 bookmod.updatefromremote(repo.ui, repo, remotebookmarks,
1887 bookmod.updatefromremote(repo.ui, repo, remotebookmarks,
1886 pullop.remote.url(),
1888 pullop.remote.url(),
1887 pullop.gettransaction,
1889 pullop.gettransaction,
1888 explicit=pullop.explicitbookmarks)
1890 explicit=pullop.explicitbookmarks)
1889
1891
1890 def _pullobsolete(pullop):
1892 def _pullobsolete(pullop):
1891 """utility function to pull obsolete markers from a remote
1893 """utility function to pull obsolete markers from a remote
1892
1894
1893 The `gettransaction` is function that return the pull transaction, creating
1895 The `gettransaction` is function that return the pull transaction, creating
1894 one if necessary. We return the transaction to inform the calling code that
1896 one if necessary. We return the transaction to inform the calling code that
1895 a new transaction have been created (when applicable).
1897 a new transaction have been created (when applicable).
1896
1898
1897 Exists mostly to allow overriding for experimentation purpose"""
1899 Exists mostly to allow overriding for experimentation purpose"""
1898 if 'obsmarkers' in pullop.stepsdone:
1900 if 'obsmarkers' in pullop.stepsdone:
1899 return
1901 return
1900 pullop.stepsdone.add('obsmarkers')
1902 pullop.stepsdone.add('obsmarkers')
1901 tr = None
1903 tr = None
1902 if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
1904 if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
1903 pullop.repo.ui.debug('fetching remote obsolete markers\n')
1905 pullop.repo.ui.debug('fetching remote obsolete markers\n')
1904 remoteobs = listkeys(pullop.remote, 'obsolete')
1906 remoteobs = listkeys(pullop.remote, 'obsolete')
1905 if 'dump0' in remoteobs:
1907 if 'dump0' in remoteobs:
1906 tr = pullop.gettransaction()
1908 tr = pullop.gettransaction()
1907 markers = []
1909 markers = []
1908 for key in sorted(remoteobs, reverse=True):
1910 for key in sorted(remoteobs, reverse=True):
1909 if key.startswith('dump'):
1911 if key.startswith('dump'):
1910 data = util.b85decode(remoteobs[key])
1912 data = util.b85decode(remoteobs[key])
1911 version, newmarks = obsolete._readmarkers(data)
1913 version, newmarks = obsolete._readmarkers(data)
1912 markers += newmarks
1914 markers += newmarks
1913 if markers:
1915 if markers:
1914 pullop.repo.obsstore.add(tr, markers)
1916 pullop.repo.obsstore.add(tr, markers)
1915 pullop.repo.invalidatevolatilesets()
1917 pullop.repo.invalidatevolatilesets()
1916 return tr
1918 return tr
1917
1919
1918 def applynarrowacl(repo, kwargs):
1920 def applynarrowacl(repo, kwargs):
1919 """Apply narrow fetch access control.
1921 """Apply narrow fetch access control.
1920
1922
1921 This massages the named arguments for getbundle wire protocol commands
1923 This massages the named arguments for getbundle wire protocol commands
1922 so requested data is filtered through access control rules.
1924 so requested data is filtered through access control rules.
1923 """
1925 """
1924 ui = repo.ui
1926 ui = repo.ui
1925 # TODO this assumes existence of HTTP and is a layering violation.
1927 # TODO this assumes existence of HTTP and is a layering violation.
1926 username = ui.shortuser(ui.environ.get('REMOTE_USER') or ui.username())
1928 username = ui.shortuser(ui.environ.get('REMOTE_USER') or ui.username())
1927 user_includes = ui.configlist(
1929 user_includes = ui.configlist(
1928 _NARROWACL_SECTION, username + '.includes',
1930 _NARROWACL_SECTION, username + '.includes',
1929 ui.configlist(_NARROWACL_SECTION, 'default.includes'))
1931 ui.configlist(_NARROWACL_SECTION, 'default.includes'))
1930 user_excludes = ui.configlist(
1932 user_excludes = ui.configlist(
1931 _NARROWACL_SECTION, username + '.excludes',
1933 _NARROWACL_SECTION, username + '.excludes',
1932 ui.configlist(_NARROWACL_SECTION, 'default.excludes'))
1934 ui.configlist(_NARROWACL_SECTION, 'default.excludes'))
1933 if not user_includes:
1935 if not user_includes:
1934 raise error.Abort(_("{} configuration for user {} is empty")
1936 raise error.Abort(_("{} configuration for user {} is empty")
1935 .format(_NARROWACL_SECTION, username))
1937 .format(_NARROWACL_SECTION, username))
1936
1938
1937 user_includes = [
1939 user_includes = [
1938 'path:.' if p == '*' else 'path:' + p for p in user_includes]
1940 'path:.' if p == '*' else 'path:' + p for p in user_includes]
1939 user_excludes = [
1941 user_excludes = [
1940 'path:.' if p == '*' else 'path:' + p for p in user_excludes]
1942 'path:.' if p == '*' else 'path:' + p for p in user_excludes]
1941
1943
1942 req_includes = set(kwargs.get(r'includepats', []))
1944 req_includes = set(kwargs.get(r'includepats', []))
1943 req_excludes = set(kwargs.get(r'excludepats', []))
1945 req_excludes = set(kwargs.get(r'excludepats', []))
1944
1946
1945 req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
1947 req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
1946 req_includes, req_excludes, user_includes, user_excludes)
1948 req_includes, req_excludes, user_includes, user_excludes)
1947
1949
1948 if invalid_includes:
1950 if invalid_includes:
1949 raise error.Abort(
1951 raise error.Abort(
1950 _("The following includes are not accessible for {}: {}")
1952 _("The following includes are not accessible for {}: {}")
1951 .format(username, invalid_includes))
1953 .format(username, invalid_includes))
1952
1954
1953 new_args = {}
1955 new_args = {}
1954 new_args.update(kwargs)
1956 new_args.update(kwargs)
1955 new_args[r'narrow'] = True
1957 new_args[r'narrow'] = True
1956 new_args[r'narrow_acl'] = True
1958 new_args[r'narrow_acl'] = True
1957 new_args[r'includepats'] = req_includes
1959 new_args[r'includepats'] = req_includes
1958 if req_excludes:
1960 if req_excludes:
1959 new_args[r'excludepats'] = req_excludes
1961 new_args[r'excludepats'] = req_excludes
1960
1962
1961 return new_args
1963 return new_args
1962
1964
1963 def _computeellipsis(repo, common, heads, known, match, depth=None):
1965 def _computeellipsis(repo, common, heads, known, match, depth=None):
1964 """Compute the shape of a narrowed DAG.
1966 """Compute the shape of a narrowed DAG.
1965
1967
1966 Args:
1968 Args:
1967 repo: The repository we're transferring.
1969 repo: The repository we're transferring.
1968 common: The roots of the DAG range we're transferring.
1970 common: The roots of the DAG range we're transferring.
1969 May be just [nullid], which means all ancestors of heads.
1971 May be just [nullid], which means all ancestors of heads.
1970 heads: The heads of the DAG range we're transferring.
1972 heads: The heads of the DAG range we're transferring.
1971 match: The narrowmatcher that allows us to identify relevant changes.
1973 match: The narrowmatcher that allows us to identify relevant changes.
1972 depth: If not None, only consider nodes to be full nodes if they are at
1974 depth: If not None, only consider nodes to be full nodes if they are at
1973 most depth changesets away from one of heads.
1975 most depth changesets away from one of heads.
1974
1976
1975 Returns:
1977 Returns:
1976 A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
1978 A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
1977
1979
1978 visitnodes: The list of nodes (either full or ellipsis) which
1980 visitnodes: The list of nodes (either full or ellipsis) which
1979 need to be sent to the client.
1981 need to be sent to the client.
1980 relevant_nodes: The set of changelog nodes which change a file inside
1982 relevant_nodes: The set of changelog nodes which change a file inside
1981 the narrowspec. The client needs these as non-ellipsis nodes.
1983 the narrowspec. The client needs these as non-ellipsis nodes.
1982 ellipsisroots: A dict of {rev: parents} that is used in
1984 ellipsisroots: A dict of {rev: parents} that is used in
1983 narrowchangegroup to produce ellipsis nodes with the
1985 narrowchangegroup to produce ellipsis nodes with the
1984 correct parents.
1986 correct parents.
1985 """
1987 """
1986 cl = repo.changelog
1988 cl = repo.changelog
1987 mfl = repo.manifestlog
1989 mfl = repo.manifestlog
1988
1990
1989 clrev = cl.rev
1991 clrev = cl.rev
1990
1992
1991 commonrevs = {clrev(n) for n in common} | {nullrev}
1993 commonrevs = {clrev(n) for n in common} | {nullrev}
1992 headsrevs = {clrev(n) for n in heads}
1994 headsrevs = {clrev(n) for n in heads}
1993
1995
1994 if depth:
1996 if depth:
1995 revdepth = {h: 0 for h in headsrevs}
1997 revdepth = {h: 0 for h in headsrevs}
1996
1998
1997 ellipsisheads = collections.defaultdict(set)
1999 ellipsisheads = collections.defaultdict(set)
1998 ellipsisroots = collections.defaultdict(set)
2000 ellipsisroots = collections.defaultdict(set)
1999
2001
2000 def addroot(head, curchange):
2002 def addroot(head, curchange):
2001 """Add a root to an ellipsis head, splitting heads with 3 roots."""
2003 """Add a root to an ellipsis head, splitting heads with 3 roots."""
2002 ellipsisroots[head].add(curchange)
2004 ellipsisroots[head].add(curchange)
2003 # Recursively split ellipsis heads with 3 roots by finding the
2005 # Recursively split ellipsis heads with 3 roots by finding the
2004 # roots' youngest common descendant which is an elided merge commit.
2006 # roots' youngest common descendant which is an elided merge commit.
2005 # That descendant takes 2 of the 3 roots as its own, and becomes a
2007 # That descendant takes 2 of the 3 roots as its own, and becomes a
2006 # root of the head.
2008 # root of the head.
2007 while len(ellipsisroots[head]) > 2:
2009 while len(ellipsisroots[head]) > 2:
2008 child, roots = splithead(head)
2010 child, roots = splithead(head)
2009 splitroots(head, child, roots)
2011 splitroots(head, child, roots)
2010 head = child # Recurse in case we just added a 3rd root
2012 head = child # Recurse in case we just added a 3rd root
2011
2013
2012 def splitroots(head, child, roots):
2014 def splitroots(head, child, roots):
2013 ellipsisroots[head].difference_update(roots)
2015 ellipsisroots[head].difference_update(roots)
2014 ellipsisroots[head].add(child)
2016 ellipsisroots[head].add(child)
2015 ellipsisroots[child].update(roots)
2017 ellipsisroots[child].update(roots)
2016 ellipsisroots[child].discard(child)
2018 ellipsisroots[child].discard(child)
2017
2019
2018 def splithead(head):
2020 def splithead(head):
2019 r1, r2, r3 = sorted(ellipsisroots[head])
2021 r1, r2, r3 = sorted(ellipsisroots[head])
2020 for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
2022 for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
2021 mid = repo.revs('sort(merge() & %d::%d & %d::%d, -rev)',
2023 mid = repo.revs('sort(merge() & %d::%d & %d::%d, -rev)',
2022 nr1, head, nr2, head)
2024 nr1, head, nr2, head)
2023 for j in mid:
2025 for j in mid:
2024 if j == nr2:
2026 if j == nr2:
2025 return nr2, (nr1, nr2)
2027 return nr2, (nr1, nr2)
2026 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
2028 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
2027 return j, (nr1, nr2)
2029 return j, (nr1, nr2)
2028 raise error.Abort(_('Failed to split up ellipsis node! head: %d, '
2030 raise error.Abort(_('Failed to split up ellipsis node! head: %d, '
2029 'roots: %d %d %d') % (head, r1, r2, r3))
2031 'roots: %d %d %d') % (head, r1, r2, r3))
2030
2032
2031 missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
2033 missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
2032 visit = reversed(missing)
2034 visit = reversed(missing)
2033 relevant_nodes = set()
2035 relevant_nodes = set()
2034 visitnodes = [cl.node(m) for m in missing]
2036 visitnodes = [cl.node(m) for m in missing]
2035 required = set(headsrevs) | known
2037 required = set(headsrevs) | known
2036 for rev in visit:
2038 for rev in visit:
2037 clrev = cl.changelogrevision(rev)
2039 clrev = cl.changelogrevision(rev)
2038 ps = [prev for prev in cl.parentrevs(rev) if prev != nullrev]
2040 ps = [prev for prev in cl.parentrevs(rev) if prev != nullrev]
2039 if depth is not None:
2041 if depth is not None:
2040 curdepth = revdepth[rev]
2042 curdepth = revdepth[rev]
2041 for p in ps:
2043 for p in ps:
2042 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
2044 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
2043 needed = False
2045 needed = False
2044 shallow_enough = depth is None or revdepth[rev] <= depth
2046 shallow_enough = depth is None or revdepth[rev] <= depth
2045 if shallow_enough:
2047 if shallow_enough:
2046 curmf = mfl[clrev.manifest].read()
2048 curmf = mfl[clrev.manifest].read()
2047 if ps:
2049 if ps:
2048 # We choose to not trust the changed files list in
2050 # We choose to not trust the changed files list in
2049 # changesets because it's not always correct. TODO: could
2051 # changesets because it's not always correct. TODO: could
2050 # we trust it for the non-merge case?
2052 # we trust it for the non-merge case?
2051 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
2053 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
2052 needed = bool(curmf.diff(p1mf, match))
2054 needed = bool(curmf.diff(p1mf, match))
2053 if not needed and len(ps) > 1:
2055 if not needed and len(ps) > 1:
2054 # For merge changes, the list of changed files is not
2056 # For merge changes, the list of changed files is not
2055 # helpful, since we need to emit the merge if a file
2057 # helpful, since we need to emit the merge if a file
2056 # in the narrow spec has changed on either side of the
2058 # in the narrow spec has changed on either side of the
2057 # merge. As a result, we do a manifest diff to check.
2059 # merge. As a result, we do a manifest diff to check.
2058 p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
2060 p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
2059 needed = bool(curmf.diff(p2mf, match))
2061 needed = bool(curmf.diff(p2mf, match))
2060 else:
2062 else:
2061 # For a root node, we need to include the node if any
2063 # For a root node, we need to include the node if any
2062 # files in the node match the narrowspec.
2064 # files in the node match the narrowspec.
2063 needed = any(curmf.walk(match))
2065 needed = any(curmf.walk(match))
2064
2066
2065 if needed:
2067 if needed:
2066 for head in ellipsisheads[rev]:
2068 for head in ellipsisheads[rev]:
2067 addroot(head, rev)
2069 addroot(head, rev)
2068 for p in ps:
2070 for p in ps:
2069 required.add(p)
2071 required.add(p)
2070 relevant_nodes.add(cl.node(rev))
2072 relevant_nodes.add(cl.node(rev))
2071 else:
2073 else:
2072 if not ps:
2074 if not ps:
2073 ps = [nullrev]
2075 ps = [nullrev]
2074 if rev in required:
2076 if rev in required:
2075 for head in ellipsisheads[rev]:
2077 for head in ellipsisheads[rev]:
2076 addroot(head, rev)
2078 addroot(head, rev)
2077 for p in ps:
2079 for p in ps:
2078 ellipsisheads[p].add(rev)
2080 ellipsisheads[p].add(rev)
2079 else:
2081 else:
2080 for p in ps:
2082 for p in ps:
2081 ellipsisheads[p] |= ellipsisheads[rev]
2083 ellipsisheads[p] |= ellipsisheads[rev]
2082
2084
2083 # add common changesets as roots of their reachable ellipsis heads
2085 # add common changesets as roots of their reachable ellipsis heads
2084 for c in commonrevs:
2086 for c in commonrevs:
2085 for head in ellipsisheads[c]:
2087 for head in ellipsisheads[c]:
2086 addroot(head, c)
2088 addroot(head, c)
2087 return visitnodes, relevant_nodes, ellipsisroots
2089 return visitnodes, relevant_nodes, ellipsisroots
2088
2090
2089 def caps20to10(repo, role):
2091 def caps20to10(repo, role):
2090 """return a set with appropriate options to use bundle20 during getbundle"""
2092 """return a set with appropriate options to use bundle20 during getbundle"""
2091 caps = {'HG20'}
2093 caps = {'HG20'}
2092 capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
2094 capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
2093 caps.add('bundle2=' + urlreq.quote(capsblob))
2095 caps.add('bundle2=' + urlreq.quote(capsblob))
2094 return caps
2096 return caps
2095
2097
2096 # List of names of steps to perform for a bundle2 for getbundle, order matters.
2098 # List of names of steps to perform for a bundle2 for getbundle, order matters.
2097 getbundle2partsorder = []
2099 getbundle2partsorder = []
2098
2100
2099 # Mapping between step name and function
2101 # Mapping between step name and function
2100 #
2102 #
2101 # This exists to help extensions wrap steps if necessary
2103 # This exists to help extensions wrap steps if necessary
2102 getbundle2partsmapping = {}
2104 getbundle2partsmapping = {}
2103
2105
2104 def getbundle2partsgenerator(stepname, idx=None):
2106 def getbundle2partsgenerator(stepname, idx=None):
2105 """decorator for function generating bundle2 part for getbundle
2107 """decorator for function generating bundle2 part for getbundle
2106
2108
2107 The function is added to the step -> function mapping and appended to the
2109 The function is added to the step -> function mapping and appended to the
2108 list of steps. Beware that decorated functions will be added in order
2110 list of steps. Beware that decorated functions will be added in order
2109 (this may matter).
2111 (this may matter).
2110
2112
2111 You can only use this decorator for new steps, if you want to wrap a step
2113 You can only use this decorator for new steps, if you want to wrap a step
2112 from an extension, attack the getbundle2partsmapping dictionary directly."""
2114 from an extension, attack the getbundle2partsmapping dictionary directly."""
2113 def dec(func):
2115 def dec(func):
2114 assert stepname not in getbundle2partsmapping
2116 assert stepname not in getbundle2partsmapping
2115 getbundle2partsmapping[stepname] = func
2117 getbundle2partsmapping[stepname] = func
2116 if idx is None:
2118 if idx is None:
2117 getbundle2partsorder.append(stepname)
2119 getbundle2partsorder.append(stepname)
2118 else:
2120 else:
2119 getbundle2partsorder.insert(idx, stepname)
2121 getbundle2partsorder.insert(idx, stepname)
2120 return func
2122 return func
2121 return dec
2123 return dec
2122
2124
2123 def bundle2requested(bundlecaps):
2125 def bundle2requested(bundlecaps):
2124 if bundlecaps is not None:
2126 if bundlecaps is not None:
2125 return any(cap.startswith('HG2') for cap in bundlecaps)
2127 return any(cap.startswith('HG2') for cap in bundlecaps)
2126 return False
2128 return False
2127
2129
2128 def getbundlechunks(repo, source, heads=None, common=None, bundlecaps=None,
2130 def getbundlechunks(repo, source, heads=None, common=None, bundlecaps=None,
2129 **kwargs):
2131 **kwargs):
2130 """Return chunks constituting a bundle's raw data.
2132 """Return chunks constituting a bundle's raw data.
2131
2133
2132 Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
2134 Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
2133 passed.
2135 passed.
2134
2136
2135 Returns a 2-tuple of a dict with metadata about the generated bundle
2137 Returns a 2-tuple of a dict with metadata about the generated bundle
2136 and an iterator over raw chunks (of varying sizes).
2138 and an iterator over raw chunks (of varying sizes).
2137 """
2139 """
2138 kwargs = pycompat.byteskwargs(kwargs)
2140 kwargs = pycompat.byteskwargs(kwargs)
2139 info = {}
2141 info = {}
2140 usebundle2 = bundle2requested(bundlecaps)
2142 usebundle2 = bundle2requested(bundlecaps)
2141 # bundle10 case
2143 # bundle10 case
2142 if not usebundle2:
2144 if not usebundle2:
2143 if bundlecaps and not kwargs.get('cg', True):
2145 if bundlecaps and not kwargs.get('cg', True):
2144 raise ValueError(_('request for bundle10 must include changegroup'))
2146 raise ValueError(_('request for bundle10 must include changegroup'))
2145
2147
2146 if kwargs:
2148 if kwargs:
2147 raise ValueError(_('unsupported getbundle arguments: %s')
2149 raise ValueError(_('unsupported getbundle arguments: %s')
2148 % ', '.join(sorted(kwargs.keys())))
2150 % ', '.join(sorted(kwargs.keys())))
2149 outgoing = _computeoutgoing(repo, heads, common)
2151 outgoing = _computeoutgoing(repo, heads, common)
2150 info['bundleversion'] = 1
2152 info['bundleversion'] = 1
2151 return info, changegroup.makestream(repo, outgoing, '01', source,
2153 return info, changegroup.makestream(repo, outgoing, '01', source,
2152 bundlecaps=bundlecaps)
2154 bundlecaps=bundlecaps)
2153
2155
2154 # bundle20 case
2156 # bundle20 case
2155 info['bundleversion'] = 2
2157 info['bundleversion'] = 2
2156 b2caps = {}
2158 b2caps = {}
2157 for bcaps in bundlecaps:
2159 for bcaps in bundlecaps:
2158 if bcaps.startswith('bundle2='):
2160 if bcaps.startswith('bundle2='):
2159 blob = urlreq.unquote(bcaps[len('bundle2='):])
2161 blob = urlreq.unquote(bcaps[len('bundle2='):])
2160 b2caps.update(bundle2.decodecaps(blob))
2162 b2caps.update(bundle2.decodecaps(blob))
2161 bundler = bundle2.bundle20(repo.ui, b2caps)
2163 bundler = bundle2.bundle20(repo.ui, b2caps)
2162
2164
2163 kwargs['heads'] = heads
2165 kwargs['heads'] = heads
2164 kwargs['common'] = common
2166 kwargs['common'] = common
2165
2167
2166 for name in getbundle2partsorder:
2168 for name in getbundle2partsorder:
2167 func = getbundle2partsmapping[name]
2169 func = getbundle2partsmapping[name]
2168 func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps,
2170 func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps,
2169 **pycompat.strkwargs(kwargs))
2171 **pycompat.strkwargs(kwargs))
2170
2172
2171 info['prefercompressed'] = bundler.prefercompressed
2173 info['prefercompressed'] = bundler.prefercompressed
2172
2174
2173 return info, bundler.getchunks()
2175 return info, bundler.getchunks()
2174
2176
2175 @getbundle2partsgenerator('stream2')
2177 @getbundle2partsgenerator('stream2')
2176 def _getbundlestream2(bundler, repo, *args, **kwargs):
2178 def _getbundlestream2(bundler, repo, *args, **kwargs):
2177 return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
2179 return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
2178
2180
2179 @getbundle2partsgenerator('changegroup')
2181 @getbundle2partsgenerator('changegroup')
2180 def _getbundlechangegrouppart(bundler, repo, source, bundlecaps=None,
2182 def _getbundlechangegrouppart(bundler, repo, source, bundlecaps=None,
2181 b2caps=None, heads=None, common=None, **kwargs):
2183 b2caps=None, heads=None, common=None, **kwargs):
2182 """add a changegroup part to the requested bundle"""
2184 """add a changegroup part to the requested bundle"""
2183 if not kwargs.get(r'cg', True):
2185 if not kwargs.get(r'cg', True):
2184 return
2186 return
2185
2187
2186 version = '01'
2188 version = '01'
2187 cgversions = b2caps.get('changegroup')
2189 cgversions = b2caps.get('changegroup')
2188 if cgversions: # 3.1 and 3.2 ship with an empty value
2190 if cgversions: # 3.1 and 3.2 ship with an empty value
2189 cgversions = [v for v in cgversions
2191 cgversions = [v for v in cgversions
2190 if v in changegroup.supportedoutgoingversions(repo)]
2192 if v in changegroup.supportedoutgoingversions(repo)]
2191 if not cgversions:
2193 if not cgversions:
2192 raise error.Abort(_('no common changegroup version'))
2194 raise error.Abort(_('no common changegroup version'))
2193 version = max(cgversions)
2195 version = max(cgversions)
2194
2196
2195 outgoing = _computeoutgoing(repo, heads, common)
2197 outgoing = _computeoutgoing(repo, heads, common)
2196 if not outgoing.missing:
2198 if not outgoing.missing:
2197 return
2199 return
2198
2200
2199 if kwargs.get(r'narrow', False):
2201 if kwargs.get(r'narrow', False):
2200 include = sorted(filter(bool, kwargs.get(r'includepats', [])))
2202 include = sorted(filter(bool, kwargs.get(r'includepats', [])))
2201 exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
2203 exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
2202 matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
2204 matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
2203 else:
2205 else:
2204 matcher = None
2206 matcher = None
2205
2207
2206 cgstream = changegroup.makestream(repo, outgoing, version, source,
2208 cgstream = changegroup.makestream(repo, outgoing, version, source,
2207 bundlecaps=bundlecaps, matcher=matcher)
2209 bundlecaps=bundlecaps, matcher=matcher)
2208
2210
2209 part = bundler.newpart('changegroup', data=cgstream)
2211 part = bundler.newpart('changegroup', data=cgstream)
2210 if cgversions:
2212 if cgversions:
2211 part.addparam('version', version)
2213 part.addparam('version', version)
2212
2214
2213 part.addparam('nbchanges', '%d' % len(outgoing.missing),
2215 part.addparam('nbchanges', '%d' % len(outgoing.missing),
2214 mandatory=False)
2216 mandatory=False)
2215
2217
2216 if 'treemanifest' in repo.requirements:
2218 if 'treemanifest' in repo.requirements:
2217 part.addparam('treemanifest', '1')
2219 part.addparam('treemanifest', '1')
2218
2220
2219 if (kwargs.get(r'narrow', False) and kwargs.get(r'narrow_acl', False)
2221 if (kwargs.get(r'narrow', False) and kwargs.get(r'narrow_acl', False)
2220 and (include or exclude)):
2222 and (include or exclude)):
2221 # this is mandatory because otherwise ACL clients won't work
2223 # this is mandatory because otherwise ACL clients won't work
2222 narrowspecpart = bundler.newpart('Narrow:responsespec')
2224 narrowspecpart = bundler.newpart('Narrow:responsespec')
2223 narrowspecpart.data = '%s\0%s' % ('\n'.join(include),
2225 narrowspecpart.data = '%s\0%s' % ('\n'.join(include),
2224 '\n'.join(exclude))
2226 '\n'.join(exclude))
2225
2227
2226 @getbundle2partsgenerator('bookmarks')
2228 @getbundle2partsgenerator('bookmarks')
2227 def _getbundlebookmarkpart(bundler, repo, source, bundlecaps=None,
2229 def _getbundlebookmarkpart(bundler, repo, source, bundlecaps=None,
2228 b2caps=None, **kwargs):
2230 b2caps=None, **kwargs):
2229 """add a bookmark part to the requested bundle"""
2231 """add a bookmark part to the requested bundle"""
2230 if not kwargs.get(r'bookmarks', False):
2232 if not kwargs.get(r'bookmarks', False):
2231 return
2233 return
2232 if 'bookmarks' not in b2caps:
2234 if 'bookmarks' not in b2caps:
2233 raise error.Abort(_('no common bookmarks exchange method'))
2235 raise error.Abort(_('no common bookmarks exchange method'))
2234 books = bookmod.listbinbookmarks(repo)
2236 books = bookmod.listbinbookmarks(repo)
2235 data = bookmod.binaryencode(books)
2237 data = bookmod.binaryencode(books)
2236 if data:
2238 if data:
2237 bundler.newpart('bookmarks', data=data)
2239 bundler.newpart('bookmarks', data=data)
2238
2240
2239 @getbundle2partsgenerator('listkeys')
2241 @getbundle2partsgenerator('listkeys')
2240 def _getbundlelistkeysparts(bundler, repo, source, bundlecaps=None,
2242 def _getbundlelistkeysparts(bundler, repo, source, bundlecaps=None,
2241 b2caps=None, **kwargs):
2243 b2caps=None, **kwargs):
2242 """add parts containing listkeys namespaces to the requested bundle"""
2244 """add parts containing listkeys namespaces to the requested bundle"""
2243 listkeys = kwargs.get(r'listkeys', ())
2245 listkeys = kwargs.get(r'listkeys', ())
2244 for namespace in listkeys:
2246 for namespace in listkeys:
2245 part = bundler.newpart('listkeys')
2247 part = bundler.newpart('listkeys')
2246 part.addparam('namespace', namespace)
2248 part.addparam('namespace', namespace)
2247 keys = repo.listkeys(namespace).items()
2249 keys = repo.listkeys(namespace).items()
2248 part.data = pushkey.encodekeys(keys)
2250 part.data = pushkey.encodekeys(keys)
2249
2251
2250 @getbundle2partsgenerator('obsmarkers')
2252 @getbundle2partsgenerator('obsmarkers')
2251 def _getbundleobsmarkerpart(bundler, repo, source, bundlecaps=None,
2253 def _getbundleobsmarkerpart(bundler, repo, source, bundlecaps=None,
2252 b2caps=None, heads=None, **kwargs):
2254 b2caps=None, heads=None, **kwargs):
2253 """add an obsolescence markers part to the requested bundle"""
2255 """add an obsolescence markers part to the requested bundle"""
2254 if kwargs.get(r'obsmarkers', False):
2256 if kwargs.get(r'obsmarkers', False):
2255 if heads is None:
2257 if heads is None:
2256 heads = repo.heads()
2258 heads = repo.heads()
2257 subset = [c.node() for c in repo.set('::%ln', heads)]
2259 subset = [c.node() for c in repo.set('::%ln', heads)]
2258 markers = repo.obsstore.relevantmarkers(subset)
2260 markers = repo.obsstore.relevantmarkers(subset)
2259 markers = sorted(markers)
2261 markers = sorted(markers)
2260 bundle2.buildobsmarkerspart(bundler, markers)
2262 bundle2.buildobsmarkerspart(bundler, markers)
2261
2263
2262 @getbundle2partsgenerator('phases')
2264 @getbundle2partsgenerator('phases')
2263 def _getbundlephasespart(bundler, repo, source, bundlecaps=None,
2265 def _getbundlephasespart(bundler, repo, source, bundlecaps=None,
2264 b2caps=None, heads=None, **kwargs):
2266 b2caps=None, heads=None, **kwargs):
2265 """add phase heads part to the requested bundle"""
2267 """add phase heads part to the requested bundle"""
2266 if kwargs.get(r'phases', False):
2268 if kwargs.get(r'phases', False):
2267 if not 'heads' in b2caps.get('phases'):
2269 if not 'heads' in b2caps.get('phases'):
2268 raise error.Abort(_('no common phases exchange method'))
2270 raise error.Abort(_('no common phases exchange method'))
2269 if heads is None:
2271 if heads is None:
2270 heads = repo.heads()
2272 heads = repo.heads()
2271
2273
2272 headsbyphase = collections.defaultdict(set)
2274 headsbyphase = collections.defaultdict(set)
2273 if repo.publishing():
2275 if repo.publishing():
2274 headsbyphase[phases.public] = heads
2276 headsbyphase[phases.public] = heads
2275 else:
2277 else:
2276 # find the appropriate heads to move
2278 # find the appropriate heads to move
2277
2279
2278 phase = repo._phasecache.phase
2280 phase = repo._phasecache.phase
2279 node = repo.changelog.node
2281 node = repo.changelog.node
2280 rev = repo.changelog.rev
2282 rev = repo.changelog.rev
2281 for h in heads:
2283 for h in heads:
2282 headsbyphase[phase(repo, rev(h))].add(h)
2284 headsbyphase[phase(repo, rev(h))].add(h)
2283 seenphases = list(headsbyphase.keys())
2285 seenphases = list(headsbyphase.keys())
2284
2286
2285 # We do not handle anything but public and draft phase for now)
2287 # We do not handle anything but public and draft phase for now)
2286 if seenphases:
2288 if seenphases:
2287 assert max(seenphases) <= phases.draft
2289 assert max(seenphases) <= phases.draft
2288
2290
2289 # if client is pulling non-public changesets, we need to find
2291 # if client is pulling non-public changesets, we need to find
2290 # intermediate public heads.
2292 # intermediate public heads.
2291 draftheads = headsbyphase.get(phases.draft, set())
2293 draftheads = headsbyphase.get(phases.draft, set())
2292 if draftheads:
2294 if draftheads:
2293 publicheads = headsbyphase.get(phases.public, set())
2295 publicheads = headsbyphase.get(phases.public, set())
2294
2296
2295 revset = 'heads(only(%ln, %ln) and public())'
2297 revset = 'heads(only(%ln, %ln) and public())'
2296 extraheads = repo.revs(revset, draftheads, publicheads)
2298 extraheads = repo.revs(revset, draftheads, publicheads)
2297 for r in extraheads:
2299 for r in extraheads:
2298 headsbyphase[phases.public].add(node(r))
2300 headsbyphase[phases.public].add(node(r))
2299
2301
2300 # transform data in a format used by the encoding function
2302 # transform data in a format used by the encoding function
2301 phasemapping = []
2303 phasemapping = []
2302 for phase in phases.allphases:
2304 for phase in phases.allphases:
2303 phasemapping.append(sorted(headsbyphase[phase]))
2305 phasemapping.append(sorted(headsbyphase[phase]))
2304
2306
2305 # generate the actual part
2307 # generate the actual part
2306 phasedata = phases.binaryencode(phasemapping)
2308 phasedata = phases.binaryencode(phasemapping)
2307 bundler.newpart('phase-heads', data=phasedata)
2309 bundler.newpart('phase-heads', data=phasedata)
2308
2310
2309 @getbundle2partsgenerator('hgtagsfnodes')
2311 @getbundle2partsgenerator('hgtagsfnodes')
2310 def _getbundletagsfnodes(bundler, repo, source, bundlecaps=None,
2312 def _getbundletagsfnodes(bundler, repo, source, bundlecaps=None,
2311 b2caps=None, heads=None, common=None,
2313 b2caps=None, heads=None, common=None,
2312 **kwargs):
2314 **kwargs):
2313 """Transfer the .hgtags filenodes mapping.
2315 """Transfer the .hgtags filenodes mapping.
2314
2316
2315 Only values for heads in this bundle will be transferred.
2317 Only values for heads in this bundle will be transferred.
2316
2318
2317 The part data consists of pairs of 20 byte changeset node and .hgtags
2319 The part data consists of pairs of 20 byte changeset node and .hgtags
2318 filenodes raw values.
2320 filenodes raw values.
2319 """
2321 """
2320 # Don't send unless:
2322 # Don't send unless:
2321 # - changeset are being exchanged,
2323 # - changeset are being exchanged,
2322 # - the client supports it.
2324 # - the client supports it.
2323 if not (kwargs.get(r'cg', True) and 'hgtagsfnodes' in b2caps):
2325 if not (kwargs.get(r'cg', True) and 'hgtagsfnodes' in b2caps):
2324 return
2326 return
2325
2327
2326 outgoing = _computeoutgoing(repo, heads, common)
2328 outgoing = _computeoutgoing(repo, heads, common)
2327 bundle2.addparttagsfnodescache(repo, bundler, outgoing)
2329 bundle2.addparttagsfnodescache(repo, bundler, outgoing)
2328
2330
2329 @getbundle2partsgenerator('cache:rev-branch-cache')
2331 @getbundle2partsgenerator('cache:rev-branch-cache')
2330 def _getbundlerevbranchcache(bundler, repo, source, bundlecaps=None,
2332 def _getbundlerevbranchcache(bundler, repo, source, bundlecaps=None,
2331 b2caps=None, heads=None, common=None,
2333 b2caps=None, heads=None, common=None,
2332 **kwargs):
2334 **kwargs):
2333 """Transfer the rev-branch-cache mapping
2335 """Transfer the rev-branch-cache mapping
2334
2336
2335 The payload is a series of data related to each branch
2337 The payload is a series of data related to each branch
2336
2338
2337 1) branch name length
2339 1) branch name length
2338 2) number of open heads
2340 2) number of open heads
2339 3) number of closed heads
2341 3) number of closed heads
2340 4) open heads nodes
2342 4) open heads nodes
2341 5) closed heads nodes
2343 5) closed heads nodes
2342 """
2344 """
2343 # Don't send unless:
2345 # Don't send unless:
2344 # - changeset are being exchanged,
2346 # - changeset are being exchanged,
2345 # - the client supports it.
2347 # - the client supports it.
2346 # - narrow bundle isn't in play (not currently compatible).
2348 # - narrow bundle isn't in play (not currently compatible).
2347 if (not kwargs.get(r'cg', True)
2349 if (not kwargs.get(r'cg', True)
2348 or 'rev-branch-cache' not in b2caps
2350 or 'rev-branch-cache' not in b2caps
2349 or kwargs.get(r'narrow', False)
2351 or kwargs.get(r'narrow', False)
2350 or repo.ui.has_section(_NARROWACL_SECTION)):
2352 or repo.ui.has_section(_NARROWACL_SECTION)):
2351 return
2353 return
2352
2354
2353 outgoing = _computeoutgoing(repo, heads, common)
2355 outgoing = _computeoutgoing(repo, heads, common)
2354 bundle2.addpartrevbranchcache(repo, bundler, outgoing)
2356 bundle2.addpartrevbranchcache(repo, bundler, outgoing)
2355
2357
2356 def check_heads(repo, their_heads, context):
2358 def check_heads(repo, their_heads, context):
2357 """check if the heads of a repo have been modified
2359 """check if the heads of a repo have been modified
2358
2360
2359 Used by peer for unbundling.
2361 Used by peer for unbundling.
2360 """
2362 """
2361 heads = repo.heads()
2363 heads = repo.heads()
2362 heads_hash = hashlib.sha1(''.join(sorted(heads))).digest()
2364 heads_hash = hashlib.sha1(''.join(sorted(heads))).digest()
2363 if not (their_heads == ['force'] or their_heads == heads or
2365 if not (their_heads == ['force'] or their_heads == heads or
2364 their_heads == ['hashed', heads_hash]):
2366 their_heads == ['hashed', heads_hash]):
2365 # someone else committed/pushed/unbundled while we
2367 # someone else committed/pushed/unbundled while we
2366 # were transferring data
2368 # were transferring data
2367 raise error.PushRaced('repository changed while %s - '
2369 raise error.PushRaced('repository changed while %s - '
2368 'please try again' % context)
2370 'please try again' % context)
2369
2371
2370 def unbundle(repo, cg, heads, source, url):
2372 def unbundle(repo, cg, heads, source, url):
2371 """Apply a bundle to a repo.
2373 """Apply a bundle to a repo.
2372
2374
2373 this function makes sure the repo is locked during the application and have
2375 this function makes sure the repo is locked during the application and have
2374 mechanism to check that no push race occurred between the creation of the
2376 mechanism to check that no push race occurred between the creation of the
2375 bundle and its application.
2377 bundle and its application.
2376
2378
2377 If the push was raced as PushRaced exception is raised."""
2379 If the push was raced as PushRaced exception is raised."""
2378 r = 0
2380 r = 0
2379 # need a transaction when processing a bundle2 stream
2381 # need a transaction when processing a bundle2 stream
2380 # [wlock, lock, tr] - needs to be an array so nested functions can modify it
2382 # [wlock, lock, tr] - needs to be an array so nested functions can modify it
2381 lockandtr = [None, None, None]
2383 lockandtr = [None, None, None]
2382 recordout = None
2384 recordout = None
2383 # quick fix for output mismatch with bundle2 in 3.4
2385 # quick fix for output mismatch with bundle2 in 3.4
2384 captureoutput = repo.ui.configbool('experimental', 'bundle2-output-capture')
2386 captureoutput = repo.ui.configbool('experimental', 'bundle2-output-capture')
2385 if url.startswith('remote:http:') or url.startswith('remote:https:'):
2387 if url.startswith('remote:http:') or url.startswith('remote:https:'):
2386 captureoutput = True
2388 captureoutput = True
2387 try:
2389 try:
2388 # note: outside bundle1, 'heads' is expected to be empty and this
2390 # note: outside bundle1, 'heads' is expected to be empty and this
2389 # 'check_heads' call wil be a no-op
2391 # 'check_heads' call wil be a no-op
2390 check_heads(repo, heads, 'uploading changes')
2392 check_heads(repo, heads, 'uploading changes')
2391 # push can proceed
2393 # push can proceed
2392 if not isinstance(cg, bundle2.unbundle20):
2394 if not isinstance(cg, bundle2.unbundle20):
2393 # legacy case: bundle1 (changegroup 01)
2395 # legacy case: bundle1 (changegroup 01)
2394 txnname = "\n".join([source, util.hidepassword(url)])
2396 txnname = "\n".join([source, util.hidepassword(url)])
2395 with repo.lock(), repo.transaction(txnname) as tr:
2397 with repo.lock(), repo.transaction(txnname) as tr:
2396 op = bundle2.applybundle(repo, cg, tr, source, url)
2398 op = bundle2.applybundle(repo, cg, tr, source, url)
2397 r = bundle2.combinechangegroupresults(op)
2399 r = bundle2.combinechangegroupresults(op)
2398 else:
2400 else:
2399 r = None
2401 r = None
2400 try:
2402 try:
2401 def gettransaction():
2403 def gettransaction():
2402 if not lockandtr[2]:
2404 if not lockandtr[2]:
2403 if not bookmod.bookmarksinstore(repo):
2405 if not bookmod.bookmarksinstore(repo):
2404 lockandtr[0] = repo.wlock()
2406 lockandtr[0] = repo.wlock()
2405 lockandtr[1] = repo.lock()
2407 lockandtr[1] = repo.lock()
2406 lockandtr[2] = repo.transaction(source)
2408 lockandtr[2] = repo.transaction(source)
2407 lockandtr[2].hookargs['source'] = source
2409 lockandtr[2].hookargs['source'] = source
2408 lockandtr[2].hookargs['url'] = url
2410 lockandtr[2].hookargs['url'] = url
2409 lockandtr[2].hookargs['bundle2'] = '1'
2411 lockandtr[2].hookargs['bundle2'] = '1'
2410 return lockandtr[2]
2412 return lockandtr[2]
2411
2413
2412 # Do greedy locking by default until we're satisfied with lazy
2414 # Do greedy locking by default until we're satisfied with lazy
2413 # locking.
2415 # locking.
2414 if not repo.ui.configbool('experimental', 'bundle2lazylocking'):
2416 if not repo.ui.configbool('experimental', 'bundle2lazylocking'):
2415 gettransaction()
2417 gettransaction()
2416
2418
2417 op = bundle2.bundleoperation(repo, gettransaction,
2419 op = bundle2.bundleoperation(repo, gettransaction,
2418 captureoutput=captureoutput,
2420 captureoutput=captureoutput,
2419 source='push')
2421 source='push')
2420 try:
2422 try:
2421 op = bundle2.processbundle(repo, cg, op=op)
2423 op = bundle2.processbundle(repo, cg, op=op)
2422 finally:
2424 finally:
2423 r = op.reply
2425 r = op.reply
2424 if captureoutput and r is not None:
2426 if captureoutput and r is not None:
2425 repo.ui.pushbuffer(error=True, subproc=True)
2427 repo.ui.pushbuffer(error=True, subproc=True)
2426 def recordout(output):
2428 def recordout(output):
2427 r.newpart('output', data=output, mandatory=False)
2429 r.newpart('output', data=output, mandatory=False)
2428 if lockandtr[2] is not None:
2430 if lockandtr[2] is not None:
2429 lockandtr[2].close()
2431 lockandtr[2].close()
2430 except BaseException as exc:
2432 except BaseException as exc:
2431 exc.duringunbundle2 = True
2433 exc.duringunbundle2 = True
2432 if captureoutput and r is not None:
2434 if captureoutput and r is not None:
2433 parts = exc._bundle2salvagedoutput = r.salvageoutput()
2435 parts = exc._bundle2salvagedoutput = r.salvageoutput()
2434 def recordout(output):
2436 def recordout(output):
2435 part = bundle2.bundlepart('output', data=output,
2437 part = bundle2.bundlepart('output', data=output,
2436 mandatory=False)
2438 mandatory=False)
2437 parts.append(part)
2439 parts.append(part)
2438 raise
2440 raise
2439 finally:
2441 finally:
2440 lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
2442 lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
2441 if recordout is not None:
2443 if recordout is not None:
2442 recordout(repo.ui.popbuffer())
2444 recordout(repo.ui.popbuffer())
2443 return r
2445 return r
2444
2446
2445 def _maybeapplyclonebundle(pullop):
2447 def _maybeapplyclonebundle(pullop):
2446 """Apply a clone bundle from a remote, if possible."""
2448 """Apply a clone bundle from a remote, if possible."""
2447
2449
2448 repo = pullop.repo
2450 repo = pullop.repo
2449 remote = pullop.remote
2451 remote = pullop.remote
2450
2452
2451 if not repo.ui.configbool('ui', 'clonebundles'):
2453 if not repo.ui.configbool('ui', 'clonebundles'):
2452 return
2454 return
2453
2455
2454 # Only run if local repo is empty.
2456 # Only run if local repo is empty.
2455 if len(repo):
2457 if len(repo):
2456 return
2458 return
2457
2459
2458 if pullop.heads:
2460 if pullop.heads:
2459 return
2461 return
2460
2462
2461 if not remote.capable('clonebundles'):
2463 if not remote.capable('clonebundles'):
2462 return
2464 return
2463
2465
2464 with remote.commandexecutor() as e:
2466 with remote.commandexecutor() as e:
2465 res = e.callcommand('clonebundles', {}).result()
2467 res = e.callcommand('clonebundles', {}).result()
2466
2468
2467 # If we call the wire protocol command, that's good enough to record the
2469 # If we call the wire protocol command, that's good enough to record the
2468 # attempt.
2470 # attempt.
2469 pullop.clonebundleattempted = True
2471 pullop.clonebundleattempted = True
2470
2472
2471 entries = parseclonebundlesmanifest(repo, res)
2473 entries = parseclonebundlesmanifest(repo, res)
2472 if not entries:
2474 if not entries:
2473 repo.ui.note(_('no clone bundles available on remote; '
2475 repo.ui.note(_('no clone bundles available on remote; '
2474 'falling back to regular clone\n'))
2476 'falling back to regular clone\n'))
2475 return
2477 return
2476
2478
2477 entries = filterclonebundleentries(
2479 entries = filterclonebundleentries(
2478 repo, entries, streamclonerequested=pullop.streamclonerequested)
2480 repo, entries, streamclonerequested=pullop.streamclonerequested)
2479
2481
2480 if not entries:
2482 if not entries:
2481 # There is a thundering herd concern here. However, if a server
2483 # There is a thundering herd concern here. However, if a server
2482 # operator doesn't advertise bundles appropriate for its clients,
2484 # operator doesn't advertise bundles appropriate for its clients,
2483 # they deserve what's coming. Furthermore, from a client's
2485 # they deserve what's coming. Furthermore, from a client's
2484 # perspective, no automatic fallback would mean not being able to
2486 # perspective, no automatic fallback would mean not being able to
2485 # clone!
2487 # clone!
2486 repo.ui.warn(_('no compatible clone bundles available on server; '
2488 repo.ui.warn(_('no compatible clone bundles available on server; '
2487 'falling back to regular clone\n'))
2489 'falling back to regular clone\n'))
2488 repo.ui.warn(_('(you may want to report this to the server '
2490 repo.ui.warn(_('(you may want to report this to the server '
2489 'operator)\n'))
2491 'operator)\n'))
2490 return
2492 return
2491
2493
2492 entries = sortclonebundleentries(repo.ui, entries)
2494 entries = sortclonebundleentries(repo.ui, entries)
2493
2495
2494 url = entries[0]['URL']
2496 url = entries[0]['URL']
2495 repo.ui.status(_('applying clone bundle from %s\n') % url)
2497 repo.ui.status(_('applying clone bundle from %s\n') % url)
2496 if trypullbundlefromurl(repo.ui, repo, url):
2498 if trypullbundlefromurl(repo.ui, repo, url):
2497 repo.ui.status(_('finished applying clone bundle\n'))
2499 repo.ui.status(_('finished applying clone bundle\n'))
2498 # Bundle failed.
2500 # Bundle failed.
2499 #
2501 #
2500 # We abort by default to avoid the thundering herd of
2502 # We abort by default to avoid the thundering herd of
2501 # clients flooding a server that was expecting expensive
2503 # clients flooding a server that was expecting expensive
2502 # clone load to be offloaded.
2504 # clone load to be offloaded.
2503 elif repo.ui.configbool('ui', 'clonebundlefallback'):
2505 elif repo.ui.configbool('ui', 'clonebundlefallback'):
2504 repo.ui.warn(_('falling back to normal clone\n'))
2506 repo.ui.warn(_('falling back to normal clone\n'))
2505 else:
2507 else:
2506 raise error.Abort(_('error applying bundle'),
2508 raise error.Abort(_('error applying bundle'),
2507 hint=_('if this error persists, consider contacting '
2509 hint=_('if this error persists, consider contacting '
2508 'the server operator or disable clone '
2510 'the server operator or disable clone '
2509 'bundles via '
2511 'bundles via '
2510 '"--config ui.clonebundles=false"'))
2512 '"--config ui.clonebundles=false"'))
2511
2513
2512 def parseclonebundlesmanifest(repo, s):
2514 def parseclonebundlesmanifest(repo, s):
2513 """Parses the raw text of a clone bundles manifest.
2515 """Parses the raw text of a clone bundles manifest.
2514
2516
2515 Returns a list of dicts. The dicts have a ``URL`` key corresponding
2517 Returns a list of dicts. The dicts have a ``URL`` key corresponding
2516 to the URL and other keys are the attributes for the entry.
2518 to the URL and other keys are the attributes for the entry.
2517 """
2519 """
2518 m = []
2520 m = []
2519 for line in s.splitlines():
2521 for line in s.splitlines():
2520 fields = line.split()
2522 fields = line.split()
2521 if not fields:
2523 if not fields:
2522 continue
2524 continue
2523 attrs = {'URL': fields[0]}
2525 attrs = {'URL': fields[0]}
2524 for rawattr in fields[1:]:
2526 for rawattr in fields[1:]:
2525 key, value = rawattr.split('=', 1)
2527 key, value = rawattr.split('=', 1)
2526 key = urlreq.unquote(key)
2528 key = urlreq.unquote(key)
2527 value = urlreq.unquote(value)
2529 value = urlreq.unquote(value)
2528 attrs[key] = value
2530 attrs[key] = value
2529
2531
2530 # Parse BUNDLESPEC into components. This makes client-side
2532 # Parse BUNDLESPEC into components. This makes client-side
2531 # preferences easier to specify since you can prefer a single
2533 # preferences easier to specify since you can prefer a single
2532 # component of the BUNDLESPEC.
2534 # component of the BUNDLESPEC.
2533 if key == 'BUNDLESPEC':
2535 if key == 'BUNDLESPEC':
2534 try:
2536 try:
2535 bundlespec = parsebundlespec(repo, value)
2537 bundlespec = parsebundlespec(repo, value)
2536 attrs['COMPRESSION'] = bundlespec.compression
2538 attrs['COMPRESSION'] = bundlespec.compression
2537 attrs['VERSION'] = bundlespec.version
2539 attrs['VERSION'] = bundlespec.version
2538 except error.InvalidBundleSpecification:
2540 except error.InvalidBundleSpecification:
2539 pass
2541 pass
2540 except error.UnsupportedBundleSpecification:
2542 except error.UnsupportedBundleSpecification:
2541 pass
2543 pass
2542
2544
2543 m.append(attrs)
2545 m.append(attrs)
2544
2546
2545 return m
2547 return m
2546
2548
2547 def isstreamclonespec(bundlespec):
2549 def isstreamclonespec(bundlespec):
2548 # Stream clone v1
2550 # Stream clone v1
2549 if (bundlespec.wirecompression == 'UN' and bundlespec.wireversion == 's1'):
2551 if (bundlespec.wirecompression == 'UN' and bundlespec.wireversion == 's1'):
2550 return True
2552 return True
2551
2553
2552 # Stream clone v2
2554 # Stream clone v2
2553 if (bundlespec.wirecompression == 'UN' and
2555 if (bundlespec.wirecompression == 'UN' and
2554 bundlespec.wireversion == '02' and
2556 bundlespec.wireversion == '02' and
2555 bundlespec.contentopts.get('streamv2')):
2557 bundlespec.contentopts.get('streamv2')):
2556 return True
2558 return True
2557
2559
2558 return False
2560 return False
2559
2561
2560 def filterclonebundleentries(repo, entries, streamclonerequested=False):
2562 def filterclonebundleentries(repo, entries, streamclonerequested=False):
2561 """Remove incompatible clone bundle manifest entries.
2563 """Remove incompatible clone bundle manifest entries.
2562
2564
2563 Accepts a list of entries parsed with ``parseclonebundlesmanifest``
2565 Accepts a list of entries parsed with ``parseclonebundlesmanifest``
2564 and returns a new list consisting of only the entries that this client
2566 and returns a new list consisting of only the entries that this client
2565 should be able to apply.
2567 should be able to apply.
2566
2568
2567 There is no guarantee we'll be able to apply all returned entries because
2569 There is no guarantee we'll be able to apply all returned entries because
2568 the metadata we use to filter on may be missing or wrong.
2570 the metadata we use to filter on may be missing or wrong.
2569 """
2571 """
2570 newentries = []
2572 newentries = []
2571 for entry in entries:
2573 for entry in entries:
2572 spec = entry.get('BUNDLESPEC')
2574 spec = entry.get('BUNDLESPEC')
2573 if spec:
2575 if spec:
2574 try:
2576 try:
2575 bundlespec = parsebundlespec(repo, spec, strict=True)
2577 bundlespec = parsebundlespec(repo, spec, strict=True)
2576
2578
2577 # If a stream clone was requested, filter out non-streamclone
2579 # If a stream clone was requested, filter out non-streamclone
2578 # entries.
2580 # entries.
2579 if streamclonerequested and not isstreamclonespec(bundlespec):
2581 if streamclonerequested and not isstreamclonespec(bundlespec):
2580 repo.ui.debug('filtering %s because not a stream clone\n' %
2582 repo.ui.debug('filtering %s because not a stream clone\n' %
2581 entry['URL'])
2583 entry['URL'])
2582 continue
2584 continue
2583
2585
2584 except error.InvalidBundleSpecification as e:
2586 except error.InvalidBundleSpecification as e:
2585 repo.ui.debug(stringutil.forcebytestr(e) + '\n')
2587 repo.ui.debug(stringutil.forcebytestr(e) + '\n')
2586 continue
2588 continue
2587 except error.UnsupportedBundleSpecification as e:
2589 except error.UnsupportedBundleSpecification as e:
2588 repo.ui.debug('filtering %s because unsupported bundle '
2590 repo.ui.debug('filtering %s because unsupported bundle '
2589 'spec: %s\n' % (
2591 'spec: %s\n' % (
2590 entry['URL'], stringutil.forcebytestr(e)))
2592 entry['URL'], stringutil.forcebytestr(e)))
2591 continue
2593 continue
2592 # If we don't have a spec and requested a stream clone, we don't know
2594 # If we don't have a spec and requested a stream clone, we don't know
2593 # what the entry is so don't attempt to apply it.
2595 # what the entry is so don't attempt to apply it.
2594 elif streamclonerequested:
2596 elif streamclonerequested:
2595 repo.ui.debug('filtering %s because cannot determine if a stream '
2597 repo.ui.debug('filtering %s because cannot determine if a stream '
2596 'clone bundle\n' % entry['URL'])
2598 'clone bundle\n' % entry['URL'])
2597 continue
2599 continue
2598
2600
2599 if 'REQUIRESNI' in entry and not sslutil.hassni:
2601 if 'REQUIRESNI' in entry and not sslutil.hassni:
2600 repo.ui.debug('filtering %s because SNI not supported\n' %
2602 repo.ui.debug('filtering %s because SNI not supported\n' %
2601 entry['URL'])
2603 entry['URL'])
2602 continue
2604 continue
2603
2605
2604 newentries.append(entry)
2606 newentries.append(entry)
2605
2607
2606 return newentries
2608 return newentries
2607
2609
2608 class clonebundleentry(object):
2610 class clonebundleentry(object):
2609 """Represents an item in a clone bundles manifest.
2611 """Represents an item in a clone bundles manifest.
2610
2612
2611 This rich class is needed to support sorting since sorted() in Python 3
2613 This rich class is needed to support sorting since sorted() in Python 3
2612 doesn't support ``cmp`` and our comparison is complex enough that ``key=``
2614 doesn't support ``cmp`` and our comparison is complex enough that ``key=``
2613 won't work.
2615 won't work.
2614 """
2616 """
2615
2617
2616 def __init__(self, value, prefers):
2618 def __init__(self, value, prefers):
2617 self.value = value
2619 self.value = value
2618 self.prefers = prefers
2620 self.prefers = prefers
2619
2621
2620 def _cmp(self, other):
2622 def _cmp(self, other):
2621 for prefkey, prefvalue in self.prefers:
2623 for prefkey, prefvalue in self.prefers:
2622 avalue = self.value.get(prefkey)
2624 avalue = self.value.get(prefkey)
2623 bvalue = other.value.get(prefkey)
2625 bvalue = other.value.get(prefkey)
2624
2626
2625 # Special case for b missing attribute and a matches exactly.
2627 # Special case for b missing attribute and a matches exactly.
2626 if avalue is not None and bvalue is None and avalue == prefvalue:
2628 if avalue is not None and bvalue is None and avalue == prefvalue:
2627 return -1
2629 return -1
2628
2630
2629 # Special case for a missing attribute and b matches exactly.
2631 # Special case for a missing attribute and b matches exactly.
2630 if bvalue is not None and avalue is None and bvalue == prefvalue:
2632 if bvalue is not None and avalue is None and bvalue == prefvalue:
2631 return 1
2633 return 1
2632
2634
2633 # We can't compare unless attribute present on both.
2635 # We can't compare unless attribute present on both.
2634 if avalue is None or bvalue is None:
2636 if avalue is None or bvalue is None:
2635 continue
2637 continue
2636
2638
2637 # Same values should fall back to next attribute.
2639 # Same values should fall back to next attribute.
2638 if avalue == bvalue:
2640 if avalue == bvalue:
2639 continue
2641 continue
2640
2642
2641 # Exact matches come first.
2643 # Exact matches come first.
2642 if avalue == prefvalue:
2644 if avalue == prefvalue:
2643 return -1
2645 return -1
2644 if bvalue == prefvalue:
2646 if bvalue == prefvalue:
2645 return 1
2647 return 1
2646
2648
2647 # Fall back to next attribute.
2649 # Fall back to next attribute.
2648 continue
2650 continue
2649
2651
2650 # If we got here we couldn't sort by attributes and prefers. Fall
2652 # If we got here we couldn't sort by attributes and prefers. Fall
2651 # back to index order.
2653 # back to index order.
2652 return 0
2654 return 0
2653
2655
2654 def __lt__(self, other):
2656 def __lt__(self, other):
2655 return self._cmp(other) < 0
2657 return self._cmp(other) < 0
2656
2658
2657 def __gt__(self, other):
2659 def __gt__(self, other):
2658 return self._cmp(other) > 0
2660 return self._cmp(other) > 0
2659
2661
2660 def __eq__(self, other):
2662 def __eq__(self, other):
2661 return self._cmp(other) == 0
2663 return self._cmp(other) == 0
2662
2664
2663 def __le__(self, other):
2665 def __le__(self, other):
2664 return self._cmp(other) <= 0
2666 return self._cmp(other) <= 0
2665
2667
2666 def __ge__(self, other):
2668 def __ge__(self, other):
2667 return self._cmp(other) >= 0
2669 return self._cmp(other) >= 0
2668
2670
2669 def __ne__(self, other):
2671 def __ne__(self, other):
2670 return self._cmp(other) != 0
2672 return self._cmp(other) != 0
2671
2673
2672 def sortclonebundleentries(ui, entries):
2674 def sortclonebundleentries(ui, entries):
2673 prefers = ui.configlist('ui', 'clonebundleprefers')
2675 prefers = ui.configlist('ui', 'clonebundleprefers')
2674 if not prefers:
2676 if not prefers:
2675 return list(entries)
2677 return list(entries)
2676
2678
2677 prefers = [p.split('=', 1) for p in prefers]
2679 prefers = [p.split('=', 1) for p in prefers]
2678
2680
2679 items = sorted(clonebundleentry(v, prefers) for v in entries)
2681 items = sorted(clonebundleentry(v, prefers) for v in entries)
2680 return [i.value for i in items]
2682 return [i.value for i in items]
2681
2683
2682 def trypullbundlefromurl(ui, repo, url):
2684 def trypullbundlefromurl(ui, repo, url):
2683 """Attempt to apply a bundle from a URL."""
2685 """Attempt to apply a bundle from a URL."""
2684 with repo.lock(), repo.transaction('bundleurl') as tr:
2686 with repo.lock(), repo.transaction('bundleurl') as tr:
2685 try:
2687 try:
2686 fh = urlmod.open(ui, url)
2688 fh = urlmod.open(ui, url)
2687 cg = readbundle(ui, fh, 'stream')
2689 cg = readbundle(ui, fh, 'stream')
2688
2690
2689 if isinstance(cg, streamclone.streamcloneapplier):
2691 if isinstance(cg, streamclone.streamcloneapplier):
2690 cg.apply(repo)
2692 cg.apply(repo)
2691 else:
2693 else:
2692 bundle2.applybundle(repo, cg, tr, 'clonebundles', url)
2694 bundle2.applybundle(repo, cg, tr, 'clonebundles', url)
2693 return True
2695 return True
2694 except urlerr.httperror as e:
2696 except urlerr.httperror as e:
2695 ui.warn(_('HTTP error fetching bundle: %s\n') %
2697 ui.warn(_('HTTP error fetching bundle: %s\n') %
2696 stringutil.forcebytestr(e))
2698 stringutil.forcebytestr(e))
2697 except urlerr.urlerror as e:
2699 except urlerr.urlerror as e:
2698 ui.warn(_('error fetching bundle: %s\n') %
2700 ui.warn(_('error fetching bundle: %s\n') %
2699 stringutil.forcebytestr(e.reason))
2701 stringutil.forcebytestr(e.reason))
2700
2702
2701 return False
2703 return False
@@ -1,697 +1,699
1 # exchangev2.py - repository exchange for wire protocol version 2
1 # exchangev2.py - repository exchange for wire protocol version 2
2 #
2 #
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import collections
10 import collections
11 import weakref
11 import weakref
12
12
13 from .i18n import _
13 from .i18n import _
14 from .node import (
14 from .node import (
15 nullid,
15 nullid,
16 short,
16 short,
17 )
17 )
18 from . import (
18 from . import (
19 bookmarks,
19 bookmarks,
20 error,
20 error,
21 mdiff,
21 mdiff,
22 narrowspec,
22 narrowspec,
23 phases,
23 phases,
24 pycompat,
24 pycompat,
25 setdiscovery,
26 )
27 from .interfaces import (
25 repository,
28 repository,
26 setdiscovery,
27 )
29 )
28
30
29 def pull(pullop):
31 def pull(pullop):
30 """Pull using wire protocol version 2."""
32 """Pull using wire protocol version 2."""
31 repo = pullop.repo
33 repo = pullop.repo
32 remote = pullop.remote
34 remote = pullop.remote
33
35
34 usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
36 usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
35
37
36 # If this is a clone and it was requested to perform a "stream clone",
38 # If this is a clone and it was requested to perform a "stream clone",
37 # we obtain the raw files data from the remote then fall back to an
39 # we obtain the raw files data from the remote then fall back to an
38 # incremental pull. This is somewhat hacky and is not nearly robust enough
40 # incremental pull. This is somewhat hacky and is not nearly robust enough
39 # for long-term usage.
41 # for long-term usage.
40 if usingrawchangelogandmanifest:
42 if usingrawchangelogandmanifest:
41 with repo.transaction('clone'):
43 with repo.transaction('clone'):
42 _fetchrawstorefiles(repo, remote)
44 _fetchrawstorefiles(repo, remote)
43 repo.invalidate(clearfilecache=True)
45 repo.invalidate(clearfilecache=True)
44
46
45 tr = pullop.trmanager.transaction()
47 tr = pullop.trmanager.transaction()
46
48
47 # We don't use the repo's narrow matcher here because the patterns passed
49 # We don't use the repo's narrow matcher here because the patterns passed
48 # to exchange.pull() could be different.
50 # to exchange.pull() could be different.
49 narrowmatcher = narrowspec.match(repo.root,
51 narrowmatcher = narrowspec.match(repo.root,
50 # Empty maps to nevermatcher. So always
52 # Empty maps to nevermatcher. So always
51 # set includes if missing.
53 # set includes if missing.
52 pullop.includepats or {'path:.'},
54 pullop.includepats or {'path:.'},
53 pullop.excludepats)
55 pullop.excludepats)
54
56
55 if pullop.includepats or pullop.excludepats:
57 if pullop.includepats or pullop.excludepats:
56 pathfilter = {}
58 pathfilter = {}
57 if pullop.includepats:
59 if pullop.includepats:
58 pathfilter[b'include'] = sorted(pullop.includepats)
60 pathfilter[b'include'] = sorted(pullop.includepats)
59 if pullop.excludepats:
61 if pullop.excludepats:
60 pathfilter[b'exclude'] = sorted(pullop.excludepats)
62 pathfilter[b'exclude'] = sorted(pullop.excludepats)
61 else:
63 else:
62 pathfilter = None
64 pathfilter = None
63
65
64 # Figure out what needs to be fetched.
66 # Figure out what needs to be fetched.
65 common, fetch, remoteheads = _pullchangesetdiscovery(
67 common, fetch, remoteheads = _pullchangesetdiscovery(
66 repo, remote, pullop.heads, abortwhenunrelated=pullop.force)
68 repo, remote, pullop.heads, abortwhenunrelated=pullop.force)
67
69
68 # And fetch the data.
70 # And fetch the data.
69 pullheads = pullop.heads or remoteheads
71 pullheads = pullop.heads or remoteheads
70 csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads)
72 csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads)
71
73
72 # New revisions are written to the changelog. But all other updates
74 # New revisions are written to the changelog. But all other updates
73 # are deferred. Do those now.
75 # are deferred. Do those now.
74
76
75 # Ensure all new changesets are draft by default. If the repo is
77 # Ensure all new changesets are draft by default. If the repo is
76 # publishing, the phase will be adjusted by the loop below.
78 # publishing, the phase will be adjusted by the loop below.
77 if csetres['added']:
79 if csetres['added']:
78 phases.registernew(repo, tr, phases.draft, csetres['added'])
80 phases.registernew(repo, tr, phases.draft, csetres['added'])
79
81
80 # And adjust the phase of all changesets accordingly.
82 # And adjust the phase of all changesets accordingly.
81 for phase in phases.phasenames:
83 for phase in phases.phasenames:
82 if phase == b'secret' or not csetres['nodesbyphase'][phase]:
84 if phase == b'secret' or not csetres['nodesbyphase'][phase]:
83 continue
85 continue
84
86
85 phases.advanceboundary(repo, tr, phases.phasenames.index(phase),
87 phases.advanceboundary(repo, tr, phases.phasenames.index(phase),
86 csetres['nodesbyphase'][phase])
88 csetres['nodesbyphase'][phase])
87
89
88 # Write bookmark updates.
90 # Write bookmark updates.
89 bookmarks.updatefromremote(repo.ui, repo, csetres['bookmarks'],
91 bookmarks.updatefromremote(repo.ui, repo, csetres['bookmarks'],
90 remote.url(), pullop.gettransaction,
92 remote.url(), pullop.gettransaction,
91 explicit=pullop.explicitbookmarks)
93 explicit=pullop.explicitbookmarks)
92
94
93 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
95 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
94
96
95 # We don't properly support shallow changeset and manifest yet. So we apply
97 # We don't properly support shallow changeset and manifest yet. So we apply
96 # depth limiting locally.
98 # depth limiting locally.
97 if pullop.depth:
99 if pullop.depth:
98 relevantcsetnodes = set()
100 relevantcsetnodes = set()
99 clnode = repo.changelog.node
101 clnode = repo.changelog.node
100
102
101 for rev in repo.revs(b'ancestors(%ln, %s)',
103 for rev in repo.revs(b'ancestors(%ln, %s)',
102 pullheads, pullop.depth - 1):
104 pullheads, pullop.depth - 1):
103 relevantcsetnodes.add(clnode(rev))
105 relevantcsetnodes.add(clnode(rev))
104
106
105 csetrelevantfilter = lambda n: n in relevantcsetnodes
107 csetrelevantfilter = lambda n: n in relevantcsetnodes
106
108
107 else:
109 else:
108 csetrelevantfilter = lambda n: True
110 csetrelevantfilter = lambda n: True
109
111
110 # If obtaining the raw store files, we need to scan the full repo to
112 # If obtaining the raw store files, we need to scan the full repo to
111 # derive all the changesets, manifests, and linkrevs.
113 # derive all the changesets, manifests, and linkrevs.
112 if usingrawchangelogandmanifest:
114 if usingrawchangelogandmanifest:
113 csetsforfiles = []
115 csetsforfiles = []
114 mnodesforfiles = []
116 mnodesforfiles = []
115 manifestlinkrevs = {}
117 manifestlinkrevs = {}
116
118
117 for rev in repo:
119 for rev in repo:
118 ctx = repo[rev]
120 ctx = repo[rev]
119 node = ctx.node()
121 node = ctx.node()
120
122
121 if not csetrelevantfilter(node):
123 if not csetrelevantfilter(node):
122 continue
124 continue
123
125
124 mnode = ctx.manifestnode()
126 mnode = ctx.manifestnode()
125
127
126 csetsforfiles.append(node)
128 csetsforfiles.append(node)
127 mnodesforfiles.append(mnode)
129 mnodesforfiles.append(mnode)
128 manifestlinkrevs[mnode] = rev
130 manifestlinkrevs[mnode] = rev
129
131
130 else:
132 else:
131 csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)]
133 csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)]
132 mnodesforfiles = manres['added']
134 mnodesforfiles = manres['added']
133 manifestlinkrevs = manres['linkrevs']
135 manifestlinkrevs = manres['linkrevs']
134
136
135 # Find all file nodes referenced by added manifests and fetch those
137 # Find all file nodes referenced by added manifests and fetch those
136 # revisions.
138 # revisions.
137 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
139 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
138 _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles,
140 _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles,
139 manifestlinkrevs, shallow=bool(pullop.depth))
141 manifestlinkrevs, shallow=bool(pullop.depth))
140
142
141 def _checkuserawstorefiledata(pullop):
143 def _checkuserawstorefiledata(pullop):
142 """Check whether we should use rawstorefiledata command to retrieve data."""
144 """Check whether we should use rawstorefiledata command to retrieve data."""
143
145
144 repo = pullop.repo
146 repo = pullop.repo
145 remote = pullop.remote
147 remote = pullop.remote
146
148
147 # Command to obtain raw store data isn't available.
149 # Command to obtain raw store data isn't available.
148 if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
150 if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
149 return False
151 return False
150
152
151 # Only honor if user requested stream clone operation.
153 # Only honor if user requested stream clone operation.
152 if not pullop.streamclonerequested:
154 if not pullop.streamclonerequested:
153 return False
155 return False
154
156
155 # Only works on empty repos.
157 # Only works on empty repos.
156 if len(repo):
158 if len(repo):
157 return False
159 return False
158
160
159 # TODO This is super hacky. There needs to be a storage API for this. We
161 # TODO This is super hacky. There needs to be a storage API for this. We
160 # also need to check for compatibility with the remote.
162 # also need to check for compatibility with the remote.
161 if b'revlogv1' not in repo.requirements:
163 if b'revlogv1' not in repo.requirements:
162 return False
164 return False
163
165
164 return True
166 return True
165
167
166 def _fetchrawstorefiles(repo, remote):
168 def _fetchrawstorefiles(repo, remote):
167 with remote.commandexecutor() as e:
169 with remote.commandexecutor() as e:
168 objs = e.callcommand(b'rawstorefiledata', {
170 objs = e.callcommand(b'rawstorefiledata', {
169 b'files': [b'changelog', b'manifestlog'],
171 b'files': [b'changelog', b'manifestlog'],
170 }).result()
172 }).result()
171
173
172 # First object is a summary of files data that follows.
174 # First object is a summary of files data that follows.
173 overall = next(objs)
175 overall = next(objs)
174
176
175 progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'],
177 progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'],
176 unit=_('bytes'))
178 unit=_('bytes'))
177 with progress:
179 with progress:
178 progress.update(0)
180 progress.update(0)
179
181
180 # Next are pairs of file metadata, data.
182 # Next are pairs of file metadata, data.
181 while True:
183 while True:
182 try:
184 try:
183 filemeta = next(objs)
185 filemeta = next(objs)
184 except StopIteration:
186 except StopIteration:
185 break
187 break
186
188
187 for k in (b'location', b'path', b'size'):
189 for k in (b'location', b'path', b'size'):
188 if k not in filemeta:
190 if k not in filemeta:
189 raise error.Abort(_(b'remote file data missing key: %s')
191 raise error.Abort(_(b'remote file data missing key: %s')
190 % k)
192 % k)
191
193
192 if filemeta[b'location'] == b'store':
194 if filemeta[b'location'] == b'store':
193 vfs = repo.svfs
195 vfs = repo.svfs
194 else:
196 else:
195 raise error.Abort(_(b'invalid location for raw file data: '
197 raise error.Abort(_(b'invalid location for raw file data: '
196 b'%s') % filemeta[b'location'])
198 b'%s') % filemeta[b'location'])
197
199
198 bytesremaining = filemeta[b'size']
200 bytesremaining = filemeta[b'size']
199
201
200 with vfs.open(filemeta[b'path'], b'wb') as fh:
202 with vfs.open(filemeta[b'path'], b'wb') as fh:
201 while True:
203 while True:
202 try:
204 try:
203 chunk = next(objs)
205 chunk = next(objs)
204 except StopIteration:
206 except StopIteration:
205 break
207 break
206
208
207 bytesremaining -= len(chunk)
209 bytesremaining -= len(chunk)
208
210
209 if bytesremaining < 0:
211 if bytesremaining < 0:
210 raise error.Abort(_(
212 raise error.Abort(_(
211 b'received invalid number of bytes for file '
213 b'received invalid number of bytes for file '
212 b'data; expected %d, got extra') %
214 b'data; expected %d, got extra') %
213 filemeta[b'size'])
215 filemeta[b'size'])
214
216
215 progress.increment(step=len(chunk))
217 progress.increment(step=len(chunk))
216 fh.write(chunk)
218 fh.write(chunk)
217
219
218 try:
220 try:
219 if chunk.islast:
221 if chunk.islast:
220 break
222 break
221 except AttributeError:
223 except AttributeError:
222 raise error.Abort(_(
224 raise error.Abort(_(
223 b'did not receive indefinite length bytestring '
225 b'did not receive indefinite length bytestring '
224 b'for file data'))
226 b'for file data'))
225
227
226 if bytesremaining:
228 if bytesremaining:
227 raise error.Abort(_(b'received invalid number of bytes for'
229 raise error.Abort(_(b'received invalid number of bytes for'
228 b'file data; expected %d got %d') %
230 b'file data; expected %d got %d') %
229 (filemeta[b'size'],
231 (filemeta[b'size'],
230 filemeta[b'size'] - bytesremaining))
232 filemeta[b'size'] - bytesremaining))
231
233
232 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
234 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
233 """Determine which changesets need to be pulled."""
235 """Determine which changesets need to be pulled."""
234
236
235 if heads:
237 if heads:
236 knownnode = repo.changelog.hasnode
238 knownnode = repo.changelog.hasnode
237 if all(knownnode(head) for head in heads):
239 if all(knownnode(head) for head in heads):
238 return heads, False, heads
240 return heads, False, heads
239
241
240 # TODO wire protocol version 2 is capable of more efficient discovery
242 # TODO wire protocol version 2 is capable of more efficient discovery
241 # than setdiscovery. Consider implementing something better.
243 # than setdiscovery. Consider implementing something better.
242 common, fetch, remoteheads = setdiscovery.findcommonheads(
244 common, fetch, remoteheads = setdiscovery.findcommonheads(
243 repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated)
245 repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated)
244
246
245 common = set(common)
247 common = set(common)
246 remoteheads = set(remoteheads)
248 remoteheads = set(remoteheads)
247
249
248 # If a remote head is filtered locally, put it back in the common set.
250 # If a remote head is filtered locally, put it back in the common set.
249 # See the comment in exchange._pulldiscoverychangegroup() for more.
251 # See the comment in exchange._pulldiscoverychangegroup() for more.
250
252
251 if fetch and remoteheads:
253 if fetch and remoteheads:
252 nodemap = repo.unfiltered().changelog.nodemap
254 nodemap = repo.unfiltered().changelog.nodemap
253
255
254 common |= {head for head in remoteheads if head in nodemap}
256 common |= {head for head in remoteheads if head in nodemap}
255
257
256 if set(remoteheads).issubset(common):
258 if set(remoteheads).issubset(common):
257 fetch = []
259 fetch = []
258
260
259 common.discard(nullid)
261 common.discard(nullid)
260
262
261 return common, fetch, remoteheads
263 return common, fetch, remoteheads
262
264
263 def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads):
265 def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads):
264 # TODO consider adding a step here where we obtain the DAG shape first
266 # TODO consider adding a step here where we obtain the DAG shape first
265 # (or ask the server to slice changesets into chunks for us) so that
267 # (or ask the server to slice changesets into chunks for us) so that
266 # we can perform multiple fetches in batches. This will facilitate
268 # we can perform multiple fetches in batches. This will facilitate
267 # resuming interrupted clones, higher server-side cache hit rates due
269 # resuming interrupted clones, higher server-side cache hit rates due
268 # to smaller segments, etc.
270 # to smaller segments, etc.
269 with remote.commandexecutor() as e:
271 with remote.commandexecutor() as e:
270 objs = e.callcommand(b'changesetdata', {
272 objs = e.callcommand(b'changesetdata', {
271 b'revisions': [{
273 b'revisions': [{
272 b'type': b'changesetdagrange',
274 b'type': b'changesetdagrange',
273 b'roots': sorted(common),
275 b'roots': sorted(common),
274 b'heads': sorted(remoteheads),
276 b'heads': sorted(remoteheads),
275 }],
277 }],
276 b'fields': {b'bookmarks', b'parents', b'phase', b'revision'},
278 b'fields': {b'bookmarks', b'parents', b'phase', b'revision'},
277 }).result()
279 }).result()
278
280
279 # The context manager waits on all response data when exiting. So
281 # The context manager waits on all response data when exiting. So
280 # we need to remain in the context manager in order to stream data.
282 # we need to remain in the context manager in order to stream data.
281 return _processchangesetdata(repo, tr, objs)
283 return _processchangesetdata(repo, tr, objs)
282
284
283 def _processchangesetdata(repo, tr, objs):
285 def _processchangesetdata(repo, tr, objs):
284 repo.hook('prechangegroup', throw=True,
286 repo.hook('prechangegroup', throw=True,
285 **pycompat.strkwargs(tr.hookargs))
287 **pycompat.strkwargs(tr.hookargs))
286
288
287 urepo = repo.unfiltered()
289 urepo = repo.unfiltered()
288 cl = urepo.changelog
290 cl = urepo.changelog
289
291
290 cl.delayupdate(tr)
292 cl.delayupdate(tr)
291
293
292 # The first emitted object is a header describing the data that
294 # The first emitted object is a header describing the data that
293 # follows.
295 # follows.
294 meta = next(objs)
296 meta = next(objs)
295
297
296 progress = repo.ui.makeprogress(_('changesets'),
298 progress = repo.ui.makeprogress(_('changesets'),
297 unit=_('chunks'),
299 unit=_('chunks'),
298 total=meta.get(b'totalitems'))
300 total=meta.get(b'totalitems'))
299
301
300 manifestnodes = {}
302 manifestnodes = {}
301
303
302 def linkrev(node):
304 def linkrev(node):
303 repo.ui.debug('add changeset %s\n' % short(node))
305 repo.ui.debug('add changeset %s\n' % short(node))
304 # Linkrev for changelog is always self.
306 # Linkrev for changelog is always self.
305 return len(cl)
307 return len(cl)
306
308
307 def onchangeset(cl, node):
309 def onchangeset(cl, node):
308 progress.increment()
310 progress.increment()
309
311
310 revision = cl.changelogrevision(node)
312 revision = cl.changelogrevision(node)
311
313
312 # We need to preserve the mapping of changelog revision to node
314 # We need to preserve the mapping of changelog revision to node
313 # so we can set the linkrev accordingly when manifests are added.
315 # so we can set the linkrev accordingly when manifests are added.
314 manifestnodes[cl.rev(node)] = revision.manifest
316 manifestnodes[cl.rev(node)] = revision.manifest
315
317
316 nodesbyphase = {phase: set() for phase in phases.phasenames}
318 nodesbyphase = {phase: set() for phase in phases.phasenames}
317 remotebookmarks = {}
319 remotebookmarks = {}
318
320
319 # addgroup() expects a 7-tuple describing revisions. This normalizes
321 # addgroup() expects a 7-tuple describing revisions. This normalizes
320 # the wire data to that format.
322 # the wire data to that format.
321 #
323 #
322 # This loop also aggregates non-revision metadata, such as phase
324 # This loop also aggregates non-revision metadata, such as phase
323 # data.
325 # data.
324 def iterrevisions():
326 def iterrevisions():
325 for cset in objs:
327 for cset in objs:
326 node = cset[b'node']
328 node = cset[b'node']
327
329
328 if b'phase' in cset:
330 if b'phase' in cset:
329 nodesbyphase[cset[b'phase']].add(node)
331 nodesbyphase[cset[b'phase']].add(node)
330
332
331 for mark in cset.get(b'bookmarks', []):
333 for mark in cset.get(b'bookmarks', []):
332 remotebookmarks[mark] = node
334 remotebookmarks[mark] = node
333
335
334 # TODO add mechanism for extensions to examine records so they
336 # TODO add mechanism for extensions to examine records so they
335 # can siphon off custom data fields.
337 # can siphon off custom data fields.
336
338
337 extrafields = {}
339 extrafields = {}
338
340
339 for field, size in cset.get(b'fieldsfollowing', []):
341 for field, size in cset.get(b'fieldsfollowing', []):
340 extrafields[field] = next(objs)
342 extrafields[field] = next(objs)
341
343
342 # Some entries might only be metadata only updates.
344 # Some entries might only be metadata only updates.
343 if b'revision' not in extrafields:
345 if b'revision' not in extrafields:
344 continue
346 continue
345
347
346 data = extrafields[b'revision']
348 data = extrafields[b'revision']
347
349
348 yield (
350 yield (
349 node,
351 node,
350 cset[b'parents'][0],
352 cset[b'parents'][0],
351 cset[b'parents'][1],
353 cset[b'parents'][1],
352 # Linknode is always itself for changesets.
354 # Linknode is always itself for changesets.
353 cset[b'node'],
355 cset[b'node'],
354 # We always send full revisions. So delta base is not set.
356 # We always send full revisions. So delta base is not set.
355 nullid,
357 nullid,
356 mdiff.trivialdiffheader(len(data)) + data,
358 mdiff.trivialdiffheader(len(data)) + data,
357 # Flags not yet supported.
359 # Flags not yet supported.
358 0,
360 0,
359 )
361 )
360
362
361 added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr),
363 added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr),
362 addrevisioncb=onchangeset)
364 addrevisioncb=onchangeset)
363
365
364 progress.complete()
366 progress.complete()
365
367
366 return {
368 return {
367 'added': added,
369 'added': added,
368 'nodesbyphase': nodesbyphase,
370 'nodesbyphase': nodesbyphase,
369 'bookmarks': remotebookmarks,
371 'bookmarks': remotebookmarks,
370 'manifestnodes': manifestnodes,
372 'manifestnodes': manifestnodes,
371 }
373 }
372
374
373 def _fetchmanifests(repo, tr, remote, manifestnodes):
375 def _fetchmanifests(repo, tr, remote, manifestnodes):
374 rootmanifest = repo.manifestlog.getstorage(b'')
376 rootmanifest = repo.manifestlog.getstorage(b'')
375
377
376 # Some manifests can be shared between changesets. Filter out revisions
378 # Some manifests can be shared between changesets. Filter out revisions
377 # we already know about.
379 # we already know about.
378 fetchnodes = []
380 fetchnodes = []
379 linkrevs = {}
381 linkrevs = {}
380 seen = set()
382 seen = set()
381
383
382 for clrev, node in sorted(manifestnodes.iteritems()):
384 for clrev, node in sorted(manifestnodes.iteritems()):
383 if node in seen:
385 if node in seen:
384 continue
386 continue
385
387
386 try:
388 try:
387 rootmanifest.rev(node)
389 rootmanifest.rev(node)
388 except error.LookupError:
390 except error.LookupError:
389 fetchnodes.append(node)
391 fetchnodes.append(node)
390 linkrevs[node] = clrev
392 linkrevs[node] = clrev
391
393
392 seen.add(node)
394 seen.add(node)
393
395
394 # TODO handle tree manifests
396 # TODO handle tree manifests
395
397
396 # addgroup() expects 7-tuple describing revisions. This normalizes
398 # addgroup() expects 7-tuple describing revisions. This normalizes
397 # the wire data to that format.
399 # the wire data to that format.
398 def iterrevisions(objs, progress):
400 def iterrevisions(objs, progress):
399 for manifest in objs:
401 for manifest in objs:
400 node = manifest[b'node']
402 node = manifest[b'node']
401
403
402 extrafields = {}
404 extrafields = {}
403
405
404 for field, size in manifest.get(b'fieldsfollowing', []):
406 for field, size in manifest.get(b'fieldsfollowing', []):
405 extrafields[field] = next(objs)
407 extrafields[field] = next(objs)
406
408
407 if b'delta' in extrafields:
409 if b'delta' in extrafields:
408 basenode = manifest[b'deltabasenode']
410 basenode = manifest[b'deltabasenode']
409 delta = extrafields[b'delta']
411 delta = extrafields[b'delta']
410 elif b'revision' in extrafields:
412 elif b'revision' in extrafields:
411 basenode = nullid
413 basenode = nullid
412 revision = extrafields[b'revision']
414 revision = extrafields[b'revision']
413 delta = mdiff.trivialdiffheader(len(revision)) + revision
415 delta = mdiff.trivialdiffheader(len(revision)) + revision
414 else:
416 else:
415 continue
417 continue
416
418
417 yield (
419 yield (
418 node,
420 node,
419 manifest[b'parents'][0],
421 manifest[b'parents'][0],
420 manifest[b'parents'][1],
422 manifest[b'parents'][1],
421 # The value passed in is passed to the lookup function passed
423 # The value passed in is passed to the lookup function passed
422 # to addgroup(). We already have a map of manifest node to
424 # to addgroup(). We already have a map of manifest node to
423 # changelog revision number. So we just pass in the
425 # changelog revision number. So we just pass in the
424 # manifest node here and use linkrevs.__getitem__ as the
426 # manifest node here and use linkrevs.__getitem__ as the
425 # resolution function.
427 # resolution function.
426 node,
428 node,
427 basenode,
429 basenode,
428 delta,
430 delta,
429 # Flags not yet supported.
431 # Flags not yet supported.
430 0
432 0
431 )
433 )
432
434
433 progress.increment()
435 progress.increment()
434
436
435 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
437 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
436 total=len(fetchnodes))
438 total=len(fetchnodes))
437
439
438 commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
440 commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
439 batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
441 batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
440 # TODO make size configurable on client?
442 # TODO make size configurable on client?
441
443
442 # We send commands 1 at a time to the remote. This is not the most
444 # We send commands 1 at a time to the remote. This is not the most
443 # efficient because we incur a round trip at the end of each batch.
445 # efficient because we incur a round trip at the end of each batch.
444 # However, the existing frame-based reactor keeps consuming server
446 # However, the existing frame-based reactor keeps consuming server
445 # data in the background. And this results in response data buffering
447 # data in the background. And this results in response data buffering
446 # in memory. This can consume gigabytes of memory.
448 # in memory. This can consume gigabytes of memory.
447 # TODO send multiple commands in a request once background buffering
449 # TODO send multiple commands in a request once background buffering
448 # issues are resolved.
450 # issues are resolved.
449
451
450 added = []
452 added = []
451
453
452 for i in pycompat.xrange(0, len(fetchnodes), batchsize):
454 for i in pycompat.xrange(0, len(fetchnodes), batchsize):
453 batch = [node for node in fetchnodes[i:i + batchsize]]
455 batch = [node for node in fetchnodes[i:i + batchsize]]
454 if not batch:
456 if not batch:
455 continue
457 continue
456
458
457 with remote.commandexecutor() as e:
459 with remote.commandexecutor() as e:
458 objs = e.callcommand(b'manifestdata', {
460 objs = e.callcommand(b'manifestdata', {
459 b'tree': b'',
461 b'tree': b'',
460 b'nodes': batch,
462 b'nodes': batch,
461 b'fields': {b'parents', b'revision'},
463 b'fields': {b'parents', b'revision'},
462 b'haveparents': True,
464 b'haveparents': True,
463 }).result()
465 }).result()
464
466
465 # Chomp off header object.
467 # Chomp off header object.
466 next(objs)
468 next(objs)
467
469
468 added.extend(rootmanifest.addgroup(
470 added.extend(rootmanifest.addgroup(
469 iterrevisions(objs, progress),
471 iterrevisions(objs, progress),
470 linkrevs.__getitem__,
472 linkrevs.__getitem__,
471 weakref.proxy(tr)))
473 weakref.proxy(tr)))
472
474
473 progress.complete()
475 progress.complete()
474
476
475 return {
477 return {
476 'added': added,
478 'added': added,
477 'linkrevs': linkrevs,
479 'linkrevs': linkrevs,
478 }
480 }
479
481
480 def _derivefilesfrommanifests(repo, matcher, manifestnodes):
482 def _derivefilesfrommanifests(repo, matcher, manifestnodes):
481 """Determine what file nodes are relevant given a set of manifest nodes.
483 """Determine what file nodes are relevant given a set of manifest nodes.
482
484
483 Returns a dict mapping file paths to dicts of file node to first manifest
485 Returns a dict mapping file paths to dicts of file node to first manifest
484 node.
486 node.
485 """
487 """
486 ml = repo.manifestlog
488 ml = repo.manifestlog
487 fnodes = collections.defaultdict(dict)
489 fnodes = collections.defaultdict(dict)
488
490
489 progress = repo.ui.makeprogress(
491 progress = repo.ui.makeprogress(
490 _('scanning manifests'), total=len(manifestnodes))
492 _('scanning manifests'), total=len(manifestnodes))
491
493
492 with progress:
494 with progress:
493 for manifestnode in manifestnodes:
495 for manifestnode in manifestnodes:
494 m = ml.get(b'', manifestnode)
496 m = ml.get(b'', manifestnode)
495
497
496 # TODO this will pull in unwanted nodes because it takes the storage
498 # TODO this will pull in unwanted nodes because it takes the storage
497 # delta into consideration. What we really want is something that
499 # delta into consideration. What we really want is something that
498 # takes the delta between the manifest's parents. And ideally we
500 # takes the delta between the manifest's parents. And ideally we
499 # would ignore file nodes that are known locally. For now, ignore
501 # would ignore file nodes that are known locally. For now, ignore
500 # both these limitations. This will result in incremental fetches
502 # both these limitations. This will result in incremental fetches
501 # requesting data we already have. So this is far from ideal.
503 # requesting data we already have. So this is far from ideal.
502 md = m.readfast()
504 md = m.readfast()
503
505
504 for path, fnode in md.items():
506 for path, fnode in md.items():
505 if matcher(path):
507 if matcher(path):
506 fnodes[path].setdefault(fnode, manifestnode)
508 fnodes[path].setdefault(fnode, manifestnode)
507
509
508 progress.increment()
510 progress.increment()
509
511
510 return fnodes
512 return fnodes
511
513
512 def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
514 def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
513 """Fetch file data from explicit file revisions."""
515 """Fetch file data from explicit file revisions."""
514 def iterrevisions(objs, progress):
516 def iterrevisions(objs, progress):
515 for filerevision in objs:
517 for filerevision in objs:
516 node = filerevision[b'node']
518 node = filerevision[b'node']
517
519
518 extrafields = {}
520 extrafields = {}
519
521
520 for field, size in filerevision.get(b'fieldsfollowing', []):
522 for field, size in filerevision.get(b'fieldsfollowing', []):
521 extrafields[field] = next(objs)
523 extrafields[field] = next(objs)
522
524
523 if b'delta' in extrafields:
525 if b'delta' in extrafields:
524 basenode = filerevision[b'deltabasenode']
526 basenode = filerevision[b'deltabasenode']
525 delta = extrafields[b'delta']
527 delta = extrafields[b'delta']
526 elif b'revision' in extrafields:
528 elif b'revision' in extrafields:
527 basenode = nullid
529 basenode = nullid
528 revision = extrafields[b'revision']
530 revision = extrafields[b'revision']
529 delta = mdiff.trivialdiffheader(len(revision)) + revision
531 delta = mdiff.trivialdiffheader(len(revision)) + revision
530 else:
532 else:
531 continue
533 continue
532
534
533 yield (
535 yield (
534 node,
536 node,
535 filerevision[b'parents'][0],
537 filerevision[b'parents'][0],
536 filerevision[b'parents'][1],
538 filerevision[b'parents'][1],
537 node,
539 node,
538 basenode,
540 basenode,
539 delta,
541 delta,
540 # Flags not yet supported.
542 # Flags not yet supported.
541 0,
543 0,
542 )
544 )
543
545
544 progress.increment()
546 progress.increment()
545
547
546 progress = repo.ui.makeprogress(
548 progress = repo.ui.makeprogress(
547 _('files'), unit=_('chunks'),
549 _('files'), unit=_('chunks'),
548 total=sum(len(v) for v in fnodes.itervalues()))
550 total=sum(len(v) for v in fnodes.itervalues()))
549
551
550 # TODO make batch size configurable
552 # TODO make batch size configurable
551 batchsize = 10000
553 batchsize = 10000
552 fnodeslist = [x for x in sorted(fnodes.items())]
554 fnodeslist = [x for x in sorted(fnodes.items())]
553
555
554 for i in pycompat.xrange(0, len(fnodeslist), batchsize):
556 for i in pycompat.xrange(0, len(fnodeslist), batchsize):
555 batch = [x for x in fnodeslist[i:i + batchsize]]
557 batch = [x for x in fnodeslist[i:i + batchsize]]
556 if not batch:
558 if not batch:
557 continue
559 continue
558
560
559 with remote.commandexecutor() as e:
561 with remote.commandexecutor() as e:
560 fs = []
562 fs = []
561 locallinkrevs = {}
563 locallinkrevs = {}
562
564
563 for path, nodes in batch:
565 for path, nodes in batch:
564 fs.append((path, e.callcommand(b'filedata', {
566 fs.append((path, e.callcommand(b'filedata', {
565 b'path': path,
567 b'path': path,
566 b'nodes': sorted(nodes),
568 b'nodes': sorted(nodes),
567 b'fields': {b'parents', b'revision'},
569 b'fields': {b'parents', b'revision'},
568 b'haveparents': True,
570 b'haveparents': True,
569 })))
571 })))
570
572
571 locallinkrevs[path] = {
573 locallinkrevs[path] = {
572 node: linkrevs[manifestnode]
574 node: linkrevs[manifestnode]
573 for node, manifestnode in nodes.iteritems()}
575 for node, manifestnode in nodes.iteritems()}
574
576
575 for path, f in fs:
577 for path, f in fs:
576 objs = f.result()
578 objs = f.result()
577
579
578 # Chomp off header objects.
580 # Chomp off header objects.
579 next(objs)
581 next(objs)
580
582
581 store = repo.file(path)
583 store = repo.file(path)
582 store.addgroup(
584 store.addgroup(
583 iterrevisions(objs, progress),
585 iterrevisions(objs, progress),
584 locallinkrevs[path].__getitem__,
586 locallinkrevs[path].__getitem__,
585 weakref.proxy(tr))
587 weakref.proxy(tr))
586
588
587 def _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csets,
589 def _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csets,
588 manlinkrevs, shallow=False):
590 manlinkrevs, shallow=False):
589 """Fetch file data from explicit changeset revisions."""
591 """Fetch file data from explicit changeset revisions."""
590
592
591 def iterrevisions(objs, remaining, progress):
593 def iterrevisions(objs, remaining, progress):
592 while remaining:
594 while remaining:
593 filerevision = next(objs)
595 filerevision = next(objs)
594
596
595 node = filerevision[b'node']
597 node = filerevision[b'node']
596
598
597 extrafields = {}
599 extrafields = {}
598
600
599 for field, size in filerevision.get(b'fieldsfollowing', []):
601 for field, size in filerevision.get(b'fieldsfollowing', []):
600 extrafields[field] = next(objs)
602 extrafields[field] = next(objs)
601
603
602 if b'delta' in extrafields:
604 if b'delta' in extrafields:
603 basenode = filerevision[b'deltabasenode']
605 basenode = filerevision[b'deltabasenode']
604 delta = extrafields[b'delta']
606 delta = extrafields[b'delta']
605 elif b'revision' in extrafields:
607 elif b'revision' in extrafields:
606 basenode = nullid
608 basenode = nullid
607 revision = extrafields[b'revision']
609 revision = extrafields[b'revision']
608 delta = mdiff.trivialdiffheader(len(revision)) + revision
610 delta = mdiff.trivialdiffheader(len(revision)) + revision
609 else:
611 else:
610 continue
612 continue
611
613
612 if b'linknode' in filerevision:
614 if b'linknode' in filerevision:
613 linknode = filerevision[b'linknode']
615 linknode = filerevision[b'linknode']
614 else:
616 else:
615 linknode = node
617 linknode = node
616
618
617 yield (
619 yield (
618 node,
620 node,
619 filerevision[b'parents'][0],
621 filerevision[b'parents'][0],
620 filerevision[b'parents'][1],
622 filerevision[b'parents'][1],
621 linknode,
623 linknode,
622 basenode,
624 basenode,
623 delta,
625 delta,
624 # Flags not yet supported.
626 # Flags not yet supported.
625 0,
627 0,
626 )
628 )
627
629
628 progress.increment()
630 progress.increment()
629 remaining -= 1
631 remaining -= 1
630
632
631 progress = repo.ui.makeprogress(
633 progress = repo.ui.makeprogress(
632 _('files'), unit=_('chunks'),
634 _('files'), unit=_('chunks'),
633 total=sum(len(v) for v in fnodes.itervalues()))
635 total=sum(len(v) for v in fnodes.itervalues()))
634
636
635 commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
637 commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
636 batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
638 batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
637
639
638 shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features
640 shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features
639 fields = {b'parents', b'revision'}
641 fields = {b'parents', b'revision'}
640 clrev = repo.changelog.rev
642 clrev = repo.changelog.rev
641
643
642 # There are no guarantees that we'll have ancestor revisions if
644 # There are no guarantees that we'll have ancestor revisions if
643 # a) this repo has shallow file storage b) shallow data fetching is enabled.
645 # a) this repo has shallow file storage b) shallow data fetching is enabled.
644 # Force remote to not delta against possibly unknown revisions when these
646 # Force remote to not delta against possibly unknown revisions when these
645 # conditions hold.
647 # conditions hold.
646 haveparents = not (shallowfiles or shallow)
648 haveparents = not (shallowfiles or shallow)
647
649
648 # Similarly, we may not have calculated linkrevs for all incoming file
650 # Similarly, we may not have calculated linkrevs for all incoming file
649 # revisions. Ask the remote to do work for us in this case.
651 # revisions. Ask the remote to do work for us in this case.
650 if not haveparents:
652 if not haveparents:
651 fields.add(b'linknode')
653 fields.add(b'linknode')
652
654
653 for i in pycompat.xrange(0, len(csets), batchsize):
655 for i in pycompat.xrange(0, len(csets), batchsize):
654 batch = [x for x in csets[i:i + batchsize]]
656 batch = [x for x in csets[i:i + batchsize]]
655 if not batch:
657 if not batch:
656 continue
658 continue
657
659
658 with remote.commandexecutor() as e:
660 with remote.commandexecutor() as e:
659 args = {
661 args = {
660 b'revisions': [{
662 b'revisions': [{
661 b'type': b'changesetexplicit',
663 b'type': b'changesetexplicit',
662 b'nodes': batch,
664 b'nodes': batch,
663 }],
665 }],
664 b'fields': fields,
666 b'fields': fields,
665 b'haveparents': haveparents,
667 b'haveparents': haveparents,
666 }
668 }
667
669
668 if pathfilter:
670 if pathfilter:
669 args[b'pathfilter'] = pathfilter
671 args[b'pathfilter'] = pathfilter
670
672
671 objs = e.callcommand(b'filesdata', args).result()
673 objs = e.callcommand(b'filesdata', args).result()
672
674
673 # First object is an overall header.
675 # First object is an overall header.
674 overall = next(objs)
676 overall = next(objs)
675
677
676 # We have overall['totalpaths'] segments.
678 # We have overall['totalpaths'] segments.
677 for i in pycompat.xrange(overall[b'totalpaths']):
679 for i in pycompat.xrange(overall[b'totalpaths']):
678 header = next(objs)
680 header = next(objs)
679
681
680 path = header[b'path']
682 path = header[b'path']
681 store = repo.file(path)
683 store = repo.file(path)
682
684
683 linkrevs = {
685 linkrevs = {
684 fnode: manlinkrevs[mnode]
686 fnode: manlinkrevs[mnode]
685 for fnode, mnode in fnodes[path].iteritems()}
687 for fnode, mnode in fnodes[path].iteritems()}
686
688
687 def getlinkrev(node):
689 def getlinkrev(node):
688 if node in linkrevs:
690 if node in linkrevs:
689 return linkrevs[node]
691 return linkrevs[node]
690 else:
692 else:
691 return clrev(node)
693 return clrev(node)
692
694
693 store.addgroup(iterrevisions(objs, header[b'totalitems'],
695 store.addgroup(iterrevisions(objs, header[b'totalitems'],
694 progress),
696 progress),
695 getlinkrev,
697 getlinkrev,
696 weakref.proxy(tr),
698 weakref.proxy(tr),
697 maybemissingparents=shallow)
699 maybemissingparents=shallow)
@@ -1,242 +1,244
1 # filelog.py - file history class for mercurial
1 # filelog.py - file history class for mercurial
2 #
2 #
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 from .i18n import _
10 from .i18n import _
11 from .node import (
11 from .node import (
12 nullid,
12 nullid,
13 nullrev,
13 nullrev,
14 )
14 )
15 from . import (
15 from . import (
16 error,
16 error,
17 revlog,
18 )
19 from .interfaces import (
17 repository,
20 repository,
18 revlog,
19 )
21 )
20 from .utils import (
22 from .utils import (
21 interfaceutil,
23 interfaceutil,
22 storageutil,
24 storageutil,
23 )
25 )
24
26
25 @interfaceutil.implementer(repository.ifilestorage)
27 @interfaceutil.implementer(repository.ifilestorage)
26 class filelog(object):
28 class filelog(object):
27 def __init__(self, opener, path):
29 def __init__(self, opener, path):
28 self._revlog = revlog.revlog(opener,
30 self._revlog = revlog.revlog(opener,
29 '/'.join(('data', path + '.i')),
31 '/'.join(('data', path + '.i')),
30 censorable=True)
32 censorable=True)
31 # Full name of the user visible file, relative to the repository root.
33 # Full name of the user visible file, relative to the repository root.
32 # Used by LFS.
34 # Used by LFS.
33 self._revlog.filename = path
35 self._revlog.filename = path
34
36
35 def __len__(self):
37 def __len__(self):
36 return len(self._revlog)
38 return len(self._revlog)
37
39
38 def __iter__(self):
40 def __iter__(self):
39 return self._revlog.__iter__()
41 return self._revlog.__iter__()
40
42
41 def hasnode(self, node):
43 def hasnode(self, node):
42 if node in (nullid, nullrev):
44 if node in (nullid, nullrev):
43 return False
45 return False
44
46
45 try:
47 try:
46 self._revlog.rev(node)
48 self._revlog.rev(node)
47 return True
49 return True
48 except (TypeError, ValueError, IndexError, error.LookupError):
50 except (TypeError, ValueError, IndexError, error.LookupError):
49 return False
51 return False
50
52
51 def revs(self, start=0, stop=None):
53 def revs(self, start=0, stop=None):
52 return self._revlog.revs(start=start, stop=stop)
54 return self._revlog.revs(start=start, stop=stop)
53
55
54 def parents(self, node):
56 def parents(self, node):
55 return self._revlog.parents(node)
57 return self._revlog.parents(node)
56
58
57 def parentrevs(self, rev):
59 def parentrevs(self, rev):
58 return self._revlog.parentrevs(rev)
60 return self._revlog.parentrevs(rev)
59
61
60 def rev(self, node):
62 def rev(self, node):
61 return self._revlog.rev(node)
63 return self._revlog.rev(node)
62
64
63 def node(self, rev):
65 def node(self, rev):
64 return self._revlog.node(rev)
66 return self._revlog.node(rev)
65
67
66 def lookup(self, node):
68 def lookup(self, node):
67 return storageutil.fileidlookup(self._revlog, node,
69 return storageutil.fileidlookup(self._revlog, node,
68 self._revlog.indexfile)
70 self._revlog.indexfile)
69
71
70 def linkrev(self, rev):
72 def linkrev(self, rev):
71 return self._revlog.linkrev(rev)
73 return self._revlog.linkrev(rev)
72
74
73 def commonancestorsheads(self, node1, node2):
75 def commonancestorsheads(self, node1, node2):
74 return self._revlog.commonancestorsheads(node1, node2)
76 return self._revlog.commonancestorsheads(node1, node2)
75
77
76 # Used by dagop.blockdescendants().
78 # Used by dagop.blockdescendants().
77 def descendants(self, revs):
79 def descendants(self, revs):
78 return self._revlog.descendants(revs)
80 return self._revlog.descendants(revs)
79
81
80 def heads(self, start=None, stop=None):
82 def heads(self, start=None, stop=None):
81 return self._revlog.heads(start, stop)
83 return self._revlog.heads(start, stop)
82
84
83 # Used by hgweb, children extension.
85 # Used by hgweb, children extension.
84 def children(self, node):
86 def children(self, node):
85 return self._revlog.children(node)
87 return self._revlog.children(node)
86
88
87 def iscensored(self, rev):
89 def iscensored(self, rev):
88 return self._revlog.iscensored(rev)
90 return self._revlog.iscensored(rev)
89
91
90 def revision(self, node, _df=None, raw=False):
92 def revision(self, node, _df=None, raw=False):
91 return self._revlog.revision(node, _df=_df, raw=raw)
93 return self._revlog.revision(node, _df=_df, raw=raw)
92
94
93 def rawdata(self, node, _df=None):
95 def rawdata(self, node, _df=None):
94 return self._revlog.rawdata(node, _df=_df)
96 return self._revlog.rawdata(node, _df=_df)
95
97
96 def emitrevisions(self, nodes, nodesorder=None,
98 def emitrevisions(self, nodes, nodesorder=None,
97 revisiondata=False, assumehaveparentrevisions=False,
99 revisiondata=False, assumehaveparentrevisions=False,
98 deltamode=repository.CG_DELTAMODE_STD):
100 deltamode=repository.CG_DELTAMODE_STD):
99 return self._revlog.emitrevisions(
101 return self._revlog.emitrevisions(
100 nodes, nodesorder=nodesorder, revisiondata=revisiondata,
102 nodes, nodesorder=nodesorder, revisiondata=revisiondata,
101 assumehaveparentrevisions=assumehaveparentrevisions,
103 assumehaveparentrevisions=assumehaveparentrevisions,
102 deltamode=deltamode)
104 deltamode=deltamode)
103
105
104 def addrevision(self, revisiondata, transaction, linkrev, p1, p2,
106 def addrevision(self, revisiondata, transaction, linkrev, p1, p2,
105 node=None, flags=revlog.REVIDX_DEFAULT_FLAGS,
107 node=None, flags=revlog.REVIDX_DEFAULT_FLAGS,
106 cachedelta=None):
108 cachedelta=None):
107 return self._revlog.addrevision(revisiondata, transaction, linkrev,
109 return self._revlog.addrevision(revisiondata, transaction, linkrev,
108 p1, p2, node=node, flags=flags,
110 p1, p2, node=node, flags=flags,
109 cachedelta=cachedelta)
111 cachedelta=cachedelta)
110
112
111 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None,
113 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None,
112 maybemissingparents=False):
114 maybemissingparents=False):
113 if maybemissingparents:
115 if maybemissingparents:
114 raise error.Abort(_('revlog storage does not support missing '
116 raise error.Abort(_('revlog storage does not support missing '
115 'parents write mode'))
117 'parents write mode'))
116
118
117 return self._revlog.addgroup(deltas, linkmapper, transaction,
119 return self._revlog.addgroup(deltas, linkmapper, transaction,
118 addrevisioncb=addrevisioncb)
120 addrevisioncb=addrevisioncb)
119
121
120 def getstrippoint(self, minlink):
122 def getstrippoint(self, minlink):
121 return self._revlog.getstrippoint(minlink)
123 return self._revlog.getstrippoint(minlink)
122
124
123 def strip(self, minlink, transaction):
125 def strip(self, minlink, transaction):
124 return self._revlog.strip(minlink, transaction)
126 return self._revlog.strip(minlink, transaction)
125
127
126 def censorrevision(self, tr, node, tombstone=b''):
128 def censorrevision(self, tr, node, tombstone=b''):
127 return self._revlog.censorrevision(tr, node, tombstone=tombstone)
129 return self._revlog.censorrevision(tr, node, tombstone=tombstone)
128
130
129 def files(self):
131 def files(self):
130 return self._revlog.files()
132 return self._revlog.files()
131
133
132 def read(self, node):
134 def read(self, node):
133 return storageutil.filtermetadata(self.revision(node))
135 return storageutil.filtermetadata(self.revision(node))
134
136
135 def add(self, text, meta, transaction, link, p1=None, p2=None):
137 def add(self, text, meta, transaction, link, p1=None, p2=None):
136 if meta or text.startswith('\1\n'):
138 if meta or text.startswith('\1\n'):
137 text = storageutil.packmeta(meta, text)
139 text = storageutil.packmeta(meta, text)
138 return self.addrevision(text, transaction, link, p1, p2)
140 return self.addrevision(text, transaction, link, p1, p2)
139
141
140 def renamed(self, node):
142 def renamed(self, node):
141 return storageutil.filerevisioncopied(self, node)
143 return storageutil.filerevisioncopied(self, node)
142
144
143 def size(self, rev):
145 def size(self, rev):
144 """return the size of a given revision"""
146 """return the size of a given revision"""
145
147
146 # for revisions with renames, we have to go the slow way
148 # for revisions with renames, we have to go the slow way
147 node = self.node(rev)
149 node = self.node(rev)
148 if self.renamed(node):
150 if self.renamed(node):
149 return len(self.read(node))
151 return len(self.read(node))
150 if self.iscensored(rev):
152 if self.iscensored(rev):
151 return 0
153 return 0
152
154
153 # XXX if self.read(node).startswith("\1\n"), this returns (size+4)
155 # XXX if self.read(node).startswith("\1\n"), this returns (size+4)
154 return self._revlog.size(rev)
156 return self._revlog.size(rev)
155
157
156 def cmp(self, node, text):
158 def cmp(self, node, text):
157 """compare text with a given file revision
159 """compare text with a given file revision
158
160
159 returns True if text is different than what is stored.
161 returns True if text is different than what is stored.
160 """
162 """
161 return not storageutil.filedataequivalent(self, node, text)
163 return not storageutil.filedataequivalent(self, node, text)
162
164
163 def verifyintegrity(self, state):
165 def verifyintegrity(self, state):
164 return self._revlog.verifyintegrity(state)
166 return self._revlog.verifyintegrity(state)
165
167
166 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
168 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
167 revisionscount=False, trackedsize=False,
169 revisionscount=False, trackedsize=False,
168 storedsize=False):
170 storedsize=False):
169 return self._revlog.storageinfo(
171 return self._revlog.storageinfo(
170 exclusivefiles=exclusivefiles, sharedfiles=sharedfiles,
172 exclusivefiles=exclusivefiles, sharedfiles=sharedfiles,
171 revisionscount=revisionscount, trackedsize=trackedsize,
173 revisionscount=revisionscount, trackedsize=trackedsize,
172 storedsize=storedsize)
174 storedsize=storedsize)
173
175
174 # TODO these aren't part of the interface and aren't internal methods.
176 # TODO these aren't part of the interface and aren't internal methods.
175 # Callers should be fixed to not use them.
177 # Callers should be fixed to not use them.
176
178
177 # Used by bundlefilelog, unionfilelog.
179 # Used by bundlefilelog, unionfilelog.
178 @property
180 @property
179 def indexfile(self):
181 def indexfile(self):
180 return self._revlog.indexfile
182 return self._revlog.indexfile
181
183
182 @indexfile.setter
184 @indexfile.setter
183 def indexfile(self, value):
185 def indexfile(self, value):
184 self._revlog.indexfile = value
186 self._revlog.indexfile = value
185
187
186 # Used by repo upgrade.
188 # Used by repo upgrade.
187 def clone(self, tr, destrevlog, **kwargs):
189 def clone(self, tr, destrevlog, **kwargs):
188 if not isinstance(destrevlog, filelog):
190 if not isinstance(destrevlog, filelog):
189 raise error.ProgrammingError('expected filelog to clone()')
191 raise error.ProgrammingError('expected filelog to clone()')
190
192
191 return self._revlog.clone(tr, destrevlog._revlog, **kwargs)
193 return self._revlog.clone(tr, destrevlog._revlog, **kwargs)
192
194
193 class narrowfilelog(filelog):
195 class narrowfilelog(filelog):
194 """Filelog variation to be used with narrow stores."""
196 """Filelog variation to be used with narrow stores."""
195
197
196 def __init__(self, opener, path, narrowmatch):
198 def __init__(self, opener, path, narrowmatch):
197 super(narrowfilelog, self).__init__(opener, path)
199 super(narrowfilelog, self).__init__(opener, path)
198 self._narrowmatch = narrowmatch
200 self._narrowmatch = narrowmatch
199
201
200 def renamed(self, node):
202 def renamed(self, node):
201 res = super(narrowfilelog, self).renamed(node)
203 res = super(narrowfilelog, self).renamed(node)
202
204
203 # Renames that come from outside the narrowspec are problematic
205 # Renames that come from outside the narrowspec are problematic
204 # because we may lack the base text for the rename. This can result
206 # because we may lack the base text for the rename. This can result
205 # in code attempting to walk the ancestry or compute a diff
207 # in code attempting to walk the ancestry or compute a diff
206 # encountering a missing revision. We address this by silently
208 # encountering a missing revision. We address this by silently
207 # removing rename metadata if the source file is outside the
209 # removing rename metadata if the source file is outside the
208 # narrow spec.
210 # narrow spec.
209 #
211 #
210 # A better solution would be to see if the base revision is available,
212 # A better solution would be to see if the base revision is available,
211 # rather than assuming it isn't.
213 # rather than assuming it isn't.
212 #
214 #
213 # An even better solution would be to teach all consumers of rename
215 # An even better solution would be to teach all consumers of rename
214 # metadata that the base revision may not be available.
216 # metadata that the base revision may not be available.
215 #
217 #
216 # TODO consider better ways of doing this.
218 # TODO consider better ways of doing this.
217 if res and not self._narrowmatch(res[0]):
219 if res and not self._narrowmatch(res[0]):
218 return None
220 return None
219
221
220 return res
222 return res
221
223
222 def size(self, rev):
224 def size(self, rev):
223 # Because we have a custom renamed() that may lie, we need to call
225 # Because we have a custom renamed() that may lie, we need to call
224 # the base renamed() to report accurate results.
226 # the base renamed() to report accurate results.
225 node = self.node(rev)
227 node = self.node(rev)
226 if super(narrowfilelog, self).renamed(node):
228 if super(narrowfilelog, self).renamed(node):
227 return len(self.read(node))
229 return len(self.read(node))
228 else:
230 else:
229 return super(narrowfilelog, self).size(rev)
231 return super(narrowfilelog, self).size(rev)
230
232
231 def cmp(self, node, text):
233 def cmp(self, node, text):
232 different = super(narrowfilelog, self).cmp(node, text)
234 different = super(narrowfilelog, self).cmp(node, text)
233
235
234 # Because renamed() may lie, we may get false positives for
236 # Because renamed() may lie, we may get false positives for
235 # different content. Check for this by comparing against the original
237 # different content. Check for this by comparing against the original
236 # renamed() implementation.
238 # renamed() implementation.
237 if different:
239 if different:
238 if super(narrowfilelog, self).renamed(node):
240 if super(narrowfilelog, self).renamed(node):
239 t2 = self.read(node)
241 t2 = self.read(node)
240 return t2 != text
242 return t2 != text
241
243
242 return different
244 return different
@@ -1,1237 +1,1240
1 # hg.py - repository classes for mercurial
1 # hg.py - repository classes for mercurial
2 #
2 #
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8
8
9 from __future__ import absolute_import
9 from __future__ import absolute_import
10
10
11 import errno
11 import errno
12 import hashlib
12 import hashlib
13 import os
13 import os
14 import shutil
14 import shutil
15 import stat
15 import stat
16
16
17 from .i18n import _
17 from .i18n import _
18 from .node import (
18 from .node import (
19 nullid,
19 nullid,
20 )
20 )
21
21
22 from . import (
22 from . import (
23 bookmarks,
23 bookmarks,
24 bundlerepo,
24 bundlerepo,
25 cacheutil,
25 cacheutil,
26 cmdutil,
26 cmdutil,
27 destutil,
27 destutil,
28 discovery,
28 discovery,
29 error,
29 error,
30 exchange,
30 exchange,
31 extensions,
31 extensions,
32 httppeer,
32 httppeer,
33 localrepo,
33 localrepo,
34 lock,
34 lock,
35 logcmdutil,
35 logcmdutil,
36 logexchange,
36 logexchange,
37 merge as mergemod,
37 merge as mergemod,
38 narrowspec,
38 narrowspec,
39 node,
39 node,
40 phases,
40 phases,
41 pycompat,
41 pycompat,
42 repository as repositorymod,
43 scmutil,
42 scmutil,
44 sshpeer,
43 sshpeer,
45 statichttprepo,
44 statichttprepo,
46 ui as uimod,
45 ui as uimod,
47 unionrepo,
46 unionrepo,
48 url,
47 url,
49 util,
48 util,
50 verify as verifymod,
49 verify as verifymod,
51 vfs as vfsmod,
50 vfs as vfsmod,
52 )
51 )
53
52
53 from .interfaces import (
54 repository as repositorymod,
55 )
56
54 release = lock.release
57 release = lock.release
55
58
56 # shared features
59 # shared features
57 sharedbookmarks = 'bookmarks'
60 sharedbookmarks = 'bookmarks'
58
61
59 def _local(path):
62 def _local(path):
60 path = util.expandpath(util.urllocalpath(path))
63 path = util.expandpath(util.urllocalpath(path))
61
64
62 try:
65 try:
63 isfile = os.path.isfile(path)
66 isfile = os.path.isfile(path)
64 # Python 2 raises TypeError, Python 3 ValueError.
67 # Python 2 raises TypeError, Python 3 ValueError.
65 except (TypeError, ValueError) as e:
68 except (TypeError, ValueError) as e:
66 raise error.Abort(_('invalid path %s: %s') % (
69 raise error.Abort(_('invalid path %s: %s') % (
67 path, pycompat.bytestr(e)))
70 path, pycompat.bytestr(e)))
68
71
69 return isfile and bundlerepo or localrepo
72 return isfile and bundlerepo or localrepo
70
73
71 def addbranchrevs(lrepo, other, branches, revs):
74 def addbranchrevs(lrepo, other, branches, revs):
72 peer = other.peer() # a courtesy to callers using a localrepo for other
75 peer = other.peer() # a courtesy to callers using a localrepo for other
73 hashbranch, branches = branches
76 hashbranch, branches = branches
74 if not hashbranch and not branches:
77 if not hashbranch and not branches:
75 x = revs or None
78 x = revs or None
76 if revs:
79 if revs:
77 y = revs[0]
80 y = revs[0]
78 else:
81 else:
79 y = None
82 y = None
80 return x, y
83 return x, y
81 if revs:
84 if revs:
82 revs = list(revs)
85 revs = list(revs)
83 else:
86 else:
84 revs = []
87 revs = []
85
88
86 if not peer.capable('branchmap'):
89 if not peer.capable('branchmap'):
87 if branches:
90 if branches:
88 raise error.Abort(_("remote branch lookup not supported"))
91 raise error.Abort(_("remote branch lookup not supported"))
89 revs.append(hashbranch)
92 revs.append(hashbranch)
90 return revs, revs[0]
93 return revs, revs[0]
91
94
92 with peer.commandexecutor() as e:
95 with peer.commandexecutor() as e:
93 branchmap = e.callcommand('branchmap', {}).result()
96 branchmap = e.callcommand('branchmap', {}).result()
94
97
95 def primary(branch):
98 def primary(branch):
96 if branch == '.':
99 if branch == '.':
97 if not lrepo:
100 if not lrepo:
98 raise error.Abort(_("dirstate branch not accessible"))
101 raise error.Abort(_("dirstate branch not accessible"))
99 branch = lrepo.dirstate.branch()
102 branch = lrepo.dirstate.branch()
100 if branch in branchmap:
103 if branch in branchmap:
101 revs.extend(node.hex(r) for r in reversed(branchmap[branch]))
104 revs.extend(node.hex(r) for r in reversed(branchmap[branch]))
102 return True
105 return True
103 else:
106 else:
104 return False
107 return False
105
108
106 for branch in branches:
109 for branch in branches:
107 if not primary(branch):
110 if not primary(branch):
108 raise error.RepoLookupError(_("unknown branch '%s'") % branch)
111 raise error.RepoLookupError(_("unknown branch '%s'") % branch)
109 if hashbranch:
112 if hashbranch:
110 if not primary(hashbranch):
113 if not primary(hashbranch):
111 revs.append(hashbranch)
114 revs.append(hashbranch)
112 return revs, revs[0]
115 return revs, revs[0]
113
116
114 def parseurl(path, branches=None):
117 def parseurl(path, branches=None):
115 '''parse url#branch, returning (url, (branch, branches))'''
118 '''parse url#branch, returning (url, (branch, branches))'''
116
119
117 u = util.url(path)
120 u = util.url(path)
118 branch = None
121 branch = None
119 if u.fragment:
122 if u.fragment:
120 branch = u.fragment
123 branch = u.fragment
121 u.fragment = None
124 u.fragment = None
122 return bytes(u), (branch, branches or [])
125 return bytes(u), (branch, branches or [])
123
126
124 schemes = {
127 schemes = {
125 'bundle': bundlerepo,
128 'bundle': bundlerepo,
126 'union': unionrepo,
129 'union': unionrepo,
127 'file': _local,
130 'file': _local,
128 'http': httppeer,
131 'http': httppeer,
129 'https': httppeer,
132 'https': httppeer,
130 'ssh': sshpeer,
133 'ssh': sshpeer,
131 'static-http': statichttprepo,
134 'static-http': statichttprepo,
132 }
135 }
133
136
134 def _peerlookup(path):
137 def _peerlookup(path):
135 u = util.url(path)
138 u = util.url(path)
136 scheme = u.scheme or 'file'
139 scheme = u.scheme or 'file'
137 thing = schemes.get(scheme) or schemes['file']
140 thing = schemes.get(scheme) or schemes['file']
138 try:
141 try:
139 return thing(path)
142 return thing(path)
140 except TypeError:
143 except TypeError:
141 # we can't test callable(thing) because 'thing' can be an unloaded
144 # we can't test callable(thing) because 'thing' can be an unloaded
142 # module that implements __call__
145 # module that implements __call__
143 if not util.safehasattr(thing, 'instance'):
146 if not util.safehasattr(thing, 'instance'):
144 raise
147 raise
145 return thing
148 return thing
146
149
147 def islocal(repo):
150 def islocal(repo):
148 '''return true if repo (or path pointing to repo) is local'''
151 '''return true if repo (or path pointing to repo) is local'''
149 if isinstance(repo, bytes):
152 if isinstance(repo, bytes):
150 try:
153 try:
151 return _peerlookup(repo).islocal(repo)
154 return _peerlookup(repo).islocal(repo)
152 except AttributeError:
155 except AttributeError:
153 return False
156 return False
154 return repo.local()
157 return repo.local()
155
158
156 def openpath(ui, path, sendaccept=True):
159 def openpath(ui, path, sendaccept=True):
157 '''open path with open if local, url.open if remote'''
160 '''open path with open if local, url.open if remote'''
158 pathurl = util.url(path, parsequery=False, parsefragment=False)
161 pathurl = util.url(path, parsequery=False, parsefragment=False)
159 if pathurl.islocal():
162 if pathurl.islocal():
160 return util.posixfile(pathurl.localpath(), 'rb')
163 return util.posixfile(pathurl.localpath(), 'rb')
161 else:
164 else:
162 return url.open(ui, path, sendaccept=sendaccept)
165 return url.open(ui, path, sendaccept=sendaccept)
163
166
164 # a list of (ui, repo) functions called for wire peer initialization
167 # a list of (ui, repo) functions called for wire peer initialization
165 wirepeersetupfuncs = []
168 wirepeersetupfuncs = []
166
169
167 def _peerorrepo(ui, path, create=False, presetupfuncs=None,
170 def _peerorrepo(ui, path, create=False, presetupfuncs=None,
168 intents=None, createopts=None):
171 intents=None, createopts=None):
169 """return a repository object for the specified path"""
172 """return a repository object for the specified path"""
170 obj = _peerlookup(path).instance(ui, path, create, intents=intents,
173 obj = _peerlookup(path).instance(ui, path, create, intents=intents,
171 createopts=createopts)
174 createopts=createopts)
172 ui = getattr(obj, "ui", ui)
175 ui = getattr(obj, "ui", ui)
173 for f in presetupfuncs or []:
176 for f in presetupfuncs or []:
174 f(ui, obj)
177 f(ui, obj)
175 ui.log(b'extension', b'- executing reposetup hooks\n')
178 ui.log(b'extension', b'- executing reposetup hooks\n')
176 with util.timedcm('all reposetup') as allreposetupstats:
179 with util.timedcm('all reposetup') as allreposetupstats:
177 for name, module in extensions.extensions(ui):
180 for name, module in extensions.extensions(ui):
178 ui.log(b'extension', b' - running reposetup for %s\n', name)
181 ui.log(b'extension', b' - running reposetup for %s\n', name)
179 hook = getattr(module, 'reposetup', None)
182 hook = getattr(module, 'reposetup', None)
180 if hook:
183 if hook:
181 with util.timedcm('reposetup %r', name) as stats:
184 with util.timedcm('reposetup %r', name) as stats:
182 hook(ui, obj)
185 hook(ui, obj)
183 ui.log(b'extension', b' > reposetup for %s took %s\n',
186 ui.log(b'extension', b' > reposetup for %s took %s\n',
184 name, stats)
187 name, stats)
185 ui.log(b'extension', b'> all reposetup took %s\n', allreposetupstats)
188 ui.log(b'extension', b'> all reposetup took %s\n', allreposetupstats)
186 if not obj.local():
189 if not obj.local():
187 for f in wirepeersetupfuncs:
190 for f in wirepeersetupfuncs:
188 f(ui, obj)
191 f(ui, obj)
189 return obj
192 return obj
190
193
191 def repository(ui, path='', create=False, presetupfuncs=None, intents=None,
194 def repository(ui, path='', create=False, presetupfuncs=None, intents=None,
192 createopts=None):
195 createopts=None):
193 """return a repository object for the specified path"""
196 """return a repository object for the specified path"""
194 peer = _peerorrepo(ui, path, create, presetupfuncs=presetupfuncs,
197 peer = _peerorrepo(ui, path, create, presetupfuncs=presetupfuncs,
195 intents=intents, createopts=createopts)
198 intents=intents, createopts=createopts)
196 repo = peer.local()
199 repo = peer.local()
197 if not repo:
200 if not repo:
198 raise error.Abort(_("repository '%s' is not local") %
201 raise error.Abort(_("repository '%s' is not local") %
199 (path or peer.url()))
202 (path or peer.url()))
200 return repo.filtered('visible')
203 return repo.filtered('visible')
201
204
202 def peer(uiorrepo, opts, path, create=False, intents=None, createopts=None):
205 def peer(uiorrepo, opts, path, create=False, intents=None, createopts=None):
203 '''return a repository peer for the specified path'''
206 '''return a repository peer for the specified path'''
204 rui = remoteui(uiorrepo, opts)
207 rui = remoteui(uiorrepo, opts)
205 return _peerorrepo(rui, path, create, intents=intents,
208 return _peerorrepo(rui, path, create, intents=intents,
206 createopts=createopts).peer()
209 createopts=createopts).peer()
207
210
208 def defaultdest(source):
211 def defaultdest(source):
209 '''return default destination of clone if none is given
212 '''return default destination of clone if none is given
210
213
211 >>> defaultdest(b'foo')
214 >>> defaultdest(b'foo')
212 'foo'
215 'foo'
213 >>> defaultdest(b'/foo/bar')
216 >>> defaultdest(b'/foo/bar')
214 'bar'
217 'bar'
215 >>> defaultdest(b'/')
218 >>> defaultdest(b'/')
216 ''
219 ''
217 >>> defaultdest(b'')
220 >>> defaultdest(b'')
218 ''
221 ''
219 >>> defaultdest(b'http://example.org/')
222 >>> defaultdest(b'http://example.org/')
220 ''
223 ''
221 >>> defaultdest(b'http://example.org/foo/')
224 >>> defaultdest(b'http://example.org/foo/')
222 'foo'
225 'foo'
223 '''
226 '''
224 path = util.url(source).path
227 path = util.url(source).path
225 if not path:
228 if not path:
226 return ''
229 return ''
227 return os.path.basename(os.path.normpath(path))
230 return os.path.basename(os.path.normpath(path))
228
231
229 def sharedreposource(repo):
232 def sharedreposource(repo):
230 """Returns repository object for source repository of a shared repo.
233 """Returns repository object for source repository of a shared repo.
231
234
232 If repo is not a shared repository, returns None.
235 If repo is not a shared repository, returns None.
233 """
236 """
234 if repo.sharedpath == repo.path:
237 if repo.sharedpath == repo.path:
235 return None
238 return None
236
239
237 if util.safehasattr(repo, 'srcrepo') and repo.srcrepo:
240 if util.safehasattr(repo, 'srcrepo') and repo.srcrepo:
238 return repo.srcrepo
241 return repo.srcrepo
239
242
240 # the sharedpath always ends in the .hg; we want the path to the repo
243 # the sharedpath always ends in the .hg; we want the path to the repo
241 source = repo.vfs.split(repo.sharedpath)[0]
244 source = repo.vfs.split(repo.sharedpath)[0]
242 srcurl, branches = parseurl(source)
245 srcurl, branches = parseurl(source)
243 srcrepo = repository(repo.ui, srcurl)
246 srcrepo = repository(repo.ui, srcurl)
244 repo.srcrepo = srcrepo
247 repo.srcrepo = srcrepo
245 return srcrepo
248 return srcrepo
246
249
247 def share(ui, source, dest=None, update=True, bookmarks=True, defaultpath=None,
250 def share(ui, source, dest=None, update=True, bookmarks=True, defaultpath=None,
248 relative=False):
251 relative=False):
249 '''create a shared repository'''
252 '''create a shared repository'''
250
253
251 if not islocal(source):
254 if not islocal(source):
252 raise error.Abort(_('can only share local repositories'))
255 raise error.Abort(_('can only share local repositories'))
253
256
254 if not dest:
257 if not dest:
255 dest = defaultdest(source)
258 dest = defaultdest(source)
256 else:
259 else:
257 dest = ui.expandpath(dest)
260 dest = ui.expandpath(dest)
258
261
259 if isinstance(source, bytes):
262 if isinstance(source, bytes):
260 origsource = ui.expandpath(source)
263 origsource = ui.expandpath(source)
261 source, branches = parseurl(origsource)
264 source, branches = parseurl(origsource)
262 srcrepo = repository(ui, source)
265 srcrepo = repository(ui, source)
263 rev, checkout = addbranchrevs(srcrepo, srcrepo, branches, None)
266 rev, checkout = addbranchrevs(srcrepo, srcrepo, branches, None)
264 else:
267 else:
265 srcrepo = source.local()
268 srcrepo = source.local()
266 checkout = None
269 checkout = None
267
270
268 shareditems = set()
271 shareditems = set()
269 if bookmarks:
272 if bookmarks:
270 shareditems.add(sharedbookmarks)
273 shareditems.add(sharedbookmarks)
271
274
272 r = repository(ui, dest, create=True, createopts={
275 r = repository(ui, dest, create=True, createopts={
273 'sharedrepo': srcrepo,
276 'sharedrepo': srcrepo,
274 'sharedrelative': relative,
277 'sharedrelative': relative,
275 'shareditems': shareditems,
278 'shareditems': shareditems,
276 })
279 })
277
280
278 postshare(srcrepo, r, defaultpath=defaultpath)
281 postshare(srcrepo, r, defaultpath=defaultpath)
279 r = repository(ui, dest)
282 r = repository(ui, dest)
280 _postshareupdate(r, update, checkout=checkout)
283 _postshareupdate(r, update, checkout=checkout)
281 return r
284 return r
282
285
283 def unshare(ui, repo):
286 def unshare(ui, repo):
284 """convert a shared repository to a normal one
287 """convert a shared repository to a normal one
285
288
286 Copy the store data to the repo and remove the sharedpath data.
289 Copy the store data to the repo and remove the sharedpath data.
287
290
288 Returns a new repository object representing the unshared repository.
291 Returns a new repository object representing the unshared repository.
289
292
290 The passed repository object is not usable after this function is
293 The passed repository object is not usable after this function is
291 called.
294 called.
292 """
295 """
293
296
294 with repo.lock():
297 with repo.lock():
295 # we use locks here because if we race with commit, we
298 # we use locks here because if we race with commit, we
296 # can end up with extra data in the cloned revlogs that's
299 # can end up with extra data in the cloned revlogs that's
297 # not pointed to by changesets, thus causing verify to
300 # not pointed to by changesets, thus causing verify to
298 # fail
301 # fail
299 destlock = copystore(ui, repo, repo.path)
302 destlock = copystore(ui, repo, repo.path)
300 with destlock or util.nullcontextmanager():
303 with destlock or util.nullcontextmanager():
301
304
302 sharefile = repo.vfs.join('sharedpath')
305 sharefile = repo.vfs.join('sharedpath')
303 util.rename(sharefile, sharefile + '.old')
306 util.rename(sharefile, sharefile + '.old')
304
307
305 repo.requirements.discard('shared')
308 repo.requirements.discard('shared')
306 repo.requirements.discard('relshared')
309 repo.requirements.discard('relshared')
307 repo._writerequirements()
310 repo._writerequirements()
308
311
309 # Removing share changes some fundamental properties of the repo instance.
312 # Removing share changes some fundamental properties of the repo instance.
310 # So we instantiate a new repo object and operate on it rather than
313 # So we instantiate a new repo object and operate on it rather than
311 # try to keep the existing repo usable.
314 # try to keep the existing repo usable.
312 newrepo = repository(repo.baseui, repo.root, create=False)
315 newrepo = repository(repo.baseui, repo.root, create=False)
313
316
314 # TODO: figure out how to access subrepos that exist, but were previously
317 # TODO: figure out how to access subrepos that exist, but were previously
315 # removed from .hgsub
318 # removed from .hgsub
316 c = newrepo['.']
319 c = newrepo['.']
317 subs = c.substate
320 subs = c.substate
318 for s in sorted(subs):
321 for s in sorted(subs):
319 c.sub(s).unshare()
322 c.sub(s).unshare()
320
323
321 localrepo.poisonrepository(repo)
324 localrepo.poisonrepository(repo)
322
325
323 return newrepo
326 return newrepo
324
327
325 def postshare(sourcerepo, destrepo, defaultpath=None):
328 def postshare(sourcerepo, destrepo, defaultpath=None):
326 """Called after a new shared repo is created.
329 """Called after a new shared repo is created.
327
330
328 The new repo only has a requirements file and pointer to the source.
331 The new repo only has a requirements file and pointer to the source.
329 This function configures additional shared data.
332 This function configures additional shared data.
330
333
331 Extensions can wrap this function and write additional entries to
334 Extensions can wrap this function and write additional entries to
332 destrepo/.hg/shared to indicate additional pieces of data to be shared.
335 destrepo/.hg/shared to indicate additional pieces of data to be shared.
333 """
336 """
334 default = defaultpath or sourcerepo.ui.config('paths', 'default')
337 default = defaultpath or sourcerepo.ui.config('paths', 'default')
335 if default:
338 if default:
336 template = ('[paths]\n'
339 template = ('[paths]\n'
337 'default = %s\n')
340 'default = %s\n')
338 destrepo.vfs.write('hgrc', util.tonativeeol(template % default))
341 destrepo.vfs.write('hgrc', util.tonativeeol(template % default))
339 if repositorymod.NARROW_REQUIREMENT in sourcerepo.requirements:
342 if repositorymod.NARROW_REQUIREMENT in sourcerepo.requirements:
340 with destrepo.wlock():
343 with destrepo.wlock():
341 narrowspec.copytoworkingcopy(destrepo)
344 narrowspec.copytoworkingcopy(destrepo)
342
345
343 def _postshareupdate(repo, update, checkout=None):
346 def _postshareupdate(repo, update, checkout=None):
344 """Maybe perform a working directory update after a shared repo is created.
347 """Maybe perform a working directory update after a shared repo is created.
345
348
346 ``update`` can be a boolean or a revision to update to.
349 ``update`` can be a boolean or a revision to update to.
347 """
350 """
348 if not update:
351 if not update:
349 return
352 return
350
353
351 repo.ui.status(_("updating working directory\n"))
354 repo.ui.status(_("updating working directory\n"))
352 if update is not True:
355 if update is not True:
353 checkout = update
356 checkout = update
354 for test in (checkout, 'default', 'tip'):
357 for test in (checkout, 'default', 'tip'):
355 if test is None:
358 if test is None:
356 continue
359 continue
357 try:
360 try:
358 uprev = repo.lookup(test)
361 uprev = repo.lookup(test)
359 break
362 break
360 except error.RepoLookupError:
363 except error.RepoLookupError:
361 continue
364 continue
362 _update(repo, uprev)
365 _update(repo, uprev)
363
366
364 def copystore(ui, srcrepo, destpath):
367 def copystore(ui, srcrepo, destpath):
365 '''copy files from store of srcrepo in destpath
368 '''copy files from store of srcrepo in destpath
366
369
367 returns destlock
370 returns destlock
368 '''
371 '''
369 destlock = None
372 destlock = None
370 try:
373 try:
371 hardlink = None
374 hardlink = None
372 topic = _('linking') if hardlink else _('copying')
375 topic = _('linking') if hardlink else _('copying')
373 with ui.makeprogress(topic, unit=_('files')) as progress:
376 with ui.makeprogress(topic, unit=_('files')) as progress:
374 num = 0
377 num = 0
375 srcpublishing = srcrepo.publishing()
378 srcpublishing = srcrepo.publishing()
376 srcvfs = vfsmod.vfs(srcrepo.sharedpath)
379 srcvfs = vfsmod.vfs(srcrepo.sharedpath)
377 dstvfs = vfsmod.vfs(destpath)
380 dstvfs = vfsmod.vfs(destpath)
378 for f in srcrepo.store.copylist():
381 for f in srcrepo.store.copylist():
379 if srcpublishing and f.endswith('phaseroots'):
382 if srcpublishing and f.endswith('phaseroots'):
380 continue
383 continue
381 dstbase = os.path.dirname(f)
384 dstbase = os.path.dirname(f)
382 if dstbase and not dstvfs.exists(dstbase):
385 if dstbase and not dstvfs.exists(dstbase):
383 dstvfs.mkdir(dstbase)
386 dstvfs.mkdir(dstbase)
384 if srcvfs.exists(f):
387 if srcvfs.exists(f):
385 if f.endswith('data'):
388 if f.endswith('data'):
386 # 'dstbase' may be empty (e.g. revlog format 0)
389 # 'dstbase' may be empty (e.g. revlog format 0)
387 lockfile = os.path.join(dstbase, "lock")
390 lockfile = os.path.join(dstbase, "lock")
388 # lock to avoid premature writing to the target
391 # lock to avoid premature writing to the target
389 destlock = lock.lock(dstvfs, lockfile)
392 destlock = lock.lock(dstvfs, lockfile)
390 hardlink, n = util.copyfiles(srcvfs.join(f), dstvfs.join(f),
393 hardlink, n = util.copyfiles(srcvfs.join(f), dstvfs.join(f),
391 hardlink, progress)
394 hardlink, progress)
392 num += n
395 num += n
393 if hardlink:
396 if hardlink:
394 ui.debug("linked %d files\n" % num)
397 ui.debug("linked %d files\n" % num)
395 else:
398 else:
396 ui.debug("copied %d files\n" % num)
399 ui.debug("copied %d files\n" % num)
397 return destlock
400 return destlock
398 except: # re-raises
401 except: # re-raises
399 release(destlock)
402 release(destlock)
400 raise
403 raise
401
404
402 def clonewithshare(ui, peeropts, sharepath, source, srcpeer, dest, pull=False,
405 def clonewithshare(ui, peeropts, sharepath, source, srcpeer, dest, pull=False,
403 rev=None, update=True, stream=False):
406 rev=None, update=True, stream=False):
404 """Perform a clone using a shared repo.
407 """Perform a clone using a shared repo.
405
408
406 The store for the repository will be located at <sharepath>/.hg. The
409 The store for the repository will be located at <sharepath>/.hg. The
407 specified revisions will be cloned or pulled from "source". A shared repo
410 specified revisions will be cloned or pulled from "source". A shared repo
408 will be created at "dest" and a working copy will be created if "update" is
411 will be created at "dest" and a working copy will be created if "update" is
409 True.
412 True.
410 """
413 """
411 revs = None
414 revs = None
412 if rev:
415 if rev:
413 if not srcpeer.capable('lookup'):
416 if not srcpeer.capable('lookup'):
414 raise error.Abort(_("src repository does not support "
417 raise error.Abort(_("src repository does not support "
415 "revision lookup and so doesn't "
418 "revision lookup and so doesn't "
416 "support clone by revision"))
419 "support clone by revision"))
417
420
418 # TODO this is batchable.
421 # TODO this is batchable.
419 remoterevs = []
422 remoterevs = []
420 for r in rev:
423 for r in rev:
421 with srcpeer.commandexecutor() as e:
424 with srcpeer.commandexecutor() as e:
422 remoterevs.append(e.callcommand('lookup', {
425 remoterevs.append(e.callcommand('lookup', {
423 'key': r,
426 'key': r,
424 }).result())
427 }).result())
425 revs = remoterevs
428 revs = remoterevs
426
429
427 # Obtain a lock before checking for or cloning the pooled repo otherwise
430 # Obtain a lock before checking for or cloning the pooled repo otherwise
428 # 2 clients may race creating or populating it.
431 # 2 clients may race creating or populating it.
429 pooldir = os.path.dirname(sharepath)
432 pooldir = os.path.dirname(sharepath)
430 # lock class requires the directory to exist.
433 # lock class requires the directory to exist.
431 try:
434 try:
432 util.makedir(pooldir, False)
435 util.makedir(pooldir, False)
433 except OSError as e:
436 except OSError as e:
434 if e.errno != errno.EEXIST:
437 if e.errno != errno.EEXIST:
435 raise
438 raise
436
439
437 poolvfs = vfsmod.vfs(pooldir)
440 poolvfs = vfsmod.vfs(pooldir)
438 basename = os.path.basename(sharepath)
441 basename = os.path.basename(sharepath)
439
442
440 with lock.lock(poolvfs, '%s.lock' % basename):
443 with lock.lock(poolvfs, '%s.lock' % basename):
441 if os.path.exists(sharepath):
444 if os.path.exists(sharepath):
442 ui.status(_('(sharing from existing pooled repository %s)\n') %
445 ui.status(_('(sharing from existing pooled repository %s)\n') %
443 basename)
446 basename)
444 else:
447 else:
445 ui.status(_('(sharing from new pooled repository %s)\n') % basename)
448 ui.status(_('(sharing from new pooled repository %s)\n') % basename)
446 # Always use pull mode because hardlinks in share mode don't work
449 # Always use pull mode because hardlinks in share mode don't work
447 # well. Never update because working copies aren't necessary in
450 # well. Never update because working copies aren't necessary in
448 # share mode.
451 # share mode.
449 clone(ui, peeropts, source, dest=sharepath, pull=True,
452 clone(ui, peeropts, source, dest=sharepath, pull=True,
450 revs=rev, update=False, stream=stream)
453 revs=rev, update=False, stream=stream)
451
454
452 # Resolve the value to put in [paths] section for the source.
455 # Resolve the value to put in [paths] section for the source.
453 if islocal(source):
456 if islocal(source):
454 defaultpath = os.path.abspath(util.urllocalpath(source))
457 defaultpath = os.path.abspath(util.urllocalpath(source))
455 else:
458 else:
456 defaultpath = source
459 defaultpath = source
457
460
458 sharerepo = repository(ui, path=sharepath)
461 sharerepo = repository(ui, path=sharepath)
459 destrepo = share(ui, sharerepo, dest=dest, update=False, bookmarks=False,
462 destrepo = share(ui, sharerepo, dest=dest, update=False, bookmarks=False,
460 defaultpath=defaultpath)
463 defaultpath=defaultpath)
461
464
462 # We need to perform a pull against the dest repo to fetch bookmarks
465 # We need to perform a pull against the dest repo to fetch bookmarks
463 # and other non-store data that isn't shared by default. In the case of
466 # and other non-store data that isn't shared by default. In the case of
464 # non-existing shared repo, this means we pull from the remote twice. This
467 # non-existing shared repo, this means we pull from the remote twice. This
465 # is a bit weird. But at the time it was implemented, there wasn't an easy
468 # is a bit weird. But at the time it was implemented, there wasn't an easy
466 # way to pull just non-changegroup data.
469 # way to pull just non-changegroup data.
467 exchange.pull(destrepo, srcpeer, heads=revs)
470 exchange.pull(destrepo, srcpeer, heads=revs)
468
471
469 _postshareupdate(destrepo, update)
472 _postshareupdate(destrepo, update)
470
473
471 return srcpeer, peer(ui, peeropts, dest)
474 return srcpeer, peer(ui, peeropts, dest)
472
475
473 # Recomputing branch cache might be slow on big repos,
476 # Recomputing branch cache might be slow on big repos,
474 # so just copy it
477 # so just copy it
475 def _copycache(srcrepo, dstcachedir, fname):
478 def _copycache(srcrepo, dstcachedir, fname):
476 """copy a cache from srcrepo to destcachedir (if it exists)"""
479 """copy a cache from srcrepo to destcachedir (if it exists)"""
477 srcbranchcache = srcrepo.vfs.join('cache/%s' % fname)
480 srcbranchcache = srcrepo.vfs.join('cache/%s' % fname)
478 dstbranchcache = os.path.join(dstcachedir, fname)
481 dstbranchcache = os.path.join(dstcachedir, fname)
479 if os.path.exists(srcbranchcache):
482 if os.path.exists(srcbranchcache):
480 if not os.path.exists(dstcachedir):
483 if not os.path.exists(dstcachedir):
481 os.mkdir(dstcachedir)
484 os.mkdir(dstcachedir)
482 util.copyfile(srcbranchcache, dstbranchcache)
485 util.copyfile(srcbranchcache, dstbranchcache)
483
486
484 def clone(ui, peeropts, source, dest=None, pull=False, revs=None,
487 def clone(ui, peeropts, source, dest=None, pull=False, revs=None,
485 update=True, stream=False, branch=None, shareopts=None,
488 update=True, stream=False, branch=None, shareopts=None,
486 storeincludepats=None, storeexcludepats=None, depth=None):
489 storeincludepats=None, storeexcludepats=None, depth=None):
487 """Make a copy of an existing repository.
490 """Make a copy of an existing repository.
488
491
489 Create a copy of an existing repository in a new directory. The
492 Create a copy of an existing repository in a new directory. The
490 source and destination are URLs, as passed to the repository
493 source and destination are URLs, as passed to the repository
491 function. Returns a pair of repository peers, the source and
494 function. Returns a pair of repository peers, the source and
492 newly created destination.
495 newly created destination.
493
496
494 The location of the source is added to the new repository's
497 The location of the source is added to the new repository's
495 .hg/hgrc file, as the default to be used for future pulls and
498 .hg/hgrc file, as the default to be used for future pulls and
496 pushes.
499 pushes.
497
500
498 If an exception is raised, the partly cloned/updated destination
501 If an exception is raised, the partly cloned/updated destination
499 repository will be deleted.
502 repository will be deleted.
500
503
501 Arguments:
504 Arguments:
502
505
503 source: repository object or URL
506 source: repository object or URL
504
507
505 dest: URL of destination repository to create (defaults to base
508 dest: URL of destination repository to create (defaults to base
506 name of source repository)
509 name of source repository)
507
510
508 pull: always pull from source repository, even in local case or if the
511 pull: always pull from source repository, even in local case or if the
509 server prefers streaming
512 server prefers streaming
510
513
511 stream: stream raw data uncompressed from repository (fast over
514 stream: stream raw data uncompressed from repository (fast over
512 LAN, slow over WAN)
515 LAN, slow over WAN)
513
516
514 revs: revision to clone up to (implies pull=True)
517 revs: revision to clone up to (implies pull=True)
515
518
516 update: update working directory after clone completes, if
519 update: update working directory after clone completes, if
517 destination is local repository (True means update to default rev,
520 destination is local repository (True means update to default rev,
518 anything else is treated as a revision)
521 anything else is treated as a revision)
519
522
520 branch: branches to clone
523 branch: branches to clone
521
524
522 shareopts: dict of options to control auto sharing behavior. The "pool" key
525 shareopts: dict of options to control auto sharing behavior. The "pool" key
523 activates auto sharing mode and defines the directory for stores. The
526 activates auto sharing mode and defines the directory for stores. The
524 "mode" key determines how to construct the directory name of the shared
527 "mode" key determines how to construct the directory name of the shared
525 repository. "identity" means the name is derived from the node of the first
528 repository. "identity" means the name is derived from the node of the first
526 changeset in the repository. "remote" means the name is derived from the
529 changeset in the repository. "remote" means the name is derived from the
527 remote's path/URL. Defaults to "identity."
530 remote's path/URL. Defaults to "identity."
528
531
529 storeincludepats and storeexcludepats: sets of file patterns to include and
532 storeincludepats and storeexcludepats: sets of file patterns to include and
530 exclude in the repository copy, respectively. If not defined, all files
533 exclude in the repository copy, respectively. If not defined, all files
531 will be included (a "full" clone). Otherwise a "narrow" clone containing
534 will be included (a "full" clone). Otherwise a "narrow" clone containing
532 only the requested files will be performed. If ``storeincludepats`` is not
535 only the requested files will be performed. If ``storeincludepats`` is not
533 defined but ``storeexcludepats`` is, ``storeincludepats`` is assumed to be
536 defined but ``storeexcludepats`` is, ``storeincludepats`` is assumed to be
534 ``path:.``. If both are empty sets, no files will be cloned.
537 ``path:.``. If both are empty sets, no files will be cloned.
535 """
538 """
536
539
537 if isinstance(source, bytes):
540 if isinstance(source, bytes):
538 origsource = ui.expandpath(source)
541 origsource = ui.expandpath(source)
539 source, branches = parseurl(origsource, branch)
542 source, branches = parseurl(origsource, branch)
540 srcpeer = peer(ui, peeropts, source)
543 srcpeer = peer(ui, peeropts, source)
541 else:
544 else:
542 srcpeer = source.peer() # in case we were called with a localrepo
545 srcpeer = source.peer() # in case we were called with a localrepo
543 branches = (None, branch or [])
546 branches = (None, branch or [])
544 origsource = source = srcpeer.url()
547 origsource = source = srcpeer.url()
545 revs, checkout = addbranchrevs(srcpeer, srcpeer, branches, revs)
548 revs, checkout = addbranchrevs(srcpeer, srcpeer, branches, revs)
546
549
547 if dest is None:
550 if dest is None:
548 dest = defaultdest(source)
551 dest = defaultdest(source)
549 if dest:
552 if dest:
550 ui.status(_("destination directory: %s\n") % dest)
553 ui.status(_("destination directory: %s\n") % dest)
551 else:
554 else:
552 dest = ui.expandpath(dest)
555 dest = ui.expandpath(dest)
553
556
554 dest = util.urllocalpath(dest)
557 dest = util.urllocalpath(dest)
555 source = util.urllocalpath(source)
558 source = util.urllocalpath(source)
556
559
557 if not dest:
560 if not dest:
558 raise error.Abort(_("empty destination path is not valid"))
561 raise error.Abort(_("empty destination path is not valid"))
559
562
560 destvfs = vfsmod.vfs(dest, expandpath=True)
563 destvfs = vfsmod.vfs(dest, expandpath=True)
561 if destvfs.lexists():
564 if destvfs.lexists():
562 if not destvfs.isdir():
565 if not destvfs.isdir():
563 raise error.Abort(_("destination '%s' already exists") % dest)
566 raise error.Abort(_("destination '%s' already exists") % dest)
564 elif destvfs.listdir():
567 elif destvfs.listdir():
565 raise error.Abort(_("destination '%s' is not empty") % dest)
568 raise error.Abort(_("destination '%s' is not empty") % dest)
566
569
567 createopts = {}
570 createopts = {}
568 narrow = False
571 narrow = False
569
572
570 if storeincludepats is not None:
573 if storeincludepats is not None:
571 narrowspec.validatepatterns(storeincludepats)
574 narrowspec.validatepatterns(storeincludepats)
572 narrow = True
575 narrow = True
573
576
574 if storeexcludepats is not None:
577 if storeexcludepats is not None:
575 narrowspec.validatepatterns(storeexcludepats)
578 narrowspec.validatepatterns(storeexcludepats)
576 narrow = True
579 narrow = True
577
580
578 if narrow:
581 if narrow:
579 # Include everything by default if only exclusion patterns defined.
582 # Include everything by default if only exclusion patterns defined.
580 if storeexcludepats and not storeincludepats:
583 if storeexcludepats and not storeincludepats:
581 storeincludepats = {'path:.'}
584 storeincludepats = {'path:.'}
582
585
583 createopts['narrowfiles'] = True
586 createopts['narrowfiles'] = True
584
587
585 if depth:
588 if depth:
586 createopts['shallowfilestore'] = True
589 createopts['shallowfilestore'] = True
587
590
588 if srcpeer.capable(b'lfs-serve'):
591 if srcpeer.capable(b'lfs-serve'):
589 # Repository creation honors the config if it disabled the extension, so
592 # Repository creation honors the config if it disabled the extension, so
590 # we can't just announce that lfs will be enabled. This check avoids
593 # we can't just announce that lfs will be enabled. This check avoids
591 # saying that lfs will be enabled, and then saying it's an unknown
594 # saying that lfs will be enabled, and then saying it's an unknown
592 # feature. The lfs creation option is set in either case so that a
595 # feature. The lfs creation option is set in either case so that a
593 # requirement is added. If the extension is explicitly disabled but the
596 # requirement is added. If the extension is explicitly disabled but the
594 # requirement is set, the clone aborts early, before transferring any
597 # requirement is set, the clone aborts early, before transferring any
595 # data.
598 # data.
596 createopts['lfs'] = True
599 createopts['lfs'] = True
597
600
598 if extensions.disabledext('lfs'):
601 if extensions.disabledext('lfs'):
599 ui.status(_('(remote is using large file support (lfs), but it is '
602 ui.status(_('(remote is using large file support (lfs), but it is '
600 'explicitly disabled in the local configuration)\n'))
603 'explicitly disabled in the local configuration)\n'))
601 else:
604 else:
602 ui.status(_('(remote is using large file support (lfs); lfs will '
605 ui.status(_('(remote is using large file support (lfs); lfs will '
603 'be enabled for this repository)\n'))
606 'be enabled for this repository)\n'))
604
607
605 shareopts = shareopts or {}
608 shareopts = shareopts or {}
606 sharepool = shareopts.get('pool')
609 sharepool = shareopts.get('pool')
607 sharenamemode = shareopts.get('mode')
610 sharenamemode = shareopts.get('mode')
608 if sharepool and islocal(dest):
611 if sharepool and islocal(dest):
609 sharepath = None
612 sharepath = None
610 if sharenamemode == 'identity':
613 if sharenamemode == 'identity':
611 # Resolve the name from the initial changeset in the remote
614 # Resolve the name from the initial changeset in the remote
612 # repository. This returns nullid when the remote is empty. It
615 # repository. This returns nullid when the remote is empty. It
613 # raises RepoLookupError if revision 0 is filtered or otherwise
616 # raises RepoLookupError if revision 0 is filtered or otherwise
614 # not available. If we fail to resolve, sharing is not enabled.
617 # not available. If we fail to resolve, sharing is not enabled.
615 try:
618 try:
616 with srcpeer.commandexecutor() as e:
619 with srcpeer.commandexecutor() as e:
617 rootnode = e.callcommand('lookup', {
620 rootnode = e.callcommand('lookup', {
618 'key': '0',
621 'key': '0',
619 }).result()
622 }).result()
620
623
621 if rootnode != node.nullid:
624 if rootnode != node.nullid:
622 sharepath = os.path.join(sharepool, node.hex(rootnode))
625 sharepath = os.path.join(sharepool, node.hex(rootnode))
623 else:
626 else:
624 ui.status(_('(not using pooled storage: '
627 ui.status(_('(not using pooled storage: '
625 'remote appears to be empty)\n'))
628 'remote appears to be empty)\n'))
626 except error.RepoLookupError:
629 except error.RepoLookupError:
627 ui.status(_('(not using pooled storage: '
630 ui.status(_('(not using pooled storage: '
628 'unable to resolve identity of remote)\n'))
631 'unable to resolve identity of remote)\n'))
629 elif sharenamemode == 'remote':
632 elif sharenamemode == 'remote':
630 sharepath = os.path.join(
633 sharepath = os.path.join(
631 sharepool, node.hex(hashlib.sha1(source).digest()))
634 sharepool, node.hex(hashlib.sha1(source).digest()))
632 else:
635 else:
633 raise error.Abort(_('unknown share naming mode: %s') %
636 raise error.Abort(_('unknown share naming mode: %s') %
634 sharenamemode)
637 sharenamemode)
635
638
636 # TODO this is a somewhat arbitrary restriction.
639 # TODO this is a somewhat arbitrary restriction.
637 if narrow:
640 if narrow:
638 ui.status(_('(pooled storage not supported for narrow clones)\n'))
641 ui.status(_('(pooled storage not supported for narrow clones)\n'))
639 sharepath = None
642 sharepath = None
640
643
641 if sharepath:
644 if sharepath:
642 return clonewithshare(ui, peeropts, sharepath, source, srcpeer,
645 return clonewithshare(ui, peeropts, sharepath, source, srcpeer,
643 dest, pull=pull, rev=revs, update=update,
646 dest, pull=pull, rev=revs, update=update,
644 stream=stream)
647 stream=stream)
645
648
646 srclock = destlock = cleandir = None
649 srclock = destlock = cleandir = None
647 srcrepo = srcpeer.local()
650 srcrepo = srcpeer.local()
648 try:
651 try:
649 abspath = origsource
652 abspath = origsource
650 if islocal(origsource):
653 if islocal(origsource):
651 abspath = os.path.abspath(util.urllocalpath(origsource))
654 abspath = os.path.abspath(util.urllocalpath(origsource))
652
655
653 if islocal(dest):
656 if islocal(dest):
654 cleandir = dest
657 cleandir = dest
655
658
656 copy = False
659 copy = False
657 if (srcrepo and srcrepo.cancopy() and islocal(dest)
660 if (srcrepo and srcrepo.cancopy() and islocal(dest)
658 and not phases.hassecret(srcrepo)):
661 and not phases.hassecret(srcrepo)):
659 copy = not pull and not revs
662 copy = not pull and not revs
660
663
661 # TODO this is a somewhat arbitrary restriction.
664 # TODO this is a somewhat arbitrary restriction.
662 if narrow:
665 if narrow:
663 copy = False
666 copy = False
664
667
665 if copy:
668 if copy:
666 try:
669 try:
667 # we use a lock here because if we race with commit, we
670 # we use a lock here because if we race with commit, we
668 # can end up with extra data in the cloned revlogs that's
671 # can end up with extra data in the cloned revlogs that's
669 # not pointed to by changesets, thus causing verify to
672 # not pointed to by changesets, thus causing verify to
670 # fail
673 # fail
671 srclock = srcrepo.lock(wait=False)
674 srclock = srcrepo.lock(wait=False)
672 except error.LockError:
675 except error.LockError:
673 copy = False
676 copy = False
674
677
675 if copy:
678 if copy:
676 srcrepo.hook('preoutgoing', throw=True, source='clone')
679 srcrepo.hook('preoutgoing', throw=True, source='clone')
677 hgdir = os.path.realpath(os.path.join(dest, ".hg"))
680 hgdir = os.path.realpath(os.path.join(dest, ".hg"))
678 if not os.path.exists(dest):
681 if not os.path.exists(dest):
679 util.makedirs(dest)
682 util.makedirs(dest)
680 else:
683 else:
681 # only clean up directories we create ourselves
684 # only clean up directories we create ourselves
682 cleandir = hgdir
685 cleandir = hgdir
683 try:
686 try:
684 destpath = hgdir
687 destpath = hgdir
685 util.makedir(destpath, notindexed=True)
688 util.makedir(destpath, notindexed=True)
686 except OSError as inst:
689 except OSError as inst:
687 if inst.errno == errno.EEXIST:
690 if inst.errno == errno.EEXIST:
688 cleandir = None
691 cleandir = None
689 raise error.Abort(_("destination '%s' already exists")
692 raise error.Abort(_("destination '%s' already exists")
690 % dest)
693 % dest)
691 raise
694 raise
692
695
693 destlock = copystore(ui, srcrepo, destpath)
696 destlock = copystore(ui, srcrepo, destpath)
694 # copy bookmarks over
697 # copy bookmarks over
695 srcbookmarks = srcrepo.vfs.join('bookmarks')
698 srcbookmarks = srcrepo.vfs.join('bookmarks')
696 dstbookmarks = os.path.join(destpath, 'bookmarks')
699 dstbookmarks = os.path.join(destpath, 'bookmarks')
697 if os.path.exists(srcbookmarks):
700 if os.path.exists(srcbookmarks):
698 util.copyfile(srcbookmarks, dstbookmarks)
701 util.copyfile(srcbookmarks, dstbookmarks)
699
702
700 dstcachedir = os.path.join(destpath, 'cache')
703 dstcachedir = os.path.join(destpath, 'cache')
701 for cache in cacheutil.cachetocopy(srcrepo):
704 for cache in cacheutil.cachetocopy(srcrepo):
702 _copycache(srcrepo, dstcachedir, cache)
705 _copycache(srcrepo, dstcachedir, cache)
703
706
704 # we need to re-init the repo after manually copying the data
707 # we need to re-init the repo after manually copying the data
705 # into it
708 # into it
706 destpeer = peer(srcrepo, peeropts, dest)
709 destpeer = peer(srcrepo, peeropts, dest)
707 srcrepo.hook('outgoing', source='clone',
710 srcrepo.hook('outgoing', source='clone',
708 node=node.hex(node.nullid))
711 node=node.hex(node.nullid))
709 else:
712 else:
710 try:
713 try:
711 # only pass ui when no srcrepo
714 # only pass ui when no srcrepo
712 destpeer = peer(srcrepo or ui, peeropts, dest, create=True,
715 destpeer = peer(srcrepo or ui, peeropts, dest, create=True,
713 createopts=createopts)
716 createopts=createopts)
714 except OSError as inst:
717 except OSError as inst:
715 if inst.errno == errno.EEXIST:
718 if inst.errno == errno.EEXIST:
716 cleandir = None
719 cleandir = None
717 raise error.Abort(_("destination '%s' already exists")
720 raise error.Abort(_("destination '%s' already exists")
718 % dest)
721 % dest)
719 raise
722 raise
720
723
721 if revs:
724 if revs:
722 if not srcpeer.capable('lookup'):
725 if not srcpeer.capable('lookup'):
723 raise error.Abort(_("src repository does not support "
726 raise error.Abort(_("src repository does not support "
724 "revision lookup and so doesn't "
727 "revision lookup and so doesn't "
725 "support clone by revision"))
728 "support clone by revision"))
726
729
727 # TODO this is batchable.
730 # TODO this is batchable.
728 remoterevs = []
731 remoterevs = []
729 for rev in revs:
732 for rev in revs:
730 with srcpeer.commandexecutor() as e:
733 with srcpeer.commandexecutor() as e:
731 remoterevs.append(e.callcommand('lookup', {
734 remoterevs.append(e.callcommand('lookup', {
732 'key': rev,
735 'key': rev,
733 }).result())
736 }).result())
734 revs = remoterevs
737 revs = remoterevs
735
738
736 checkout = revs[0]
739 checkout = revs[0]
737 else:
740 else:
738 revs = None
741 revs = None
739 local = destpeer.local()
742 local = destpeer.local()
740 if local:
743 if local:
741 if narrow:
744 if narrow:
742 with local.wlock(), local.lock():
745 with local.wlock(), local.lock():
743 local.setnarrowpats(storeincludepats, storeexcludepats)
746 local.setnarrowpats(storeincludepats, storeexcludepats)
744 narrowspec.copytoworkingcopy(local)
747 narrowspec.copytoworkingcopy(local)
745
748
746 u = util.url(abspath)
749 u = util.url(abspath)
747 defaulturl = bytes(u)
750 defaulturl = bytes(u)
748 local.ui.setconfig('paths', 'default', defaulturl, 'clone')
751 local.ui.setconfig('paths', 'default', defaulturl, 'clone')
749 if not stream:
752 if not stream:
750 if pull:
753 if pull:
751 stream = False
754 stream = False
752 else:
755 else:
753 stream = None
756 stream = None
754 # internal config: ui.quietbookmarkmove
757 # internal config: ui.quietbookmarkmove
755 overrides = {('ui', 'quietbookmarkmove'): True}
758 overrides = {('ui', 'quietbookmarkmove'): True}
756 with local.ui.configoverride(overrides, 'clone'):
759 with local.ui.configoverride(overrides, 'clone'):
757 exchange.pull(local, srcpeer, revs,
760 exchange.pull(local, srcpeer, revs,
758 streamclonerequested=stream,
761 streamclonerequested=stream,
759 includepats=storeincludepats,
762 includepats=storeincludepats,
760 excludepats=storeexcludepats,
763 excludepats=storeexcludepats,
761 depth=depth)
764 depth=depth)
762 elif srcrepo:
765 elif srcrepo:
763 # TODO lift restriction once exchange.push() accepts narrow
766 # TODO lift restriction once exchange.push() accepts narrow
764 # push.
767 # push.
765 if narrow:
768 if narrow:
766 raise error.Abort(_('narrow clone not available for '
769 raise error.Abort(_('narrow clone not available for '
767 'remote destinations'))
770 'remote destinations'))
768
771
769 exchange.push(srcrepo, destpeer, revs=revs,
772 exchange.push(srcrepo, destpeer, revs=revs,
770 bookmarks=srcrepo._bookmarks.keys())
773 bookmarks=srcrepo._bookmarks.keys())
771 else:
774 else:
772 raise error.Abort(_("clone from remote to remote not supported")
775 raise error.Abort(_("clone from remote to remote not supported")
773 )
776 )
774
777
775 cleandir = None
778 cleandir = None
776
779
777 destrepo = destpeer.local()
780 destrepo = destpeer.local()
778 if destrepo:
781 if destrepo:
779 template = uimod.samplehgrcs['cloned']
782 template = uimod.samplehgrcs['cloned']
780 u = util.url(abspath)
783 u = util.url(abspath)
781 u.passwd = None
784 u.passwd = None
782 defaulturl = bytes(u)
785 defaulturl = bytes(u)
783 destrepo.vfs.write('hgrc', util.tonativeeol(template % defaulturl))
786 destrepo.vfs.write('hgrc', util.tonativeeol(template % defaulturl))
784 destrepo.ui.setconfig('paths', 'default', defaulturl, 'clone')
787 destrepo.ui.setconfig('paths', 'default', defaulturl, 'clone')
785
788
786 if ui.configbool('experimental', 'remotenames'):
789 if ui.configbool('experimental', 'remotenames'):
787 logexchange.pullremotenames(destrepo, srcpeer)
790 logexchange.pullremotenames(destrepo, srcpeer)
788
791
789 if update:
792 if update:
790 if update is not True:
793 if update is not True:
791 with srcpeer.commandexecutor() as e:
794 with srcpeer.commandexecutor() as e:
792 checkout = e.callcommand('lookup', {
795 checkout = e.callcommand('lookup', {
793 'key': update,
796 'key': update,
794 }).result()
797 }).result()
795
798
796 uprev = None
799 uprev = None
797 status = None
800 status = None
798 if checkout is not None:
801 if checkout is not None:
799 # Some extensions (at least hg-git and hg-subversion) have
802 # Some extensions (at least hg-git and hg-subversion) have
800 # a peer.lookup() implementation that returns a name instead
803 # a peer.lookup() implementation that returns a name instead
801 # of a nodeid. We work around it here until we've figured
804 # of a nodeid. We work around it here until we've figured
802 # out a better solution.
805 # out a better solution.
803 if len(checkout) == 20 and checkout in destrepo:
806 if len(checkout) == 20 and checkout in destrepo:
804 uprev = checkout
807 uprev = checkout
805 elif scmutil.isrevsymbol(destrepo, checkout):
808 elif scmutil.isrevsymbol(destrepo, checkout):
806 uprev = scmutil.revsymbol(destrepo, checkout).node()
809 uprev = scmutil.revsymbol(destrepo, checkout).node()
807 else:
810 else:
808 if update is not True:
811 if update is not True:
809 try:
812 try:
810 uprev = destrepo.lookup(update)
813 uprev = destrepo.lookup(update)
811 except error.RepoLookupError:
814 except error.RepoLookupError:
812 pass
815 pass
813 if uprev is None:
816 if uprev is None:
814 try:
817 try:
815 uprev = destrepo._bookmarks['@']
818 uprev = destrepo._bookmarks['@']
816 update = '@'
819 update = '@'
817 bn = destrepo[uprev].branch()
820 bn = destrepo[uprev].branch()
818 if bn == 'default':
821 if bn == 'default':
819 status = _("updating to bookmark @\n")
822 status = _("updating to bookmark @\n")
820 else:
823 else:
821 status = (_("updating to bookmark @ on branch %s\n")
824 status = (_("updating to bookmark @ on branch %s\n")
822 % bn)
825 % bn)
823 except KeyError:
826 except KeyError:
824 try:
827 try:
825 uprev = destrepo.branchtip('default')
828 uprev = destrepo.branchtip('default')
826 except error.RepoLookupError:
829 except error.RepoLookupError:
827 uprev = destrepo.lookup('tip')
830 uprev = destrepo.lookup('tip')
828 if not status:
831 if not status:
829 bn = destrepo[uprev].branch()
832 bn = destrepo[uprev].branch()
830 status = _("updating to branch %s\n") % bn
833 status = _("updating to branch %s\n") % bn
831 destrepo.ui.status(status)
834 destrepo.ui.status(status)
832 _update(destrepo, uprev)
835 _update(destrepo, uprev)
833 if update in destrepo._bookmarks:
836 if update in destrepo._bookmarks:
834 bookmarks.activate(destrepo, update)
837 bookmarks.activate(destrepo, update)
835 finally:
838 finally:
836 release(srclock, destlock)
839 release(srclock, destlock)
837 if cleandir is not None:
840 if cleandir is not None:
838 shutil.rmtree(cleandir, True)
841 shutil.rmtree(cleandir, True)
839 if srcpeer is not None:
842 if srcpeer is not None:
840 srcpeer.close()
843 srcpeer.close()
841 return srcpeer, destpeer
844 return srcpeer, destpeer
842
845
843 def _showstats(repo, stats, quietempty=False):
846 def _showstats(repo, stats, quietempty=False):
844 if quietempty and stats.isempty():
847 if quietempty and stats.isempty():
845 return
848 return
846 repo.ui.status(_("%d files updated, %d files merged, "
849 repo.ui.status(_("%d files updated, %d files merged, "
847 "%d files removed, %d files unresolved\n") % (
850 "%d files removed, %d files unresolved\n") % (
848 stats.updatedcount, stats.mergedcount,
851 stats.updatedcount, stats.mergedcount,
849 stats.removedcount, stats.unresolvedcount))
852 stats.removedcount, stats.unresolvedcount))
850
853
851 def updaterepo(repo, node, overwrite, updatecheck=None):
854 def updaterepo(repo, node, overwrite, updatecheck=None):
852 """Update the working directory to node.
855 """Update the working directory to node.
853
856
854 When overwrite is set, changes are clobbered, merged else
857 When overwrite is set, changes are clobbered, merged else
855
858
856 returns stats (see pydoc mercurial.merge.applyupdates)"""
859 returns stats (see pydoc mercurial.merge.applyupdates)"""
857 return mergemod.update(repo, node, branchmerge=False, force=overwrite,
860 return mergemod.update(repo, node, branchmerge=False, force=overwrite,
858 labels=['working copy', 'destination'],
861 labels=['working copy', 'destination'],
859 updatecheck=updatecheck)
862 updatecheck=updatecheck)
860
863
861 def update(repo, node, quietempty=False, updatecheck=None):
864 def update(repo, node, quietempty=False, updatecheck=None):
862 """update the working directory to node"""
865 """update the working directory to node"""
863 stats = updaterepo(repo, node, False, updatecheck=updatecheck)
866 stats = updaterepo(repo, node, False, updatecheck=updatecheck)
864 _showstats(repo, stats, quietempty)
867 _showstats(repo, stats, quietempty)
865 if stats.unresolvedcount:
868 if stats.unresolvedcount:
866 repo.ui.status(_("use 'hg resolve' to retry unresolved file merges\n"))
869 repo.ui.status(_("use 'hg resolve' to retry unresolved file merges\n"))
867 return stats.unresolvedcount > 0
870 return stats.unresolvedcount > 0
868
871
869 # naming conflict in clone()
872 # naming conflict in clone()
870 _update = update
873 _update = update
871
874
872 def clean(repo, node, show_stats=True, quietempty=False):
875 def clean(repo, node, show_stats=True, quietempty=False):
873 """forcibly switch the working directory to node, clobbering changes"""
876 """forcibly switch the working directory to node, clobbering changes"""
874 stats = updaterepo(repo, node, True)
877 stats = updaterepo(repo, node, True)
875 repo.vfs.unlinkpath('graftstate', ignoremissing=True)
878 repo.vfs.unlinkpath('graftstate', ignoremissing=True)
876 if show_stats:
879 if show_stats:
877 _showstats(repo, stats, quietempty)
880 _showstats(repo, stats, quietempty)
878 return stats.unresolvedcount > 0
881 return stats.unresolvedcount > 0
879
882
880 # naming conflict in updatetotally()
883 # naming conflict in updatetotally()
881 _clean = clean
884 _clean = clean
882
885
883 def updatetotally(ui, repo, checkout, brev, clean=False, updatecheck=None):
886 def updatetotally(ui, repo, checkout, brev, clean=False, updatecheck=None):
884 """Update the working directory with extra care for non-file components
887 """Update the working directory with extra care for non-file components
885
888
886 This takes care of non-file components below:
889 This takes care of non-file components below:
887
890
888 :bookmark: might be advanced or (in)activated
891 :bookmark: might be advanced or (in)activated
889
892
890 This takes arguments below:
893 This takes arguments below:
891
894
892 :checkout: to which revision the working directory is updated
895 :checkout: to which revision the working directory is updated
893 :brev: a name, which might be a bookmark to be activated after updating
896 :brev: a name, which might be a bookmark to be activated after updating
894 :clean: whether changes in the working directory can be discarded
897 :clean: whether changes in the working directory can be discarded
895 :updatecheck: how to deal with a dirty working directory
898 :updatecheck: how to deal with a dirty working directory
896
899
897 Valid values for updatecheck are (None => linear):
900 Valid values for updatecheck are (None => linear):
898
901
899 * abort: abort if the working directory is dirty
902 * abort: abort if the working directory is dirty
900 * none: don't check (merge working directory changes into destination)
903 * none: don't check (merge working directory changes into destination)
901 * linear: check that update is linear before merging working directory
904 * linear: check that update is linear before merging working directory
902 changes into destination
905 changes into destination
903 * noconflict: check that the update does not result in file merges
906 * noconflict: check that the update does not result in file merges
904
907
905 This returns whether conflict is detected at updating or not.
908 This returns whether conflict is detected at updating or not.
906 """
909 """
907 if updatecheck is None:
910 if updatecheck is None:
908 updatecheck = ui.config('commands', 'update.check')
911 updatecheck = ui.config('commands', 'update.check')
909 if updatecheck not in ('abort', 'none', 'linear', 'noconflict'):
912 if updatecheck not in ('abort', 'none', 'linear', 'noconflict'):
910 # If not configured, or invalid value configured
913 # If not configured, or invalid value configured
911 updatecheck = 'linear'
914 updatecheck = 'linear'
912 with repo.wlock():
915 with repo.wlock():
913 movemarkfrom = None
916 movemarkfrom = None
914 warndest = False
917 warndest = False
915 if checkout is None:
918 if checkout is None:
916 updata = destutil.destupdate(repo, clean=clean)
919 updata = destutil.destupdate(repo, clean=clean)
917 checkout, movemarkfrom, brev = updata
920 checkout, movemarkfrom, brev = updata
918 warndest = True
921 warndest = True
919
922
920 if clean:
923 if clean:
921 ret = _clean(repo, checkout)
924 ret = _clean(repo, checkout)
922 else:
925 else:
923 if updatecheck == 'abort':
926 if updatecheck == 'abort':
924 cmdutil.bailifchanged(repo, merge=False)
927 cmdutil.bailifchanged(repo, merge=False)
925 updatecheck = 'none'
928 updatecheck = 'none'
926 ret = _update(repo, checkout, updatecheck=updatecheck)
929 ret = _update(repo, checkout, updatecheck=updatecheck)
927
930
928 if not ret and movemarkfrom:
931 if not ret and movemarkfrom:
929 if movemarkfrom == repo['.'].node():
932 if movemarkfrom == repo['.'].node():
930 pass # no-op update
933 pass # no-op update
931 elif bookmarks.update(repo, [movemarkfrom], repo['.'].node()):
934 elif bookmarks.update(repo, [movemarkfrom], repo['.'].node()):
932 b = ui.label(repo._activebookmark, 'bookmarks.active')
935 b = ui.label(repo._activebookmark, 'bookmarks.active')
933 ui.status(_("updating bookmark %s\n") % b)
936 ui.status(_("updating bookmark %s\n") % b)
934 else:
937 else:
935 # this can happen with a non-linear update
938 # this can happen with a non-linear update
936 b = ui.label(repo._activebookmark, 'bookmarks')
939 b = ui.label(repo._activebookmark, 'bookmarks')
937 ui.status(_("(leaving bookmark %s)\n") % b)
940 ui.status(_("(leaving bookmark %s)\n") % b)
938 bookmarks.deactivate(repo)
941 bookmarks.deactivate(repo)
939 elif brev in repo._bookmarks:
942 elif brev in repo._bookmarks:
940 if brev != repo._activebookmark:
943 if brev != repo._activebookmark:
941 b = ui.label(brev, 'bookmarks.active')
944 b = ui.label(brev, 'bookmarks.active')
942 ui.status(_("(activating bookmark %s)\n") % b)
945 ui.status(_("(activating bookmark %s)\n") % b)
943 bookmarks.activate(repo, brev)
946 bookmarks.activate(repo, brev)
944 elif brev:
947 elif brev:
945 if repo._activebookmark:
948 if repo._activebookmark:
946 b = ui.label(repo._activebookmark, 'bookmarks')
949 b = ui.label(repo._activebookmark, 'bookmarks')
947 ui.status(_("(leaving bookmark %s)\n") % b)
950 ui.status(_("(leaving bookmark %s)\n") % b)
948 bookmarks.deactivate(repo)
951 bookmarks.deactivate(repo)
949
952
950 if warndest:
953 if warndest:
951 destutil.statusotherdests(ui, repo)
954 destutil.statusotherdests(ui, repo)
952
955
953 return ret
956 return ret
954
957
955 def merge(repo, node, force=None, remind=True, mergeforce=False, labels=None,
958 def merge(repo, node, force=None, remind=True, mergeforce=False, labels=None,
956 abort=False):
959 abort=False):
957 """Branch merge with node, resolving changes. Return true if any
960 """Branch merge with node, resolving changes. Return true if any
958 unresolved conflicts."""
961 unresolved conflicts."""
959 if abort:
962 if abort:
960 return abortmerge(repo.ui, repo)
963 return abortmerge(repo.ui, repo)
961
964
962 stats = mergemod.update(repo, node, branchmerge=True, force=force,
965 stats = mergemod.update(repo, node, branchmerge=True, force=force,
963 mergeforce=mergeforce, labels=labels)
966 mergeforce=mergeforce, labels=labels)
964 _showstats(repo, stats)
967 _showstats(repo, stats)
965 if stats.unresolvedcount:
968 if stats.unresolvedcount:
966 repo.ui.status(_("use 'hg resolve' to retry unresolved file merges "
969 repo.ui.status(_("use 'hg resolve' to retry unresolved file merges "
967 "or 'hg merge --abort' to abandon\n"))
970 "or 'hg merge --abort' to abandon\n"))
968 elif remind:
971 elif remind:
969 repo.ui.status(_("(branch merge, don't forget to commit)\n"))
972 repo.ui.status(_("(branch merge, don't forget to commit)\n"))
970 return stats.unresolvedcount > 0
973 return stats.unresolvedcount > 0
971
974
972 def abortmerge(ui, repo):
975 def abortmerge(ui, repo):
973 ms = mergemod.mergestate.read(repo)
976 ms = mergemod.mergestate.read(repo)
974 if ms.active():
977 if ms.active():
975 # there were conflicts
978 # there were conflicts
976 node = ms.localctx.hex()
979 node = ms.localctx.hex()
977 else:
980 else:
978 # there were no conficts, mergestate was not stored
981 # there were no conficts, mergestate was not stored
979 node = repo['.'].hex()
982 node = repo['.'].hex()
980
983
981 repo.ui.status(_("aborting the merge, updating back to"
984 repo.ui.status(_("aborting the merge, updating back to"
982 " %s\n") % node[:12])
985 " %s\n") % node[:12])
983 stats = mergemod.update(repo, node, branchmerge=False, force=True)
986 stats = mergemod.update(repo, node, branchmerge=False, force=True)
984 _showstats(repo, stats)
987 _showstats(repo, stats)
985 return stats.unresolvedcount > 0
988 return stats.unresolvedcount > 0
986
989
987 def _incoming(displaychlist, subreporecurse, ui, repo, source,
990 def _incoming(displaychlist, subreporecurse, ui, repo, source,
988 opts, buffered=False):
991 opts, buffered=False):
989 """
992 """
990 Helper for incoming / gincoming.
993 Helper for incoming / gincoming.
991 displaychlist gets called with
994 displaychlist gets called with
992 (remoterepo, incomingchangesetlist, displayer) parameters,
995 (remoterepo, incomingchangesetlist, displayer) parameters,
993 and is supposed to contain only code that can't be unified.
996 and is supposed to contain only code that can't be unified.
994 """
997 """
995 source, branches = parseurl(ui.expandpath(source), opts.get('branch'))
998 source, branches = parseurl(ui.expandpath(source), opts.get('branch'))
996 other = peer(repo, opts, source)
999 other = peer(repo, opts, source)
997 ui.status(_('comparing with %s\n') % util.hidepassword(source))
1000 ui.status(_('comparing with %s\n') % util.hidepassword(source))
998 revs, checkout = addbranchrevs(repo, other, branches, opts.get('rev'))
1001 revs, checkout = addbranchrevs(repo, other, branches, opts.get('rev'))
999
1002
1000 if revs:
1003 if revs:
1001 revs = [other.lookup(rev) for rev in revs]
1004 revs = [other.lookup(rev) for rev in revs]
1002 other, chlist, cleanupfn = bundlerepo.getremotechanges(ui, repo, other,
1005 other, chlist, cleanupfn = bundlerepo.getremotechanges(ui, repo, other,
1003 revs, opts["bundle"], opts["force"])
1006 revs, opts["bundle"], opts["force"])
1004 try:
1007 try:
1005 if not chlist:
1008 if not chlist:
1006 ui.status(_("no changes found\n"))
1009 ui.status(_("no changes found\n"))
1007 return subreporecurse()
1010 return subreporecurse()
1008 ui.pager('incoming')
1011 ui.pager('incoming')
1009 displayer = logcmdutil.changesetdisplayer(ui, other, opts,
1012 displayer = logcmdutil.changesetdisplayer(ui, other, opts,
1010 buffered=buffered)
1013 buffered=buffered)
1011 displaychlist(other, chlist, displayer)
1014 displaychlist(other, chlist, displayer)
1012 displayer.close()
1015 displayer.close()
1013 finally:
1016 finally:
1014 cleanupfn()
1017 cleanupfn()
1015 subreporecurse()
1018 subreporecurse()
1016 return 0 # exit code is zero since we found incoming changes
1019 return 0 # exit code is zero since we found incoming changes
1017
1020
1018 def incoming(ui, repo, source, opts):
1021 def incoming(ui, repo, source, opts):
1019 def subreporecurse():
1022 def subreporecurse():
1020 ret = 1
1023 ret = 1
1021 if opts.get('subrepos'):
1024 if opts.get('subrepos'):
1022 ctx = repo[None]
1025 ctx = repo[None]
1023 for subpath in sorted(ctx.substate):
1026 for subpath in sorted(ctx.substate):
1024 sub = ctx.sub(subpath)
1027 sub = ctx.sub(subpath)
1025 ret = min(ret, sub.incoming(ui, source, opts))
1028 ret = min(ret, sub.incoming(ui, source, opts))
1026 return ret
1029 return ret
1027
1030
1028 def display(other, chlist, displayer):
1031 def display(other, chlist, displayer):
1029 limit = logcmdutil.getlimit(opts)
1032 limit = logcmdutil.getlimit(opts)
1030 if opts.get('newest_first'):
1033 if opts.get('newest_first'):
1031 chlist.reverse()
1034 chlist.reverse()
1032 count = 0
1035 count = 0
1033 for n in chlist:
1036 for n in chlist:
1034 if limit is not None and count >= limit:
1037 if limit is not None and count >= limit:
1035 break
1038 break
1036 parents = [p for p in other.changelog.parents(n) if p != nullid]
1039 parents = [p for p in other.changelog.parents(n) if p != nullid]
1037 if opts.get('no_merges') and len(parents) == 2:
1040 if opts.get('no_merges') and len(parents) == 2:
1038 continue
1041 continue
1039 count += 1
1042 count += 1
1040 displayer.show(other[n])
1043 displayer.show(other[n])
1041 return _incoming(display, subreporecurse, ui, repo, source, opts)
1044 return _incoming(display, subreporecurse, ui, repo, source, opts)
1042
1045
1043 def _outgoing(ui, repo, dest, opts):
1046 def _outgoing(ui, repo, dest, opts):
1044 path = ui.paths.getpath(dest, default=('default-push', 'default'))
1047 path = ui.paths.getpath(dest, default=('default-push', 'default'))
1045 if not path:
1048 if not path:
1046 raise error.Abort(_('default repository not configured!'),
1049 raise error.Abort(_('default repository not configured!'),
1047 hint=_("see 'hg help config.paths'"))
1050 hint=_("see 'hg help config.paths'"))
1048 dest = path.pushloc or path.loc
1051 dest = path.pushloc or path.loc
1049 branches = path.branch, opts.get('branch') or []
1052 branches = path.branch, opts.get('branch') or []
1050
1053
1051 ui.status(_('comparing with %s\n') % util.hidepassword(dest))
1054 ui.status(_('comparing with %s\n') % util.hidepassword(dest))
1052 revs, checkout = addbranchrevs(repo, repo, branches, opts.get('rev'))
1055 revs, checkout = addbranchrevs(repo, repo, branches, opts.get('rev'))
1053 if revs:
1056 if revs:
1054 revs = [repo[rev].node() for rev in scmutil.revrange(repo, revs)]
1057 revs = [repo[rev].node() for rev in scmutil.revrange(repo, revs)]
1055
1058
1056 other = peer(repo, opts, dest)
1059 other = peer(repo, opts, dest)
1057 outgoing = discovery.findcommonoutgoing(repo, other, revs,
1060 outgoing = discovery.findcommonoutgoing(repo, other, revs,
1058 force=opts.get('force'))
1061 force=opts.get('force'))
1059 o = outgoing.missing
1062 o = outgoing.missing
1060 if not o:
1063 if not o:
1061 scmutil.nochangesfound(repo.ui, repo, outgoing.excluded)
1064 scmutil.nochangesfound(repo.ui, repo, outgoing.excluded)
1062 return o, other
1065 return o, other
1063
1066
1064 def outgoing(ui, repo, dest, opts):
1067 def outgoing(ui, repo, dest, opts):
1065 def recurse():
1068 def recurse():
1066 ret = 1
1069 ret = 1
1067 if opts.get('subrepos'):
1070 if opts.get('subrepos'):
1068 ctx = repo[None]
1071 ctx = repo[None]
1069 for subpath in sorted(ctx.substate):
1072 for subpath in sorted(ctx.substate):
1070 sub = ctx.sub(subpath)
1073 sub = ctx.sub(subpath)
1071 ret = min(ret, sub.outgoing(ui, dest, opts))
1074 ret = min(ret, sub.outgoing(ui, dest, opts))
1072 return ret
1075 return ret
1073
1076
1074 limit = logcmdutil.getlimit(opts)
1077 limit = logcmdutil.getlimit(opts)
1075 o, other = _outgoing(ui, repo, dest, opts)
1078 o, other = _outgoing(ui, repo, dest, opts)
1076 if not o:
1079 if not o:
1077 cmdutil.outgoinghooks(ui, repo, other, opts, o)
1080 cmdutil.outgoinghooks(ui, repo, other, opts, o)
1078 return recurse()
1081 return recurse()
1079
1082
1080 if opts.get('newest_first'):
1083 if opts.get('newest_first'):
1081 o.reverse()
1084 o.reverse()
1082 ui.pager('outgoing')
1085 ui.pager('outgoing')
1083 displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
1086 displayer = logcmdutil.changesetdisplayer(ui, repo, opts)
1084 count = 0
1087 count = 0
1085 for n in o:
1088 for n in o:
1086 if limit is not None and count >= limit:
1089 if limit is not None and count >= limit:
1087 break
1090 break
1088 parents = [p for p in repo.changelog.parents(n) if p != nullid]
1091 parents = [p for p in repo.changelog.parents(n) if p != nullid]
1089 if opts.get('no_merges') and len(parents) == 2:
1092 if opts.get('no_merges') and len(parents) == 2:
1090 continue
1093 continue
1091 count += 1
1094 count += 1
1092 displayer.show(repo[n])
1095 displayer.show(repo[n])
1093 displayer.close()
1096 displayer.close()
1094 cmdutil.outgoinghooks(ui, repo, other, opts, o)
1097 cmdutil.outgoinghooks(ui, repo, other, opts, o)
1095 recurse()
1098 recurse()
1096 return 0 # exit code is zero since we found outgoing changes
1099 return 0 # exit code is zero since we found outgoing changes
1097
1100
1098 def verify(repo, level=None):
1101 def verify(repo, level=None):
1099 """verify the consistency of a repository"""
1102 """verify the consistency of a repository"""
1100 ret = verifymod.verify(repo, level=level)
1103 ret = verifymod.verify(repo, level=level)
1101
1104
1102 # Broken subrepo references in hidden csets don't seem worth worrying about,
1105 # Broken subrepo references in hidden csets don't seem worth worrying about,
1103 # since they can't be pushed/pulled, and --hidden can be used if they are a
1106 # since they can't be pushed/pulled, and --hidden can be used if they are a
1104 # concern.
1107 # concern.
1105
1108
1106 # pathto() is needed for -R case
1109 # pathto() is needed for -R case
1107 revs = repo.revs("filelog(%s)",
1110 revs = repo.revs("filelog(%s)",
1108 util.pathto(repo.root, repo.getcwd(), '.hgsubstate'))
1111 util.pathto(repo.root, repo.getcwd(), '.hgsubstate'))
1109
1112
1110 if revs:
1113 if revs:
1111 repo.ui.status(_('checking subrepo links\n'))
1114 repo.ui.status(_('checking subrepo links\n'))
1112 for rev in revs:
1115 for rev in revs:
1113 ctx = repo[rev]
1116 ctx = repo[rev]
1114 try:
1117 try:
1115 for subpath in ctx.substate:
1118 for subpath in ctx.substate:
1116 try:
1119 try:
1117 ret = (ctx.sub(subpath, allowcreate=False).verify()
1120 ret = (ctx.sub(subpath, allowcreate=False).verify()
1118 or ret)
1121 or ret)
1119 except error.RepoError as e:
1122 except error.RepoError as e:
1120 repo.ui.warn(('%d: %s\n') % (rev, e))
1123 repo.ui.warn(('%d: %s\n') % (rev, e))
1121 except Exception:
1124 except Exception:
1122 repo.ui.warn(_('.hgsubstate is corrupt in revision %s\n') %
1125 repo.ui.warn(_('.hgsubstate is corrupt in revision %s\n') %
1123 node.short(ctx.node()))
1126 node.short(ctx.node()))
1124
1127
1125 return ret
1128 return ret
1126
1129
1127 def remoteui(src, opts):
1130 def remoteui(src, opts):
1128 'build a remote ui from ui or repo and opts'
1131 'build a remote ui from ui or repo and opts'
1129 if util.safehasattr(src, 'baseui'): # looks like a repository
1132 if util.safehasattr(src, 'baseui'): # looks like a repository
1130 dst = src.baseui.copy() # drop repo-specific config
1133 dst = src.baseui.copy() # drop repo-specific config
1131 src = src.ui # copy target options from repo
1134 src = src.ui # copy target options from repo
1132 else: # assume it's a global ui object
1135 else: # assume it's a global ui object
1133 dst = src.copy() # keep all global options
1136 dst = src.copy() # keep all global options
1134
1137
1135 # copy ssh-specific options
1138 # copy ssh-specific options
1136 for o in 'ssh', 'remotecmd':
1139 for o in 'ssh', 'remotecmd':
1137 v = opts.get(o) or src.config('ui', o)
1140 v = opts.get(o) or src.config('ui', o)
1138 if v:
1141 if v:
1139 dst.setconfig("ui", o, v, 'copied')
1142 dst.setconfig("ui", o, v, 'copied')
1140
1143
1141 # copy bundle-specific options
1144 # copy bundle-specific options
1142 r = src.config('bundle', 'mainreporoot')
1145 r = src.config('bundle', 'mainreporoot')
1143 if r:
1146 if r:
1144 dst.setconfig('bundle', 'mainreporoot', r, 'copied')
1147 dst.setconfig('bundle', 'mainreporoot', r, 'copied')
1145
1148
1146 # copy selected local settings to the remote ui
1149 # copy selected local settings to the remote ui
1147 for sect in ('auth', 'hostfingerprints', 'hostsecurity', 'http_proxy'):
1150 for sect in ('auth', 'hostfingerprints', 'hostsecurity', 'http_proxy'):
1148 for key, val in src.configitems(sect):
1151 for key, val in src.configitems(sect):
1149 dst.setconfig(sect, key, val, 'copied')
1152 dst.setconfig(sect, key, val, 'copied')
1150 v = src.config('web', 'cacerts')
1153 v = src.config('web', 'cacerts')
1151 if v:
1154 if v:
1152 dst.setconfig('web', 'cacerts', util.expandpath(v), 'copied')
1155 dst.setconfig('web', 'cacerts', util.expandpath(v), 'copied')
1153
1156
1154 return dst
1157 return dst
1155
1158
1156 # Files of interest
1159 # Files of interest
1157 # Used to check if the repository has changed looking at mtime and size of
1160 # Used to check if the repository has changed looking at mtime and size of
1158 # these files.
1161 # these files.
1159 foi = [('spath', '00changelog.i'),
1162 foi = [('spath', '00changelog.i'),
1160 ('spath', 'phaseroots'), # ! phase can change content at the same size
1163 ('spath', 'phaseroots'), # ! phase can change content at the same size
1161 ('spath', 'obsstore'),
1164 ('spath', 'obsstore'),
1162 ('path', 'bookmarks'), # ! bookmark can change content at the same size
1165 ('path', 'bookmarks'), # ! bookmark can change content at the same size
1163 ]
1166 ]
1164
1167
1165 class cachedlocalrepo(object):
1168 class cachedlocalrepo(object):
1166 """Holds a localrepository that can be cached and reused."""
1169 """Holds a localrepository that can be cached and reused."""
1167
1170
1168 def __init__(self, repo):
1171 def __init__(self, repo):
1169 """Create a new cached repo from an existing repo.
1172 """Create a new cached repo from an existing repo.
1170
1173
1171 We assume the passed in repo was recently created. If the
1174 We assume the passed in repo was recently created. If the
1172 repo has changed between when it was created and when it was
1175 repo has changed between when it was created and when it was
1173 turned into a cache, it may not refresh properly.
1176 turned into a cache, it may not refresh properly.
1174 """
1177 """
1175 assert isinstance(repo, localrepo.localrepository)
1178 assert isinstance(repo, localrepo.localrepository)
1176 self._repo = repo
1179 self._repo = repo
1177 self._state, self.mtime = self._repostate()
1180 self._state, self.mtime = self._repostate()
1178 self._filtername = repo.filtername
1181 self._filtername = repo.filtername
1179
1182
1180 def fetch(self):
1183 def fetch(self):
1181 """Refresh (if necessary) and return a repository.
1184 """Refresh (if necessary) and return a repository.
1182
1185
1183 If the cached instance is out of date, it will be recreated
1186 If the cached instance is out of date, it will be recreated
1184 automatically and returned.
1187 automatically and returned.
1185
1188
1186 Returns a tuple of the repo and a boolean indicating whether a new
1189 Returns a tuple of the repo and a boolean indicating whether a new
1187 repo instance was created.
1190 repo instance was created.
1188 """
1191 """
1189 # We compare the mtimes and sizes of some well-known files to
1192 # We compare the mtimes and sizes of some well-known files to
1190 # determine if the repo changed. This is not precise, as mtimes
1193 # determine if the repo changed. This is not precise, as mtimes
1191 # are susceptible to clock skew and imprecise filesystems and
1194 # are susceptible to clock skew and imprecise filesystems and
1192 # file content can change while maintaining the same size.
1195 # file content can change while maintaining the same size.
1193
1196
1194 state, mtime = self._repostate()
1197 state, mtime = self._repostate()
1195 if state == self._state:
1198 if state == self._state:
1196 return self._repo, False
1199 return self._repo, False
1197
1200
1198 repo = repository(self._repo.baseui, self._repo.url())
1201 repo = repository(self._repo.baseui, self._repo.url())
1199 if self._filtername:
1202 if self._filtername:
1200 self._repo = repo.filtered(self._filtername)
1203 self._repo = repo.filtered(self._filtername)
1201 else:
1204 else:
1202 self._repo = repo.unfiltered()
1205 self._repo = repo.unfiltered()
1203 self._state = state
1206 self._state = state
1204 self.mtime = mtime
1207 self.mtime = mtime
1205
1208
1206 return self._repo, True
1209 return self._repo, True
1207
1210
1208 def _repostate(self):
1211 def _repostate(self):
1209 state = []
1212 state = []
1210 maxmtime = -1
1213 maxmtime = -1
1211 for attr, fname in foi:
1214 for attr, fname in foi:
1212 prefix = getattr(self._repo, attr)
1215 prefix = getattr(self._repo, attr)
1213 p = os.path.join(prefix, fname)
1216 p = os.path.join(prefix, fname)
1214 try:
1217 try:
1215 st = os.stat(p)
1218 st = os.stat(p)
1216 except OSError:
1219 except OSError:
1217 st = os.stat(prefix)
1220 st = os.stat(prefix)
1218 state.append((st[stat.ST_MTIME], st.st_size))
1221 state.append((st[stat.ST_MTIME], st.st_size))
1219 maxmtime = max(maxmtime, st[stat.ST_MTIME])
1222 maxmtime = max(maxmtime, st[stat.ST_MTIME])
1220
1223
1221 return tuple(state), maxmtime
1224 return tuple(state), maxmtime
1222
1225
1223 def copy(self):
1226 def copy(self):
1224 """Obtain a copy of this class instance.
1227 """Obtain a copy of this class instance.
1225
1228
1226 A new localrepository instance is obtained. The new instance should be
1229 A new localrepository instance is obtained. The new instance should be
1227 completely independent of the original.
1230 completely independent of the original.
1228 """
1231 """
1229 repo = repository(self._repo.baseui, self._repo.origroot)
1232 repo = repository(self._repo.baseui, self._repo.origroot)
1230 if self._filtername:
1233 if self._filtername:
1231 repo = repo.filtered(self._filtername)
1234 repo = repo.filtered(self._filtername)
1232 else:
1235 else:
1233 repo = repo.unfiltered()
1236 repo = repo.unfiltered()
1234 c = cachedlocalrepo(repo)
1237 c = cachedlocalrepo(repo)
1235 c._state = self._state
1238 c._state = self._state
1236 c.mtime = self.mtime
1239 c.mtime = self.mtime
1237 return c
1240 return c
@@ -1,1010 +1,1012
1 # httppeer.py - HTTP repository proxy classes for mercurial
1 # httppeer.py - HTTP repository proxy classes for mercurial
2 #
2 #
3 # Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
4 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8
8
9 from __future__ import absolute_import
9 from __future__ import absolute_import
10
10
11 import errno
11 import errno
12 import io
12 import io
13 import os
13 import os
14 import socket
14 import socket
15 import struct
15 import struct
16 import weakref
16 import weakref
17
17
18 from .i18n import _
18 from .i18n import _
19 from .interfaces import (
20 repository,
21 )
19 from . import (
22 from . import (
20 bundle2,
23 bundle2,
21 error,
24 error,
22 httpconnection,
25 httpconnection,
23 pycompat,
26 pycompat,
24 repository,
25 statichttprepo,
27 statichttprepo,
26 url as urlmod,
28 url as urlmod,
27 util,
29 util,
28 wireprotoframing,
30 wireprotoframing,
29 wireprototypes,
31 wireprototypes,
30 wireprotov1peer,
32 wireprotov1peer,
31 wireprotov2peer,
33 wireprotov2peer,
32 wireprotov2server,
34 wireprotov2server,
33 )
35 )
34 from .utils import (
36 from .utils import (
35 cborutil,
37 cborutil,
36 interfaceutil,
38 interfaceutil,
37 stringutil,
39 stringutil,
38 )
40 )
39
41
40 httplib = util.httplib
42 httplib = util.httplib
41 urlerr = util.urlerr
43 urlerr = util.urlerr
42 urlreq = util.urlreq
44 urlreq = util.urlreq
43
45
44 def encodevalueinheaders(value, header, limit):
46 def encodevalueinheaders(value, header, limit):
45 """Encode a string value into multiple HTTP headers.
47 """Encode a string value into multiple HTTP headers.
46
48
47 ``value`` will be encoded into 1 or more HTTP headers with the names
49 ``value`` will be encoded into 1 or more HTTP headers with the names
48 ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
50 ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
49 name + value will be at most ``limit`` bytes long.
51 name + value will be at most ``limit`` bytes long.
50
52
51 Returns an iterable of 2-tuples consisting of header names and
53 Returns an iterable of 2-tuples consisting of header names and
52 values as native strings.
54 values as native strings.
53 """
55 """
54 # HTTP Headers are ASCII. Python 3 requires them to be unicodes,
56 # HTTP Headers are ASCII. Python 3 requires them to be unicodes,
55 # not bytes. This function always takes bytes in as arguments.
57 # not bytes. This function always takes bytes in as arguments.
56 fmt = pycompat.strurl(header) + r'-%s'
58 fmt = pycompat.strurl(header) + r'-%s'
57 # Note: it is *NOT* a bug that the last bit here is a bytestring
59 # Note: it is *NOT* a bug that the last bit here is a bytestring
58 # and not a unicode: we're just getting the encoded length anyway,
60 # and not a unicode: we're just getting the encoded length anyway,
59 # and using an r-string to make it portable between Python 2 and 3
61 # and using an r-string to make it portable between Python 2 and 3
60 # doesn't work because then the \r is a literal backslash-r
62 # doesn't work because then the \r is a literal backslash-r
61 # instead of a carriage return.
63 # instead of a carriage return.
62 valuelen = limit - len(fmt % r'000') - len(': \r\n')
64 valuelen = limit - len(fmt % r'000') - len(': \r\n')
63 result = []
65 result = []
64
66
65 n = 0
67 n = 0
66 for i in pycompat.xrange(0, len(value), valuelen):
68 for i in pycompat.xrange(0, len(value), valuelen):
67 n += 1
69 n += 1
68 result.append((fmt % str(n), pycompat.strurl(value[i:i + valuelen])))
70 result.append((fmt % str(n), pycompat.strurl(value[i:i + valuelen])))
69
71
70 return result
72 return result
71
73
72 class _multifile(object):
74 class _multifile(object):
73 def __init__(self, *fileobjs):
75 def __init__(self, *fileobjs):
74 for f in fileobjs:
76 for f in fileobjs:
75 if not util.safehasattr(f, 'length'):
77 if not util.safehasattr(f, 'length'):
76 raise ValueError(
78 raise ValueError(
77 '_multifile only supports file objects that '
79 '_multifile only supports file objects that '
78 'have a length but this one does not:', type(f), f)
80 'have a length but this one does not:', type(f), f)
79 self._fileobjs = fileobjs
81 self._fileobjs = fileobjs
80 self._index = 0
82 self._index = 0
81
83
82 @property
84 @property
83 def length(self):
85 def length(self):
84 return sum(f.length for f in self._fileobjs)
86 return sum(f.length for f in self._fileobjs)
85
87
86 def read(self, amt=None):
88 def read(self, amt=None):
87 if amt <= 0:
89 if amt <= 0:
88 return ''.join(f.read() for f in self._fileobjs)
90 return ''.join(f.read() for f in self._fileobjs)
89 parts = []
91 parts = []
90 while amt and self._index < len(self._fileobjs):
92 while amt and self._index < len(self._fileobjs):
91 parts.append(self._fileobjs[self._index].read(amt))
93 parts.append(self._fileobjs[self._index].read(amt))
92 got = len(parts[-1])
94 got = len(parts[-1])
93 if got < amt:
95 if got < amt:
94 self._index += 1
96 self._index += 1
95 amt -= got
97 amt -= got
96 return ''.join(parts)
98 return ''.join(parts)
97
99
98 def seek(self, offset, whence=os.SEEK_SET):
100 def seek(self, offset, whence=os.SEEK_SET):
99 if whence != os.SEEK_SET:
101 if whence != os.SEEK_SET:
100 raise NotImplementedError(
102 raise NotImplementedError(
101 '_multifile does not support anything other'
103 '_multifile does not support anything other'
102 ' than os.SEEK_SET for whence on seek()')
104 ' than os.SEEK_SET for whence on seek()')
103 if offset != 0:
105 if offset != 0:
104 raise NotImplementedError(
106 raise NotImplementedError(
105 '_multifile only supports seeking to start, but that '
107 '_multifile only supports seeking to start, but that '
106 'could be fixed if you need it')
108 'could be fixed if you need it')
107 for f in self._fileobjs:
109 for f in self._fileobjs:
108 f.seek(0)
110 f.seek(0)
109 self._index = 0
111 self._index = 0
110
112
111 def makev1commandrequest(ui, requestbuilder, caps, capablefn,
113 def makev1commandrequest(ui, requestbuilder, caps, capablefn,
112 repobaseurl, cmd, args):
114 repobaseurl, cmd, args):
113 """Make an HTTP request to run a command for a version 1 client.
115 """Make an HTTP request to run a command for a version 1 client.
114
116
115 ``caps`` is a set of known server capabilities. The value may be
117 ``caps`` is a set of known server capabilities. The value may be
116 None if capabilities are not yet known.
118 None if capabilities are not yet known.
117
119
118 ``capablefn`` is a function to evaluate a capability.
120 ``capablefn`` is a function to evaluate a capability.
119
121
120 ``cmd``, ``args``, and ``data`` define the command, its arguments, and
122 ``cmd``, ``args``, and ``data`` define the command, its arguments, and
121 raw data to pass to it.
123 raw data to pass to it.
122 """
124 """
123 if cmd == 'pushkey':
125 if cmd == 'pushkey':
124 args['data'] = ''
126 args['data'] = ''
125 data = args.pop('data', None)
127 data = args.pop('data', None)
126 headers = args.pop('headers', {})
128 headers = args.pop('headers', {})
127
129
128 ui.debug("sending %s command\n" % cmd)
130 ui.debug("sending %s command\n" % cmd)
129 q = [('cmd', cmd)]
131 q = [('cmd', cmd)]
130 headersize = 0
132 headersize = 0
131 # Important: don't use self.capable() here or else you end up
133 # Important: don't use self.capable() here or else you end up
132 # with infinite recursion when trying to look up capabilities
134 # with infinite recursion when trying to look up capabilities
133 # for the first time.
135 # for the first time.
134 postargsok = caps is not None and 'httppostargs' in caps
136 postargsok = caps is not None and 'httppostargs' in caps
135
137
136 # Send arguments via POST.
138 # Send arguments via POST.
137 if postargsok and args:
139 if postargsok and args:
138 strargs = urlreq.urlencode(sorted(args.items()))
140 strargs = urlreq.urlencode(sorted(args.items()))
139 if not data:
141 if not data:
140 data = strargs
142 data = strargs
141 else:
143 else:
142 if isinstance(data, bytes):
144 if isinstance(data, bytes):
143 i = io.BytesIO(data)
145 i = io.BytesIO(data)
144 i.length = len(data)
146 i.length = len(data)
145 data = i
147 data = i
146 argsio = io.BytesIO(strargs)
148 argsio = io.BytesIO(strargs)
147 argsio.length = len(strargs)
149 argsio.length = len(strargs)
148 data = _multifile(argsio, data)
150 data = _multifile(argsio, data)
149 headers[r'X-HgArgs-Post'] = len(strargs)
151 headers[r'X-HgArgs-Post'] = len(strargs)
150 elif args:
152 elif args:
151 # Calling self.capable() can infinite loop if we are calling
153 # Calling self.capable() can infinite loop if we are calling
152 # "capabilities". But that command should never accept wire
154 # "capabilities". But that command should never accept wire
153 # protocol arguments. So this should never happen.
155 # protocol arguments. So this should never happen.
154 assert cmd != 'capabilities'
156 assert cmd != 'capabilities'
155 httpheader = capablefn('httpheader')
157 httpheader = capablefn('httpheader')
156 if httpheader:
158 if httpheader:
157 headersize = int(httpheader.split(',', 1)[0])
159 headersize = int(httpheader.split(',', 1)[0])
158
160
159 # Send arguments via HTTP headers.
161 # Send arguments via HTTP headers.
160 if headersize > 0:
162 if headersize > 0:
161 # The headers can typically carry more data than the URL.
163 # The headers can typically carry more data than the URL.
162 encargs = urlreq.urlencode(sorted(args.items()))
164 encargs = urlreq.urlencode(sorted(args.items()))
163 for header, value in encodevalueinheaders(encargs, 'X-HgArg',
165 for header, value in encodevalueinheaders(encargs, 'X-HgArg',
164 headersize):
166 headersize):
165 headers[header] = value
167 headers[header] = value
166 # Send arguments via query string (Mercurial <1.9).
168 # Send arguments via query string (Mercurial <1.9).
167 else:
169 else:
168 q += sorted(args.items())
170 q += sorted(args.items())
169
171
170 qs = '?%s' % urlreq.urlencode(q)
172 qs = '?%s' % urlreq.urlencode(q)
171 cu = "%s%s" % (repobaseurl, qs)
173 cu = "%s%s" % (repobaseurl, qs)
172 size = 0
174 size = 0
173 if util.safehasattr(data, 'length'):
175 if util.safehasattr(data, 'length'):
174 size = data.length
176 size = data.length
175 elif data is not None:
177 elif data is not None:
176 size = len(data)
178 size = len(data)
177 if data is not None and r'Content-Type' not in headers:
179 if data is not None and r'Content-Type' not in headers:
178 headers[r'Content-Type'] = r'application/mercurial-0.1'
180 headers[r'Content-Type'] = r'application/mercurial-0.1'
179
181
180 # Tell the server we accept application/mercurial-0.2 and multiple
182 # Tell the server we accept application/mercurial-0.2 and multiple
181 # compression formats if the server is capable of emitting those
183 # compression formats if the server is capable of emitting those
182 # payloads.
184 # payloads.
183 # Note: Keep this set empty by default, as client advertisement of
185 # Note: Keep this set empty by default, as client advertisement of
184 # protocol parameters should only occur after the handshake.
186 # protocol parameters should only occur after the handshake.
185 protoparams = set()
187 protoparams = set()
186
188
187 mediatypes = set()
189 mediatypes = set()
188 if caps is not None:
190 if caps is not None:
189 mt = capablefn('httpmediatype')
191 mt = capablefn('httpmediatype')
190 if mt:
192 if mt:
191 protoparams.add('0.1')
193 protoparams.add('0.1')
192 mediatypes = set(mt.split(','))
194 mediatypes = set(mt.split(','))
193
195
194 protoparams.add('partial-pull')
196 protoparams.add('partial-pull')
195
197
196 if '0.2tx' in mediatypes:
198 if '0.2tx' in mediatypes:
197 protoparams.add('0.2')
199 protoparams.add('0.2')
198
200
199 if '0.2tx' in mediatypes and capablefn('compression'):
201 if '0.2tx' in mediatypes and capablefn('compression'):
200 # We /could/ compare supported compression formats and prune
202 # We /could/ compare supported compression formats and prune
201 # non-mutually supported or error if nothing is mutually supported.
203 # non-mutually supported or error if nothing is mutually supported.
202 # For now, send the full list to the server and have it error.
204 # For now, send the full list to the server and have it error.
203 comps = [e.wireprotosupport().name for e in
205 comps = [e.wireprotosupport().name for e in
204 util.compengines.supportedwireengines(util.CLIENTROLE)]
206 util.compengines.supportedwireengines(util.CLIENTROLE)]
205 protoparams.add('comp=%s' % ','.join(comps))
207 protoparams.add('comp=%s' % ','.join(comps))
206
208
207 if protoparams:
209 if protoparams:
208 protoheaders = encodevalueinheaders(' '.join(sorted(protoparams)),
210 protoheaders = encodevalueinheaders(' '.join(sorted(protoparams)),
209 'X-HgProto',
211 'X-HgProto',
210 headersize or 1024)
212 headersize or 1024)
211 for header, value in protoheaders:
213 for header, value in protoheaders:
212 headers[header] = value
214 headers[header] = value
213
215
214 varyheaders = []
216 varyheaders = []
215 for header in headers:
217 for header in headers:
216 if header.lower().startswith(r'x-hg'):
218 if header.lower().startswith(r'x-hg'):
217 varyheaders.append(header)
219 varyheaders.append(header)
218
220
219 if varyheaders:
221 if varyheaders:
220 headers[r'Vary'] = r','.join(sorted(varyheaders))
222 headers[r'Vary'] = r','.join(sorted(varyheaders))
221
223
222 req = requestbuilder(pycompat.strurl(cu), data, headers)
224 req = requestbuilder(pycompat.strurl(cu), data, headers)
223
225
224 if data is not None:
226 if data is not None:
225 ui.debug("sending %d bytes\n" % size)
227 ui.debug("sending %d bytes\n" % size)
226 req.add_unredirected_header(r'Content-Length', r'%d' % size)
228 req.add_unredirected_header(r'Content-Length', r'%d' % size)
227
229
228 return req, cu, qs
230 return req, cu, qs
229
231
230 def _reqdata(req):
232 def _reqdata(req):
231 """Get request data, if any. If no data, returns None."""
233 """Get request data, if any. If no data, returns None."""
232 if pycompat.ispy3:
234 if pycompat.ispy3:
233 return req.data
235 return req.data
234 if not req.has_data():
236 if not req.has_data():
235 return None
237 return None
236 return req.get_data()
238 return req.get_data()
237
239
238 def sendrequest(ui, opener, req):
240 def sendrequest(ui, opener, req):
239 """Send a prepared HTTP request.
241 """Send a prepared HTTP request.
240
242
241 Returns the response object.
243 Returns the response object.
242 """
244 """
243 dbg = ui.debug
245 dbg = ui.debug
244 if (ui.debugflag
246 if (ui.debugflag
245 and ui.configbool('devel', 'debug.peer-request')):
247 and ui.configbool('devel', 'debug.peer-request')):
246 line = 'devel-peer-request: %s\n'
248 line = 'devel-peer-request: %s\n'
247 dbg(line % '%s %s' % (pycompat.bytesurl(req.get_method()),
249 dbg(line % '%s %s' % (pycompat.bytesurl(req.get_method()),
248 pycompat.bytesurl(req.get_full_url())))
250 pycompat.bytesurl(req.get_full_url())))
249 hgargssize = None
251 hgargssize = None
250
252
251 for header, value in sorted(req.header_items()):
253 for header, value in sorted(req.header_items()):
252 header = pycompat.bytesurl(header)
254 header = pycompat.bytesurl(header)
253 value = pycompat.bytesurl(value)
255 value = pycompat.bytesurl(value)
254 if header.startswith('X-hgarg-'):
256 if header.startswith('X-hgarg-'):
255 if hgargssize is None:
257 if hgargssize is None:
256 hgargssize = 0
258 hgargssize = 0
257 hgargssize += len(value)
259 hgargssize += len(value)
258 else:
260 else:
259 dbg(line % ' %s %s' % (header, value))
261 dbg(line % ' %s %s' % (header, value))
260
262
261 if hgargssize is not None:
263 if hgargssize is not None:
262 dbg(line % ' %d bytes of commands arguments in headers'
264 dbg(line % ' %d bytes of commands arguments in headers'
263 % hgargssize)
265 % hgargssize)
264 data = _reqdata(req)
266 data = _reqdata(req)
265 if data is not None:
267 if data is not None:
266 length = getattr(data, 'length', None)
268 length = getattr(data, 'length', None)
267 if length is None:
269 if length is None:
268 length = len(data)
270 length = len(data)
269 dbg(line % ' %d bytes of data' % length)
271 dbg(line % ' %d bytes of data' % length)
270
272
271 start = util.timer()
273 start = util.timer()
272
274
273 res = None
275 res = None
274 try:
276 try:
275 res = opener.open(req)
277 res = opener.open(req)
276 except urlerr.httperror as inst:
278 except urlerr.httperror as inst:
277 if inst.code == 401:
279 if inst.code == 401:
278 raise error.Abort(_('authorization failed'))
280 raise error.Abort(_('authorization failed'))
279 raise
281 raise
280 except httplib.HTTPException as inst:
282 except httplib.HTTPException as inst:
281 ui.debug('http error requesting %s\n' %
283 ui.debug('http error requesting %s\n' %
282 util.hidepassword(req.get_full_url()))
284 util.hidepassword(req.get_full_url()))
283 ui.traceback()
285 ui.traceback()
284 raise IOError(None, inst)
286 raise IOError(None, inst)
285 finally:
287 finally:
286 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'):
288 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'):
287 code = res.code if res else -1
289 code = res.code if res else -1
288 dbg(line % ' finished in %.4f seconds (%d)'
290 dbg(line % ' finished in %.4f seconds (%d)'
289 % (util.timer() - start, code))
291 % (util.timer() - start, code))
290
292
291 # Insert error handlers for common I/O failures.
293 # Insert error handlers for common I/O failures.
292 urlmod.wrapresponse(res)
294 urlmod.wrapresponse(res)
293
295
294 return res
296 return res
295
297
296 class RedirectedRepoError(error.RepoError):
298 class RedirectedRepoError(error.RepoError):
297 def __init__(self, msg, respurl):
299 def __init__(self, msg, respurl):
298 super(RedirectedRepoError, self).__init__(msg)
300 super(RedirectedRepoError, self).__init__(msg)
299 self.respurl = respurl
301 self.respurl = respurl
300
302
301 def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible,
303 def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible,
302 allowcbor=False):
304 allowcbor=False):
303 # record the url we got redirected to
305 # record the url we got redirected to
304 redirected = False
306 redirected = False
305 respurl = pycompat.bytesurl(resp.geturl())
307 respurl = pycompat.bytesurl(resp.geturl())
306 if respurl.endswith(qs):
308 if respurl.endswith(qs):
307 respurl = respurl[:-len(qs)]
309 respurl = respurl[:-len(qs)]
308 qsdropped = False
310 qsdropped = False
309 else:
311 else:
310 qsdropped = True
312 qsdropped = True
311
313
312 if baseurl.rstrip('/') != respurl.rstrip('/'):
314 if baseurl.rstrip('/') != respurl.rstrip('/'):
313 redirected = True
315 redirected = True
314 if not ui.quiet:
316 if not ui.quiet:
315 ui.warn(_('real URL is %s\n') % respurl)
317 ui.warn(_('real URL is %s\n') % respurl)
316
318
317 try:
319 try:
318 proto = pycompat.bytesurl(resp.getheader(r'content-type', r''))
320 proto = pycompat.bytesurl(resp.getheader(r'content-type', r''))
319 except AttributeError:
321 except AttributeError:
320 proto = pycompat.bytesurl(resp.headers.get(r'content-type', r''))
322 proto = pycompat.bytesurl(resp.headers.get(r'content-type', r''))
321
323
322 safeurl = util.hidepassword(baseurl)
324 safeurl = util.hidepassword(baseurl)
323 if proto.startswith('application/hg-error'):
325 if proto.startswith('application/hg-error'):
324 raise error.OutOfBandError(resp.read())
326 raise error.OutOfBandError(resp.read())
325
327
326 # Pre 1.0 versions of Mercurial used text/plain and
328 # Pre 1.0 versions of Mercurial used text/plain and
327 # application/hg-changegroup. We don't support such old servers.
329 # application/hg-changegroup. We don't support such old servers.
328 if not proto.startswith('application/mercurial-'):
330 if not proto.startswith('application/mercurial-'):
329 ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl))
331 ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl))
330 msg = _("'%s' does not appear to be an hg repository:\n"
332 msg = _("'%s' does not appear to be an hg repository:\n"
331 "---%%<--- (%s)\n%s\n---%%<---\n") % (
333 "---%%<--- (%s)\n%s\n---%%<---\n") % (
332 safeurl, proto or 'no content-type', resp.read(1024))
334 safeurl, proto or 'no content-type', resp.read(1024))
333
335
334 # Some servers may strip the query string from the redirect. We
336 # Some servers may strip the query string from the redirect. We
335 # raise a special error type so callers can react to this specially.
337 # raise a special error type so callers can react to this specially.
336 if redirected and qsdropped:
338 if redirected and qsdropped:
337 raise RedirectedRepoError(msg, respurl)
339 raise RedirectedRepoError(msg, respurl)
338 else:
340 else:
339 raise error.RepoError(msg)
341 raise error.RepoError(msg)
340
342
341 try:
343 try:
342 subtype = proto.split('-', 1)[1]
344 subtype = proto.split('-', 1)[1]
343
345
344 # Unless we end up supporting CBOR in the legacy wire protocol,
346 # Unless we end up supporting CBOR in the legacy wire protocol,
345 # this should ONLY be encountered for the initial capabilities
347 # this should ONLY be encountered for the initial capabilities
346 # request during handshake.
348 # request during handshake.
347 if subtype == 'cbor':
349 if subtype == 'cbor':
348 if allowcbor:
350 if allowcbor:
349 return respurl, proto, resp
351 return respurl, proto, resp
350 else:
352 else:
351 raise error.RepoError(_('unexpected CBOR response from '
353 raise error.RepoError(_('unexpected CBOR response from '
352 'server'))
354 'server'))
353
355
354 version_info = tuple([int(n) for n in subtype.split('.')])
356 version_info = tuple([int(n) for n in subtype.split('.')])
355 except ValueError:
357 except ValueError:
356 raise error.RepoError(_("'%s' sent a broken Content-Type "
358 raise error.RepoError(_("'%s' sent a broken Content-Type "
357 "header (%s)") % (safeurl, proto))
359 "header (%s)") % (safeurl, proto))
358
360
359 # TODO consider switching to a decompression reader that uses
361 # TODO consider switching to a decompression reader that uses
360 # generators.
362 # generators.
361 if version_info == (0, 1):
363 if version_info == (0, 1):
362 if compressible:
364 if compressible:
363 resp = util.compengines['zlib'].decompressorreader(resp)
365 resp = util.compengines['zlib'].decompressorreader(resp)
364
366
365 elif version_info == (0, 2):
367 elif version_info == (0, 2):
366 # application/mercurial-0.2 always identifies the compression
368 # application/mercurial-0.2 always identifies the compression
367 # engine in the payload header.
369 # engine in the payload header.
368 elen = struct.unpack('B', util.readexactly(resp, 1))[0]
370 elen = struct.unpack('B', util.readexactly(resp, 1))[0]
369 ename = util.readexactly(resp, elen)
371 ename = util.readexactly(resp, elen)
370 engine = util.compengines.forwiretype(ename)
372 engine = util.compengines.forwiretype(ename)
371
373
372 resp = engine.decompressorreader(resp)
374 resp = engine.decompressorreader(resp)
373 else:
375 else:
374 raise error.RepoError(_("'%s' uses newer protocol %s") %
376 raise error.RepoError(_("'%s' uses newer protocol %s") %
375 (safeurl, subtype))
377 (safeurl, subtype))
376
378
377 return respurl, proto, resp
379 return respurl, proto, resp
378
380
379 class httppeer(wireprotov1peer.wirepeer):
381 class httppeer(wireprotov1peer.wirepeer):
380 def __init__(self, ui, path, url, opener, requestbuilder, caps):
382 def __init__(self, ui, path, url, opener, requestbuilder, caps):
381 self.ui = ui
383 self.ui = ui
382 self._path = path
384 self._path = path
383 self._url = url
385 self._url = url
384 self._caps = caps
386 self._caps = caps
385 self.limitedarguments = caps is not None and 'httppostargs' not in caps
387 self.limitedarguments = caps is not None and 'httppostargs' not in caps
386 self._urlopener = opener
388 self._urlopener = opener
387 self._requestbuilder = requestbuilder
389 self._requestbuilder = requestbuilder
388
390
389 def __del__(self):
391 def __del__(self):
390 for h in self._urlopener.handlers:
392 for h in self._urlopener.handlers:
391 h.close()
393 h.close()
392 getattr(h, "close_all", lambda: None)()
394 getattr(h, "close_all", lambda: None)()
393
395
394 # Begin of ipeerconnection interface.
396 # Begin of ipeerconnection interface.
395
397
396 def url(self):
398 def url(self):
397 return self._path
399 return self._path
398
400
399 def local(self):
401 def local(self):
400 return None
402 return None
401
403
402 def peer(self):
404 def peer(self):
403 return self
405 return self
404
406
405 def canpush(self):
407 def canpush(self):
406 return True
408 return True
407
409
408 def close(self):
410 def close(self):
409 try:
411 try:
410 reqs, sent, recv = (self._urlopener.requestscount,
412 reqs, sent, recv = (self._urlopener.requestscount,
411 self._urlopener.sentbytescount,
413 self._urlopener.sentbytescount,
412 self._urlopener.receivedbytescount)
414 self._urlopener.receivedbytescount)
413 except AttributeError:
415 except AttributeError:
414 return
416 return
415 self.ui.note(_('(sent %d HTTP requests and %d bytes; '
417 self.ui.note(_('(sent %d HTTP requests and %d bytes; '
416 'received %d bytes in responses)\n') %
418 'received %d bytes in responses)\n') %
417 (reqs, sent, recv))
419 (reqs, sent, recv))
418
420
419 # End of ipeerconnection interface.
421 # End of ipeerconnection interface.
420
422
421 # Begin of ipeercommands interface.
423 # Begin of ipeercommands interface.
422
424
423 def capabilities(self):
425 def capabilities(self):
424 return self._caps
426 return self._caps
425
427
426 # End of ipeercommands interface.
428 # End of ipeercommands interface.
427
429
428 def _callstream(self, cmd, _compressible=False, **args):
430 def _callstream(self, cmd, _compressible=False, **args):
429 args = pycompat.byteskwargs(args)
431 args = pycompat.byteskwargs(args)
430
432
431 req, cu, qs = makev1commandrequest(self.ui, self._requestbuilder,
433 req, cu, qs = makev1commandrequest(self.ui, self._requestbuilder,
432 self._caps, self.capable,
434 self._caps, self.capable,
433 self._url, cmd, args)
435 self._url, cmd, args)
434
436
435 resp = sendrequest(self.ui, self._urlopener, req)
437 resp = sendrequest(self.ui, self._urlopener, req)
436
438
437 self._url, ct, resp = parsev1commandresponse(self.ui, self._url, cu, qs,
439 self._url, ct, resp = parsev1commandresponse(self.ui, self._url, cu, qs,
438 resp, _compressible)
440 resp, _compressible)
439
441
440 return resp
442 return resp
441
443
442 def _call(self, cmd, **args):
444 def _call(self, cmd, **args):
443 fp = self._callstream(cmd, **args)
445 fp = self._callstream(cmd, **args)
444 try:
446 try:
445 return fp.read()
447 return fp.read()
446 finally:
448 finally:
447 # if using keepalive, allow connection to be reused
449 # if using keepalive, allow connection to be reused
448 fp.close()
450 fp.close()
449
451
450 def _callpush(self, cmd, cg, **args):
452 def _callpush(self, cmd, cg, **args):
451 # have to stream bundle to a temp file because we do not have
453 # have to stream bundle to a temp file because we do not have
452 # http 1.1 chunked transfer.
454 # http 1.1 chunked transfer.
453
455
454 types = self.capable('unbundle')
456 types = self.capable('unbundle')
455 try:
457 try:
456 types = types.split(',')
458 types = types.split(',')
457 except AttributeError:
459 except AttributeError:
458 # servers older than d1b16a746db6 will send 'unbundle' as a
460 # servers older than d1b16a746db6 will send 'unbundle' as a
459 # boolean capability. They only support headerless/uncompressed
461 # boolean capability. They only support headerless/uncompressed
460 # bundles.
462 # bundles.
461 types = [""]
463 types = [""]
462 for x in types:
464 for x in types:
463 if x in bundle2.bundletypes:
465 if x in bundle2.bundletypes:
464 type = x
466 type = x
465 break
467 break
466
468
467 tempname = bundle2.writebundle(self.ui, cg, None, type)
469 tempname = bundle2.writebundle(self.ui, cg, None, type)
468 fp = httpconnection.httpsendfile(self.ui, tempname, "rb")
470 fp = httpconnection.httpsendfile(self.ui, tempname, "rb")
469 headers = {r'Content-Type': r'application/mercurial-0.1'}
471 headers = {r'Content-Type': r'application/mercurial-0.1'}
470
472
471 try:
473 try:
472 r = self._call(cmd, data=fp, headers=headers, **args)
474 r = self._call(cmd, data=fp, headers=headers, **args)
473 vals = r.split('\n', 1)
475 vals = r.split('\n', 1)
474 if len(vals) < 2:
476 if len(vals) < 2:
475 raise error.ResponseError(_("unexpected response:"), r)
477 raise error.ResponseError(_("unexpected response:"), r)
476 return vals
478 return vals
477 except urlerr.httperror:
479 except urlerr.httperror:
478 # Catch and re-raise these so we don't try and treat them
480 # Catch and re-raise these so we don't try and treat them
479 # like generic socket errors. They lack any values in
481 # like generic socket errors. They lack any values in
480 # .args on Python 3 which breaks our socket.error block.
482 # .args on Python 3 which breaks our socket.error block.
481 raise
483 raise
482 except socket.error as err:
484 except socket.error as err:
483 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
485 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
484 raise error.Abort(_('push failed: %s') % err.args[1])
486 raise error.Abort(_('push failed: %s') % err.args[1])
485 raise error.Abort(err.args[1])
487 raise error.Abort(err.args[1])
486 finally:
488 finally:
487 fp.close()
489 fp.close()
488 os.unlink(tempname)
490 os.unlink(tempname)
489
491
490 def _calltwowaystream(self, cmd, fp, **args):
492 def _calltwowaystream(self, cmd, fp, **args):
491 fh = None
493 fh = None
492 fp_ = None
494 fp_ = None
493 filename = None
495 filename = None
494 try:
496 try:
495 # dump bundle to disk
497 # dump bundle to disk
496 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
498 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
497 fh = os.fdopen(fd, r"wb")
499 fh = os.fdopen(fd, r"wb")
498 d = fp.read(4096)
500 d = fp.read(4096)
499 while d:
501 while d:
500 fh.write(d)
502 fh.write(d)
501 d = fp.read(4096)
503 d = fp.read(4096)
502 fh.close()
504 fh.close()
503 # start http push
505 # start http push
504 fp_ = httpconnection.httpsendfile(self.ui, filename, "rb")
506 fp_ = httpconnection.httpsendfile(self.ui, filename, "rb")
505 headers = {r'Content-Type': r'application/mercurial-0.1'}
507 headers = {r'Content-Type': r'application/mercurial-0.1'}
506 return self._callstream(cmd, data=fp_, headers=headers, **args)
508 return self._callstream(cmd, data=fp_, headers=headers, **args)
507 finally:
509 finally:
508 if fp_ is not None:
510 if fp_ is not None:
509 fp_.close()
511 fp_.close()
510 if fh is not None:
512 if fh is not None:
511 fh.close()
513 fh.close()
512 os.unlink(filename)
514 os.unlink(filename)
513
515
514 def _callcompressable(self, cmd, **args):
516 def _callcompressable(self, cmd, **args):
515 return self._callstream(cmd, _compressible=True, **args)
517 return self._callstream(cmd, _compressible=True, **args)
516
518
517 def _abort(self, exception):
519 def _abort(self, exception):
518 raise exception
520 raise exception
519
521
520 def sendv2request(ui, opener, requestbuilder, apiurl, permission, requests,
522 def sendv2request(ui, opener, requestbuilder, apiurl, permission, requests,
521 redirect):
523 redirect):
522 wireprotoframing.populatestreamencoders()
524 wireprotoframing.populatestreamencoders()
523
525
524 uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order')
526 uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order')
525
527
526 if uiencoders:
528 if uiencoders:
527 encoders = []
529 encoders = []
528
530
529 for encoder in uiencoders:
531 for encoder in uiencoders:
530 if encoder not in wireprotoframing.STREAM_ENCODERS:
532 if encoder not in wireprotoframing.STREAM_ENCODERS:
531 ui.warn(_(b'wire protocol version 2 encoder referenced in '
533 ui.warn(_(b'wire protocol version 2 encoder referenced in '
532 b'config (%s) is not known; ignoring\n') % encoder)
534 b'config (%s) is not known; ignoring\n') % encoder)
533 else:
535 else:
534 encoders.append(encoder)
536 encoders.append(encoder)
535
537
536 else:
538 else:
537 encoders = wireprotoframing.STREAM_ENCODERS_ORDER
539 encoders = wireprotoframing.STREAM_ENCODERS_ORDER
538
540
539 reactor = wireprotoframing.clientreactor(ui,
541 reactor = wireprotoframing.clientreactor(ui,
540 hasmultiplesend=False,
542 hasmultiplesend=False,
541 buffersends=True,
543 buffersends=True,
542 clientcontentencoders=encoders)
544 clientcontentencoders=encoders)
543
545
544 handler = wireprotov2peer.clienthandler(ui, reactor,
546 handler = wireprotov2peer.clienthandler(ui, reactor,
545 opener=opener,
547 opener=opener,
546 requestbuilder=requestbuilder)
548 requestbuilder=requestbuilder)
547
549
548 url = '%s/%s' % (apiurl, permission)
550 url = '%s/%s' % (apiurl, permission)
549
551
550 if len(requests) > 1:
552 if len(requests) > 1:
551 url += '/multirequest'
553 url += '/multirequest'
552 else:
554 else:
553 url += '/%s' % requests[0][0]
555 url += '/%s' % requests[0][0]
554
556
555 ui.debug('sending %d commands\n' % len(requests))
557 ui.debug('sending %d commands\n' % len(requests))
556 for command, args, f in requests:
558 for command, args, f in requests:
557 ui.debug('sending command %s: %s\n' % (
559 ui.debug('sending command %s: %s\n' % (
558 command, stringutil.pprint(args, indent=2)))
560 command, stringutil.pprint(args, indent=2)))
559 assert not list(handler.callcommand(command, args, f,
561 assert not list(handler.callcommand(command, args, f,
560 redirect=redirect))
562 redirect=redirect))
561
563
562 # TODO stream this.
564 # TODO stream this.
563 body = b''.join(map(bytes, handler.flushcommands()))
565 body = b''.join(map(bytes, handler.flushcommands()))
564
566
565 # TODO modify user-agent to reflect v2
567 # TODO modify user-agent to reflect v2
566 headers = {
568 headers = {
567 r'Accept': wireprotov2server.FRAMINGTYPE,
569 r'Accept': wireprotov2server.FRAMINGTYPE,
568 r'Content-Type': wireprotov2server.FRAMINGTYPE,
570 r'Content-Type': wireprotov2server.FRAMINGTYPE,
569 }
571 }
570
572
571 req = requestbuilder(pycompat.strurl(url), body, headers)
573 req = requestbuilder(pycompat.strurl(url), body, headers)
572 req.add_unredirected_header(r'Content-Length', r'%d' % len(body))
574 req.add_unredirected_header(r'Content-Length', r'%d' % len(body))
573
575
574 try:
576 try:
575 res = opener.open(req)
577 res = opener.open(req)
576 except urlerr.httperror as e:
578 except urlerr.httperror as e:
577 if e.code == 401:
579 if e.code == 401:
578 raise error.Abort(_('authorization failed'))
580 raise error.Abort(_('authorization failed'))
579
581
580 raise
582 raise
581 except httplib.HTTPException as e:
583 except httplib.HTTPException as e:
582 ui.traceback()
584 ui.traceback()
583 raise IOError(None, e)
585 raise IOError(None, e)
584
586
585 return handler, res
587 return handler, res
586
588
587 class queuedcommandfuture(pycompat.futures.Future):
589 class queuedcommandfuture(pycompat.futures.Future):
588 """Wraps result() on command futures to trigger submission on call."""
590 """Wraps result() on command futures to trigger submission on call."""
589
591
590 def result(self, timeout=None):
592 def result(self, timeout=None):
591 if self.done():
593 if self.done():
592 return pycompat.futures.Future.result(self, timeout)
594 return pycompat.futures.Future.result(self, timeout)
593
595
594 self._peerexecutor.sendcommands()
596 self._peerexecutor.sendcommands()
595
597
596 # sendcommands() will restore the original __class__ and self.result
598 # sendcommands() will restore the original __class__ and self.result
597 # will resolve to Future.result.
599 # will resolve to Future.result.
598 return self.result(timeout)
600 return self.result(timeout)
599
601
600 @interfaceutil.implementer(repository.ipeercommandexecutor)
602 @interfaceutil.implementer(repository.ipeercommandexecutor)
601 class httpv2executor(object):
603 class httpv2executor(object):
602 def __init__(self, ui, opener, requestbuilder, apiurl, descriptor,
604 def __init__(self, ui, opener, requestbuilder, apiurl, descriptor,
603 redirect):
605 redirect):
604 self._ui = ui
606 self._ui = ui
605 self._opener = opener
607 self._opener = opener
606 self._requestbuilder = requestbuilder
608 self._requestbuilder = requestbuilder
607 self._apiurl = apiurl
609 self._apiurl = apiurl
608 self._descriptor = descriptor
610 self._descriptor = descriptor
609 self._redirect = redirect
611 self._redirect = redirect
610 self._sent = False
612 self._sent = False
611 self._closed = False
613 self._closed = False
612 self._neededpermissions = set()
614 self._neededpermissions = set()
613 self._calls = []
615 self._calls = []
614 self._futures = weakref.WeakSet()
616 self._futures = weakref.WeakSet()
615 self._responseexecutor = None
617 self._responseexecutor = None
616 self._responsef = None
618 self._responsef = None
617
619
618 def __enter__(self):
620 def __enter__(self):
619 return self
621 return self
620
622
621 def __exit__(self, exctype, excvalue, exctb):
623 def __exit__(self, exctype, excvalue, exctb):
622 self.close()
624 self.close()
623
625
624 def callcommand(self, command, args):
626 def callcommand(self, command, args):
625 if self._sent:
627 if self._sent:
626 raise error.ProgrammingError('callcommand() cannot be used after '
628 raise error.ProgrammingError('callcommand() cannot be used after '
627 'commands are sent')
629 'commands are sent')
628
630
629 if self._closed:
631 if self._closed:
630 raise error.ProgrammingError('callcommand() cannot be used after '
632 raise error.ProgrammingError('callcommand() cannot be used after '
631 'close()')
633 'close()')
632
634
633 # The service advertises which commands are available. So if we attempt
635 # The service advertises which commands are available. So if we attempt
634 # to call an unknown command or pass an unknown argument, we can screen
636 # to call an unknown command or pass an unknown argument, we can screen
635 # for this.
637 # for this.
636 if command not in self._descriptor['commands']:
638 if command not in self._descriptor['commands']:
637 raise error.ProgrammingError(
639 raise error.ProgrammingError(
638 'wire protocol command %s is not available' % command)
640 'wire protocol command %s is not available' % command)
639
641
640 cmdinfo = self._descriptor['commands'][command]
642 cmdinfo = self._descriptor['commands'][command]
641 unknownargs = set(args.keys()) - set(cmdinfo.get('args', {}))
643 unknownargs = set(args.keys()) - set(cmdinfo.get('args', {}))
642
644
643 if unknownargs:
645 if unknownargs:
644 raise error.ProgrammingError(
646 raise error.ProgrammingError(
645 'wire protocol command %s does not accept argument: %s' % (
647 'wire protocol command %s does not accept argument: %s' % (
646 command, ', '.join(sorted(unknownargs))))
648 command, ', '.join(sorted(unknownargs))))
647
649
648 self._neededpermissions |= set(cmdinfo['permissions'])
650 self._neededpermissions |= set(cmdinfo['permissions'])
649
651
650 # TODO we /could/ also validate types here, since the API descriptor
652 # TODO we /could/ also validate types here, since the API descriptor
651 # includes types...
653 # includes types...
652
654
653 f = pycompat.futures.Future()
655 f = pycompat.futures.Future()
654
656
655 # Monkeypatch it so result() triggers sendcommands(), otherwise result()
657 # Monkeypatch it so result() triggers sendcommands(), otherwise result()
656 # could deadlock.
658 # could deadlock.
657 f.__class__ = queuedcommandfuture
659 f.__class__ = queuedcommandfuture
658 f._peerexecutor = self
660 f._peerexecutor = self
659
661
660 self._futures.add(f)
662 self._futures.add(f)
661 self._calls.append((command, args, f))
663 self._calls.append((command, args, f))
662
664
663 return f
665 return f
664
666
665 def sendcommands(self):
667 def sendcommands(self):
666 if self._sent:
668 if self._sent:
667 return
669 return
668
670
669 if not self._calls:
671 if not self._calls:
670 return
672 return
671
673
672 self._sent = True
674 self._sent = True
673
675
674 # Unhack any future types so caller sees a clean type and so we
676 # Unhack any future types so caller sees a clean type and so we
675 # break reference cycle.
677 # break reference cycle.
676 for f in self._futures:
678 for f in self._futures:
677 if isinstance(f, queuedcommandfuture):
679 if isinstance(f, queuedcommandfuture):
678 f.__class__ = pycompat.futures.Future
680 f.__class__ = pycompat.futures.Future
679 f._peerexecutor = None
681 f._peerexecutor = None
680
682
681 # Mark the future as running and filter out cancelled futures.
683 # Mark the future as running and filter out cancelled futures.
682 calls = [(command, args, f)
684 calls = [(command, args, f)
683 for command, args, f in self._calls
685 for command, args, f in self._calls
684 if f.set_running_or_notify_cancel()]
686 if f.set_running_or_notify_cancel()]
685
687
686 # Clear out references, prevent improper object usage.
688 # Clear out references, prevent improper object usage.
687 self._calls = None
689 self._calls = None
688
690
689 if not calls:
691 if not calls:
690 return
692 return
691
693
692 permissions = set(self._neededpermissions)
694 permissions = set(self._neededpermissions)
693
695
694 if 'push' in permissions and 'pull' in permissions:
696 if 'push' in permissions and 'pull' in permissions:
695 permissions.remove('pull')
697 permissions.remove('pull')
696
698
697 if len(permissions) > 1:
699 if len(permissions) > 1:
698 raise error.RepoError(_('cannot make request requiring multiple '
700 raise error.RepoError(_('cannot make request requiring multiple '
699 'permissions: %s') %
701 'permissions: %s') %
700 _(', ').join(sorted(permissions)))
702 _(', ').join(sorted(permissions)))
701
703
702 permission = {
704 permission = {
703 'push': 'rw',
705 'push': 'rw',
704 'pull': 'ro',
706 'pull': 'ro',
705 }[permissions.pop()]
707 }[permissions.pop()]
706
708
707 handler, resp = sendv2request(
709 handler, resp = sendv2request(
708 self._ui, self._opener, self._requestbuilder, self._apiurl,
710 self._ui, self._opener, self._requestbuilder, self._apiurl,
709 permission, calls, self._redirect)
711 permission, calls, self._redirect)
710
712
711 # TODO we probably want to validate the HTTP code, media type, etc.
713 # TODO we probably want to validate the HTTP code, media type, etc.
712
714
713 self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
715 self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
714 self._responsef = self._responseexecutor.submit(self._handleresponse,
716 self._responsef = self._responseexecutor.submit(self._handleresponse,
715 handler, resp)
717 handler, resp)
716
718
717 def close(self):
719 def close(self):
718 if self._closed:
720 if self._closed:
719 return
721 return
720
722
721 self.sendcommands()
723 self.sendcommands()
722
724
723 self._closed = True
725 self._closed = True
724
726
725 if not self._responsef:
727 if not self._responsef:
726 return
728 return
727
729
728 # TODO ^C here may not result in immediate program termination.
730 # TODO ^C here may not result in immediate program termination.
729
731
730 try:
732 try:
731 self._responsef.result()
733 self._responsef.result()
732 finally:
734 finally:
733 self._responseexecutor.shutdown(wait=True)
735 self._responseexecutor.shutdown(wait=True)
734 self._responsef = None
736 self._responsef = None
735 self._responseexecutor = None
737 self._responseexecutor = None
736
738
737 # If any of our futures are still in progress, mark them as
739 # If any of our futures are still in progress, mark them as
738 # errored, otherwise a result() could wait indefinitely.
740 # errored, otherwise a result() could wait indefinitely.
739 for f in self._futures:
741 for f in self._futures:
740 if not f.done():
742 if not f.done():
741 f.set_exception(error.ResponseError(
743 f.set_exception(error.ResponseError(
742 _('unfulfilled command response')))
744 _('unfulfilled command response')))
743
745
744 self._futures = None
746 self._futures = None
745
747
746 def _handleresponse(self, handler, resp):
748 def _handleresponse(self, handler, resp):
747 # Called in a thread to read the response.
749 # Called in a thread to read the response.
748
750
749 while handler.readdata(resp):
751 while handler.readdata(resp):
750 pass
752 pass
751
753
752 @interfaceutil.implementer(repository.ipeerv2)
754 @interfaceutil.implementer(repository.ipeerv2)
753 class httpv2peer(object):
755 class httpv2peer(object):
754
756
755 limitedarguments = False
757 limitedarguments = False
756
758
757 def __init__(self, ui, repourl, apipath, opener, requestbuilder,
759 def __init__(self, ui, repourl, apipath, opener, requestbuilder,
758 apidescriptor):
760 apidescriptor):
759 self.ui = ui
761 self.ui = ui
760 self.apidescriptor = apidescriptor
762 self.apidescriptor = apidescriptor
761
763
762 if repourl.endswith('/'):
764 if repourl.endswith('/'):
763 repourl = repourl[:-1]
765 repourl = repourl[:-1]
764
766
765 self._url = repourl
767 self._url = repourl
766 self._apipath = apipath
768 self._apipath = apipath
767 self._apiurl = '%s/%s' % (repourl, apipath)
769 self._apiurl = '%s/%s' % (repourl, apipath)
768 self._opener = opener
770 self._opener = opener
769 self._requestbuilder = requestbuilder
771 self._requestbuilder = requestbuilder
770
772
771 self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
773 self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
772
774
773 # Start of ipeerconnection.
775 # Start of ipeerconnection.
774
776
775 def url(self):
777 def url(self):
776 return self._url
778 return self._url
777
779
778 def local(self):
780 def local(self):
779 return None
781 return None
780
782
781 def peer(self):
783 def peer(self):
782 return self
784 return self
783
785
784 def canpush(self):
786 def canpush(self):
785 # TODO change once implemented.
787 # TODO change once implemented.
786 return False
788 return False
787
789
788 def close(self):
790 def close(self):
789 self.ui.note(_('(sent %d HTTP requests and %d bytes; '
791 self.ui.note(_('(sent %d HTTP requests and %d bytes; '
790 'received %d bytes in responses)\n') %
792 'received %d bytes in responses)\n') %
791 (self._opener.requestscount,
793 (self._opener.requestscount,
792 self._opener.sentbytescount,
794 self._opener.sentbytescount,
793 self._opener.receivedbytescount))
795 self._opener.receivedbytescount))
794
796
795 # End of ipeerconnection.
797 # End of ipeerconnection.
796
798
797 # Start of ipeercapabilities.
799 # Start of ipeercapabilities.
798
800
799 def capable(self, name):
801 def capable(self, name):
800 # The capabilities used internally historically map to capabilities
802 # The capabilities used internally historically map to capabilities
801 # advertised from the "capabilities" wire protocol command. However,
803 # advertised from the "capabilities" wire protocol command. However,
802 # version 2 of that command works differently.
804 # version 2 of that command works differently.
803
805
804 # Maps to commands that are available.
806 # Maps to commands that are available.
805 if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'):
807 if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'):
806 return True
808 return True
807
809
808 # Other concepts.
810 # Other concepts.
809 if name in ('bundle2'):
811 if name in ('bundle2'):
810 return True
812 return True
811
813
812 # Alias command-* to presence of command of that name.
814 # Alias command-* to presence of command of that name.
813 if name.startswith('command-'):
815 if name.startswith('command-'):
814 return name[len('command-'):] in self.apidescriptor['commands']
816 return name[len('command-'):] in self.apidescriptor['commands']
815
817
816 return False
818 return False
817
819
818 def requirecap(self, name, purpose):
820 def requirecap(self, name, purpose):
819 if self.capable(name):
821 if self.capable(name):
820 return
822 return
821
823
822 raise error.CapabilityError(
824 raise error.CapabilityError(
823 _('cannot %s; client or remote repository does not support the '
825 _('cannot %s; client or remote repository does not support the '
824 '\'%s\' capability') % (purpose, name))
826 '\'%s\' capability') % (purpose, name))
825
827
826 # End of ipeercapabilities.
828 # End of ipeercapabilities.
827
829
828 def _call(self, name, **args):
830 def _call(self, name, **args):
829 with self.commandexecutor() as e:
831 with self.commandexecutor() as e:
830 return e.callcommand(name, args).result()
832 return e.callcommand(name, args).result()
831
833
832 def commandexecutor(self):
834 def commandexecutor(self):
833 return httpv2executor(self.ui, self._opener, self._requestbuilder,
835 return httpv2executor(self.ui, self._opener, self._requestbuilder,
834 self._apiurl, self.apidescriptor, self._redirect)
836 self._apiurl, self.apidescriptor, self._redirect)
835
837
836 # Registry of API service names to metadata about peers that handle it.
838 # Registry of API service names to metadata about peers that handle it.
837 #
839 #
838 # The following keys are meaningful:
840 # The following keys are meaningful:
839 #
841 #
840 # init
842 # init
841 # Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
843 # Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
842 # apidescriptor) to create a peer.
844 # apidescriptor) to create a peer.
843 #
845 #
844 # priority
846 # priority
845 # Integer priority for the service. If we could choose from multiple
847 # Integer priority for the service. If we could choose from multiple
846 # services, we choose the one with the highest priority.
848 # services, we choose the one with the highest priority.
847 API_PEERS = {
849 API_PEERS = {
848 wireprototypes.HTTP_WIREPROTO_V2: {
850 wireprototypes.HTTP_WIREPROTO_V2: {
849 'init': httpv2peer,
851 'init': httpv2peer,
850 'priority': 50,
852 'priority': 50,
851 },
853 },
852 }
854 }
853
855
854 def performhandshake(ui, url, opener, requestbuilder):
856 def performhandshake(ui, url, opener, requestbuilder):
855 # The handshake is a request to the capabilities command.
857 # The handshake is a request to the capabilities command.
856
858
857 caps = None
859 caps = None
858 def capable(x):
860 def capable(x):
859 raise error.ProgrammingError('should not be called')
861 raise error.ProgrammingError('should not be called')
860
862
861 args = {}
863 args = {}
862
864
863 # The client advertises support for newer protocols by adding an
865 # The client advertises support for newer protocols by adding an
864 # X-HgUpgrade-* header with a list of supported APIs and an
866 # X-HgUpgrade-* header with a list of supported APIs and an
865 # X-HgProto-* header advertising which serializing formats it supports.
867 # X-HgProto-* header advertising which serializing formats it supports.
866 # We only support the HTTP version 2 transport and CBOR responses for
868 # We only support the HTTP version 2 transport and CBOR responses for
867 # now.
869 # now.
868 advertisev2 = ui.configbool('experimental', 'httppeer.advertise-v2')
870 advertisev2 = ui.configbool('experimental', 'httppeer.advertise-v2')
869
871
870 if advertisev2:
872 if advertisev2:
871 args['headers'] = {
873 args['headers'] = {
872 r'X-HgProto-1': r'cbor',
874 r'X-HgProto-1': r'cbor',
873 }
875 }
874
876
875 args['headers'].update(
877 args['headers'].update(
876 encodevalueinheaders(' '.join(sorted(API_PEERS)),
878 encodevalueinheaders(' '.join(sorted(API_PEERS)),
877 'X-HgUpgrade',
879 'X-HgUpgrade',
878 # We don't know the header limit this early.
880 # We don't know the header limit this early.
879 # So make it small.
881 # So make it small.
880 1024))
882 1024))
881
883
882 req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
884 req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
883 capable, url, 'capabilities',
885 capable, url, 'capabilities',
884 args)
886 args)
885 resp = sendrequest(ui, opener, req)
887 resp = sendrequest(ui, opener, req)
886
888
887 # The server may redirect us to the repo root, stripping the
889 # The server may redirect us to the repo root, stripping the
888 # ?cmd=capabilities query string from the URL. The server would likely
890 # ?cmd=capabilities query string from the URL. The server would likely
889 # return HTML in this case and ``parsev1commandresponse()`` would raise.
891 # return HTML in this case and ``parsev1commandresponse()`` would raise.
890 # We catch this special case and re-issue the capabilities request against
892 # We catch this special case and re-issue the capabilities request against
891 # the new URL.
893 # the new URL.
892 #
894 #
893 # We should ideally not do this, as a redirect that drops the query
895 # We should ideally not do this, as a redirect that drops the query
894 # string from the URL is arguably a server bug. (Garbage in, garbage out).
896 # string from the URL is arguably a server bug. (Garbage in, garbage out).
895 # However, Mercurial clients for several years appeared to handle this
897 # However, Mercurial clients for several years appeared to handle this
896 # issue without behavior degradation. And according to issue 5860, it may
898 # issue without behavior degradation. And according to issue 5860, it may
897 # be a longstanding bug in some server implementations. So we allow a
899 # be a longstanding bug in some server implementations. So we allow a
898 # redirect that drops the query string to "just work."
900 # redirect that drops the query string to "just work."
899 try:
901 try:
900 respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
902 respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
901 compressible=False,
903 compressible=False,
902 allowcbor=advertisev2)
904 allowcbor=advertisev2)
903 except RedirectedRepoError as e:
905 except RedirectedRepoError as e:
904 req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
906 req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
905 capable, e.respurl,
907 capable, e.respurl,
906 'capabilities', args)
908 'capabilities', args)
907 resp = sendrequest(ui, opener, req)
909 resp = sendrequest(ui, opener, req)
908 respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
910 respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
909 compressible=False,
911 compressible=False,
910 allowcbor=advertisev2)
912 allowcbor=advertisev2)
911
913
912 try:
914 try:
913 rawdata = resp.read()
915 rawdata = resp.read()
914 finally:
916 finally:
915 resp.close()
917 resp.close()
916
918
917 if not ct.startswith('application/mercurial-'):
919 if not ct.startswith('application/mercurial-'):
918 raise error.ProgrammingError('unexpected content-type: %s' % ct)
920 raise error.ProgrammingError('unexpected content-type: %s' % ct)
919
921
920 if advertisev2:
922 if advertisev2:
921 if ct == 'application/mercurial-cbor':
923 if ct == 'application/mercurial-cbor':
922 try:
924 try:
923 info = cborutil.decodeall(rawdata)[0]
925 info = cborutil.decodeall(rawdata)[0]
924 except cborutil.CBORDecodeError:
926 except cborutil.CBORDecodeError:
925 raise error.Abort(_('error decoding CBOR from remote server'),
927 raise error.Abort(_('error decoding CBOR from remote server'),
926 hint=_('try again and consider contacting '
928 hint=_('try again and consider contacting '
927 'the server operator'))
929 'the server operator'))
928
930
929 # We got a legacy response. That's fine.
931 # We got a legacy response. That's fine.
930 elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'):
932 elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'):
931 info = {
933 info = {
932 'v1capabilities': set(rawdata.split())
934 'v1capabilities': set(rawdata.split())
933 }
935 }
934
936
935 else:
937 else:
936 raise error.RepoError(
938 raise error.RepoError(
937 _('unexpected response type from server: %s') % ct)
939 _('unexpected response type from server: %s') % ct)
938 else:
940 else:
939 info = {
941 info = {
940 'v1capabilities': set(rawdata.split())
942 'v1capabilities': set(rawdata.split())
941 }
943 }
942
944
943 return respurl, info
945 return respurl, info
944
946
945 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
947 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
946 """Construct an appropriate HTTP peer instance.
948 """Construct an appropriate HTTP peer instance.
947
949
948 ``opener`` is an ``url.opener`` that should be used to establish
950 ``opener`` is an ``url.opener`` that should be used to establish
949 connections, perform HTTP requests.
951 connections, perform HTTP requests.
950
952
951 ``requestbuilder`` is the type used for constructing HTTP requests.
953 ``requestbuilder`` is the type used for constructing HTTP requests.
952 It exists as an argument so extensions can override the default.
954 It exists as an argument so extensions can override the default.
953 """
955 """
954 u = util.url(path)
956 u = util.url(path)
955 if u.query or u.fragment:
957 if u.query or u.fragment:
956 raise error.Abort(_('unsupported URL component: "%s"') %
958 raise error.Abort(_('unsupported URL component: "%s"') %
957 (u.query or u.fragment))
959 (u.query or u.fragment))
958
960
959 # urllib cannot handle URLs with embedded user or passwd.
961 # urllib cannot handle URLs with embedded user or passwd.
960 url, authinfo = u.authinfo()
962 url, authinfo = u.authinfo()
961 ui.debug('using %s\n' % url)
963 ui.debug('using %s\n' % url)
962
964
963 opener = opener or urlmod.opener(ui, authinfo)
965 opener = opener or urlmod.opener(ui, authinfo)
964
966
965 respurl, info = performhandshake(ui, url, opener, requestbuilder)
967 respurl, info = performhandshake(ui, url, opener, requestbuilder)
966
968
967 # Given the intersection of APIs that both we and the server support,
969 # Given the intersection of APIs that both we and the server support,
968 # sort by their advertised priority and pick the first one.
970 # sort by their advertised priority and pick the first one.
969 #
971 #
970 # TODO consider making this request-based and interface driven. For
972 # TODO consider making this request-based and interface driven. For
971 # example, the caller could say "I want a peer that does X." It's quite
973 # example, the caller could say "I want a peer that does X." It's quite
972 # possible that not all peers would do that. Since we know the service
974 # possible that not all peers would do that. Since we know the service
973 # capabilities, we could filter out services not meeting the
975 # capabilities, we could filter out services not meeting the
974 # requirements. Possibly by consulting the interfaces defined by the
976 # requirements. Possibly by consulting the interfaces defined by the
975 # peer type.
977 # peer type.
976 apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys())
978 apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys())
977
979
978 preferredchoices = sorted(apipeerchoices,
980 preferredchoices = sorted(apipeerchoices,
979 key=lambda x: API_PEERS[x]['priority'],
981 key=lambda x: API_PEERS[x]['priority'],
980 reverse=True)
982 reverse=True)
981
983
982 for service in preferredchoices:
984 for service in preferredchoices:
983 apipath = '%s/%s' % (info['apibase'].rstrip('/'), service)
985 apipath = '%s/%s' % (info['apibase'].rstrip('/'), service)
984
986
985 return API_PEERS[service]['init'](ui, respurl, apipath, opener,
987 return API_PEERS[service]['init'](ui, respurl, apipath, opener,
986 requestbuilder,
988 requestbuilder,
987 info['apis'][service])
989 info['apis'][service])
988
990
989 # Failed to construct an API peer. Fall back to legacy.
991 # Failed to construct an API peer. Fall back to legacy.
990 return httppeer(ui, path, respurl, opener, requestbuilder,
992 return httppeer(ui, path, respurl, opener, requestbuilder,
991 info['v1capabilities'])
993 info['v1capabilities'])
992
994
993 def instance(ui, path, create, intents=None, createopts=None):
995 def instance(ui, path, create, intents=None, createopts=None):
994 if create:
996 if create:
995 raise error.Abort(_('cannot create new http repository'))
997 raise error.Abort(_('cannot create new http repository'))
996 try:
998 try:
997 if path.startswith('https:') and not urlmod.has_https:
999 if path.startswith('https:') and not urlmod.has_https:
998 raise error.Abort(_('Python support for SSL and HTTPS '
1000 raise error.Abort(_('Python support for SSL and HTTPS '
999 'is not installed'))
1001 'is not installed'))
1000
1002
1001 inst = makepeer(ui, path)
1003 inst = makepeer(ui, path)
1002
1004
1003 return inst
1005 return inst
1004 except error.RepoError as httpexception:
1006 except error.RepoError as httpexception:
1005 try:
1007 try:
1006 r = statichttprepo.instance(ui, "static-" + path, create)
1008 r = statichttprepo.instance(ui, "static-" + path, create)
1007 ui.note(_('(falling back to static-http)\n'))
1009 ui.note(_('(falling back to static-http)\n'))
1008 return r
1010 return r
1009 except error.RepoError:
1011 except error.RepoError:
1010 raise httpexception # use the original http RepoError instead
1012 raise httpexception # use the original http RepoError instead
@@ -1,1877 +1,1877
1 # repository.py - Interfaces and base classes for repositories and peers.
1 # repository.py - Interfaces and base classes for repositories and peers.
2 #
2 #
3 # Copyright 2017 Gregory Szorc <gregory.szorc@gmail.com>
3 # Copyright 2017 Gregory Szorc <gregory.szorc@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 from .i18n import _
10 from ..i18n import _
11 from . import (
11 from .. import (
12 error,
12 error,
13 )
13 )
14 from .utils import (
14 from ..utils import (
15 interfaceutil,
15 interfaceutil,
16 )
16 )
17
17
18 # When narrowing is finalized and no longer subject to format changes,
18 # When narrowing is finalized and no longer subject to format changes,
19 # we should move this to just "narrow" or similar.
19 # we should move this to just "narrow" or similar.
20 NARROW_REQUIREMENT = 'narrowhg-experimental'
20 NARROW_REQUIREMENT = 'narrowhg-experimental'
21
21
22 # Local repository feature string.
22 # Local repository feature string.
23
23
24 # Revlogs are being used for file storage.
24 # Revlogs are being used for file storage.
25 REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage'
25 REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage'
26 # The storage part of the repository is shared from an external source.
26 # The storage part of the repository is shared from an external source.
27 REPO_FEATURE_SHARED_STORAGE = b'sharedstore'
27 REPO_FEATURE_SHARED_STORAGE = b'sharedstore'
28 # LFS supported for backing file storage.
28 # LFS supported for backing file storage.
29 REPO_FEATURE_LFS = b'lfs'
29 REPO_FEATURE_LFS = b'lfs'
30 # Repository supports being stream cloned.
30 # Repository supports being stream cloned.
31 REPO_FEATURE_STREAM_CLONE = b'streamclone'
31 REPO_FEATURE_STREAM_CLONE = b'streamclone'
32 # Files storage may lack data for all ancestors.
32 # Files storage may lack data for all ancestors.
33 REPO_FEATURE_SHALLOW_FILE_STORAGE = b'shallowfilestorage'
33 REPO_FEATURE_SHALLOW_FILE_STORAGE = b'shallowfilestorage'
34
34
35 REVISION_FLAG_CENSORED = 1 << 15
35 REVISION_FLAG_CENSORED = 1 << 15
36 REVISION_FLAG_ELLIPSIS = 1 << 14
36 REVISION_FLAG_ELLIPSIS = 1 << 14
37 REVISION_FLAG_EXTSTORED = 1 << 13
37 REVISION_FLAG_EXTSTORED = 1 << 13
38
38
39 REVISION_FLAGS_KNOWN = (
39 REVISION_FLAGS_KNOWN = (
40 REVISION_FLAG_CENSORED | REVISION_FLAG_ELLIPSIS | REVISION_FLAG_EXTSTORED)
40 REVISION_FLAG_CENSORED | REVISION_FLAG_ELLIPSIS | REVISION_FLAG_EXTSTORED)
41
41
42 CG_DELTAMODE_STD = b'default'
42 CG_DELTAMODE_STD = b'default'
43 CG_DELTAMODE_PREV = b'previous'
43 CG_DELTAMODE_PREV = b'previous'
44 CG_DELTAMODE_FULL = b'fulltext'
44 CG_DELTAMODE_FULL = b'fulltext'
45 CG_DELTAMODE_P1 = b'p1'
45 CG_DELTAMODE_P1 = b'p1'
46
46
47 class ipeerconnection(interfaceutil.Interface):
47 class ipeerconnection(interfaceutil.Interface):
48 """Represents a "connection" to a repository.
48 """Represents a "connection" to a repository.
49
49
50 This is the base interface for representing a connection to a repository.
50 This is the base interface for representing a connection to a repository.
51 It holds basic properties and methods applicable to all peer types.
51 It holds basic properties and methods applicable to all peer types.
52
52
53 This is not a complete interface definition and should not be used
53 This is not a complete interface definition and should not be used
54 outside of this module.
54 outside of this module.
55 """
55 """
56 ui = interfaceutil.Attribute("""ui.ui instance""")
56 ui = interfaceutil.Attribute("""ui.ui instance""")
57
57
58 def url():
58 def url():
59 """Returns a URL string representing this peer.
59 """Returns a URL string representing this peer.
60
60
61 Currently, implementations expose the raw URL used to construct the
61 Currently, implementations expose the raw URL used to construct the
62 instance. It may contain credentials as part of the URL. The
62 instance. It may contain credentials as part of the URL. The
63 expectations of the value aren't well-defined and this could lead to
63 expectations of the value aren't well-defined and this could lead to
64 data leakage.
64 data leakage.
65
65
66 TODO audit/clean consumers and more clearly define the contents of this
66 TODO audit/clean consumers and more clearly define the contents of this
67 value.
67 value.
68 """
68 """
69
69
70 def local():
70 def local():
71 """Returns a local repository instance.
71 """Returns a local repository instance.
72
72
73 If the peer represents a local repository, returns an object that
73 If the peer represents a local repository, returns an object that
74 can be used to interface with it. Otherwise returns ``None``.
74 can be used to interface with it. Otherwise returns ``None``.
75 """
75 """
76
76
77 def peer():
77 def peer():
78 """Returns an object conforming to this interface.
78 """Returns an object conforming to this interface.
79
79
80 Most implementations will ``return self``.
80 Most implementations will ``return self``.
81 """
81 """
82
82
83 def canpush():
83 def canpush():
84 """Returns a boolean indicating if this peer can be pushed to."""
84 """Returns a boolean indicating if this peer can be pushed to."""
85
85
86 def close():
86 def close():
87 """Close the connection to this peer.
87 """Close the connection to this peer.
88
88
89 This is called when the peer will no longer be used. Resources
89 This is called when the peer will no longer be used. Resources
90 associated with the peer should be cleaned up.
90 associated with the peer should be cleaned up.
91 """
91 """
92
92
93 class ipeercapabilities(interfaceutil.Interface):
93 class ipeercapabilities(interfaceutil.Interface):
94 """Peer sub-interface related to capabilities."""
94 """Peer sub-interface related to capabilities."""
95
95
96 def capable(name):
96 def capable(name):
97 """Determine support for a named capability.
97 """Determine support for a named capability.
98
98
99 Returns ``False`` if capability not supported.
99 Returns ``False`` if capability not supported.
100
100
101 Returns ``True`` if boolean capability is supported. Returns a string
101 Returns ``True`` if boolean capability is supported. Returns a string
102 if capability support is non-boolean.
102 if capability support is non-boolean.
103
103
104 Capability strings may or may not map to wire protocol capabilities.
104 Capability strings may or may not map to wire protocol capabilities.
105 """
105 """
106
106
107 def requirecap(name, purpose):
107 def requirecap(name, purpose):
108 """Require a capability to be present.
108 """Require a capability to be present.
109
109
110 Raises a ``CapabilityError`` if the capability isn't present.
110 Raises a ``CapabilityError`` if the capability isn't present.
111 """
111 """
112
112
113 class ipeercommands(interfaceutil.Interface):
113 class ipeercommands(interfaceutil.Interface):
114 """Client-side interface for communicating over the wire protocol.
114 """Client-side interface for communicating over the wire protocol.
115
115
116 This interface is used as a gateway to the Mercurial wire protocol.
116 This interface is used as a gateway to the Mercurial wire protocol.
117 methods commonly call wire protocol commands of the same name.
117 methods commonly call wire protocol commands of the same name.
118 """
118 """
119
119
120 def branchmap():
120 def branchmap():
121 """Obtain heads in named branches.
121 """Obtain heads in named branches.
122
122
123 Returns a dict mapping branch name to an iterable of nodes that are
123 Returns a dict mapping branch name to an iterable of nodes that are
124 heads on that branch.
124 heads on that branch.
125 """
125 """
126
126
127 def capabilities():
127 def capabilities():
128 """Obtain capabilities of the peer.
128 """Obtain capabilities of the peer.
129
129
130 Returns a set of string capabilities.
130 Returns a set of string capabilities.
131 """
131 """
132
132
133 def clonebundles():
133 def clonebundles():
134 """Obtains the clone bundles manifest for the repo.
134 """Obtains the clone bundles manifest for the repo.
135
135
136 Returns the manifest as unparsed bytes.
136 Returns the manifest as unparsed bytes.
137 """
137 """
138
138
139 def debugwireargs(one, two, three=None, four=None, five=None):
139 def debugwireargs(one, two, three=None, four=None, five=None):
140 """Used to facilitate debugging of arguments passed over the wire."""
140 """Used to facilitate debugging of arguments passed over the wire."""
141
141
142 def getbundle(source, **kwargs):
142 def getbundle(source, **kwargs):
143 """Obtain remote repository data as a bundle.
143 """Obtain remote repository data as a bundle.
144
144
145 This command is how the bulk of repository data is transferred from
145 This command is how the bulk of repository data is transferred from
146 the peer to the local repository
146 the peer to the local repository
147
147
148 Returns a generator of bundle data.
148 Returns a generator of bundle data.
149 """
149 """
150
150
151 def heads():
151 def heads():
152 """Determine all known head revisions in the peer.
152 """Determine all known head revisions in the peer.
153
153
154 Returns an iterable of binary nodes.
154 Returns an iterable of binary nodes.
155 """
155 """
156
156
157 def known(nodes):
157 def known(nodes):
158 """Determine whether multiple nodes are known.
158 """Determine whether multiple nodes are known.
159
159
160 Accepts an iterable of nodes whose presence to check for.
160 Accepts an iterable of nodes whose presence to check for.
161
161
162 Returns an iterable of booleans indicating of the corresponding node
162 Returns an iterable of booleans indicating of the corresponding node
163 at that index is known to the peer.
163 at that index is known to the peer.
164 """
164 """
165
165
166 def listkeys(namespace):
166 def listkeys(namespace):
167 """Obtain all keys in a pushkey namespace.
167 """Obtain all keys in a pushkey namespace.
168
168
169 Returns an iterable of key names.
169 Returns an iterable of key names.
170 """
170 """
171
171
172 def lookup(key):
172 def lookup(key):
173 """Resolve a value to a known revision.
173 """Resolve a value to a known revision.
174
174
175 Returns a binary node of the resolved revision on success.
175 Returns a binary node of the resolved revision on success.
176 """
176 """
177
177
178 def pushkey(namespace, key, old, new):
178 def pushkey(namespace, key, old, new):
179 """Set a value using the ``pushkey`` protocol.
179 """Set a value using the ``pushkey`` protocol.
180
180
181 Arguments correspond to the pushkey namespace and key to operate on and
181 Arguments correspond to the pushkey namespace and key to operate on and
182 the old and new values for that key.
182 the old and new values for that key.
183
183
184 Returns a string with the peer result. The value inside varies by the
184 Returns a string with the peer result. The value inside varies by the
185 namespace.
185 namespace.
186 """
186 """
187
187
188 def stream_out():
188 def stream_out():
189 """Obtain streaming clone data.
189 """Obtain streaming clone data.
190
190
191 Successful result should be a generator of data chunks.
191 Successful result should be a generator of data chunks.
192 """
192 """
193
193
194 def unbundle(bundle, heads, url):
194 def unbundle(bundle, heads, url):
195 """Transfer repository data to the peer.
195 """Transfer repository data to the peer.
196
196
197 This is how the bulk of data during a push is transferred.
197 This is how the bulk of data during a push is transferred.
198
198
199 Returns the integer number of heads added to the peer.
199 Returns the integer number of heads added to the peer.
200 """
200 """
201
201
202 class ipeerlegacycommands(interfaceutil.Interface):
202 class ipeerlegacycommands(interfaceutil.Interface):
203 """Interface for implementing support for legacy wire protocol commands.
203 """Interface for implementing support for legacy wire protocol commands.
204
204
205 Wire protocol commands transition to legacy status when they are no longer
205 Wire protocol commands transition to legacy status when they are no longer
206 used by modern clients. To facilitate identifying which commands are
206 used by modern clients. To facilitate identifying which commands are
207 legacy, the interfaces are split.
207 legacy, the interfaces are split.
208 """
208 """
209
209
210 def between(pairs):
210 def between(pairs):
211 """Obtain nodes between pairs of nodes.
211 """Obtain nodes between pairs of nodes.
212
212
213 ``pairs`` is an iterable of node pairs.
213 ``pairs`` is an iterable of node pairs.
214
214
215 Returns an iterable of iterables of nodes corresponding to each
215 Returns an iterable of iterables of nodes corresponding to each
216 requested pair.
216 requested pair.
217 """
217 """
218
218
219 def branches(nodes):
219 def branches(nodes):
220 """Obtain ancestor changesets of specific nodes back to a branch point.
220 """Obtain ancestor changesets of specific nodes back to a branch point.
221
221
222 For each requested node, the peer finds the first ancestor node that is
222 For each requested node, the peer finds the first ancestor node that is
223 a DAG root or is a merge.
223 a DAG root or is a merge.
224
224
225 Returns an iterable of iterables with the resolved values for each node.
225 Returns an iterable of iterables with the resolved values for each node.
226 """
226 """
227
227
228 def changegroup(nodes, source):
228 def changegroup(nodes, source):
229 """Obtain a changegroup with data for descendants of specified nodes."""
229 """Obtain a changegroup with data for descendants of specified nodes."""
230
230
231 def changegroupsubset(bases, heads, source):
231 def changegroupsubset(bases, heads, source):
232 pass
232 pass
233
233
234 class ipeercommandexecutor(interfaceutil.Interface):
234 class ipeercommandexecutor(interfaceutil.Interface):
235 """Represents a mechanism to execute remote commands.
235 """Represents a mechanism to execute remote commands.
236
236
237 This is the primary interface for requesting that wire protocol commands
237 This is the primary interface for requesting that wire protocol commands
238 be executed. Instances of this interface are active in a context manager
238 be executed. Instances of this interface are active in a context manager
239 and have a well-defined lifetime. When the context manager exits, all
239 and have a well-defined lifetime. When the context manager exits, all
240 outstanding requests are waited on.
240 outstanding requests are waited on.
241 """
241 """
242
242
243 def callcommand(name, args):
243 def callcommand(name, args):
244 """Request that a named command be executed.
244 """Request that a named command be executed.
245
245
246 Receives the command name and a dictionary of command arguments.
246 Receives the command name and a dictionary of command arguments.
247
247
248 Returns a ``concurrent.futures.Future`` that will resolve to the
248 Returns a ``concurrent.futures.Future`` that will resolve to the
249 result of that command request. That exact value is left up to
249 result of that command request. That exact value is left up to
250 the implementation and possibly varies by command.
250 the implementation and possibly varies by command.
251
251
252 Not all commands can coexist with other commands in an executor
252 Not all commands can coexist with other commands in an executor
253 instance: it depends on the underlying wire protocol transport being
253 instance: it depends on the underlying wire protocol transport being
254 used and the command itself.
254 used and the command itself.
255
255
256 Implementations MAY call ``sendcommands()`` automatically if the
256 Implementations MAY call ``sendcommands()`` automatically if the
257 requested command can not coexist with other commands in this executor.
257 requested command can not coexist with other commands in this executor.
258
258
259 Implementations MAY call ``sendcommands()`` automatically when the
259 Implementations MAY call ``sendcommands()`` automatically when the
260 future's ``result()`` is called. So, consumers using multiple
260 future's ``result()`` is called. So, consumers using multiple
261 commands with an executor MUST ensure that ``result()`` is not called
261 commands with an executor MUST ensure that ``result()`` is not called
262 until all command requests have been issued.
262 until all command requests have been issued.
263 """
263 """
264
264
265 def sendcommands():
265 def sendcommands():
266 """Trigger submission of queued command requests.
266 """Trigger submission of queued command requests.
267
267
268 Not all transports submit commands as soon as they are requested to
268 Not all transports submit commands as soon as they are requested to
269 run. When called, this method forces queued command requests to be
269 run. When called, this method forces queued command requests to be
270 issued. It will no-op if all commands have already been sent.
270 issued. It will no-op if all commands have already been sent.
271
271
272 When called, no more new commands may be issued with this executor.
272 When called, no more new commands may be issued with this executor.
273 """
273 """
274
274
275 def close():
275 def close():
276 """Signal that this command request is finished.
276 """Signal that this command request is finished.
277
277
278 When called, no more new commands may be issued. All outstanding
278 When called, no more new commands may be issued. All outstanding
279 commands that have previously been issued are waited on before
279 commands that have previously been issued are waited on before
280 returning. This not only includes waiting for the futures to resolve,
280 returning. This not only includes waiting for the futures to resolve,
281 but also waiting for all response data to arrive. In other words,
281 but also waiting for all response data to arrive. In other words,
282 calling this waits for all on-wire state for issued command requests
282 calling this waits for all on-wire state for issued command requests
283 to finish.
283 to finish.
284
284
285 When used as a context manager, this method is called when exiting the
285 When used as a context manager, this method is called when exiting the
286 context manager.
286 context manager.
287
287
288 This method may call ``sendcommands()`` if there are buffered commands.
288 This method may call ``sendcommands()`` if there are buffered commands.
289 """
289 """
290
290
291 class ipeerrequests(interfaceutil.Interface):
291 class ipeerrequests(interfaceutil.Interface):
292 """Interface for executing commands on a peer."""
292 """Interface for executing commands on a peer."""
293
293
294 limitedarguments = interfaceutil.Attribute(
294 limitedarguments = interfaceutil.Attribute(
295 """True if the peer cannot receive large argument value for commands."""
295 """True if the peer cannot receive large argument value for commands."""
296 )
296 )
297
297
298 def commandexecutor():
298 def commandexecutor():
299 """A context manager that resolves to an ipeercommandexecutor.
299 """A context manager that resolves to an ipeercommandexecutor.
300
300
301 The object this resolves to can be used to issue command requests
301 The object this resolves to can be used to issue command requests
302 to the peer.
302 to the peer.
303
303
304 Callers should call its ``callcommand`` method to issue command
304 Callers should call its ``callcommand`` method to issue command
305 requests.
305 requests.
306
306
307 A new executor should be obtained for each distinct set of commands
307 A new executor should be obtained for each distinct set of commands
308 (possibly just a single command) that the consumer wants to execute
308 (possibly just a single command) that the consumer wants to execute
309 as part of a single operation or round trip. This is because some
309 as part of a single operation or round trip. This is because some
310 peers are half-duplex and/or don't support persistent connections.
310 peers are half-duplex and/or don't support persistent connections.
311 e.g. in the case of HTTP peers, commands sent to an executor represent
311 e.g. in the case of HTTP peers, commands sent to an executor represent
312 a single HTTP request. While some peers may support multiple command
312 a single HTTP request. While some peers may support multiple command
313 sends over the wire per executor, consumers need to code to the least
313 sends over the wire per executor, consumers need to code to the least
314 capable peer. So it should be assumed that command executors buffer
314 capable peer. So it should be assumed that command executors buffer
315 called commands until they are told to send them and that each
315 called commands until they are told to send them and that each
316 command executor could result in a new connection or wire-level request
316 command executor could result in a new connection or wire-level request
317 being issued.
317 being issued.
318 """
318 """
319
319
320 class ipeerbase(ipeerconnection, ipeercapabilities, ipeerrequests):
320 class ipeerbase(ipeerconnection, ipeercapabilities, ipeerrequests):
321 """Unified interface for peer repositories.
321 """Unified interface for peer repositories.
322
322
323 All peer instances must conform to this interface.
323 All peer instances must conform to this interface.
324 """
324 """
325
325
326 class ipeerv2(ipeerconnection, ipeercapabilities, ipeerrequests):
326 class ipeerv2(ipeerconnection, ipeercapabilities, ipeerrequests):
327 """Unified peer interface for wire protocol version 2 peers."""
327 """Unified peer interface for wire protocol version 2 peers."""
328
328
329 apidescriptor = interfaceutil.Attribute(
329 apidescriptor = interfaceutil.Attribute(
330 """Data structure holding description of server API.""")
330 """Data structure holding description of server API.""")
331
331
332 @interfaceutil.implementer(ipeerbase)
332 @interfaceutil.implementer(ipeerbase)
333 class peer(object):
333 class peer(object):
334 """Base class for peer repositories."""
334 """Base class for peer repositories."""
335
335
336 limitedarguments = False
336 limitedarguments = False
337
337
338 def capable(self, name):
338 def capable(self, name):
339 caps = self.capabilities()
339 caps = self.capabilities()
340 if name in caps:
340 if name in caps:
341 return True
341 return True
342
342
343 name = '%s=' % name
343 name = '%s=' % name
344 for cap in caps:
344 for cap in caps:
345 if cap.startswith(name):
345 if cap.startswith(name):
346 return cap[len(name):]
346 return cap[len(name):]
347
347
348 return False
348 return False
349
349
350 def requirecap(self, name, purpose):
350 def requirecap(self, name, purpose):
351 if self.capable(name):
351 if self.capable(name):
352 return
352 return
353
353
354 raise error.CapabilityError(
354 raise error.CapabilityError(
355 _('cannot %s; remote repository does not support the '
355 _('cannot %s; remote repository does not support the '
356 '\'%s\' capability') % (purpose, name))
356 '\'%s\' capability') % (purpose, name))
357
357
358 class iverifyproblem(interfaceutil.Interface):
358 class iverifyproblem(interfaceutil.Interface):
359 """Represents a problem with the integrity of the repository.
359 """Represents a problem with the integrity of the repository.
360
360
361 Instances of this interface are emitted to describe an integrity issue
361 Instances of this interface are emitted to describe an integrity issue
362 with a repository (e.g. corrupt storage, missing data, etc).
362 with a repository (e.g. corrupt storage, missing data, etc).
363
363
364 Instances are essentially messages associated with severity.
364 Instances are essentially messages associated with severity.
365 """
365 """
366 warning = interfaceutil.Attribute(
366 warning = interfaceutil.Attribute(
367 """Message indicating a non-fatal problem.""")
367 """Message indicating a non-fatal problem.""")
368
368
369 error = interfaceutil.Attribute(
369 error = interfaceutil.Attribute(
370 """Message indicating a fatal problem.""")
370 """Message indicating a fatal problem.""")
371
371
372 node = interfaceutil.Attribute(
372 node = interfaceutil.Attribute(
373 """Revision encountering the problem.
373 """Revision encountering the problem.
374
374
375 ``None`` means the problem doesn't apply to a single revision.
375 ``None`` means the problem doesn't apply to a single revision.
376 """)
376 """)
377
377
378 class irevisiondelta(interfaceutil.Interface):
378 class irevisiondelta(interfaceutil.Interface):
379 """Represents a delta between one revision and another.
379 """Represents a delta between one revision and another.
380
380
381 Instances convey enough information to allow a revision to be exchanged
381 Instances convey enough information to allow a revision to be exchanged
382 with another repository.
382 with another repository.
383
383
384 Instances represent the fulltext revision data or a delta against
384 Instances represent the fulltext revision data or a delta against
385 another revision. Therefore the ``revision`` and ``delta`` attributes
385 another revision. Therefore the ``revision`` and ``delta`` attributes
386 are mutually exclusive.
386 are mutually exclusive.
387
387
388 Typically used for changegroup generation.
388 Typically used for changegroup generation.
389 """
389 """
390
390
391 node = interfaceutil.Attribute(
391 node = interfaceutil.Attribute(
392 """20 byte node of this revision.""")
392 """20 byte node of this revision.""")
393
393
394 p1node = interfaceutil.Attribute(
394 p1node = interfaceutil.Attribute(
395 """20 byte node of 1st parent of this revision.""")
395 """20 byte node of 1st parent of this revision.""")
396
396
397 p2node = interfaceutil.Attribute(
397 p2node = interfaceutil.Attribute(
398 """20 byte node of 2nd parent of this revision.""")
398 """20 byte node of 2nd parent of this revision.""")
399
399
400 linknode = interfaceutil.Attribute(
400 linknode = interfaceutil.Attribute(
401 """20 byte node of the changelog revision this node is linked to.""")
401 """20 byte node of the changelog revision this node is linked to.""")
402
402
403 flags = interfaceutil.Attribute(
403 flags = interfaceutil.Attribute(
404 """2 bytes of integer flags that apply to this revision.
404 """2 bytes of integer flags that apply to this revision.
405
405
406 This is a bitwise composition of the ``REVISION_FLAG_*`` constants.
406 This is a bitwise composition of the ``REVISION_FLAG_*`` constants.
407 """)
407 """)
408
408
409 basenode = interfaceutil.Attribute(
409 basenode = interfaceutil.Attribute(
410 """20 byte node of the revision this data is a delta against.
410 """20 byte node of the revision this data is a delta against.
411
411
412 ``nullid`` indicates that the revision is a full revision and not
412 ``nullid`` indicates that the revision is a full revision and not
413 a delta.
413 a delta.
414 """)
414 """)
415
415
416 baserevisionsize = interfaceutil.Attribute(
416 baserevisionsize = interfaceutil.Attribute(
417 """Size of base revision this delta is against.
417 """Size of base revision this delta is against.
418
418
419 May be ``None`` if ``basenode`` is ``nullid``.
419 May be ``None`` if ``basenode`` is ``nullid``.
420 """)
420 """)
421
421
422 revision = interfaceutil.Attribute(
422 revision = interfaceutil.Attribute(
423 """Raw fulltext of revision data for this node.""")
423 """Raw fulltext of revision data for this node.""")
424
424
425 delta = interfaceutil.Attribute(
425 delta = interfaceutil.Attribute(
426 """Delta between ``basenode`` and ``node``.
426 """Delta between ``basenode`` and ``node``.
427
427
428 Stored in the bdiff delta format.
428 Stored in the bdiff delta format.
429 """)
429 """)
430
430
431 class ifilerevisionssequence(interfaceutil.Interface):
431 class ifilerevisionssequence(interfaceutil.Interface):
432 """Contains index data for all revisions of a file.
432 """Contains index data for all revisions of a file.
433
433
434 Types implementing this behave like lists of tuples. The index
434 Types implementing this behave like lists of tuples. The index
435 in the list corresponds to the revision number. The values contain
435 in the list corresponds to the revision number. The values contain
436 index metadata.
436 index metadata.
437
437
438 The *null* revision (revision number -1) is always the last item
438 The *null* revision (revision number -1) is always the last item
439 in the index.
439 in the index.
440 """
440 """
441
441
442 def __len__():
442 def __len__():
443 """The total number of revisions."""
443 """The total number of revisions."""
444
444
445 def __getitem__(rev):
445 def __getitem__(rev):
446 """Returns the object having a specific revision number.
446 """Returns the object having a specific revision number.
447
447
448 Returns an 8-tuple with the following fields:
448 Returns an 8-tuple with the following fields:
449
449
450 offset+flags
450 offset+flags
451 Contains the offset and flags for the revision. 64-bit unsigned
451 Contains the offset and flags for the revision. 64-bit unsigned
452 integer where first 6 bytes are the offset and the next 2 bytes
452 integer where first 6 bytes are the offset and the next 2 bytes
453 are flags. The offset can be 0 if it is not used by the store.
453 are flags. The offset can be 0 if it is not used by the store.
454 compressed size
454 compressed size
455 Size of the revision data in the store. It can be 0 if it isn't
455 Size of the revision data in the store. It can be 0 if it isn't
456 needed by the store.
456 needed by the store.
457 uncompressed size
457 uncompressed size
458 Fulltext size. It can be 0 if it isn't needed by the store.
458 Fulltext size. It can be 0 if it isn't needed by the store.
459 base revision
459 base revision
460 Revision number of revision the delta for storage is encoded
460 Revision number of revision the delta for storage is encoded
461 against. -1 indicates not encoded against a base revision.
461 against. -1 indicates not encoded against a base revision.
462 link revision
462 link revision
463 Revision number of changelog revision this entry is related to.
463 Revision number of changelog revision this entry is related to.
464 p1 revision
464 p1 revision
465 Revision number of 1st parent. -1 if no 1st parent.
465 Revision number of 1st parent. -1 if no 1st parent.
466 p2 revision
466 p2 revision
467 Revision number of 2nd parent. -1 if no 1st parent.
467 Revision number of 2nd parent. -1 if no 1st parent.
468 node
468 node
469 Binary node value for this revision number.
469 Binary node value for this revision number.
470
470
471 Negative values should index off the end of the sequence. ``-1``
471 Negative values should index off the end of the sequence. ``-1``
472 should return the null revision. ``-2`` should return the most
472 should return the null revision. ``-2`` should return the most
473 recent revision.
473 recent revision.
474 """
474 """
475
475
476 def __contains__(rev):
476 def __contains__(rev):
477 """Whether a revision number exists."""
477 """Whether a revision number exists."""
478
478
479 def insert(self, i, entry):
479 def insert(self, i, entry):
480 """Add an item to the index at specific revision."""
480 """Add an item to the index at specific revision."""
481
481
482 class ifileindex(interfaceutil.Interface):
482 class ifileindex(interfaceutil.Interface):
483 """Storage interface for index data of a single file.
483 """Storage interface for index data of a single file.
484
484
485 File storage data is divided into index metadata and data storage.
485 File storage data is divided into index metadata and data storage.
486 This interface defines the index portion of the interface.
486 This interface defines the index portion of the interface.
487
487
488 The index logically consists of:
488 The index logically consists of:
489
489
490 * A mapping between revision numbers and nodes.
490 * A mapping between revision numbers and nodes.
491 * DAG data (storing and querying the relationship between nodes).
491 * DAG data (storing and querying the relationship between nodes).
492 * Metadata to facilitate storage.
492 * Metadata to facilitate storage.
493 """
493 """
494 def __len__():
494 def __len__():
495 """Obtain the number of revisions stored for this file."""
495 """Obtain the number of revisions stored for this file."""
496
496
497 def __iter__():
497 def __iter__():
498 """Iterate over revision numbers for this file."""
498 """Iterate over revision numbers for this file."""
499
499
500 def hasnode(node):
500 def hasnode(node):
501 """Returns a bool indicating if a node is known to this store.
501 """Returns a bool indicating if a node is known to this store.
502
502
503 Implementations must only return True for full, binary node values:
503 Implementations must only return True for full, binary node values:
504 hex nodes, revision numbers, and partial node matches must be
504 hex nodes, revision numbers, and partial node matches must be
505 rejected.
505 rejected.
506
506
507 The null node is never present.
507 The null node is never present.
508 """
508 """
509
509
510 def revs(start=0, stop=None):
510 def revs(start=0, stop=None):
511 """Iterate over revision numbers for this file, with control."""
511 """Iterate over revision numbers for this file, with control."""
512
512
513 def parents(node):
513 def parents(node):
514 """Returns a 2-tuple of parent nodes for a revision.
514 """Returns a 2-tuple of parent nodes for a revision.
515
515
516 Values will be ``nullid`` if the parent is empty.
516 Values will be ``nullid`` if the parent is empty.
517 """
517 """
518
518
519 def parentrevs(rev):
519 def parentrevs(rev):
520 """Like parents() but operates on revision numbers."""
520 """Like parents() but operates on revision numbers."""
521
521
522 def rev(node):
522 def rev(node):
523 """Obtain the revision number given a node.
523 """Obtain the revision number given a node.
524
524
525 Raises ``error.LookupError`` if the node is not known.
525 Raises ``error.LookupError`` if the node is not known.
526 """
526 """
527
527
528 def node(rev):
528 def node(rev):
529 """Obtain the node value given a revision number.
529 """Obtain the node value given a revision number.
530
530
531 Raises ``IndexError`` if the node is not known.
531 Raises ``IndexError`` if the node is not known.
532 """
532 """
533
533
534 def lookup(node):
534 def lookup(node):
535 """Attempt to resolve a value to a node.
535 """Attempt to resolve a value to a node.
536
536
537 Value can be a binary node, hex node, revision number, or a string
537 Value can be a binary node, hex node, revision number, or a string
538 that can be converted to an integer.
538 that can be converted to an integer.
539
539
540 Raises ``error.LookupError`` if a node could not be resolved.
540 Raises ``error.LookupError`` if a node could not be resolved.
541 """
541 """
542
542
543 def linkrev(rev):
543 def linkrev(rev):
544 """Obtain the changeset revision number a revision is linked to."""
544 """Obtain the changeset revision number a revision is linked to."""
545
545
546 def iscensored(rev):
546 def iscensored(rev):
547 """Return whether a revision's content has been censored."""
547 """Return whether a revision's content has been censored."""
548
548
549 def commonancestorsheads(node1, node2):
549 def commonancestorsheads(node1, node2):
550 """Obtain an iterable of nodes containing heads of common ancestors.
550 """Obtain an iterable of nodes containing heads of common ancestors.
551
551
552 See ``ancestor.commonancestorsheads()``.
552 See ``ancestor.commonancestorsheads()``.
553 """
553 """
554
554
555 def descendants(revs):
555 def descendants(revs):
556 """Obtain descendant revision numbers for a set of revision numbers.
556 """Obtain descendant revision numbers for a set of revision numbers.
557
557
558 If ``nullrev`` is in the set, this is equivalent to ``revs()``.
558 If ``nullrev`` is in the set, this is equivalent to ``revs()``.
559 """
559 """
560
560
561 def heads(start=None, stop=None):
561 def heads(start=None, stop=None):
562 """Obtain a list of nodes that are DAG heads, with control.
562 """Obtain a list of nodes that are DAG heads, with control.
563
563
564 The set of revisions examined can be limited by specifying
564 The set of revisions examined can be limited by specifying
565 ``start`` and ``stop``. ``start`` is a node. ``stop`` is an
565 ``start`` and ``stop``. ``start`` is a node. ``stop`` is an
566 iterable of nodes. DAG traversal starts at earlier revision
566 iterable of nodes. DAG traversal starts at earlier revision
567 ``start`` and iterates forward until any node in ``stop`` is
567 ``start`` and iterates forward until any node in ``stop`` is
568 encountered.
568 encountered.
569 """
569 """
570
570
571 def children(node):
571 def children(node):
572 """Obtain nodes that are children of a node.
572 """Obtain nodes that are children of a node.
573
573
574 Returns a list of nodes.
574 Returns a list of nodes.
575 """
575 """
576
576
577 class ifiledata(interfaceutil.Interface):
577 class ifiledata(interfaceutil.Interface):
578 """Storage interface for data storage of a specific file.
578 """Storage interface for data storage of a specific file.
579
579
580 This complements ``ifileindex`` and provides an interface for accessing
580 This complements ``ifileindex`` and provides an interface for accessing
581 data for a tracked file.
581 data for a tracked file.
582 """
582 """
583 def size(rev):
583 def size(rev):
584 """Obtain the fulltext size of file data.
584 """Obtain the fulltext size of file data.
585
585
586 Any metadata is excluded from size measurements.
586 Any metadata is excluded from size measurements.
587 """
587 """
588
588
589 def revision(node, raw=False):
589 def revision(node, raw=False):
590 """"Obtain fulltext data for a node.
590 """"Obtain fulltext data for a node.
591
591
592 By default, any storage transformations are applied before the data
592 By default, any storage transformations are applied before the data
593 is returned. If ``raw`` is True, non-raw storage transformations
593 is returned. If ``raw`` is True, non-raw storage transformations
594 are not applied.
594 are not applied.
595
595
596 The fulltext data may contain a header containing metadata. Most
596 The fulltext data may contain a header containing metadata. Most
597 consumers should use ``read()`` to obtain the actual file data.
597 consumers should use ``read()`` to obtain the actual file data.
598 """
598 """
599
599
600 def rawdata(node):
600 def rawdata(node):
601 """Obtain raw data for a node.
601 """Obtain raw data for a node.
602 """
602 """
603
603
604 def read(node):
604 def read(node):
605 """Resolve file fulltext data.
605 """Resolve file fulltext data.
606
606
607 This is similar to ``revision()`` except any metadata in the data
607 This is similar to ``revision()`` except any metadata in the data
608 headers is stripped.
608 headers is stripped.
609 """
609 """
610
610
611 def renamed(node):
611 def renamed(node):
612 """Obtain copy metadata for a node.
612 """Obtain copy metadata for a node.
613
613
614 Returns ``False`` if no copy metadata is stored or a 2-tuple of
614 Returns ``False`` if no copy metadata is stored or a 2-tuple of
615 (path, node) from which this revision was copied.
615 (path, node) from which this revision was copied.
616 """
616 """
617
617
618 def cmp(node, fulltext):
618 def cmp(node, fulltext):
619 """Compare fulltext to another revision.
619 """Compare fulltext to another revision.
620
620
621 Returns True if the fulltext is different from what is stored.
621 Returns True if the fulltext is different from what is stored.
622
622
623 This takes copy metadata into account.
623 This takes copy metadata into account.
624
624
625 TODO better document the copy metadata and censoring logic.
625 TODO better document the copy metadata and censoring logic.
626 """
626 """
627
627
628 def emitrevisions(nodes,
628 def emitrevisions(nodes,
629 nodesorder=None,
629 nodesorder=None,
630 revisiondata=False,
630 revisiondata=False,
631 assumehaveparentrevisions=False,
631 assumehaveparentrevisions=False,
632 deltamode=CG_DELTAMODE_STD):
632 deltamode=CG_DELTAMODE_STD):
633 """Produce ``irevisiondelta`` for revisions.
633 """Produce ``irevisiondelta`` for revisions.
634
634
635 Given an iterable of nodes, emits objects conforming to the
635 Given an iterable of nodes, emits objects conforming to the
636 ``irevisiondelta`` interface that describe revisions in storage.
636 ``irevisiondelta`` interface that describe revisions in storage.
637
637
638 This method is a generator.
638 This method is a generator.
639
639
640 The input nodes may be unordered. Implementations must ensure that a
640 The input nodes may be unordered. Implementations must ensure that a
641 node's parents are emitted before the node itself. Transitively, this
641 node's parents are emitted before the node itself. Transitively, this
642 means that a node may only be emitted once all its ancestors in
642 means that a node may only be emitted once all its ancestors in
643 ``nodes`` have also been emitted.
643 ``nodes`` have also been emitted.
644
644
645 By default, emits "index" data (the ``node``, ``p1node``, and
645 By default, emits "index" data (the ``node``, ``p1node``, and
646 ``p2node`` attributes). If ``revisiondata`` is set, revision data
646 ``p2node`` attributes). If ``revisiondata`` is set, revision data
647 will also be present on the emitted objects.
647 will also be present on the emitted objects.
648
648
649 With default argument values, implementations can choose to emit
649 With default argument values, implementations can choose to emit
650 either fulltext revision data or a delta. When emitting deltas,
650 either fulltext revision data or a delta. When emitting deltas,
651 implementations must consider whether the delta's base revision
651 implementations must consider whether the delta's base revision
652 fulltext is available to the receiver.
652 fulltext is available to the receiver.
653
653
654 The base revision fulltext is guaranteed to be available if any of
654 The base revision fulltext is guaranteed to be available if any of
655 the following are met:
655 the following are met:
656
656
657 * Its fulltext revision was emitted by this method call.
657 * Its fulltext revision was emitted by this method call.
658 * A delta for that revision was emitted by this method call.
658 * A delta for that revision was emitted by this method call.
659 * ``assumehaveparentrevisions`` is True and the base revision is a
659 * ``assumehaveparentrevisions`` is True and the base revision is a
660 parent of the node.
660 parent of the node.
661
661
662 ``nodesorder`` can be used to control the order that revisions are
662 ``nodesorder`` can be used to control the order that revisions are
663 emitted. By default, revisions can be reordered as long as they are
663 emitted. By default, revisions can be reordered as long as they are
664 in DAG topological order (see above). If the value is ``nodes``,
664 in DAG topological order (see above). If the value is ``nodes``,
665 the iteration order from ``nodes`` should be used. If the value is
665 the iteration order from ``nodes`` should be used. If the value is
666 ``storage``, then the native order from the backing storage layer
666 ``storage``, then the native order from the backing storage layer
667 is used. (Not all storage layers will have strong ordering and behavior
667 is used. (Not all storage layers will have strong ordering and behavior
668 of this mode is storage-dependent.) ``nodes`` ordering can force
668 of this mode is storage-dependent.) ``nodes`` ordering can force
669 revisions to be emitted before their ancestors, so consumers should
669 revisions to be emitted before their ancestors, so consumers should
670 use it with care.
670 use it with care.
671
671
672 The ``linknode`` attribute on the returned ``irevisiondelta`` may not
672 The ``linknode`` attribute on the returned ``irevisiondelta`` may not
673 be set and it is the caller's responsibility to resolve it, if needed.
673 be set and it is the caller's responsibility to resolve it, if needed.
674
674
675 If ``deltamode`` is CG_DELTAMODE_PREV and revision data is requested,
675 If ``deltamode`` is CG_DELTAMODE_PREV and revision data is requested,
676 all revision data should be emitted as deltas against the revision
676 all revision data should be emitted as deltas against the revision
677 emitted just prior. The initial revision should be a delta against its
677 emitted just prior. The initial revision should be a delta against its
678 1st parent.
678 1st parent.
679 """
679 """
680
680
681 class ifilemutation(interfaceutil.Interface):
681 class ifilemutation(interfaceutil.Interface):
682 """Storage interface for mutation events of a tracked file."""
682 """Storage interface for mutation events of a tracked file."""
683
683
684 def add(filedata, meta, transaction, linkrev, p1, p2):
684 def add(filedata, meta, transaction, linkrev, p1, p2):
685 """Add a new revision to the store.
685 """Add a new revision to the store.
686
686
687 Takes file data, dictionary of metadata, a transaction, linkrev,
687 Takes file data, dictionary of metadata, a transaction, linkrev,
688 and parent nodes.
688 and parent nodes.
689
689
690 Returns the node that was added.
690 Returns the node that was added.
691
691
692 May no-op if a revision matching the supplied data is already stored.
692 May no-op if a revision matching the supplied data is already stored.
693 """
693 """
694
694
695 def addrevision(revisiondata, transaction, linkrev, p1, p2, node=None,
695 def addrevision(revisiondata, transaction, linkrev, p1, p2, node=None,
696 flags=0, cachedelta=None):
696 flags=0, cachedelta=None):
697 """Add a new revision to the store.
697 """Add a new revision to the store.
698
698
699 This is similar to ``add()`` except it operates at a lower level.
699 This is similar to ``add()`` except it operates at a lower level.
700
700
701 The data passed in already contains a metadata header, if any.
701 The data passed in already contains a metadata header, if any.
702
702
703 ``node`` and ``flags`` can be used to define the expected node and
703 ``node`` and ``flags`` can be used to define the expected node and
704 the flags to use with storage. ``flags`` is a bitwise value composed
704 the flags to use with storage. ``flags`` is a bitwise value composed
705 of the various ``REVISION_FLAG_*`` constants.
705 of the various ``REVISION_FLAG_*`` constants.
706
706
707 ``add()`` is usually called when adding files from e.g. the working
707 ``add()`` is usually called when adding files from e.g. the working
708 directory. ``addrevision()`` is often called by ``add()`` and for
708 directory. ``addrevision()`` is often called by ``add()`` and for
709 scenarios where revision data has already been computed, such as when
709 scenarios where revision data has already been computed, such as when
710 applying raw data from a peer repo.
710 applying raw data from a peer repo.
711 """
711 """
712
712
713 def addgroup(deltas, linkmapper, transaction, addrevisioncb=None,
713 def addgroup(deltas, linkmapper, transaction, addrevisioncb=None,
714 maybemissingparents=False):
714 maybemissingparents=False):
715 """Process a series of deltas for storage.
715 """Process a series of deltas for storage.
716
716
717 ``deltas`` is an iterable of 7-tuples of
717 ``deltas`` is an iterable of 7-tuples of
718 (node, p1, p2, linknode, deltabase, delta, flags) defining revisions
718 (node, p1, p2, linknode, deltabase, delta, flags) defining revisions
719 to add.
719 to add.
720
720
721 The ``delta`` field contains ``mpatch`` data to apply to a base
721 The ``delta`` field contains ``mpatch`` data to apply to a base
722 revision, identified by ``deltabase``. The base node can be
722 revision, identified by ``deltabase``. The base node can be
723 ``nullid``, in which case the header from the delta can be ignored
723 ``nullid``, in which case the header from the delta can be ignored
724 and the delta used as the fulltext.
724 and the delta used as the fulltext.
725
725
726 ``addrevisioncb`` should be called for each node as it is committed.
726 ``addrevisioncb`` should be called for each node as it is committed.
727
727
728 ``maybemissingparents`` is a bool indicating whether the incoming
728 ``maybemissingparents`` is a bool indicating whether the incoming
729 data may reference parents/ancestor revisions that aren't present.
729 data may reference parents/ancestor revisions that aren't present.
730 This flag is set when receiving data into a "shallow" store that
730 This flag is set when receiving data into a "shallow" store that
731 doesn't hold all history.
731 doesn't hold all history.
732
732
733 Returns a list of nodes that were processed. A node will be in the list
733 Returns a list of nodes that were processed. A node will be in the list
734 even if it existed in the store previously.
734 even if it existed in the store previously.
735 """
735 """
736
736
737 def censorrevision(tr, node, tombstone=b''):
737 def censorrevision(tr, node, tombstone=b''):
738 """Remove the content of a single revision.
738 """Remove the content of a single revision.
739
739
740 The specified ``node`` will have its content purged from storage.
740 The specified ``node`` will have its content purged from storage.
741 Future attempts to access the revision data for this node will
741 Future attempts to access the revision data for this node will
742 result in failure.
742 result in failure.
743
743
744 A ``tombstone`` message can optionally be stored. This message may be
744 A ``tombstone`` message can optionally be stored. This message may be
745 displayed to users when they attempt to access the missing revision
745 displayed to users when they attempt to access the missing revision
746 data.
746 data.
747
747
748 Storage backends may have stored deltas against the previous content
748 Storage backends may have stored deltas against the previous content
749 in this revision. As part of censoring a revision, these storage
749 in this revision. As part of censoring a revision, these storage
750 backends are expected to rewrite any internally stored deltas such
750 backends are expected to rewrite any internally stored deltas such
751 that they no longer reference the deleted content.
751 that they no longer reference the deleted content.
752 """
752 """
753
753
754 def getstrippoint(minlink):
754 def getstrippoint(minlink):
755 """Find the minimum revision that must be stripped to strip a linkrev.
755 """Find the minimum revision that must be stripped to strip a linkrev.
756
756
757 Returns a 2-tuple containing the minimum revision number and a set
757 Returns a 2-tuple containing the minimum revision number and a set
758 of all revisions numbers that would be broken by this strip.
758 of all revisions numbers that would be broken by this strip.
759
759
760 TODO this is highly revlog centric and should be abstracted into
760 TODO this is highly revlog centric and should be abstracted into
761 a higher-level deletion API. ``repair.strip()`` relies on this.
761 a higher-level deletion API. ``repair.strip()`` relies on this.
762 """
762 """
763
763
764 def strip(minlink, transaction):
764 def strip(minlink, transaction):
765 """Remove storage of items starting at a linkrev.
765 """Remove storage of items starting at a linkrev.
766
766
767 This uses ``getstrippoint()`` to determine the first node to remove.
767 This uses ``getstrippoint()`` to determine the first node to remove.
768 Then it effectively truncates storage for all revisions after that.
768 Then it effectively truncates storage for all revisions after that.
769
769
770 TODO this is highly revlog centric and should be abstracted into a
770 TODO this is highly revlog centric and should be abstracted into a
771 higher-level deletion API.
771 higher-level deletion API.
772 """
772 """
773
773
774 class ifilestorage(ifileindex, ifiledata, ifilemutation):
774 class ifilestorage(ifileindex, ifiledata, ifilemutation):
775 """Complete storage interface for a single tracked file."""
775 """Complete storage interface for a single tracked file."""
776
776
777 def files():
777 def files():
778 """Obtain paths that are backing storage for this file.
778 """Obtain paths that are backing storage for this file.
779
779
780 TODO this is used heavily by verify code and there should probably
780 TODO this is used heavily by verify code and there should probably
781 be a better API for that.
781 be a better API for that.
782 """
782 """
783
783
784 def storageinfo(exclusivefiles=False, sharedfiles=False,
784 def storageinfo(exclusivefiles=False, sharedfiles=False,
785 revisionscount=False, trackedsize=False,
785 revisionscount=False, trackedsize=False,
786 storedsize=False):
786 storedsize=False):
787 """Obtain information about storage for this file's data.
787 """Obtain information about storage for this file's data.
788
788
789 Returns a dict describing storage for this tracked path. The keys
789 Returns a dict describing storage for this tracked path. The keys
790 in the dict map to arguments of the same. The arguments are bools
790 in the dict map to arguments of the same. The arguments are bools
791 indicating whether to calculate and obtain that data.
791 indicating whether to calculate and obtain that data.
792
792
793 exclusivefiles
793 exclusivefiles
794 Iterable of (vfs, path) describing files that are exclusively
794 Iterable of (vfs, path) describing files that are exclusively
795 used to back storage for this tracked path.
795 used to back storage for this tracked path.
796
796
797 sharedfiles
797 sharedfiles
798 Iterable of (vfs, path) describing files that are used to back
798 Iterable of (vfs, path) describing files that are used to back
799 storage for this tracked path. Those files may also provide storage
799 storage for this tracked path. Those files may also provide storage
800 for other stored entities.
800 for other stored entities.
801
801
802 revisionscount
802 revisionscount
803 Number of revisions available for retrieval.
803 Number of revisions available for retrieval.
804
804
805 trackedsize
805 trackedsize
806 Total size in bytes of all tracked revisions. This is a sum of the
806 Total size in bytes of all tracked revisions. This is a sum of the
807 length of the fulltext of all revisions.
807 length of the fulltext of all revisions.
808
808
809 storedsize
809 storedsize
810 Total size in bytes used to store data for all tracked revisions.
810 Total size in bytes used to store data for all tracked revisions.
811 This is commonly less than ``trackedsize`` due to internal usage
811 This is commonly less than ``trackedsize`` due to internal usage
812 of deltas rather than fulltext revisions.
812 of deltas rather than fulltext revisions.
813
813
814 Not all storage backends may support all queries are have a reasonable
814 Not all storage backends may support all queries are have a reasonable
815 value to use. In that case, the value should be set to ``None`` and
815 value to use. In that case, the value should be set to ``None`` and
816 callers are expected to handle this special value.
816 callers are expected to handle this special value.
817 """
817 """
818
818
819 def verifyintegrity(state):
819 def verifyintegrity(state):
820 """Verifies the integrity of file storage.
820 """Verifies the integrity of file storage.
821
821
822 ``state`` is a dict holding state of the verifier process. It can be
822 ``state`` is a dict holding state of the verifier process. It can be
823 used to communicate data between invocations of multiple storage
823 used to communicate data between invocations of multiple storage
824 primitives.
824 primitives.
825
825
826 If individual revisions cannot have their revision content resolved,
826 If individual revisions cannot have their revision content resolved,
827 the method is expected to set the ``skipread`` key to a set of nodes
827 the method is expected to set the ``skipread`` key to a set of nodes
828 that encountered problems.
828 that encountered problems.
829
829
830 The method yields objects conforming to the ``iverifyproblem``
830 The method yields objects conforming to the ``iverifyproblem``
831 interface.
831 interface.
832 """
832 """
833
833
834 class idirs(interfaceutil.Interface):
834 class idirs(interfaceutil.Interface):
835 """Interface representing a collection of directories from paths.
835 """Interface representing a collection of directories from paths.
836
836
837 This interface is essentially a derived data structure representing
837 This interface is essentially a derived data structure representing
838 directories from a collection of paths.
838 directories from a collection of paths.
839 """
839 """
840
840
841 def addpath(path):
841 def addpath(path):
842 """Add a path to the collection.
842 """Add a path to the collection.
843
843
844 All directories in the path will be added to the collection.
844 All directories in the path will be added to the collection.
845 """
845 """
846
846
847 def delpath(path):
847 def delpath(path):
848 """Remove a path from the collection.
848 """Remove a path from the collection.
849
849
850 If the removal was the last path in a particular directory, the
850 If the removal was the last path in a particular directory, the
851 directory is removed from the collection.
851 directory is removed from the collection.
852 """
852 """
853
853
854 def __iter__():
854 def __iter__():
855 """Iterate over the directories in this collection of paths."""
855 """Iterate over the directories in this collection of paths."""
856
856
857 def __contains__(path):
857 def __contains__(path):
858 """Whether a specific directory is in this collection."""
858 """Whether a specific directory is in this collection."""
859
859
860 class imanifestdict(interfaceutil.Interface):
860 class imanifestdict(interfaceutil.Interface):
861 """Interface representing a manifest data structure.
861 """Interface representing a manifest data structure.
862
862
863 A manifest is effectively a dict mapping paths to entries. Each entry
863 A manifest is effectively a dict mapping paths to entries. Each entry
864 consists of a binary node and extra flags affecting that entry.
864 consists of a binary node and extra flags affecting that entry.
865 """
865 """
866
866
867 def __getitem__(path):
867 def __getitem__(path):
868 """Returns the binary node value for a path in the manifest.
868 """Returns the binary node value for a path in the manifest.
869
869
870 Raises ``KeyError`` if the path does not exist in the manifest.
870 Raises ``KeyError`` if the path does not exist in the manifest.
871
871
872 Equivalent to ``self.find(path)[0]``.
872 Equivalent to ``self.find(path)[0]``.
873 """
873 """
874
874
875 def find(path):
875 def find(path):
876 """Returns the entry for a path in the manifest.
876 """Returns the entry for a path in the manifest.
877
877
878 Returns a 2-tuple of (node, flags).
878 Returns a 2-tuple of (node, flags).
879
879
880 Raises ``KeyError`` if the path does not exist in the manifest.
880 Raises ``KeyError`` if the path does not exist in the manifest.
881 """
881 """
882
882
883 def __len__():
883 def __len__():
884 """Return the number of entries in the manifest."""
884 """Return the number of entries in the manifest."""
885
885
886 def __nonzero__():
886 def __nonzero__():
887 """Returns True if the manifest has entries, False otherwise."""
887 """Returns True if the manifest has entries, False otherwise."""
888
888
889 __bool__ = __nonzero__
889 __bool__ = __nonzero__
890
890
891 def __setitem__(path, node):
891 def __setitem__(path, node):
892 """Define the node value for a path in the manifest.
892 """Define the node value for a path in the manifest.
893
893
894 If the path is already in the manifest, its flags will be copied to
894 If the path is already in the manifest, its flags will be copied to
895 the new entry.
895 the new entry.
896 """
896 """
897
897
898 def __contains__(path):
898 def __contains__(path):
899 """Whether a path exists in the manifest."""
899 """Whether a path exists in the manifest."""
900
900
901 def __delitem__(path):
901 def __delitem__(path):
902 """Remove a path from the manifest.
902 """Remove a path from the manifest.
903
903
904 Raises ``KeyError`` if the path is not in the manifest.
904 Raises ``KeyError`` if the path is not in the manifest.
905 """
905 """
906
906
907 def __iter__():
907 def __iter__():
908 """Iterate over paths in the manifest."""
908 """Iterate over paths in the manifest."""
909
909
910 def iterkeys():
910 def iterkeys():
911 """Iterate over paths in the manifest."""
911 """Iterate over paths in the manifest."""
912
912
913 def keys():
913 def keys():
914 """Obtain a list of paths in the manifest."""
914 """Obtain a list of paths in the manifest."""
915
915
916 def filesnotin(other, match=None):
916 def filesnotin(other, match=None):
917 """Obtain the set of paths in this manifest but not in another.
917 """Obtain the set of paths in this manifest but not in another.
918
918
919 ``match`` is an optional matcher function to be applied to both
919 ``match`` is an optional matcher function to be applied to both
920 manifests.
920 manifests.
921
921
922 Returns a set of paths.
922 Returns a set of paths.
923 """
923 """
924
924
925 def dirs():
925 def dirs():
926 """Returns an object implementing the ``idirs`` interface."""
926 """Returns an object implementing the ``idirs`` interface."""
927
927
928 def hasdir(dir):
928 def hasdir(dir):
929 """Returns a bool indicating if a directory is in this manifest."""
929 """Returns a bool indicating if a directory is in this manifest."""
930
930
931 def matches(match):
931 def matches(match):
932 """Generate a new manifest filtered through a matcher.
932 """Generate a new manifest filtered through a matcher.
933
933
934 Returns an object conforming to the ``imanifestdict`` interface.
934 Returns an object conforming to the ``imanifestdict`` interface.
935 """
935 """
936
936
937 def walk(match):
937 def walk(match):
938 """Generator of paths in manifest satisfying a matcher.
938 """Generator of paths in manifest satisfying a matcher.
939
939
940 This is equivalent to ``self.matches(match).iterkeys()`` except a new
940 This is equivalent to ``self.matches(match).iterkeys()`` except a new
941 manifest object is not created.
941 manifest object is not created.
942
942
943 If the matcher has explicit files listed and they don't exist in
943 If the matcher has explicit files listed and they don't exist in
944 the manifest, ``match.bad()`` is called for each missing file.
944 the manifest, ``match.bad()`` is called for each missing file.
945 """
945 """
946
946
947 def diff(other, match=None, clean=False):
947 def diff(other, match=None, clean=False):
948 """Find differences between this manifest and another.
948 """Find differences between this manifest and another.
949
949
950 This manifest is compared to ``other``.
950 This manifest is compared to ``other``.
951
951
952 If ``match`` is provided, the two manifests are filtered against this
952 If ``match`` is provided, the two manifests are filtered against this
953 matcher and only entries satisfying the matcher are compared.
953 matcher and only entries satisfying the matcher are compared.
954
954
955 If ``clean`` is True, unchanged files are included in the returned
955 If ``clean`` is True, unchanged files are included in the returned
956 object.
956 object.
957
957
958 Returns a dict with paths as keys and values of 2-tuples of 2-tuples of
958 Returns a dict with paths as keys and values of 2-tuples of 2-tuples of
959 the form ``((node1, flag1), (node2, flag2))`` where ``(node1, flag1)``
959 the form ``((node1, flag1), (node2, flag2))`` where ``(node1, flag1)``
960 represents the node and flags for this manifest and ``(node2, flag2)``
960 represents the node and flags for this manifest and ``(node2, flag2)``
961 are the same for the other manifest.
961 are the same for the other manifest.
962 """
962 """
963
963
964 def setflag(path, flag):
964 def setflag(path, flag):
965 """Set the flag value for a given path.
965 """Set the flag value for a given path.
966
966
967 Raises ``KeyError`` if the path is not already in the manifest.
967 Raises ``KeyError`` if the path is not already in the manifest.
968 """
968 """
969
969
970 def get(path, default=None):
970 def get(path, default=None):
971 """Obtain the node value for a path or a default value if missing."""
971 """Obtain the node value for a path or a default value if missing."""
972
972
973 def flags(path, default=''):
973 def flags(path, default=''):
974 """Return the flags value for a path or a default value if missing."""
974 """Return the flags value for a path or a default value if missing."""
975
975
976 def copy():
976 def copy():
977 """Return a copy of this manifest."""
977 """Return a copy of this manifest."""
978
978
979 def items():
979 def items():
980 """Returns an iterable of (path, node) for items in this manifest."""
980 """Returns an iterable of (path, node) for items in this manifest."""
981
981
982 def iteritems():
982 def iteritems():
983 """Identical to items()."""
983 """Identical to items()."""
984
984
985 def iterentries():
985 def iterentries():
986 """Returns an iterable of (path, node, flags) for this manifest.
986 """Returns an iterable of (path, node, flags) for this manifest.
987
987
988 Similar to ``iteritems()`` except items are a 3-tuple and include
988 Similar to ``iteritems()`` except items are a 3-tuple and include
989 flags.
989 flags.
990 """
990 """
991
991
992 def text():
992 def text():
993 """Obtain the raw data representation for this manifest.
993 """Obtain the raw data representation for this manifest.
994
994
995 Result is used to create a manifest revision.
995 Result is used to create a manifest revision.
996 """
996 """
997
997
998 def fastdelta(base, changes):
998 def fastdelta(base, changes):
999 """Obtain a delta between this manifest and another given changes.
999 """Obtain a delta between this manifest and another given changes.
1000
1000
1001 ``base`` in the raw data representation for another manifest.
1001 ``base`` in the raw data representation for another manifest.
1002
1002
1003 ``changes`` is an iterable of ``(path, to_delete)``.
1003 ``changes`` is an iterable of ``(path, to_delete)``.
1004
1004
1005 Returns a 2-tuple containing ``bytearray(self.text())`` and the
1005 Returns a 2-tuple containing ``bytearray(self.text())`` and the
1006 delta between ``base`` and this manifest.
1006 delta between ``base`` and this manifest.
1007 """
1007 """
1008
1008
1009 class imanifestrevisionbase(interfaceutil.Interface):
1009 class imanifestrevisionbase(interfaceutil.Interface):
1010 """Base interface representing a single revision of a manifest.
1010 """Base interface representing a single revision of a manifest.
1011
1011
1012 Should not be used as a primary interface: should always be inherited
1012 Should not be used as a primary interface: should always be inherited
1013 as part of a larger interface.
1013 as part of a larger interface.
1014 """
1014 """
1015
1015
1016 def new():
1016 def new():
1017 """Obtain a new manifest instance.
1017 """Obtain a new manifest instance.
1018
1018
1019 Returns an object conforming to the ``imanifestrevisionwritable``
1019 Returns an object conforming to the ``imanifestrevisionwritable``
1020 interface. The instance will be associated with the same
1020 interface. The instance will be associated with the same
1021 ``imanifestlog`` collection as this instance.
1021 ``imanifestlog`` collection as this instance.
1022 """
1022 """
1023
1023
1024 def copy():
1024 def copy():
1025 """Obtain a copy of this manifest instance.
1025 """Obtain a copy of this manifest instance.
1026
1026
1027 Returns an object conforming to the ``imanifestrevisionwritable``
1027 Returns an object conforming to the ``imanifestrevisionwritable``
1028 interface. The instance will be associated with the same
1028 interface. The instance will be associated with the same
1029 ``imanifestlog`` collection as this instance.
1029 ``imanifestlog`` collection as this instance.
1030 """
1030 """
1031
1031
1032 def read():
1032 def read():
1033 """Obtain the parsed manifest data structure.
1033 """Obtain the parsed manifest data structure.
1034
1034
1035 The returned object conforms to the ``imanifestdict`` interface.
1035 The returned object conforms to the ``imanifestdict`` interface.
1036 """
1036 """
1037
1037
1038 class imanifestrevisionstored(imanifestrevisionbase):
1038 class imanifestrevisionstored(imanifestrevisionbase):
1039 """Interface representing a manifest revision committed to storage."""
1039 """Interface representing a manifest revision committed to storage."""
1040
1040
1041 def node():
1041 def node():
1042 """The binary node for this manifest."""
1042 """The binary node for this manifest."""
1043
1043
1044 parents = interfaceutil.Attribute(
1044 parents = interfaceutil.Attribute(
1045 """List of binary nodes that are parents for this manifest revision."""
1045 """List of binary nodes that are parents for this manifest revision."""
1046 )
1046 )
1047
1047
1048 def readdelta(shallow=False):
1048 def readdelta(shallow=False):
1049 """Obtain the manifest data structure representing changes from parent.
1049 """Obtain the manifest data structure representing changes from parent.
1050
1050
1051 This manifest is compared to its 1st parent. A new manifest representing
1051 This manifest is compared to its 1st parent. A new manifest representing
1052 those differences is constructed.
1052 those differences is constructed.
1053
1053
1054 The returned object conforms to the ``imanifestdict`` interface.
1054 The returned object conforms to the ``imanifestdict`` interface.
1055 """
1055 """
1056
1056
1057 def readfast(shallow=False):
1057 def readfast(shallow=False):
1058 """Calls either ``read()`` or ``readdelta()``.
1058 """Calls either ``read()`` or ``readdelta()``.
1059
1059
1060 The faster of the two options is called.
1060 The faster of the two options is called.
1061 """
1061 """
1062
1062
1063 def find(key):
1063 def find(key):
1064 """Calls self.read().find(key)``.
1064 """Calls self.read().find(key)``.
1065
1065
1066 Returns a 2-tuple of ``(node, flags)`` or raises ``KeyError``.
1066 Returns a 2-tuple of ``(node, flags)`` or raises ``KeyError``.
1067 """
1067 """
1068
1068
1069 class imanifestrevisionwritable(imanifestrevisionbase):
1069 class imanifestrevisionwritable(imanifestrevisionbase):
1070 """Interface representing a manifest revision that can be committed."""
1070 """Interface representing a manifest revision that can be committed."""
1071
1071
1072 def write(transaction, linkrev, p1node, p2node, added, removed, match=None):
1072 def write(transaction, linkrev, p1node, p2node, added, removed, match=None):
1073 """Add this revision to storage.
1073 """Add this revision to storage.
1074
1074
1075 Takes a transaction object, the changeset revision number it will
1075 Takes a transaction object, the changeset revision number it will
1076 be associated with, its parent nodes, and lists of added and
1076 be associated with, its parent nodes, and lists of added and
1077 removed paths.
1077 removed paths.
1078
1078
1079 If match is provided, storage can choose not to inspect or write out
1079 If match is provided, storage can choose not to inspect or write out
1080 items that do not match. Storage is still required to be able to provide
1080 items that do not match. Storage is still required to be able to provide
1081 the full manifest in the future for any directories written (these
1081 the full manifest in the future for any directories written (these
1082 manifests should not be "narrowed on disk").
1082 manifests should not be "narrowed on disk").
1083
1083
1084 Returns the binary node of the created revision.
1084 Returns the binary node of the created revision.
1085 """
1085 """
1086
1086
1087 class imanifeststorage(interfaceutil.Interface):
1087 class imanifeststorage(interfaceutil.Interface):
1088 """Storage interface for manifest data."""
1088 """Storage interface for manifest data."""
1089
1089
1090 tree = interfaceutil.Attribute(
1090 tree = interfaceutil.Attribute(
1091 """The path to the directory this manifest tracks.
1091 """The path to the directory this manifest tracks.
1092
1092
1093 The empty bytestring represents the root manifest.
1093 The empty bytestring represents the root manifest.
1094 """)
1094 """)
1095
1095
1096 index = interfaceutil.Attribute(
1096 index = interfaceutil.Attribute(
1097 """An ``ifilerevisionssequence`` instance.""")
1097 """An ``ifilerevisionssequence`` instance.""")
1098
1098
1099 indexfile = interfaceutil.Attribute(
1099 indexfile = interfaceutil.Attribute(
1100 """Path of revlog index file.
1100 """Path of revlog index file.
1101
1101
1102 TODO this is revlog specific and should not be exposed.
1102 TODO this is revlog specific and should not be exposed.
1103 """)
1103 """)
1104
1104
1105 opener = interfaceutil.Attribute(
1105 opener = interfaceutil.Attribute(
1106 """VFS opener to use to access underlying files used for storage.
1106 """VFS opener to use to access underlying files used for storage.
1107
1107
1108 TODO this is revlog specific and should not be exposed.
1108 TODO this is revlog specific and should not be exposed.
1109 """)
1109 """)
1110
1110
1111 version = interfaceutil.Attribute(
1111 version = interfaceutil.Attribute(
1112 """Revlog version number.
1112 """Revlog version number.
1113
1113
1114 TODO this is revlog specific and should not be exposed.
1114 TODO this is revlog specific and should not be exposed.
1115 """)
1115 """)
1116
1116
1117 _generaldelta = interfaceutil.Attribute(
1117 _generaldelta = interfaceutil.Attribute(
1118 """Whether generaldelta storage is being used.
1118 """Whether generaldelta storage is being used.
1119
1119
1120 TODO this is revlog specific and should not be exposed.
1120 TODO this is revlog specific and should not be exposed.
1121 """)
1121 """)
1122
1122
1123 fulltextcache = interfaceutil.Attribute(
1123 fulltextcache = interfaceutil.Attribute(
1124 """Dict with cache of fulltexts.
1124 """Dict with cache of fulltexts.
1125
1125
1126 TODO this doesn't feel appropriate for the storage interface.
1126 TODO this doesn't feel appropriate for the storage interface.
1127 """)
1127 """)
1128
1128
1129 def __len__():
1129 def __len__():
1130 """Obtain the number of revisions stored for this manifest."""
1130 """Obtain the number of revisions stored for this manifest."""
1131
1131
1132 def __iter__():
1132 def __iter__():
1133 """Iterate over revision numbers for this manifest."""
1133 """Iterate over revision numbers for this manifest."""
1134
1134
1135 def rev(node):
1135 def rev(node):
1136 """Obtain the revision number given a binary node.
1136 """Obtain the revision number given a binary node.
1137
1137
1138 Raises ``error.LookupError`` if the node is not known.
1138 Raises ``error.LookupError`` if the node is not known.
1139 """
1139 """
1140
1140
1141 def node(rev):
1141 def node(rev):
1142 """Obtain the node value given a revision number.
1142 """Obtain the node value given a revision number.
1143
1143
1144 Raises ``error.LookupError`` if the revision is not known.
1144 Raises ``error.LookupError`` if the revision is not known.
1145 """
1145 """
1146
1146
1147 def lookup(value):
1147 def lookup(value):
1148 """Attempt to resolve a value to a node.
1148 """Attempt to resolve a value to a node.
1149
1149
1150 Value can be a binary node, hex node, revision number, or a bytes
1150 Value can be a binary node, hex node, revision number, or a bytes
1151 that can be converted to an integer.
1151 that can be converted to an integer.
1152
1152
1153 Raises ``error.LookupError`` if a ndoe could not be resolved.
1153 Raises ``error.LookupError`` if a ndoe could not be resolved.
1154 """
1154 """
1155
1155
1156 def parents(node):
1156 def parents(node):
1157 """Returns a 2-tuple of parent nodes for a node.
1157 """Returns a 2-tuple of parent nodes for a node.
1158
1158
1159 Values will be ``nullid`` if the parent is empty.
1159 Values will be ``nullid`` if the parent is empty.
1160 """
1160 """
1161
1161
1162 def parentrevs(rev):
1162 def parentrevs(rev):
1163 """Like parents() but operates on revision numbers."""
1163 """Like parents() but operates on revision numbers."""
1164
1164
1165 def linkrev(rev):
1165 def linkrev(rev):
1166 """Obtain the changeset revision number a revision is linked to."""
1166 """Obtain the changeset revision number a revision is linked to."""
1167
1167
1168 def revision(node, _df=None, raw=False):
1168 def revision(node, _df=None, raw=False):
1169 """Obtain fulltext data for a node."""
1169 """Obtain fulltext data for a node."""
1170
1170
1171 def rawdata(node, _df=None):
1171 def rawdata(node, _df=None):
1172 """Obtain raw data for a node."""
1172 """Obtain raw data for a node."""
1173
1173
1174 def revdiff(rev1, rev2):
1174 def revdiff(rev1, rev2):
1175 """Obtain a delta between two revision numbers.
1175 """Obtain a delta between two revision numbers.
1176
1176
1177 The returned data is the result of ``bdiff.bdiff()`` on the raw
1177 The returned data is the result of ``bdiff.bdiff()`` on the raw
1178 revision data.
1178 revision data.
1179 """
1179 """
1180
1180
1181 def cmp(node, fulltext):
1181 def cmp(node, fulltext):
1182 """Compare fulltext to another revision.
1182 """Compare fulltext to another revision.
1183
1183
1184 Returns True if the fulltext is different from what is stored.
1184 Returns True if the fulltext is different from what is stored.
1185 """
1185 """
1186
1186
1187 def emitrevisions(nodes,
1187 def emitrevisions(nodes,
1188 nodesorder=None,
1188 nodesorder=None,
1189 revisiondata=False,
1189 revisiondata=False,
1190 assumehaveparentrevisions=False):
1190 assumehaveparentrevisions=False):
1191 """Produce ``irevisiondelta`` describing revisions.
1191 """Produce ``irevisiondelta`` describing revisions.
1192
1192
1193 See the documentation for ``ifiledata`` for more.
1193 See the documentation for ``ifiledata`` for more.
1194 """
1194 """
1195
1195
1196 def addgroup(deltas, linkmapper, transaction, addrevisioncb=None):
1196 def addgroup(deltas, linkmapper, transaction, addrevisioncb=None):
1197 """Process a series of deltas for storage.
1197 """Process a series of deltas for storage.
1198
1198
1199 See the documentation in ``ifilemutation`` for more.
1199 See the documentation in ``ifilemutation`` for more.
1200 """
1200 """
1201
1201
1202 def rawsize(rev):
1202 def rawsize(rev):
1203 """Obtain the size of tracked data.
1203 """Obtain the size of tracked data.
1204
1204
1205 Is equivalent to ``len(m.rawdata(node))``.
1205 Is equivalent to ``len(m.rawdata(node))``.
1206
1206
1207 TODO this method is only used by upgrade code and may be removed.
1207 TODO this method is only used by upgrade code and may be removed.
1208 """
1208 """
1209
1209
1210 def getstrippoint(minlink):
1210 def getstrippoint(minlink):
1211 """Find minimum revision that must be stripped to strip a linkrev.
1211 """Find minimum revision that must be stripped to strip a linkrev.
1212
1212
1213 See the documentation in ``ifilemutation`` for more.
1213 See the documentation in ``ifilemutation`` for more.
1214 """
1214 """
1215
1215
1216 def strip(minlink, transaction):
1216 def strip(minlink, transaction):
1217 """Remove storage of items starting at a linkrev.
1217 """Remove storage of items starting at a linkrev.
1218
1218
1219 See the documentation in ``ifilemutation`` for more.
1219 See the documentation in ``ifilemutation`` for more.
1220 """
1220 """
1221
1221
1222 def checksize():
1222 def checksize():
1223 """Obtain the expected sizes of backing files.
1223 """Obtain the expected sizes of backing files.
1224
1224
1225 TODO this is used by verify and it should not be part of the interface.
1225 TODO this is used by verify and it should not be part of the interface.
1226 """
1226 """
1227
1227
1228 def files():
1228 def files():
1229 """Obtain paths that are backing storage for this manifest.
1229 """Obtain paths that are backing storage for this manifest.
1230
1230
1231 TODO this is used by verify and there should probably be a better API
1231 TODO this is used by verify and there should probably be a better API
1232 for this functionality.
1232 for this functionality.
1233 """
1233 """
1234
1234
1235 def deltaparent(rev):
1235 def deltaparent(rev):
1236 """Obtain the revision that a revision is delta'd against.
1236 """Obtain the revision that a revision is delta'd against.
1237
1237
1238 TODO delta encoding is an implementation detail of storage and should
1238 TODO delta encoding is an implementation detail of storage and should
1239 not be exposed to the storage interface.
1239 not be exposed to the storage interface.
1240 """
1240 """
1241
1241
1242 def clone(tr, dest, **kwargs):
1242 def clone(tr, dest, **kwargs):
1243 """Clone this instance to another."""
1243 """Clone this instance to another."""
1244
1244
1245 def clearcaches(clear_persisted_data=False):
1245 def clearcaches(clear_persisted_data=False):
1246 """Clear any caches associated with this instance."""
1246 """Clear any caches associated with this instance."""
1247
1247
1248 def dirlog(d):
1248 def dirlog(d):
1249 """Obtain a manifest storage instance for a tree."""
1249 """Obtain a manifest storage instance for a tree."""
1250
1250
1251 def add(m, transaction, link, p1, p2, added, removed, readtree=None,
1251 def add(m, transaction, link, p1, p2, added, removed, readtree=None,
1252 match=None):
1252 match=None):
1253 """Add a revision to storage.
1253 """Add a revision to storage.
1254
1254
1255 ``m`` is an object conforming to ``imanifestdict``.
1255 ``m`` is an object conforming to ``imanifestdict``.
1256
1256
1257 ``link`` is the linkrev revision number.
1257 ``link`` is the linkrev revision number.
1258
1258
1259 ``p1`` and ``p2`` are the parent revision numbers.
1259 ``p1`` and ``p2`` are the parent revision numbers.
1260
1260
1261 ``added`` and ``removed`` are iterables of added and removed paths,
1261 ``added`` and ``removed`` are iterables of added and removed paths,
1262 respectively.
1262 respectively.
1263
1263
1264 ``readtree`` is a function that can be used to read the child tree(s)
1264 ``readtree`` is a function that can be used to read the child tree(s)
1265 when recursively writing the full tree structure when using
1265 when recursively writing the full tree structure when using
1266 treemanifets.
1266 treemanifets.
1267
1267
1268 ``match`` is a matcher that can be used to hint to storage that not all
1268 ``match`` is a matcher that can be used to hint to storage that not all
1269 paths must be inspected; this is an optimization and can be safely
1269 paths must be inspected; this is an optimization and can be safely
1270 ignored. Note that the storage must still be able to reproduce a full
1270 ignored. Note that the storage must still be able to reproduce a full
1271 manifest including files that did not match.
1271 manifest including files that did not match.
1272 """
1272 """
1273
1273
1274 def storageinfo(exclusivefiles=False, sharedfiles=False,
1274 def storageinfo(exclusivefiles=False, sharedfiles=False,
1275 revisionscount=False, trackedsize=False,
1275 revisionscount=False, trackedsize=False,
1276 storedsize=False):
1276 storedsize=False):
1277 """Obtain information about storage for this manifest's data.
1277 """Obtain information about storage for this manifest's data.
1278
1278
1279 See ``ifilestorage.storageinfo()`` for a description of this method.
1279 See ``ifilestorage.storageinfo()`` for a description of this method.
1280 This one behaves the same way, except for manifest data.
1280 This one behaves the same way, except for manifest data.
1281 """
1281 """
1282
1282
1283 class imanifestlog(interfaceutil.Interface):
1283 class imanifestlog(interfaceutil.Interface):
1284 """Interface representing a collection of manifest snapshots.
1284 """Interface representing a collection of manifest snapshots.
1285
1285
1286 Represents the root manifest in a repository.
1286 Represents the root manifest in a repository.
1287
1287
1288 Also serves as a means to access nested tree manifests and to cache
1288 Also serves as a means to access nested tree manifests and to cache
1289 tree manifests.
1289 tree manifests.
1290 """
1290 """
1291
1291
1292 def __getitem__(node):
1292 def __getitem__(node):
1293 """Obtain a manifest instance for a given binary node.
1293 """Obtain a manifest instance for a given binary node.
1294
1294
1295 Equivalent to calling ``self.get('', node)``.
1295 Equivalent to calling ``self.get('', node)``.
1296
1296
1297 The returned object conforms to the ``imanifestrevisionstored``
1297 The returned object conforms to the ``imanifestrevisionstored``
1298 interface.
1298 interface.
1299 """
1299 """
1300
1300
1301 def get(tree, node, verify=True):
1301 def get(tree, node, verify=True):
1302 """Retrieve the manifest instance for a given directory and binary node.
1302 """Retrieve the manifest instance for a given directory and binary node.
1303
1303
1304 ``node`` always refers to the node of the root manifest (which will be
1304 ``node`` always refers to the node of the root manifest (which will be
1305 the only manifest if flat manifests are being used).
1305 the only manifest if flat manifests are being used).
1306
1306
1307 If ``tree`` is the empty string, the root manifest is returned.
1307 If ``tree`` is the empty string, the root manifest is returned.
1308 Otherwise the manifest for the specified directory will be returned
1308 Otherwise the manifest for the specified directory will be returned
1309 (requires tree manifests).
1309 (requires tree manifests).
1310
1310
1311 If ``verify`` is True, ``LookupError`` is raised if the node is not
1311 If ``verify`` is True, ``LookupError`` is raised if the node is not
1312 known.
1312 known.
1313
1313
1314 The returned object conforms to the ``imanifestrevisionstored``
1314 The returned object conforms to the ``imanifestrevisionstored``
1315 interface.
1315 interface.
1316 """
1316 """
1317
1317
1318 def getstorage(tree):
1318 def getstorage(tree):
1319 """Retrieve an interface to storage for a particular tree.
1319 """Retrieve an interface to storage for a particular tree.
1320
1320
1321 If ``tree`` is the empty bytestring, storage for the root manifest will
1321 If ``tree`` is the empty bytestring, storage for the root manifest will
1322 be returned. Otherwise storage for a tree manifest is returned.
1322 be returned. Otherwise storage for a tree manifest is returned.
1323
1323
1324 TODO formalize interface for returned object.
1324 TODO formalize interface for returned object.
1325 """
1325 """
1326
1326
1327 def clearcaches():
1327 def clearcaches():
1328 """Clear caches associated with this collection."""
1328 """Clear caches associated with this collection."""
1329
1329
1330 def rev(node):
1330 def rev(node):
1331 """Obtain the revision number for a binary node.
1331 """Obtain the revision number for a binary node.
1332
1332
1333 Raises ``error.LookupError`` if the node is not known.
1333 Raises ``error.LookupError`` if the node is not known.
1334 """
1334 """
1335
1335
1336 class ilocalrepositoryfilestorage(interfaceutil.Interface):
1336 class ilocalrepositoryfilestorage(interfaceutil.Interface):
1337 """Local repository sub-interface providing access to tracked file storage.
1337 """Local repository sub-interface providing access to tracked file storage.
1338
1338
1339 This interface defines how a repository accesses storage for a single
1339 This interface defines how a repository accesses storage for a single
1340 tracked file path.
1340 tracked file path.
1341 """
1341 """
1342
1342
1343 def file(f):
1343 def file(f):
1344 """Obtain a filelog for a tracked path.
1344 """Obtain a filelog for a tracked path.
1345
1345
1346 The returned type conforms to the ``ifilestorage`` interface.
1346 The returned type conforms to the ``ifilestorage`` interface.
1347 """
1347 """
1348
1348
1349 class ilocalrepositorymain(interfaceutil.Interface):
1349 class ilocalrepositorymain(interfaceutil.Interface):
1350 """Main interface for local repositories.
1350 """Main interface for local repositories.
1351
1351
1352 This currently captures the reality of things - not how things should be.
1352 This currently captures the reality of things - not how things should be.
1353 """
1353 """
1354
1354
1355 supportedformats = interfaceutil.Attribute(
1355 supportedformats = interfaceutil.Attribute(
1356 """Set of requirements that apply to stream clone.
1356 """Set of requirements that apply to stream clone.
1357
1357
1358 This is actually a class attribute and is shared among all instances.
1358 This is actually a class attribute and is shared among all instances.
1359 """)
1359 """)
1360
1360
1361 supported = interfaceutil.Attribute(
1361 supported = interfaceutil.Attribute(
1362 """Set of requirements that this repo is capable of opening.""")
1362 """Set of requirements that this repo is capable of opening.""")
1363
1363
1364 requirements = interfaceutil.Attribute(
1364 requirements = interfaceutil.Attribute(
1365 """Set of requirements this repo uses.""")
1365 """Set of requirements this repo uses.""")
1366
1366
1367 features = interfaceutil.Attribute(
1367 features = interfaceutil.Attribute(
1368 """Set of "features" this repository supports.
1368 """Set of "features" this repository supports.
1369
1369
1370 A "feature" is a loosely-defined term. It can refer to a feature
1370 A "feature" is a loosely-defined term. It can refer to a feature
1371 in the classical sense or can describe an implementation detail
1371 in the classical sense or can describe an implementation detail
1372 of the repository. For example, a ``readonly`` feature may denote
1372 of the repository. For example, a ``readonly`` feature may denote
1373 the repository as read-only. Or a ``revlogfilestore`` feature may
1373 the repository as read-only. Or a ``revlogfilestore`` feature may
1374 denote that the repository is using revlogs for file storage.
1374 denote that the repository is using revlogs for file storage.
1375
1375
1376 The intent of features is to provide a machine-queryable mechanism
1376 The intent of features is to provide a machine-queryable mechanism
1377 for repo consumers to test for various repository characteristics.
1377 for repo consumers to test for various repository characteristics.
1378
1378
1379 Features are similar to ``requirements``. The main difference is that
1379 Features are similar to ``requirements``. The main difference is that
1380 requirements are stored on-disk and represent requirements to open the
1380 requirements are stored on-disk and represent requirements to open the
1381 repository. Features are more run-time capabilities of the repository
1381 repository. Features are more run-time capabilities of the repository
1382 and more granular capabilities (which may be derived from requirements).
1382 and more granular capabilities (which may be derived from requirements).
1383 """)
1383 """)
1384
1384
1385 filtername = interfaceutil.Attribute(
1385 filtername = interfaceutil.Attribute(
1386 """Name of the repoview that is active on this repo.""")
1386 """Name of the repoview that is active on this repo.""")
1387
1387
1388 wvfs = interfaceutil.Attribute(
1388 wvfs = interfaceutil.Attribute(
1389 """VFS used to access the working directory.""")
1389 """VFS used to access the working directory.""")
1390
1390
1391 vfs = interfaceutil.Attribute(
1391 vfs = interfaceutil.Attribute(
1392 """VFS rooted at the .hg directory.
1392 """VFS rooted at the .hg directory.
1393
1393
1394 Used to access repository data not in the store.
1394 Used to access repository data not in the store.
1395 """)
1395 """)
1396
1396
1397 svfs = interfaceutil.Attribute(
1397 svfs = interfaceutil.Attribute(
1398 """VFS rooted at the store.
1398 """VFS rooted at the store.
1399
1399
1400 Used to access repository data in the store. Typically .hg/store.
1400 Used to access repository data in the store. Typically .hg/store.
1401 But can point elsewhere if the store is shared.
1401 But can point elsewhere if the store is shared.
1402 """)
1402 """)
1403
1403
1404 root = interfaceutil.Attribute(
1404 root = interfaceutil.Attribute(
1405 """Path to the root of the working directory.""")
1405 """Path to the root of the working directory.""")
1406
1406
1407 path = interfaceutil.Attribute(
1407 path = interfaceutil.Attribute(
1408 """Path to the .hg directory.""")
1408 """Path to the .hg directory.""")
1409
1409
1410 origroot = interfaceutil.Attribute(
1410 origroot = interfaceutil.Attribute(
1411 """The filesystem path that was used to construct the repo.""")
1411 """The filesystem path that was used to construct the repo.""")
1412
1412
1413 auditor = interfaceutil.Attribute(
1413 auditor = interfaceutil.Attribute(
1414 """A pathauditor for the working directory.
1414 """A pathauditor for the working directory.
1415
1415
1416 This checks if a path refers to a nested repository.
1416 This checks if a path refers to a nested repository.
1417
1417
1418 Operates on the filesystem.
1418 Operates on the filesystem.
1419 """)
1419 """)
1420
1420
1421 nofsauditor = interfaceutil.Attribute(
1421 nofsauditor = interfaceutil.Attribute(
1422 """A pathauditor for the working directory.
1422 """A pathauditor for the working directory.
1423
1423
1424 This is like ``auditor`` except it doesn't do filesystem checks.
1424 This is like ``auditor`` except it doesn't do filesystem checks.
1425 """)
1425 """)
1426
1426
1427 baseui = interfaceutil.Attribute(
1427 baseui = interfaceutil.Attribute(
1428 """Original ui instance passed into constructor.""")
1428 """Original ui instance passed into constructor.""")
1429
1429
1430 ui = interfaceutil.Attribute(
1430 ui = interfaceutil.Attribute(
1431 """Main ui instance for this instance.""")
1431 """Main ui instance for this instance.""")
1432
1432
1433 sharedpath = interfaceutil.Attribute(
1433 sharedpath = interfaceutil.Attribute(
1434 """Path to the .hg directory of the repo this repo was shared from.""")
1434 """Path to the .hg directory of the repo this repo was shared from.""")
1435
1435
1436 store = interfaceutil.Attribute(
1436 store = interfaceutil.Attribute(
1437 """A store instance.""")
1437 """A store instance.""")
1438
1438
1439 spath = interfaceutil.Attribute(
1439 spath = interfaceutil.Attribute(
1440 """Path to the store.""")
1440 """Path to the store.""")
1441
1441
1442 sjoin = interfaceutil.Attribute(
1442 sjoin = interfaceutil.Attribute(
1443 """Alias to self.store.join.""")
1443 """Alias to self.store.join.""")
1444
1444
1445 cachevfs = interfaceutil.Attribute(
1445 cachevfs = interfaceutil.Attribute(
1446 """A VFS used to access the cache directory.
1446 """A VFS used to access the cache directory.
1447
1447
1448 Typically .hg/cache.
1448 Typically .hg/cache.
1449 """)
1449 """)
1450
1450
1451 wcachevfs = interfaceutil.Attribute(
1451 wcachevfs = interfaceutil.Attribute(
1452 """A VFS used to access the cache directory dedicated to working copy
1452 """A VFS used to access the cache directory dedicated to working copy
1453
1453
1454 Typically .hg/wcache.
1454 Typically .hg/wcache.
1455 """)
1455 """)
1456
1456
1457 filteredrevcache = interfaceutil.Attribute(
1457 filteredrevcache = interfaceutil.Attribute(
1458 """Holds sets of revisions to be filtered.""")
1458 """Holds sets of revisions to be filtered.""")
1459
1459
1460 names = interfaceutil.Attribute(
1460 names = interfaceutil.Attribute(
1461 """A ``namespaces`` instance.""")
1461 """A ``namespaces`` instance.""")
1462
1462
1463 def close():
1463 def close():
1464 """Close the handle on this repository."""
1464 """Close the handle on this repository."""
1465
1465
1466 def peer():
1466 def peer():
1467 """Obtain an object conforming to the ``peer`` interface."""
1467 """Obtain an object conforming to the ``peer`` interface."""
1468
1468
1469 def unfiltered():
1469 def unfiltered():
1470 """Obtain an unfiltered/raw view of this repo."""
1470 """Obtain an unfiltered/raw view of this repo."""
1471
1471
1472 def filtered(name, visibilityexceptions=None):
1472 def filtered(name, visibilityexceptions=None):
1473 """Obtain a named view of this repository."""
1473 """Obtain a named view of this repository."""
1474
1474
1475 obsstore = interfaceutil.Attribute(
1475 obsstore = interfaceutil.Attribute(
1476 """A store of obsolescence data.""")
1476 """A store of obsolescence data.""")
1477
1477
1478 changelog = interfaceutil.Attribute(
1478 changelog = interfaceutil.Attribute(
1479 """A handle on the changelog revlog.""")
1479 """A handle on the changelog revlog.""")
1480
1480
1481 manifestlog = interfaceutil.Attribute(
1481 manifestlog = interfaceutil.Attribute(
1482 """An instance conforming to the ``imanifestlog`` interface.
1482 """An instance conforming to the ``imanifestlog`` interface.
1483
1483
1484 Provides access to manifests for the repository.
1484 Provides access to manifests for the repository.
1485 """)
1485 """)
1486
1486
1487 dirstate = interfaceutil.Attribute(
1487 dirstate = interfaceutil.Attribute(
1488 """Working directory state.""")
1488 """Working directory state.""")
1489
1489
1490 narrowpats = interfaceutil.Attribute(
1490 narrowpats = interfaceutil.Attribute(
1491 """Matcher patterns for this repository's narrowspec.""")
1491 """Matcher patterns for this repository's narrowspec.""")
1492
1492
1493 def narrowmatch(match=None, includeexact=False):
1493 def narrowmatch(match=None, includeexact=False):
1494 """Obtain a matcher for the narrowspec."""
1494 """Obtain a matcher for the narrowspec."""
1495
1495
1496 def setnarrowpats(newincludes, newexcludes):
1496 def setnarrowpats(newincludes, newexcludes):
1497 """Define the narrowspec for this repository."""
1497 """Define the narrowspec for this repository."""
1498
1498
1499 def __getitem__(changeid):
1499 def __getitem__(changeid):
1500 """Try to resolve a changectx."""
1500 """Try to resolve a changectx."""
1501
1501
1502 def __contains__(changeid):
1502 def __contains__(changeid):
1503 """Whether a changeset exists."""
1503 """Whether a changeset exists."""
1504
1504
1505 def __nonzero__():
1505 def __nonzero__():
1506 """Always returns True."""
1506 """Always returns True."""
1507 return True
1507 return True
1508
1508
1509 __bool__ = __nonzero__
1509 __bool__ = __nonzero__
1510
1510
1511 def __len__():
1511 def __len__():
1512 """Returns the number of changesets in the repo."""
1512 """Returns the number of changesets in the repo."""
1513
1513
1514 def __iter__():
1514 def __iter__():
1515 """Iterate over revisions in the changelog."""
1515 """Iterate over revisions in the changelog."""
1516
1516
1517 def revs(expr, *args):
1517 def revs(expr, *args):
1518 """Evaluate a revset.
1518 """Evaluate a revset.
1519
1519
1520 Emits revisions.
1520 Emits revisions.
1521 """
1521 """
1522
1522
1523 def set(expr, *args):
1523 def set(expr, *args):
1524 """Evaluate a revset.
1524 """Evaluate a revset.
1525
1525
1526 Emits changectx instances.
1526 Emits changectx instances.
1527 """
1527 """
1528
1528
1529 def anyrevs(specs, user=False, localalias=None):
1529 def anyrevs(specs, user=False, localalias=None):
1530 """Find revisions matching one of the given revsets."""
1530 """Find revisions matching one of the given revsets."""
1531
1531
1532 def url():
1532 def url():
1533 """Returns a string representing the location of this repo."""
1533 """Returns a string representing the location of this repo."""
1534
1534
1535 def hook(name, throw=False, **args):
1535 def hook(name, throw=False, **args):
1536 """Call a hook."""
1536 """Call a hook."""
1537
1537
1538 def tags():
1538 def tags():
1539 """Return a mapping of tag to node."""
1539 """Return a mapping of tag to node."""
1540
1540
1541 def tagtype(tagname):
1541 def tagtype(tagname):
1542 """Return the type of a given tag."""
1542 """Return the type of a given tag."""
1543
1543
1544 def tagslist():
1544 def tagslist():
1545 """Return a list of tags ordered by revision."""
1545 """Return a list of tags ordered by revision."""
1546
1546
1547 def nodetags(node):
1547 def nodetags(node):
1548 """Return the tags associated with a node."""
1548 """Return the tags associated with a node."""
1549
1549
1550 def nodebookmarks(node):
1550 def nodebookmarks(node):
1551 """Return the list of bookmarks pointing to the specified node."""
1551 """Return the list of bookmarks pointing to the specified node."""
1552
1552
1553 def branchmap():
1553 def branchmap():
1554 """Return a mapping of branch to heads in that branch."""
1554 """Return a mapping of branch to heads in that branch."""
1555
1555
1556 def revbranchcache():
1556 def revbranchcache():
1557 pass
1557 pass
1558
1558
1559 def branchtip(branchtip, ignoremissing=False):
1559 def branchtip(branchtip, ignoremissing=False):
1560 """Return the tip node for a given branch."""
1560 """Return the tip node for a given branch."""
1561
1561
1562 def lookup(key):
1562 def lookup(key):
1563 """Resolve the node for a revision."""
1563 """Resolve the node for a revision."""
1564
1564
1565 def lookupbranch(key):
1565 def lookupbranch(key):
1566 """Look up the branch name of the given revision or branch name."""
1566 """Look up the branch name of the given revision or branch name."""
1567
1567
1568 def known(nodes):
1568 def known(nodes):
1569 """Determine whether a series of nodes is known.
1569 """Determine whether a series of nodes is known.
1570
1570
1571 Returns a list of bools.
1571 Returns a list of bools.
1572 """
1572 """
1573
1573
1574 def local():
1574 def local():
1575 """Whether the repository is local."""
1575 """Whether the repository is local."""
1576 return True
1576 return True
1577
1577
1578 def publishing():
1578 def publishing():
1579 """Whether the repository is a publishing repository."""
1579 """Whether the repository is a publishing repository."""
1580
1580
1581 def cancopy():
1581 def cancopy():
1582 pass
1582 pass
1583
1583
1584 def shared():
1584 def shared():
1585 """The type of shared repository or None."""
1585 """The type of shared repository or None."""
1586
1586
1587 def wjoin(f, *insidef):
1587 def wjoin(f, *insidef):
1588 """Calls self.vfs.reljoin(self.root, f, *insidef)"""
1588 """Calls self.vfs.reljoin(self.root, f, *insidef)"""
1589
1589
1590 def setparents(p1, p2):
1590 def setparents(p1, p2):
1591 """Set the parent nodes of the working directory."""
1591 """Set the parent nodes of the working directory."""
1592
1592
1593 def filectx(path, changeid=None, fileid=None):
1593 def filectx(path, changeid=None, fileid=None):
1594 """Obtain a filectx for the given file revision."""
1594 """Obtain a filectx for the given file revision."""
1595
1595
1596 def getcwd():
1596 def getcwd():
1597 """Obtain the current working directory from the dirstate."""
1597 """Obtain the current working directory from the dirstate."""
1598
1598
1599 def pathto(f, cwd=None):
1599 def pathto(f, cwd=None):
1600 """Obtain the relative path to a file."""
1600 """Obtain the relative path to a file."""
1601
1601
1602 def adddatafilter(name, fltr):
1602 def adddatafilter(name, fltr):
1603 pass
1603 pass
1604
1604
1605 def wread(filename):
1605 def wread(filename):
1606 """Read a file from wvfs, using data filters."""
1606 """Read a file from wvfs, using data filters."""
1607
1607
1608 def wwrite(filename, data, flags, backgroundclose=False, **kwargs):
1608 def wwrite(filename, data, flags, backgroundclose=False, **kwargs):
1609 """Write data to a file in the wvfs, using data filters."""
1609 """Write data to a file in the wvfs, using data filters."""
1610
1610
1611 def wwritedata(filename, data):
1611 def wwritedata(filename, data):
1612 """Resolve data for writing to the wvfs, using data filters."""
1612 """Resolve data for writing to the wvfs, using data filters."""
1613
1613
1614 def currenttransaction():
1614 def currenttransaction():
1615 """Obtain the current transaction instance or None."""
1615 """Obtain the current transaction instance or None."""
1616
1616
1617 def transaction(desc, report=None):
1617 def transaction(desc, report=None):
1618 """Open a new transaction to write to the repository."""
1618 """Open a new transaction to write to the repository."""
1619
1619
1620 def undofiles():
1620 def undofiles():
1621 """Returns a list of (vfs, path) for files to undo transactions."""
1621 """Returns a list of (vfs, path) for files to undo transactions."""
1622
1622
1623 def recover():
1623 def recover():
1624 """Roll back an interrupted transaction."""
1624 """Roll back an interrupted transaction."""
1625
1625
1626 def rollback(dryrun=False, force=False):
1626 def rollback(dryrun=False, force=False):
1627 """Undo the last transaction.
1627 """Undo the last transaction.
1628
1628
1629 DANGEROUS.
1629 DANGEROUS.
1630 """
1630 """
1631
1631
1632 def updatecaches(tr=None, full=False):
1632 def updatecaches(tr=None, full=False):
1633 """Warm repo caches."""
1633 """Warm repo caches."""
1634
1634
1635 def invalidatecaches():
1635 def invalidatecaches():
1636 """Invalidate cached data due to the repository mutating."""
1636 """Invalidate cached data due to the repository mutating."""
1637
1637
1638 def invalidatevolatilesets():
1638 def invalidatevolatilesets():
1639 pass
1639 pass
1640
1640
1641 def invalidatedirstate():
1641 def invalidatedirstate():
1642 """Invalidate the dirstate."""
1642 """Invalidate the dirstate."""
1643
1643
1644 def invalidate(clearfilecache=False):
1644 def invalidate(clearfilecache=False):
1645 pass
1645 pass
1646
1646
1647 def invalidateall():
1647 def invalidateall():
1648 pass
1648 pass
1649
1649
1650 def lock(wait=True):
1650 def lock(wait=True):
1651 """Lock the repository store and return a lock instance."""
1651 """Lock the repository store and return a lock instance."""
1652
1652
1653 def wlock(wait=True):
1653 def wlock(wait=True):
1654 """Lock the non-store parts of the repository."""
1654 """Lock the non-store parts of the repository."""
1655
1655
1656 def currentwlock():
1656 def currentwlock():
1657 """Return the wlock if it's held or None."""
1657 """Return the wlock if it's held or None."""
1658
1658
1659 def checkcommitpatterns(wctx, vdirs, match, status, fail):
1659 def checkcommitpatterns(wctx, vdirs, match, status, fail):
1660 pass
1660 pass
1661
1661
1662 def commit(text='', user=None, date=None, match=None, force=False,
1662 def commit(text='', user=None, date=None, match=None, force=False,
1663 editor=False, extra=None):
1663 editor=False, extra=None):
1664 """Add a new revision to the repository."""
1664 """Add a new revision to the repository."""
1665
1665
1666 def commitctx(ctx, error=False, origctx=None):
1666 def commitctx(ctx, error=False, origctx=None):
1667 """Commit a commitctx instance to the repository."""
1667 """Commit a commitctx instance to the repository."""
1668
1668
1669 def destroying():
1669 def destroying():
1670 """Inform the repository that nodes are about to be destroyed."""
1670 """Inform the repository that nodes are about to be destroyed."""
1671
1671
1672 def destroyed():
1672 def destroyed():
1673 """Inform the repository that nodes have been destroyed."""
1673 """Inform the repository that nodes have been destroyed."""
1674
1674
1675 def status(node1='.', node2=None, match=None, ignored=False,
1675 def status(node1='.', node2=None, match=None, ignored=False,
1676 clean=False, unknown=False, listsubrepos=False):
1676 clean=False, unknown=False, listsubrepos=False):
1677 """Convenience method to call repo[x].status()."""
1677 """Convenience method to call repo[x].status()."""
1678
1678
1679 def addpostdsstatus(ps):
1679 def addpostdsstatus(ps):
1680 pass
1680 pass
1681
1681
1682 def postdsstatus():
1682 def postdsstatus():
1683 pass
1683 pass
1684
1684
1685 def clearpostdsstatus():
1685 def clearpostdsstatus():
1686 pass
1686 pass
1687
1687
1688 def heads(start=None):
1688 def heads(start=None):
1689 """Obtain list of nodes that are DAG heads."""
1689 """Obtain list of nodes that are DAG heads."""
1690
1690
1691 def branchheads(branch=None, start=None, closed=False):
1691 def branchheads(branch=None, start=None, closed=False):
1692 pass
1692 pass
1693
1693
1694 def branches(nodes):
1694 def branches(nodes):
1695 pass
1695 pass
1696
1696
1697 def between(pairs):
1697 def between(pairs):
1698 pass
1698 pass
1699
1699
1700 def checkpush(pushop):
1700 def checkpush(pushop):
1701 pass
1701 pass
1702
1702
1703 prepushoutgoinghooks = interfaceutil.Attribute(
1703 prepushoutgoinghooks = interfaceutil.Attribute(
1704 """util.hooks instance.""")
1704 """util.hooks instance.""")
1705
1705
1706 def pushkey(namespace, key, old, new):
1706 def pushkey(namespace, key, old, new):
1707 pass
1707 pass
1708
1708
1709 def listkeys(namespace):
1709 def listkeys(namespace):
1710 pass
1710 pass
1711
1711
1712 def debugwireargs(one, two, three=None, four=None, five=None):
1712 def debugwireargs(one, two, three=None, four=None, five=None):
1713 pass
1713 pass
1714
1714
1715 def savecommitmessage(text):
1715 def savecommitmessage(text):
1716 pass
1716 pass
1717
1717
1718 class completelocalrepository(ilocalrepositorymain,
1718 class completelocalrepository(ilocalrepositorymain,
1719 ilocalrepositoryfilestorage):
1719 ilocalrepositoryfilestorage):
1720 """Complete interface for a local repository."""
1720 """Complete interface for a local repository."""
1721
1721
1722 class iwireprotocolcommandcacher(interfaceutil.Interface):
1722 class iwireprotocolcommandcacher(interfaceutil.Interface):
1723 """Represents a caching backend for wire protocol commands.
1723 """Represents a caching backend for wire protocol commands.
1724
1724
1725 Wire protocol version 2 supports transparent caching of many commands.
1725 Wire protocol version 2 supports transparent caching of many commands.
1726 To leverage this caching, servers can activate objects that cache
1726 To leverage this caching, servers can activate objects that cache
1727 command responses. Objects handle both cache writing and reading.
1727 command responses. Objects handle both cache writing and reading.
1728 This interface defines how that response caching mechanism works.
1728 This interface defines how that response caching mechanism works.
1729
1729
1730 Wire protocol version 2 commands emit a series of objects that are
1730 Wire protocol version 2 commands emit a series of objects that are
1731 serialized and sent to the client. The caching layer exists between
1731 serialized and sent to the client. The caching layer exists between
1732 the invocation of the command function and the sending of its output
1732 the invocation of the command function and the sending of its output
1733 objects to an output layer.
1733 objects to an output layer.
1734
1734
1735 Instances of this interface represent a binding to a cache that
1735 Instances of this interface represent a binding to a cache that
1736 can serve a response (in place of calling a command function) and/or
1736 can serve a response (in place of calling a command function) and/or
1737 write responses to a cache for subsequent use.
1737 write responses to a cache for subsequent use.
1738
1738
1739 When a command request arrives, the following happens with regards
1739 When a command request arrives, the following happens with regards
1740 to this interface:
1740 to this interface:
1741
1741
1742 1. The server determines whether the command request is cacheable.
1742 1. The server determines whether the command request is cacheable.
1743 2. If it is, an instance of this interface is spawned.
1743 2. If it is, an instance of this interface is spawned.
1744 3. The cacher is activated in a context manager (``__enter__`` is called).
1744 3. The cacher is activated in a context manager (``__enter__`` is called).
1745 4. A cache *key* for that request is derived. This will call the
1745 4. A cache *key* for that request is derived. This will call the
1746 instance's ``adjustcachekeystate()`` method so the derivation
1746 instance's ``adjustcachekeystate()`` method so the derivation
1747 can be influenced.
1747 can be influenced.
1748 5. The cacher is informed of the derived cache key via a call to
1748 5. The cacher is informed of the derived cache key via a call to
1749 ``setcachekey()``.
1749 ``setcachekey()``.
1750 6. The cacher's ``lookup()`` method is called to test for presence of
1750 6. The cacher's ``lookup()`` method is called to test for presence of
1751 the derived key in the cache.
1751 the derived key in the cache.
1752 7. If ``lookup()`` returns a hit, that cached result is used in place
1752 7. If ``lookup()`` returns a hit, that cached result is used in place
1753 of invoking the command function. ``__exit__`` is called and the instance
1753 of invoking the command function. ``__exit__`` is called and the instance
1754 is discarded.
1754 is discarded.
1755 8. The command function is invoked.
1755 8. The command function is invoked.
1756 9. ``onobject()`` is called for each object emitted by the command
1756 9. ``onobject()`` is called for each object emitted by the command
1757 function.
1757 function.
1758 10. After the final object is seen, ``onfinished()`` is called.
1758 10. After the final object is seen, ``onfinished()`` is called.
1759 11. ``__exit__`` is called to signal the end of use of the instance.
1759 11. ``__exit__`` is called to signal the end of use of the instance.
1760
1760
1761 Cache *key* derivation can be influenced by the instance.
1761 Cache *key* derivation can be influenced by the instance.
1762
1762
1763 Cache keys are initially derived by a deterministic representation of
1763 Cache keys are initially derived by a deterministic representation of
1764 the command request. This includes the command name, arguments, protocol
1764 the command request. This includes the command name, arguments, protocol
1765 version, etc. This initial key derivation is performed by CBOR-encoding a
1765 version, etc. This initial key derivation is performed by CBOR-encoding a
1766 data structure and feeding that output into a hasher.
1766 data structure and feeding that output into a hasher.
1767
1767
1768 Instances of this interface can influence this initial key derivation
1768 Instances of this interface can influence this initial key derivation
1769 via ``adjustcachekeystate()``.
1769 via ``adjustcachekeystate()``.
1770
1770
1771 The instance is informed of the derived cache key via a call to
1771 The instance is informed of the derived cache key via a call to
1772 ``setcachekey()``. The instance must store the key locally so it can
1772 ``setcachekey()``. The instance must store the key locally so it can
1773 be consulted on subsequent operations that may require it.
1773 be consulted on subsequent operations that may require it.
1774
1774
1775 When constructed, the instance has access to a callable that can be used
1775 When constructed, the instance has access to a callable that can be used
1776 for encoding response objects. This callable receives as its single
1776 for encoding response objects. This callable receives as its single
1777 argument an object emitted by a command function. It returns an iterable
1777 argument an object emitted by a command function. It returns an iterable
1778 of bytes chunks representing the encoded object. Unless the cacher is
1778 of bytes chunks representing the encoded object. Unless the cacher is
1779 caching native Python objects in memory or has a way of reconstructing
1779 caching native Python objects in memory or has a way of reconstructing
1780 the original Python objects, implementations typically call this function
1780 the original Python objects, implementations typically call this function
1781 to produce bytes from the output objects and then store those bytes in
1781 to produce bytes from the output objects and then store those bytes in
1782 the cache. When it comes time to re-emit those bytes, they are wrapped
1782 the cache. When it comes time to re-emit those bytes, they are wrapped
1783 in a ``wireprototypes.encodedresponse`` instance to tell the output
1783 in a ``wireprototypes.encodedresponse`` instance to tell the output
1784 layer that they are pre-encoded.
1784 layer that they are pre-encoded.
1785
1785
1786 When receiving the objects emitted by the command function, instances
1786 When receiving the objects emitted by the command function, instances
1787 can choose what to do with those objects. The simplest thing to do is
1787 can choose what to do with those objects. The simplest thing to do is
1788 re-emit the original objects. They will be forwarded to the output
1788 re-emit the original objects. They will be forwarded to the output
1789 layer and will be processed as if the cacher did not exist.
1789 layer and will be processed as if the cacher did not exist.
1790
1790
1791 Implementations could also choose to not emit objects - instead locally
1791 Implementations could also choose to not emit objects - instead locally
1792 buffering objects or their encoded representation. They could then emit
1792 buffering objects or their encoded representation. They could then emit
1793 a single "coalesced" object when ``onfinished()`` is called. In
1793 a single "coalesced" object when ``onfinished()`` is called. In
1794 this way, the implementation would function as a filtering layer of
1794 this way, the implementation would function as a filtering layer of
1795 sorts.
1795 sorts.
1796
1796
1797 When caching objects, typically the encoded form of the object will
1797 When caching objects, typically the encoded form of the object will
1798 be stored. Keep in mind that if the original object is forwarded to
1798 be stored. Keep in mind that if the original object is forwarded to
1799 the output layer, it will need to be encoded there as well. For large
1799 the output layer, it will need to be encoded there as well. For large
1800 output, this redundant encoding could add overhead. Implementations
1800 output, this redundant encoding could add overhead. Implementations
1801 could wrap the encoded object data in ``wireprototypes.encodedresponse``
1801 could wrap the encoded object data in ``wireprototypes.encodedresponse``
1802 instances to avoid this overhead.
1802 instances to avoid this overhead.
1803 """
1803 """
1804 def __enter__():
1804 def __enter__():
1805 """Marks the instance as active.
1805 """Marks the instance as active.
1806
1806
1807 Should return self.
1807 Should return self.
1808 """
1808 """
1809
1809
1810 def __exit__(exctype, excvalue, exctb):
1810 def __exit__(exctype, excvalue, exctb):
1811 """Called when cacher is no longer used.
1811 """Called when cacher is no longer used.
1812
1812
1813 This can be used by implementations to perform cleanup actions (e.g.
1813 This can be used by implementations to perform cleanup actions (e.g.
1814 disconnecting network sockets, aborting a partially cached response.
1814 disconnecting network sockets, aborting a partially cached response.
1815 """
1815 """
1816
1816
1817 def adjustcachekeystate(state):
1817 def adjustcachekeystate(state):
1818 """Influences cache key derivation by adjusting state to derive key.
1818 """Influences cache key derivation by adjusting state to derive key.
1819
1819
1820 A dict defining the state used to derive the cache key is passed.
1820 A dict defining the state used to derive the cache key is passed.
1821
1821
1822 Implementations can modify this dict to record additional state that
1822 Implementations can modify this dict to record additional state that
1823 is wanted to influence key derivation.
1823 is wanted to influence key derivation.
1824
1824
1825 Implementations are *highly* encouraged to not modify or delete
1825 Implementations are *highly* encouraged to not modify or delete
1826 existing keys.
1826 existing keys.
1827 """
1827 """
1828
1828
1829 def setcachekey(key):
1829 def setcachekey(key):
1830 """Record the derived cache key for this request.
1830 """Record the derived cache key for this request.
1831
1831
1832 Instances may mutate the key for internal usage, as desired. e.g.
1832 Instances may mutate the key for internal usage, as desired. e.g.
1833 instances may wish to prepend the repo name, introduce path
1833 instances may wish to prepend the repo name, introduce path
1834 components for filesystem or URL addressing, etc. Behavior is up to
1834 components for filesystem or URL addressing, etc. Behavior is up to
1835 the cache.
1835 the cache.
1836
1836
1837 Returns a bool indicating if the request is cacheable by this
1837 Returns a bool indicating if the request is cacheable by this
1838 instance.
1838 instance.
1839 """
1839 """
1840
1840
1841 def lookup():
1841 def lookup():
1842 """Attempt to resolve an entry in the cache.
1842 """Attempt to resolve an entry in the cache.
1843
1843
1844 The instance is instructed to look for the cache key that it was
1844 The instance is instructed to look for the cache key that it was
1845 informed about via the call to ``setcachekey()``.
1845 informed about via the call to ``setcachekey()``.
1846
1846
1847 If there's no cache hit or the cacher doesn't wish to use the cached
1847 If there's no cache hit or the cacher doesn't wish to use the cached
1848 entry, ``None`` should be returned.
1848 entry, ``None`` should be returned.
1849
1849
1850 Else, a dict defining the cached result should be returned. The
1850 Else, a dict defining the cached result should be returned. The
1851 dict may have the following keys:
1851 dict may have the following keys:
1852
1852
1853 objs
1853 objs
1854 An iterable of objects that should be sent to the client. That
1854 An iterable of objects that should be sent to the client. That
1855 iterable of objects is expected to be what the command function
1855 iterable of objects is expected to be what the command function
1856 would return if invoked or an equivalent representation thereof.
1856 would return if invoked or an equivalent representation thereof.
1857 """
1857 """
1858
1858
1859 def onobject(obj):
1859 def onobject(obj):
1860 """Called when a new object is emitted from the command function.
1860 """Called when a new object is emitted from the command function.
1861
1861
1862 Receives as its argument the object that was emitted from the
1862 Receives as its argument the object that was emitted from the
1863 command function.
1863 command function.
1864
1864
1865 This method returns an iterator of objects to forward to the output
1865 This method returns an iterator of objects to forward to the output
1866 layer. The easiest implementation is a generator that just
1866 layer. The easiest implementation is a generator that just
1867 ``yield obj``.
1867 ``yield obj``.
1868 """
1868 """
1869
1869
1870 def onfinished():
1870 def onfinished():
1871 """Called after all objects have been emitted from the command function.
1871 """Called after all objects have been emitted from the command function.
1872
1872
1873 Implementations should return an iterator of objects to forward to
1873 Implementations should return an iterator of objects to forward to
1874 the output layer.
1874 the output layer.
1875
1875
1876 This method can be a generator.
1876 This method can be a generator.
1877 """
1877 """
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now