##// END OF EJS Templates
py3: fix exception message encoding in infinitepush...
Emmanuel Leblond -
r43680:2ded39ef stable
parent child Browse files
Show More
@@ -1,1344 +1,1344 b''
1 # Infinite push
1 # Infinite push
2 #
2 #
3 # Copyright 2016 Facebook, Inc.
3 # Copyright 2016 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7 """ store some pushes in a remote blob store on the server (EXPERIMENTAL)
7 """ store some pushes in a remote blob store on the server (EXPERIMENTAL)
8
8
9 IMPORTANT: if you use this extension, please contact
9 IMPORTANT: if you use this extension, please contact
10 mercurial-devel@mercurial-scm.org ASAP. This extension is believed to
10 mercurial-devel@mercurial-scm.org ASAP. This extension is believed to
11 be unused and barring learning of users of this functionality, we will
11 be unused and barring learning of users of this functionality, we will
12 delete this code at the end of 2020.
12 delete this code at the end of 2020.
13
13
14 [infinitepush]
14 [infinitepush]
15 # Server-side and client-side option. Pattern of the infinitepush bookmark
15 # Server-side and client-side option. Pattern of the infinitepush bookmark
16 branchpattern = PATTERN
16 branchpattern = PATTERN
17
17
18 # Server or client
18 # Server or client
19 server = False
19 server = False
20
20
21 # Server-side option. Possible values: 'disk' or 'sql'. Fails if not set
21 # Server-side option. Possible values: 'disk' or 'sql'. Fails if not set
22 indextype = disk
22 indextype = disk
23
23
24 # Server-side option. Used only if indextype=sql.
24 # Server-side option. Used only if indextype=sql.
25 # Format: 'IP:PORT:DB_NAME:USER:PASSWORD'
25 # Format: 'IP:PORT:DB_NAME:USER:PASSWORD'
26 sqlhost = IP:PORT:DB_NAME:USER:PASSWORD
26 sqlhost = IP:PORT:DB_NAME:USER:PASSWORD
27
27
28 # Server-side option. Used only if indextype=disk.
28 # Server-side option. Used only if indextype=disk.
29 # Filesystem path to the index store
29 # Filesystem path to the index store
30 indexpath = PATH
30 indexpath = PATH
31
31
32 # Server-side option. Possible values: 'disk' or 'external'
32 # Server-side option. Possible values: 'disk' or 'external'
33 # Fails if not set
33 # Fails if not set
34 storetype = disk
34 storetype = disk
35
35
36 # Server-side option.
36 # Server-side option.
37 # Path to the binary that will save bundle to the bundlestore
37 # Path to the binary that will save bundle to the bundlestore
38 # Formatted cmd line will be passed to it (see `put_args`)
38 # Formatted cmd line will be passed to it (see `put_args`)
39 put_binary = put
39 put_binary = put
40
40
41 # Serser-side option. Used only if storetype=external.
41 # Serser-side option. Used only if storetype=external.
42 # Format cmd-line string for put binary. Placeholder: {filename}
42 # Format cmd-line string for put binary. Placeholder: {filename}
43 put_args = {filename}
43 put_args = {filename}
44
44
45 # Server-side option.
45 # Server-side option.
46 # Path to the binary that get bundle from the bundlestore.
46 # Path to the binary that get bundle from the bundlestore.
47 # Formatted cmd line will be passed to it (see `get_args`)
47 # Formatted cmd line will be passed to it (see `get_args`)
48 get_binary = get
48 get_binary = get
49
49
50 # Serser-side option. Used only if storetype=external.
50 # Serser-side option. Used only if storetype=external.
51 # Format cmd-line string for get binary. Placeholders: {filename} {handle}
51 # Format cmd-line string for get binary. Placeholders: {filename} {handle}
52 get_args = {filename} {handle}
52 get_args = {filename} {handle}
53
53
54 # Server-side option
54 # Server-side option
55 logfile = FIlE
55 logfile = FIlE
56
56
57 # Server-side option
57 # Server-side option
58 loglevel = DEBUG
58 loglevel = DEBUG
59
59
60 # Server-side option. Used only if indextype=sql.
60 # Server-side option. Used only if indextype=sql.
61 # Sets mysql wait_timeout option.
61 # Sets mysql wait_timeout option.
62 waittimeout = 300
62 waittimeout = 300
63
63
64 # Server-side option. Used only if indextype=sql.
64 # Server-side option. Used only if indextype=sql.
65 # Sets mysql innodb_lock_wait_timeout option.
65 # Sets mysql innodb_lock_wait_timeout option.
66 locktimeout = 120
66 locktimeout = 120
67
67
68 # Server-side option. Used only if indextype=sql.
68 # Server-side option. Used only if indextype=sql.
69 # Name of the repository
69 # Name of the repository
70 reponame = ''
70 reponame = ''
71
71
72 # Client-side option. Used by --list-remote option. List of remote scratch
72 # Client-side option. Used by --list-remote option. List of remote scratch
73 # patterns to list if no patterns are specified.
73 # patterns to list if no patterns are specified.
74 defaultremotepatterns = ['*']
74 defaultremotepatterns = ['*']
75
75
76 # Instructs infinitepush to forward all received bundle2 parts to the
76 # Instructs infinitepush to forward all received bundle2 parts to the
77 # bundle for storage. Defaults to False.
77 # bundle for storage. Defaults to False.
78 storeallparts = True
78 storeallparts = True
79
79
80 # routes each incoming push to the bundlestore. defaults to False
80 # routes each incoming push to the bundlestore. defaults to False
81 pushtobundlestore = True
81 pushtobundlestore = True
82
82
83 [remotenames]
83 [remotenames]
84 # Client-side option
84 # Client-side option
85 # This option should be set only if remotenames extension is enabled.
85 # This option should be set only if remotenames extension is enabled.
86 # Whether remote bookmarks are tracked by remotenames extension.
86 # Whether remote bookmarks are tracked by remotenames extension.
87 bookmarks = True
87 bookmarks = True
88 """
88 """
89
89
90 from __future__ import absolute_import
90 from __future__ import absolute_import
91
91
92 import collections
92 import collections
93 import contextlib
93 import contextlib
94 import errno
94 import errno
95 import functools
95 import functools
96 import logging
96 import logging
97 import os
97 import os
98 import random
98 import random
99 import re
99 import re
100 import socket
100 import socket
101 import subprocess
101 import subprocess
102 import time
102 import time
103
103
104 from mercurial.node import (
104 from mercurial.node import (
105 bin,
105 bin,
106 hex,
106 hex,
107 )
107 )
108
108
109 from mercurial.i18n import _
109 from mercurial.i18n import _
110
110
111 from mercurial.pycompat import (
111 from mercurial.pycompat import (
112 getattr,
112 getattr,
113 open,
113 open,
114 )
114 )
115
115
116 from mercurial.utils import (
116 from mercurial.utils import (
117 procutil,
117 procutil,
118 stringutil,
118 stringutil,
119 )
119 )
120
120
121 from mercurial import (
121 from mercurial import (
122 bundle2,
122 bundle2,
123 changegroup,
123 changegroup,
124 commands,
124 commands,
125 discovery,
125 discovery,
126 encoding,
126 encoding,
127 error,
127 error,
128 exchange,
128 exchange,
129 extensions,
129 extensions,
130 hg,
130 hg,
131 localrepo,
131 localrepo,
132 phases,
132 phases,
133 pushkey,
133 pushkey,
134 pycompat,
134 pycompat,
135 registrar,
135 registrar,
136 util,
136 util,
137 wireprototypes,
137 wireprototypes,
138 wireprotov1peer,
138 wireprotov1peer,
139 wireprotov1server,
139 wireprotov1server,
140 )
140 )
141
141
142 from . import (
142 from . import (
143 bundleparts,
143 bundleparts,
144 common,
144 common,
145 )
145 )
146
146
147 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
147 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
148 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
148 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
149 # be specifying the version(s) of Mercurial they are tested with, or
149 # be specifying the version(s) of Mercurial they are tested with, or
150 # leave the attribute unspecified.
150 # leave the attribute unspecified.
151 testedwith = b'ships-with-hg-core'
151 testedwith = b'ships-with-hg-core'
152
152
153 configtable = {}
153 configtable = {}
154 configitem = registrar.configitem(configtable)
154 configitem = registrar.configitem(configtable)
155
155
156 configitem(
156 configitem(
157 b'infinitepush', b'server', default=False,
157 b'infinitepush', b'server', default=False,
158 )
158 )
159 configitem(
159 configitem(
160 b'infinitepush', b'storetype', default=b'',
160 b'infinitepush', b'storetype', default=b'',
161 )
161 )
162 configitem(
162 configitem(
163 b'infinitepush', b'indextype', default=b'',
163 b'infinitepush', b'indextype', default=b'',
164 )
164 )
165 configitem(
165 configitem(
166 b'infinitepush', b'indexpath', default=b'',
166 b'infinitepush', b'indexpath', default=b'',
167 )
167 )
168 configitem(
168 configitem(
169 b'infinitepush', b'storeallparts', default=False,
169 b'infinitepush', b'storeallparts', default=False,
170 )
170 )
171 configitem(
171 configitem(
172 b'infinitepush', b'reponame', default=b'',
172 b'infinitepush', b'reponame', default=b'',
173 )
173 )
174 configitem(
174 configitem(
175 b'scratchbranch', b'storepath', default=b'',
175 b'scratchbranch', b'storepath', default=b'',
176 )
176 )
177 configitem(
177 configitem(
178 b'infinitepush', b'branchpattern', default=b'',
178 b'infinitepush', b'branchpattern', default=b'',
179 )
179 )
180 configitem(
180 configitem(
181 b'infinitepush', b'pushtobundlestore', default=False,
181 b'infinitepush', b'pushtobundlestore', default=False,
182 )
182 )
183 configitem(
183 configitem(
184 b'experimental', b'server-bundlestore-bookmark', default=b'',
184 b'experimental', b'server-bundlestore-bookmark', default=b'',
185 )
185 )
186 configitem(
186 configitem(
187 b'experimental', b'infinitepush-scratchpush', default=False,
187 b'experimental', b'infinitepush-scratchpush', default=False,
188 )
188 )
189
189
190 experimental = b'experimental'
190 experimental = b'experimental'
191 configbookmark = b'server-bundlestore-bookmark'
191 configbookmark = b'server-bundlestore-bookmark'
192 configscratchpush = b'infinitepush-scratchpush'
192 configscratchpush = b'infinitepush-scratchpush'
193
193
194 scratchbranchparttype = bundleparts.scratchbranchparttype
194 scratchbranchparttype = bundleparts.scratchbranchparttype
195 revsetpredicate = registrar.revsetpredicate()
195 revsetpredicate = registrar.revsetpredicate()
196 templatekeyword = registrar.templatekeyword()
196 templatekeyword = registrar.templatekeyword()
197 _scratchbranchmatcher = lambda x: False
197 _scratchbranchmatcher = lambda x: False
198 _maybehash = re.compile(r'^[a-f0-9]+$').search
198 _maybehash = re.compile(r'^[a-f0-9]+$').search
199
199
200
200
201 def _buildexternalbundlestore(ui):
201 def _buildexternalbundlestore(ui):
202 put_args = ui.configlist(b'infinitepush', b'put_args', [])
202 put_args = ui.configlist(b'infinitepush', b'put_args', [])
203 put_binary = ui.config(b'infinitepush', b'put_binary')
203 put_binary = ui.config(b'infinitepush', b'put_binary')
204 if not put_binary:
204 if not put_binary:
205 raise error.Abort(b'put binary is not specified')
205 raise error.Abort(b'put binary is not specified')
206 get_args = ui.configlist(b'infinitepush', b'get_args', [])
206 get_args = ui.configlist(b'infinitepush', b'get_args', [])
207 get_binary = ui.config(b'infinitepush', b'get_binary')
207 get_binary = ui.config(b'infinitepush', b'get_binary')
208 if not get_binary:
208 if not get_binary:
209 raise error.Abort(b'get binary is not specified')
209 raise error.Abort(b'get binary is not specified')
210 from . import store
210 from . import store
211
211
212 return store.externalbundlestore(put_binary, put_args, get_binary, get_args)
212 return store.externalbundlestore(put_binary, put_args, get_binary, get_args)
213
213
214
214
215 def _buildsqlindex(ui):
215 def _buildsqlindex(ui):
216 sqlhost = ui.config(b'infinitepush', b'sqlhost')
216 sqlhost = ui.config(b'infinitepush', b'sqlhost')
217 if not sqlhost:
217 if not sqlhost:
218 raise error.Abort(_(b'please set infinitepush.sqlhost'))
218 raise error.Abort(_(b'please set infinitepush.sqlhost'))
219 host, port, db, user, password = sqlhost.split(b':')
219 host, port, db, user, password = sqlhost.split(b':')
220 reponame = ui.config(b'infinitepush', b'reponame')
220 reponame = ui.config(b'infinitepush', b'reponame')
221 if not reponame:
221 if not reponame:
222 raise error.Abort(_(b'please set infinitepush.reponame'))
222 raise error.Abort(_(b'please set infinitepush.reponame'))
223
223
224 logfile = ui.config(b'infinitepush', b'logfile', b'')
224 logfile = ui.config(b'infinitepush', b'logfile', b'')
225 waittimeout = ui.configint(b'infinitepush', b'waittimeout', 300)
225 waittimeout = ui.configint(b'infinitepush', b'waittimeout', 300)
226 locktimeout = ui.configint(b'infinitepush', b'locktimeout', 120)
226 locktimeout = ui.configint(b'infinitepush', b'locktimeout', 120)
227 from . import sqlindexapi
227 from . import sqlindexapi
228
228
229 return sqlindexapi.sqlindexapi(
229 return sqlindexapi.sqlindexapi(
230 reponame,
230 reponame,
231 host,
231 host,
232 port,
232 port,
233 db,
233 db,
234 user,
234 user,
235 password,
235 password,
236 logfile,
236 logfile,
237 _getloglevel(ui),
237 _getloglevel(ui),
238 waittimeout=waittimeout,
238 waittimeout=waittimeout,
239 locktimeout=locktimeout,
239 locktimeout=locktimeout,
240 )
240 )
241
241
242
242
243 def _getloglevel(ui):
243 def _getloglevel(ui):
244 loglevel = ui.config(b'infinitepush', b'loglevel', b'DEBUG')
244 loglevel = ui.config(b'infinitepush', b'loglevel', b'DEBUG')
245 numeric_loglevel = getattr(logging, loglevel.upper(), None)
245 numeric_loglevel = getattr(logging, loglevel.upper(), None)
246 if not isinstance(numeric_loglevel, int):
246 if not isinstance(numeric_loglevel, int):
247 raise error.Abort(_(b'invalid log level %s') % loglevel)
247 raise error.Abort(_(b'invalid log level %s') % loglevel)
248 return numeric_loglevel
248 return numeric_loglevel
249
249
250
250
251 def _tryhoist(ui, remotebookmark):
251 def _tryhoist(ui, remotebookmark):
252 '''returns a bookmarks with hoisted part removed
252 '''returns a bookmarks with hoisted part removed
253
253
254 Remotenames extension has a 'hoist' config that allows to use remote
254 Remotenames extension has a 'hoist' config that allows to use remote
255 bookmarks without specifying remote path. For example, 'hg update master'
255 bookmarks without specifying remote path. For example, 'hg update master'
256 works as well as 'hg update remote/master'. We want to allow the same in
256 works as well as 'hg update remote/master'. We want to allow the same in
257 infinitepush.
257 infinitepush.
258 '''
258 '''
259
259
260 if common.isremotebooksenabled(ui):
260 if common.isremotebooksenabled(ui):
261 hoist = ui.config(b'remotenames', b'hoistedpeer') + b'/'
261 hoist = ui.config(b'remotenames', b'hoistedpeer') + b'/'
262 if remotebookmark.startswith(hoist):
262 if remotebookmark.startswith(hoist):
263 return remotebookmark[len(hoist) :]
263 return remotebookmark[len(hoist) :]
264 return remotebookmark
264 return remotebookmark
265
265
266
266
267 class bundlestore(object):
267 class bundlestore(object):
268 def __init__(self, repo):
268 def __init__(self, repo):
269 self._repo = repo
269 self._repo = repo
270 storetype = self._repo.ui.config(b'infinitepush', b'storetype')
270 storetype = self._repo.ui.config(b'infinitepush', b'storetype')
271 if storetype == b'disk':
271 if storetype == b'disk':
272 from . import store
272 from . import store
273
273
274 self.store = store.filebundlestore(self._repo.ui, self._repo)
274 self.store = store.filebundlestore(self._repo.ui, self._repo)
275 elif storetype == b'external':
275 elif storetype == b'external':
276 self.store = _buildexternalbundlestore(self._repo.ui)
276 self.store = _buildexternalbundlestore(self._repo.ui)
277 else:
277 else:
278 raise error.Abort(
278 raise error.Abort(
279 _(b'unknown infinitepush store type specified %s') % storetype
279 _(b'unknown infinitepush store type specified %s') % storetype
280 )
280 )
281
281
282 indextype = self._repo.ui.config(b'infinitepush', b'indextype')
282 indextype = self._repo.ui.config(b'infinitepush', b'indextype')
283 if indextype == b'disk':
283 if indextype == b'disk':
284 from . import fileindexapi
284 from . import fileindexapi
285
285
286 self.index = fileindexapi.fileindexapi(self._repo)
286 self.index = fileindexapi.fileindexapi(self._repo)
287 elif indextype == b'sql':
287 elif indextype == b'sql':
288 self.index = _buildsqlindex(self._repo.ui)
288 self.index = _buildsqlindex(self._repo.ui)
289 else:
289 else:
290 raise error.Abort(
290 raise error.Abort(
291 _(b'unknown infinitepush index type specified %s') % indextype
291 _(b'unknown infinitepush index type specified %s') % indextype
292 )
292 )
293
293
294
294
295 def _isserver(ui):
295 def _isserver(ui):
296 return ui.configbool(b'infinitepush', b'server')
296 return ui.configbool(b'infinitepush', b'server')
297
297
298
298
299 def reposetup(ui, repo):
299 def reposetup(ui, repo):
300 if _isserver(ui) and repo.local():
300 if _isserver(ui) and repo.local():
301 repo.bundlestore = bundlestore(repo)
301 repo.bundlestore = bundlestore(repo)
302
302
303
303
304 def extsetup(ui):
304 def extsetup(ui):
305 commonsetup(ui)
305 commonsetup(ui)
306 if _isserver(ui):
306 if _isserver(ui):
307 serverextsetup(ui)
307 serverextsetup(ui)
308 else:
308 else:
309 clientextsetup(ui)
309 clientextsetup(ui)
310
310
311
311
312 def commonsetup(ui):
312 def commonsetup(ui):
313 wireprotov1server.commands[b'listkeyspatterns'] = (
313 wireprotov1server.commands[b'listkeyspatterns'] = (
314 wireprotolistkeyspatterns,
314 wireprotolistkeyspatterns,
315 b'namespace patterns',
315 b'namespace patterns',
316 )
316 )
317 scratchbranchpat = ui.config(b'infinitepush', b'branchpattern')
317 scratchbranchpat = ui.config(b'infinitepush', b'branchpattern')
318 if scratchbranchpat:
318 if scratchbranchpat:
319 global _scratchbranchmatcher
319 global _scratchbranchmatcher
320 kind, pat, _scratchbranchmatcher = stringutil.stringmatcher(
320 kind, pat, _scratchbranchmatcher = stringutil.stringmatcher(
321 scratchbranchpat
321 scratchbranchpat
322 )
322 )
323
323
324
324
325 def serverextsetup(ui):
325 def serverextsetup(ui):
326 origpushkeyhandler = bundle2.parthandlermapping[b'pushkey']
326 origpushkeyhandler = bundle2.parthandlermapping[b'pushkey']
327
327
328 def newpushkeyhandler(*args, **kwargs):
328 def newpushkeyhandler(*args, **kwargs):
329 bundle2pushkey(origpushkeyhandler, *args, **kwargs)
329 bundle2pushkey(origpushkeyhandler, *args, **kwargs)
330
330
331 newpushkeyhandler.params = origpushkeyhandler.params
331 newpushkeyhandler.params = origpushkeyhandler.params
332 bundle2.parthandlermapping[b'pushkey'] = newpushkeyhandler
332 bundle2.parthandlermapping[b'pushkey'] = newpushkeyhandler
333
333
334 orighandlephasehandler = bundle2.parthandlermapping[b'phase-heads']
334 orighandlephasehandler = bundle2.parthandlermapping[b'phase-heads']
335 newphaseheadshandler = lambda *args, **kwargs: bundle2handlephases(
335 newphaseheadshandler = lambda *args, **kwargs: bundle2handlephases(
336 orighandlephasehandler, *args, **kwargs
336 orighandlephasehandler, *args, **kwargs
337 )
337 )
338 newphaseheadshandler.params = orighandlephasehandler.params
338 newphaseheadshandler.params = orighandlephasehandler.params
339 bundle2.parthandlermapping[b'phase-heads'] = newphaseheadshandler
339 bundle2.parthandlermapping[b'phase-heads'] = newphaseheadshandler
340
340
341 extensions.wrapfunction(
341 extensions.wrapfunction(
342 localrepo.localrepository, b'listkeys', localrepolistkeys
342 localrepo.localrepository, b'listkeys', localrepolistkeys
343 )
343 )
344 wireprotov1server.commands[b'lookup'] = (
344 wireprotov1server.commands[b'lookup'] = (
345 _lookupwrap(wireprotov1server.commands[b'lookup'][0]),
345 _lookupwrap(wireprotov1server.commands[b'lookup'][0]),
346 b'key',
346 b'key',
347 )
347 )
348 extensions.wrapfunction(exchange, b'getbundlechunks', getbundlechunks)
348 extensions.wrapfunction(exchange, b'getbundlechunks', getbundlechunks)
349
349
350 extensions.wrapfunction(bundle2, b'processparts', processparts)
350 extensions.wrapfunction(bundle2, b'processparts', processparts)
351
351
352
352
353 def clientextsetup(ui):
353 def clientextsetup(ui):
354 entry = extensions.wrapcommand(commands.table, b'push', _push)
354 entry = extensions.wrapcommand(commands.table, b'push', _push)
355
355
356 entry[1].append(
356 entry[1].append(
357 (
357 (
358 b'',
358 b'',
359 b'bundle-store',
359 b'bundle-store',
360 None,
360 None,
361 _(b'force push to go to bundle store (EXPERIMENTAL)'),
361 _(b'force push to go to bundle store (EXPERIMENTAL)'),
362 )
362 )
363 )
363 )
364
364
365 extensions.wrapcommand(commands.table, b'pull', _pull)
365 extensions.wrapcommand(commands.table, b'pull', _pull)
366
366
367 extensions.wrapfunction(discovery, b'checkheads', _checkheads)
367 extensions.wrapfunction(discovery, b'checkheads', _checkheads)
368
368
369 wireprotov1peer.wirepeer.listkeyspatterns = listkeyspatterns
369 wireprotov1peer.wirepeer.listkeyspatterns = listkeyspatterns
370
370
371 partorder = exchange.b2partsgenorder
371 partorder = exchange.b2partsgenorder
372 index = partorder.index(b'changeset')
372 index = partorder.index(b'changeset')
373 partorder.insert(
373 partorder.insert(
374 index, partorder.pop(partorder.index(scratchbranchparttype))
374 index, partorder.pop(partorder.index(scratchbranchparttype))
375 )
375 )
376
376
377
377
378 def _checkheads(orig, pushop):
378 def _checkheads(orig, pushop):
379 if pushop.ui.configbool(experimental, configscratchpush, False):
379 if pushop.ui.configbool(experimental, configscratchpush, False):
380 return
380 return
381 return orig(pushop)
381 return orig(pushop)
382
382
383
383
384 def wireprotolistkeyspatterns(repo, proto, namespace, patterns):
384 def wireprotolistkeyspatterns(repo, proto, namespace, patterns):
385 patterns = wireprototypes.decodelist(patterns)
385 patterns = wireprototypes.decodelist(patterns)
386 d = pycompat.iteritems(repo.listkeys(encoding.tolocal(namespace), patterns))
386 d = pycompat.iteritems(repo.listkeys(encoding.tolocal(namespace), patterns))
387 return pushkey.encodekeys(d)
387 return pushkey.encodekeys(d)
388
388
389
389
390 def localrepolistkeys(orig, self, namespace, patterns=None):
390 def localrepolistkeys(orig, self, namespace, patterns=None):
391 if namespace == b'bookmarks' and patterns:
391 if namespace == b'bookmarks' and patterns:
392 index = self.bundlestore.index
392 index = self.bundlestore.index
393 results = {}
393 results = {}
394 bookmarks = orig(self, namespace)
394 bookmarks = orig(self, namespace)
395 for pattern in patterns:
395 for pattern in patterns:
396 results.update(index.getbookmarks(pattern))
396 results.update(index.getbookmarks(pattern))
397 if pattern.endswith(b'*'):
397 if pattern.endswith(b'*'):
398 pattern = b're:^' + pattern[:-1] + b'.*'
398 pattern = b're:^' + pattern[:-1] + b'.*'
399 kind, pat, matcher = stringutil.stringmatcher(pattern)
399 kind, pat, matcher = stringutil.stringmatcher(pattern)
400 for bookmark, node in pycompat.iteritems(bookmarks):
400 for bookmark, node in pycompat.iteritems(bookmarks):
401 if matcher(bookmark):
401 if matcher(bookmark):
402 results[bookmark] = node
402 results[bookmark] = node
403 return results
403 return results
404 else:
404 else:
405 return orig(self, namespace)
405 return orig(self, namespace)
406
406
407
407
408 @wireprotov1peer.batchable
408 @wireprotov1peer.batchable
409 def listkeyspatterns(self, namespace, patterns):
409 def listkeyspatterns(self, namespace, patterns):
410 if not self.capable(b'pushkey'):
410 if not self.capable(b'pushkey'):
411 yield {}, None
411 yield {}, None
412 f = wireprotov1peer.future()
412 f = wireprotov1peer.future()
413 self.ui.debug(b'preparing listkeys for "%s"\n' % namespace)
413 self.ui.debug(b'preparing listkeys for "%s"\n' % namespace)
414 yield {
414 yield {
415 b'namespace': encoding.fromlocal(namespace),
415 b'namespace': encoding.fromlocal(namespace),
416 b'patterns': wireprototypes.encodelist(patterns),
416 b'patterns': wireprototypes.encodelist(patterns),
417 }, f
417 }, f
418 d = f.value
418 d = f.value
419 self.ui.debug(
419 self.ui.debug(
420 b'received listkey for "%s": %i bytes\n' % (namespace, len(d))
420 b'received listkey for "%s": %i bytes\n' % (namespace, len(d))
421 )
421 )
422 yield pushkey.decodekeys(d)
422 yield pushkey.decodekeys(d)
423
423
424
424
425 def _readbundlerevs(bundlerepo):
425 def _readbundlerevs(bundlerepo):
426 return list(bundlerepo.revs(b'bundle()'))
426 return list(bundlerepo.revs(b'bundle()'))
427
427
428
428
429 def _includefilelogstobundle(bundlecaps, bundlerepo, bundlerevs, ui):
429 def _includefilelogstobundle(bundlecaps, bundlerepo, bundlerevs, ui):
430 '''Tells remotefilelog to include all changed files to the changegroup
430 '''Tells remotefilelog to include all changed files to the changegroup
431
431
432 By default remotefilelog doesn't include file content to the changegroup.
432 By default remotefilelog doesn't include file content to the changegroup.
433 But we need to include it if we are fetching from bundlestore.
433 But we need to include it if we are fetching from bundlestore.
434 '''
434 '''
435 changedfiles = set()
435 changedfiles = set()
436 cl = bundlerepo.changelog
436 cl = bundlerepo.changelog
437 for r in bundlerevs:
437 for r in bundlerevs:
438 # [3] means changed files
438 # [3] means changed files
439 changedfiles.update(cl.read(r)[3])
439 changedfiles.update(cl.read(r)[3])
440 if not changedfiles:
440 if not changedfiles:
441 return bundlecaps
441 return bundlecaps
442
442
443 changedfiles = b'\0'.join(changedfiles)
443 changedfiles = b'\0'.join(changedfiles)
444 newcaps = []
444 newcaps = []
445 appended = False
445 appended = False
446 for cap in bundlecaps or []:
446 for cap in bundlecaps or []:
447 if cap.startswith(b'excludepattern='):
447 if cap.startswith(b'excludepattern='):
448 newcaps.append(b'\0'.join((cap, changedfiles)))
448 newcaps.append(b'\0'.join((cap, changedfiles)))
449 appended = True
449 appended = True
450 else:
450 else:
451 newcaps.append(cap)
451 newcaps.append(cap)
452 if not appended:
452 if not appended:
453 # Not found excludepattern cap. Just append it
453 # Not found excludepattern cap. Just append it
454 newcaps.append(b'excludepattern=' + changedfiles)
454 newcaps.append(b'excludepattern=' + changedfiles)
455
455
456 return newcaps
456 return newcaps
457
457
458
458
459 def _rebundle(bundlerepo, bundleroots, unknownhead):
459 def _rebundle(bundlerepo, bundleroots, unknownhead):
460 '''
460 '''
461 Bundle may include more revision then user requested. For example,
461 Bundle may include more revision then user requested. For example,
462 if user asks for revision but bundle also consists its descendants.
462 if user asks for revision but bundle also consists its descendants.
463 This function will filter out all revision that user is not requested.
463 This function will filter out all revision that user is not requested.
464 '''
464 '''
465 parts = []
465 parts = []
466
466
467 version = b'02'
467 version = b'02'
468 outgoing = discovery.outgoing(
468 outgoing = discovery.outgoing(
469 bundlerepo, commonheads=bundleroots, missingheads=[unknownhead]
469 bundlerepo, commonheads=bundleroots, missingheads=[unknownhead]
470 )
470 )
471 cgstream = changegroup.makestream(bundlerepo, outgoing, version, b'pull')
471 cgstream = changegroup.makestream(bundlerepo, outgoing, version, b'pull')
472 cgstream = util.chunkbuffer(cgstream).read()
472 cgstream = util.chunkbuffer(cgstream).read()
473 cgpart = bundle2.bundlepart(b'changegroup', data=cgstream)
473 cgpart = bundle2.bundlepart(b'changegroup', data=cgstream)
474 cgpart.addparam(b'version', version)
474 cgpart.addparam(b'version', version)
475 parts.append(cgpart)
475 parts.append(cgpart)
476
476
477 return parts
477 return parts
478
478
479
479
480 def _getbundleroots(oldrepo, bundlerepo, bundlerevs):
480 def _getbundleroots(oldrepo, bundlerepo, bundlerevs):
481 cl = bundlerepo.changelog
481 cl = bundlerepo.changelog
482 bundleroots = []
482 bundleroots = []
483 for rev in bundlerevs:
483 for rev in bundlerevs:
484 node = cl.node(rev)
484 node = cl.node(rev)
485 parents = cl.parents(node)
485 parents = cl.parents(node)
486 for parent in parents:
486 for parent in parents:
487 # include all revs that exist in the main repo
487 # include all revs that exist in the main repo
488 # to make sure that bundle may apply client-side
488 # to make sure that bundle may apply client-side
489 if parent in oldrepo:
489 if parent in oldrepo:
490 bundleroots.append(parent)
490 bundleroots.append(parent)
491 return bundleroots
491 return bundleroots
492
492
493
493
494 def _needsrebundling(head, bundlerepo):
494 def _needsrebundling(head, bundlerepo):
495 bundleheads = list(bundlerepo.revs(b'heads(bundle())'))
495 bundleheads = list(bundlerepo.revs(b'heads(bundle())'))
496 return not (
496 return not (
497 len(bundleheads) == 1 and bundlerepo[bundleheads[0]].node() == head
497 len(bundleheads) == 1 and bundlerepo[bundleheads[0]].node() == head
498 )
498 )
499
499
500
500
501 def _generateoutputparts(head, bundlerepo, bundleroots, bundlefile):
501 def _generateoutputparts(head, bundlerepo, bundleroots, bundlefile):
502 '''generates bundle that will be send to the user
502 '''generates bundle that will be send to the user
503
503
504 returns tuple with raw bundle string and bundle type
504 returns tuple with raw bundle string and bundle type
505 '''
505 '''
506 parts = []
506 parts = []
507 if not _needsrebundling(head, bundlerepo):
507 if not _needsrebundling(head, bundlerepo):
508 with util.posixfile(bundlefile, b"rb") as f:
508 with util.posixfile(bundlefile, b"rb") as f:
509 unbundler = exchange.readbundle(bundlerepo.ui, f, bundlefile)
509 unbundler = exchange.readbundle(bundlerepo.ui, f, bundlefile)
510 if isinstance(unbundler, changegroup.cg1unpacker):
510 if isinstance(unbundler, changegroup.cg1unpacker):
511 part = bundle2.bundlepart(
511 part = bundle2.bundlepart(
512 b'changegroup', data=unbundler._stream.read()
512 b'changegroup', data=unbundler._stream.read()
513 )
513 )
514 part.addparam(b'version', b'01')
514 part.addparam(b'version', b'01')
515 parts.append(part)
515 parts.append(part)
516 elif isinstance(unbundler, bundle2.unbundle20):
516 elif isinstance(unbundler, bundle2.unbundle20):
517 haschangegroup = False
517 haschangegroup = False
518 for part in unbundler.iterparts():
518 for part in unbundler.iterparts():
519 if part.type == b'changegroup':
519 if part.type == b'changegroup':
520 haschangegroup = True
520 haschangegroup = True
521 newpart = bundle2.bundlepart(part.type, data=part.read())
521 newpart = bundle2.bundlepart(part.type, data=part.read())
522 for key, value in pycompat.iteritems(part.params):
522 for key, value in pycompat.iteritems(part.params):
523 newpart.addparam(key, value)
523 newpart.addparam(key, value)
524 parts.append(newpart)
524 parts.append(newpart)
525
525
526 if not haschangegroup:
526 if not haschangegroup:
527 raise error.Abort(
527 raise error.Abort(
528 b'unexpected bundle without changegroup part, '
528 b'unexpected bundle without changegroup part, '
529 + b'head: %s' % hex(head),
529 + b'head: %s' % hex(head),
530 hint=b'report to administrator',
530 hint=b'report to administrator',
531 )
531 )
532 else:
532 else:
533 raise error.Abort(b'unknown bundle type')
533 raise error.Abort(b'unknown bundle type')
534 else:
534 else:
535 parts = _rebundle(bundlerepo, bundleroots, head)
535 parts = _rebundle(bundlerepo, bundleroots, head)
536
536
537 return parts
537 return parts
538
538
539
539
540 def getbundlechunks(orig, repo, source, heads=None, bundlecaps=None, **kwargs):
540 def getbundlechunks(orig, repo, source, heads=None, bundlecaps=None, **kwargs):
541 heads = heads or []
541 heads = heads or []
542 # newheads are parents of roots of scratch bundles that were requested
542 # newheads are parents of roots of scratch bundles that were requested
543 newphases = {}
543 newphases = {}
544 scratchbundles = []
544 scratchbundles = []
545 newheads = []
545 newheads = []
546 scratchheads = []
546 scratchheads = []
547 nodestobundle = {}
547 nodestobundle = {}
548 allbundlestocleanup = []
548 allbundlestocleanup = []
549 try:
549 try:
550 for head in heads:
550 for head in heads:
551 if head not in repo.changelog.nodemap:
551 if head not in repo.changelog.nodemap:
552 if head not in nodestobundle:
552 if head not in nodestobundle:
553 newbundlefile = common.downloadbundle(repo, head)
553 newbundlefile = common.downloadbundle(repo, head)
554 bundlepath = b"bundle:%s+%s" % (repo.root, newbundlefile)
554 bundlepath = b"bundle:%s+%s" % (repo.root, newbundlefile)
555 bundlerepo = hg.repository(repo.ui, bundlepath)
555 bundlerepo = hg.repository(repo.ui, bundlepath)
556
556
557 allbundlestocleanup.append((bundlerepo, newbundlefile))
557 allbundlestocleanup.append((bundlerepo, newbundlefile))
558 bundlerevs = set(_readbundlerevs(bundlerepo))
558 bundlerevs = set(_readbundlerevs(bundlerepo))
559 bundlecaps = _includefilelogstobundle(
559 bundlecaps = _includefilelogstobundle(
560 bundlecaps, bundlerepo, bundlerevs, repo.ui
560 bundlecaps, bundlerepo, bundlerevs, repo.ui
561 )
561 )
562 cl = bundlerepo.changelog
562 cl = bundlerepo.changelog
563 bundleroots = _getbundleroots(repo, bundlerepo, bundlerevs)
563 bundleroots = _getbundleroots(repo, bundlerepo, bundlerevs)
564 for rev in bundlerevs:
564 for rev in bundlerevs:
565 node = cl.node(rev)
565 node = cl.node(rev)
566 newphases[hex(node)] = str(phases.draft)
566 newphases[hex(node)] = str(phases.draft)
567 nodestobundle[node] = (
567 nodestobundle[node] = (
568 bundlerepo,
568 bundlerepo,
569 bundleroots,
569 bundleroots,
570 newbundlefile,
570 newbundlefile,
571 )
571 )
572
572
573 scratchbundles.append(
573 scratchbundles.append(
574 _generateoutputparts(head, *nodestobundle[head])
574 _generateoutputparts(head, *nodestobundle[head])
575 )
575 )
576 newheads.extend(bundleroots)
576 newheads.extend(bundleroots)
577 scratchheads.append(head)
577 scratchheads.append(head)
578 finally:
578 finally:
579 for bundlerepo, bundlefile in allbundlestocleanup:
579 for bundlerepo, bundlefile in allbundlestocleanup:
580 bundlerepo.close()
580 bundlerepo.close()
581 try:
581 try:
582 os.unlink(bundlefile)
582 os.unlink(bundlefile)
583 except (IOError, OSError):
583 except (IOError, OSError):
584 # if we can't cleanup the file then just ignore the error,
584 # if we can't cleanup the file then just ignore the error,
585 # no need to fail
585 # no need to fail
586 pass
586 pass
587
587
588 pullfrombundlestore = bool(scratchbundles)
588 pullfrombundlestore = bool(scratchbundles)
589 wrappedchangegrouppart = False
589 wrappedchangegrouppart = False
590 wrappedlistkeys = False
590 wrappedlistkeys = False
591 oldchangegrouppart = exchange.getbundle2partsmapping[b'changegroup']
591 oldchangegrouppart = exchange.getbundle2partsmapping[b'changegroup']
592 try:
592 try:
593
593
594 def _changegrouppart(bundler, *args, **kwargs):
594 def _changegrouppart(bundler, *args, **kwargs):
595 # Order is important here. First add non-scratch part
595 # Order is important here. First add non-scratch part
596 # and only then add parts with scratch bundles because
596 # and only then add parts with scratch bundles because
597 # non-scratch part contains parents of roots of scratch bundles.
597 # non-scratch part contains parents of roots of scratch bundles.
598 result = oldchangegrouppart(bundler, *args, **kwargs)
598 result = oldchangegrouppart(bundler, *args, **kwargs)
599 for bundle in scratchbundles:
599 for bundle in scratchbundles:
600 for part in bundle:
600 for part in bundle:
601 bundler.addpart(part)
601 bundler.addpart(part)
602 return result
602 return result
603
603
604 exchange.getbundle2partsmapping[b'changegroup'] = _changegrouppart
604 exchange.getbundle2partsmapping[b'changegroup'] = _changegrouppart
605 wrappedchangegrouppart = True
605 wrappedchangegrouppart = True
606
606
607 def _listkeys(orig, self, namespace):
607 def _listkeys(orig, self, namespace):
608 origvalues = orig(self, namespace)
608 origvalues = orig(self, namespace)
609 if namespace == b'phases' and pullfrombundlestore:
609 if namespace == b'phases' and pullfrombundlestore:
610 if origvalues.get(b'publishing') == b'True':
610 if origvalues.get(b'publishing') == b'True':
611 # Make repo non-publishing to preserve draft phase
611 # Make repo non-publishing to preserve draft phase
612 del origvalues[b'publishing']
612 del origvalues[b'publishing']
613 origvalues.update(newphases)
613 origvalues.update(newphases)
614 return origvalues
614 return origvalues
615
615
616 extensions.wrapfunction(
616 extensions.wrapfunction(
617 localrepo.localrepository, b'listkeys', _listkeys
617 localrepo.localrepository, b'listkeys', _listkeys
618 )
618 )
619 wrappedlistkeys = True
619 wrappedlistkeys = True
620 heads = list((set(newheads) | set(heads)) - set(scratchheads))
620 heads = list((set(newheads) | set(heads)) - set(scratchheads))
621 result = orig(
621 result = orig(
622 repo, source, heads=heads, bundlecaps=bundlecaps, **kwargs
622 repo, source, heads=heads, bundlecaps=bundlecaps, **kwargs
623 )
623 )
624 finally:
624 finally:
625 if wrappedchangegrouppart:
625 if wrappedchangegrouppart:
626 exchange.getbundle2partsmapping[b'changegroup'] = oldchangegrouppart
626 exchange.getbundle2partsmapping[b'changegroup'] = oldchangegrouppart
627 if wrappedlistkeys:
627 if wrappedlistkeys:
628 extensions.unwrapfunction(
628 extensions.unwrapfunction(
629 localrepo.localrepository, b'listkeys', _listkeys
629 localrepo.localrepository, b'listkeys', _listkeys
630 )
630 )
631 return result
631 return result
632
632
633
633
634 def _lookupwrap(orig):
634 def _lookupwrap(orig):
635 def _lookup(repo, proto, key):
635 def _lookup(repo, proto, key):
636 localkey = encoding.tolocal(key)
636 localkey = encoding.tolocal(key)
637
637
638 if isinstance(localkey, str) and _scratchbranchmatcher(localkey):
638 if isinstance(localkey, str) and _scratchbranchmatcher(localkey):
639 scratchnode = repo.bundlestore.index.getnode(localkey)
639 scratchnode = repo.bundlestore.index.getnode(localkey)
640 if scratchnode:
640 if scratchnode:
641 return b"%d %s\n" % (1, scratchnode)
641 return b"%d %s\n" % (1, scratchnode)
642 else:
642 else:
643 return b"%d %s\n" % (
643 return b"%d %s\n" % (
644 0,
644 0,
645 b'scratch branch %s not found' % localkey,
645 b'scratch branch %s not found' % localkey,
646 )
646 )
647 else:
647 else:
648 try:
648 try:
649 r = hex(repo.lookup(localkey))
649 r = hex(repo.lookup(localkey))
650 return b"%d %s\n" % (1, r)
650 return b"%d %s\n" % (1, r)
651 except Exception as inst:
651 except Exception as inst:
652 if repo.bundlestore.index.getbundle(localkey):
652 if repo.bundlestore.index.getbundle(localkey):
653 return b"%d %s\n" % (1, localkey)
653 return b"%d %s\n" % (1, localkey)
654 else:
654 else:
655 r = stringutil.forcebytestr(inst)
655 r = stringutil.forcebytestr(inst)
656 return b"%d %s\n" % (0, r)
656 return b"%d %s\n" % (0, r)
657
657
658 return _lookup
658 return _lookup
659
659
660
660
661 def _pull(orig, ui, repo, source=b"default", **opts):
661 def _pull(orig, ui, repo, source=b"default", **opts):
662 opts = pycompat.byteskwargs(opts)
662 opts = pycompat.byteskwargs(opts)
663 # Copy paste from `pull` command
663 # Copy paste from `pull` command
664 source, branches = hg.parseurl(ui.expandpath(source), opts.get(b'branch'))
664 source, branches = hg.parseurl(ui.expandpath(source), opts.get(b'branch'))
665
665
666 scratchbookmarks = {}
666 scratchbookmarks = {}
667 unfi = repo.unfiltered()
667 unfi = repo.unfiltered()
668 unknownnodes = []
668 unknownnodes = []
669 for rev in opts.get(b'rev', []):
669 for rev in opts.get(b'rev', []):
670 if rev not in unfi:
670 if rev not in unfi:
671 unknownnodes.append(rev)
671 unknownnodes.append(rev)
672 if opts.get(b'bookmark'):
672 if opts.get(b'bookmark'):
673 bookmarks = []
673 bookmarks = []
674 revs = opts.get(b'rev') or []
674 revs = opts.get(b'rev') or []
675 for bookmark in opts.get(b'bookmark'):
675 for bookmark in opts.get(b'bookmark'):
676 if _scratchbranchmatcher(bookmark):
676 if _scratchbranchmatcher(bookmark):
677 # rev is not known yet
677 # rev is not known yet
678 # it will be fetched with listkeyspatterns next
678 # it will be fetched with listkeyspatterns next
679 scratchbookmarks[bookmark] = b'REVTOFETCH'
679 scratchbookmarks[bookmark] = b'REVTOFETCH'
680 else:
680 else:
681 bookmarks.append(bookmark)
681 bookmarks.append(bookmark)
682
682
683 if scratchbookmarks:
683 if scratchbookmarks:
684 other = hg.peer(repo, opts, source)
684 other = hg.peer(repo, opts, source)
685 fetchedbookmarks = other.listkeyspatterns(
685 fetchedbookmarks = other.listkeyspatterns(
686 b'bookmarks', patterns=scratchbookmarks
686 b'bookmarks', patterns=scratchbookmarks
687 )
687 )
688 for bookmark in scratchbookmarks:
688 for bookmark in scratchbookmarks:
689 if bookmark not in fetchedbookmarks:
689 if bookmark not in fetchedbookmarks:
690 raise error.Abort(
690 raise error.Abort(
691 b'remote bookmark %s not found!' % bookmark
691 b'remote bookmark %s not found!' % bookmark
692 )
692 )
693 scratchbookmarks[bookmark] = fetchedbookmarks[bookmark]
693 scratchbookmarks[bookmark] = fetchedbookmarks[bookmark]
694 revs.append(fetchedbookmarks[bookmark])
694 revs.append(fetchedbookmarks[bookmark])
695 opts[b'bookmark'] = bookmarks
695 opts[b'bookmark'] = bookmarks
696 opts[b'rev'] = revs
696 opts[b'rev'] = revs
697
697
698 if scratchbookmarks or unknownnodes:
698 if scratchbookmarks or unknownnodes:
699 # Set anyincoming to True
699 # Set anyincoming to True
700 extensions.wrapfunction(
700 extensions.wrapfunction(
701 discovery, b'findcommonincoming', _findcommonincoming
701 discovery, b'findcommonincoming', _findcommonincoming
702 )
702 )
703 try:
703 try:
704 # Remote scratch bookmarks will be deleted because remotenames doesn't
704 # Remote scratch bookmarks will be deleted because remotenames doesn't
705 # know about them. Let's save it before pull and restore after
705 # know about them. Let's save it before pull and restore after
706 remotescratchbookmarks = _readscratchremotebookmarks(ui, repo, source)
706 remotescratchbookmarks = _readscratchremotebookmarks(ui, repo, source)
707 result = orig(ui, repo, source, **pycompat.strkwargs(opts))
707 result = orig(ui, repo, source, **pycompat.strkwargs(opts))
708 # TODO(stash): race condition is possible
708 # TODO(stash): race condition is possible
709 # if scratch bookmarks was updated right after orig.
709 # if scratch bookmarks was updated right after orig.
710 # But that's unlikely and shouldn't be harmful.
710 # But that's unlikely and shouldn't be harmful.
711 if common.isremotebooksenabled(ui):
711 if common.isremotebooksenabled(ui):
712 remotescratchbookmarks.update(scratchbookmarks)
712 remotescratchbookmarks.update(scratchbookmarks)
713 _saveremotebookmarks(repo, remotescratchbookmarks, source)
713 _saveremotebookmarks(repo, remotescratchbookmarks, source)
714 else:
714 else:
715 _savelocalbookmarks(repo, scratchbookmarks)
715 _savelocalbookmarks(repo, scratchbookmarks)
716 return result
716 return result
717 finally:
717 finally:
718 if scratchbookmarks:
718 if scratchbookmarks:
719 extensions.unwrapfunction(discovery, b'findcommonincoming')
719 extensions.unwrapfunction(discovery, b'findcommonincoming')
720
720
721
721
722 def _readscratchremotebookmarks(ui, repo, other):
722 def _readscratchremotebookmarks(ui, repo, other):
723 if common.isremotebooksenabled(ui):
723 if common.isremotebooksenabled(ui):
724 remotenamesext = extensions.find(b'remotenames')
724 remotenamesext = extensions.find(b'remotenames')
725 remotepath = remotenamesext.activepath(repo.ui, other)
725 remotepath = remotenamesext.activepath(repo.ui, other)
726 result = {}
726 result = {}
727 # Let's refresh remotenames to make sure we have it up to date
727 # Let's refresh remotenames to make sure we have it up to date
728 # Seems that `repo.names['remotebookmarks']` may return stale bookmarks
728 # Seems that `repo.names['remotebookmarks']` may return stale bookmarks
729 # and it results in deleting scratch bookmarks. Our best guess how to
729 # and it results in deleting scratch bookmarks. Our best guess how to
730 # fix it is to use `clearnames()`
730 # fix it is to use `clearnames()`
731 repo._remotenames.clearnames()
731 repo._remotenames.clearnames()
732 for remotebookmark in repo.names[b'remotebookmarks'].listnames(repo):
732 for remotebookmark in repo.names[b'remotebookmarks'].listnames(repo):
733 path, bookname = remotenamesext.splitremotename(remotebookmark)
733 path, bookname = remotenamesext.splitremotename(remotebookmark)
734 if path == remotepath and _scratchbranchmatcher(bookname):
734 if path == remotepath and _scratchbranchmatcher(bookname):
735 nodes = repo.names[b'remotebookmarks'].nodes(
735 nodes = repo.names[b'remotebookmarks'].nodes(
736 repo, remotebookmark
736 repo, remotebookmark
737 )
737 )
738 if nodes:
738 if nodes:
739 result[bookname] = hex(nodes[0])
739 result[bookname] = hex(nodes[0])
740 return result
740 return result
741 else:
741 else:
742 return {}
742 return {}
743
743
744
744
745 def _saveremotebookmarks(repo, newbookmarks, remote):
745 def _saveremotebookmarks(repo, newbookmarks, remote):
746 remotenamesext = extensions.find(b'remotenames')
746 remotenamesext = extensions.find(b'remotenames')
747 remotepath = remotenamesext.activepath(repo.ui, remote)
747 remotepath = remotenamesext.activepath(repo.ui, remote)
748 branches = collections.defaultdict(list)
748 branches = collections.defaultdict(list)
749 bookmarks = {}
749 bookmarks = {}
750 remotenames = remotenamesext.readremotenames(repo)
750 remotenames = remotenamesext.readremotenames(repo)
751 for hexnode, nametype, remote, rname in remotenames:
751 for hexnode, nametype, remote, rname in remotenames:
752 if remote != remotepath:
752 if remote != remotepath:
753 continue
753 continue
754 if nametype == b'bookmarks':
754 if nametype == b'bookmarks':
755 if rname in newbookmarks:
755 if rname in newbookmarks:
756 # It's possible if we have a normal bookmark that matches
756 # It's possible if we have a normal bookmark that matches
757 # scratch branch pattern. In this case just use the current
757 # scratch branch pattern. In this case just use the current
758 # bookmark node
758 # bookmark node
759 del newbookmarks[rname]
759 del newbookmarks[rname]
760 bookmarks[rname] = hexnode
760 bookmarks[rname] = hexnode
761 elif nametype == b'branches':
761 elif nametype == b'branches':
762 # saveremotenames expects 20 byte binary nodes for branches
762 # saveremotenames expects 20 byte binary nodes for branches
763 branches[rname].append(bin(hexnode))
763 branches[rname].append(bin(hexnode))
764
764
765 for bookmark, hexnode in pycompat.iteritems(newbookmarks):
765 for bookmark, hexnode in pycompat.iteritems(newbookmarks):
766 bookmarks[bookmark] = hexnode
766 bookmarks[bookmark] = hexnode
767 remotenamesext.saveremotenames(repo, remotepath, branches, bookmarks)
767 remotenamesext.saveremotenames(repo, remotepath, branches, bookmarks)
768
768
769
769
770 def _savelocalbookmarks(repo, bookmarks):
770 def _savelocalbookmarks(repo, bookmarks):
771 if not bookmarks:
771 if not bookmarks:
772 return
772 return
773 with repo.wlock(), repo.lock(), repo.transaction(b'bookmark') as tr:
773 with repo.wlock(), repo.lock(), repo.transaction(b'bookmark') as tr:
774 changes = []
774 changes = []
775 for scratchbook, node in pycompat.iteritems(bookmarks):
775 for scratchbook, node in pycompat.iteritems(bookmarks):
776 changectx = repo[node]
776 changectx = repo[node]
777 changes.append((scratchbook, changectx.node()))
777 changes.append((scratchbook, changectx.node()))
778 repo._bookmarks.applychanges(repo, tr, changes)
778 repo._bookmarks.applychanges(repo, tr, changes)
779
779
780
780
781 def _findcommonincoming(orig, *args, **kwargs):
781 def _findcommonincoming(orig, *args, **kwargs):
782 common, inc, remoteheads = orig(*args, **kwargs)
782 common, inc, remoteheads = orig(*args, **kwargs)
783 return common, True, remoteheads
783 return common, True, remoteheads
784
784
785
785
786 def _push(orig, ui, repo, dest=None, *args, **opts):
786 def _push(orig, ui, repo, dest=None, *args, **opts):
787 opts = pycompat.byteskwargs(opts)
787 opts = pycompat.byteskwargs(opts)
788 bookmark = opts.get(b'bookmark')
788 bookmark = opts.get(b'bookmark')
789 # we only support pushing one infinitepush bookmark at once
789 # we only support pushing one infinitepush bookmark at once
790 if len(bookmark) == 1:
790 if len(bookmark) == 1:
791 bookmark = bookmark[0]
791 bookmark = bookmark[0]
792 else:
792 else:
793 bookmark = b''
793 bookmark = b''
794
794
795 oldphasemove = None
795 oldphasemove = None
796 overrides = {(experimental, configbookmark): bookmark}
796 overrides = {(experimental, configbookmark): bookmark}
797
797
798 with ui.configoverride(overrides, b'infinitepush'):
798 with ui.configoverride(overrides, b'infinitepush'):
799 scratchpush = opts.get(b'bundle_store')
799 scratchpush = opts.get(b'bundle_store')
800 if _scratchbranchmatcher(bookmark):
800 if _scratchbranchmatcher(bookmark):
801 scratchpush = True
801 scratchpush = True
802 # bundle2 can be sent back after push (for example, bundle2
802 # bundle2 can be sent back after push (for example, bundle2
803 # containing `pushkey` part to update bookmarks)
803 # containing `pushkey` part to update bookmarks)
804 ui.setconfig(experimental, b'bundle2.pushback', True)
804 ui.setconfig(experimental, b'bundle2.pushback', True)
805
805
806 if scratchpush:
806 if scratchpush:
807 # this is an infinitepush, we don't want the bookmark to be applied
807 # this is an infinitepush, we don't want the bookmark to be applied
808 # rather that should be stored in the bundlestore
808 # rather that should be stored in the bundlestore
809 opts[b'bookmark'] = []
809 opts[b'bookmark'] = []
810 ui.setconfig(experimental, configscratchpush, True)
810 ui.setconfig(experimental, configscratchpush, True)
811 oldphasemove = extensions.wrapfunction(
811 oldphasemove = extensions.wrapfunction(
812 exchange, b'_localphasemove', _phasemove
812 exchange, b'_localphasemove', _phasemove
813 )
813 )
814 # Copy-paste from `push` command
814 # Copy-paste from `push` command
815 path = ui.paths.getpath(dest, default=(b'default-push', b'default'))
815 path = ui.paths.getpath(dest, default=(b'default-push', b'default'))
816 if not path:
816 if not path:
817 raise error.Abort(
817 raise error.Abort(
818 _(b'default repository not configured!'),
818 _(b'default repository not configured!'),
819 hint=_(b"see 'hg help config.paths'"),
819 hint=_(b"see 'hg help config.paths'"),
820 )
820 )
821 destpath = path.pushloc or path.loc
821 destpath = path.pushloc or path.loc
822 # Remote scratch bookmarks will be deleted because remotenames doesn't
822 # Remote scratch bookmarks will be deleted because remotenames doesn't
823 # know about them. Let's save it before push and restore after
823 # know about them. Let's save it before push and restore after
824 remotescratchbookmarks = _readscratchremotebookmarks(ui, repo, destpath)
824 remotescratchbookmarks = _readscratchremotebookmarks(ui, repo, destpath)
825 result = orig(ui, repo, dest, *args, **pycompat.strkwargs(opts))
825 result = orig(ui, repo, dest, *args, **pycompat.strkwargs(opts))
826 if common.isremotebooksenabled(ui):
826 if common.isremotebooksenabled(ui):
827 if bookmark and scratchpush:
827 if bookmark and scratchpush:
828 other = hg.peer(repo, opts, destpath)
828 other = hg.peer(repo, opts, destpath)
829 fetchedbookmarks = other.listkeyspatterns(
829 fetchedbookmarks = other.listkeyspatterns(
830 b'bookmarks', patterns=[bookmark]
830 b'bookmarks', patterns=[bookmark]
831 )
831 )
832 remotescratchbookmarks.update(fetchedbookmarks)
832 remotescratchbookmarks.update(fetchedbookmarks)
833 _saveremotebookmarks(repo, remotescratchbookmarks, destpath)
833 _saveremotebookmarks(repo, remotescratchbookmarks, destpath)
834 if oldphasemove:
834 if oldphasemove:
835 exchange._localphasemove = oldphasemove
835 exchange._localphasemove = oldphasemove
836 return result
836 return result
837
837
838
838
839 def _deleteinfinitepushbookmarks(ui, repo, path, names):
839 def _deleteinfinitepushbookmarks(ui, repo, path, names):
840 """Prune remote names by removing the bookmarks we don't want anymore,
840 """Prune remote names by removing the bookmarks we don't want anymore,
841 then writing the result back to disk
841 then writing the result back to disk
842 """
842 """
843 remotenamesext = extensions.find(b'remotenames')
843 remotenamesext = extensions.find(b'remotenames')
844
844
845 # remotename format is:
845 # remotename format is:
846 # (node, nametype ("branches" or "bookmarks"), remote, name)
846 # (node, nametype ("branches" or "bookmarks"), remote, name)
847 nametype_idx = 1
847 nametype_idx = 1
848 remote_idx = 2
848 remote_idx = 2
849 name_idx = 3
849 name_idx = 3
850 remotenames = [
850 remotenames = [
851 remotename
851 remotename
852 for remotename in remotenamesext.readremotenames(repo)
852 for remotename in remotenamesext.readremotenames(repo)
853 if remotename[remote_idx] == path
853 if remotename[remote_idx] == path
854 ]
854 ]
855 remote_bm_names = [
855 remote_bm_names = [
856 remotename[name_idx]
856 remotename[name_idx]
857 for remotename in remotenames
857 for remotename in remotenames
858 if remotename[nametype_idx] == b"bookmarks"
858 if remotename[nametype_idx] == b"bookmarks"
859 ]
859 ]
860
860
861 for name in names:
861 for name in names:
862 if name not in remote_bm_names:
862 if name not in remote_bm_names:
863 raise error.Abort(
863 raise error.Abort(
864 _(
864 _(
865 b"infinitepush bookmark '{}' does not exist "
865 b"infinitepush bookmark '{}' does not exist "
866 b"in path '{}'"
866 b"in path '{}'"
867 ).format(name, path)
867 ).format(name, path)
868 )
868 )
869
869
870 bookmarks = {}
870 bookmarks = {}
871 branches = collections.defaultdict(list)
871 branches = collections.defaultdict(list)
872 for node, nametype, remote, name in remotenames:
872 for node, nametype, remote, name in remotenames:
873 if nametype == b"bookmarks" and name not in names:
873 if nametype == b"bookmarks" and name not in names:
874 bookmarks[name] = node
874 bookmarks[name] = node
875 elif nametype == b"branches":
875 elif nametype == b"branches":
876 # saveremotenames wants binary nodes for branches
876 # saveremotenames wants binary nodes for branches
877 branches[name].append(bin(node))
877 branches[name].append(bin(node))
878
878
879 remotenamesext.saveremotenames(repo, path, branches, bookmarks)
879 remotenamesext.saveremotenames(repo, path, branches, bookmarks)
880
880
881
881
882 def _phasemove(orig, pushop, nodes, phase=phases.public):
882 def _phasemove(orig, pushop, nodes, phase=phases.public):
883 """prevent commits from being marked public
883 """prevent commits from being marked public
884
884
885 Since these are going to a scratch branch, they aren't really being
885 Since these are going to a scratch branch, they aren't really being
886 published."""
886 published."""
887
887
888 if phase != phases.public:
888 if phase != phases.public:
889 orig(pushop, nodes, phase)
889 orig(pushop, nodes, phase)
890
890
891
891
892 @exchange.b2partsgenerator(scratchbranchparttype)
892 @exchange.b2partsgenerator(scratchbranchparttype)
893 def partgen(pushop, bundler):
893 def partgen(pushop, bundler):
894 bookmark = pushop.ui.config(experimental, configbookmark)
894 bookmark = pushop.ui.config(experimental, configbookmark)
895 scratchpush = pushop.ui.configbool(experimental, configscratchpush)
895 scratchpush = pushop.ui.configbool(experimental, configscratchpush)
896 if b'changesets' in pushop.stepsdone or not scratchpush:
896 if b'changesets' in pushop.stepsdone or not scratchpush:
897 return
897 return
898
898
899 if scratchbranchparttype not in bundle2.bundle2caps(pushop.remote):
899 if scratchbranchparttype not in bundle2.bundle2caps(pushop.remote):
900 return
900 return
901
901
902 pushop.stepsdone.add(b'changesets')
902 pushop.stepsdone.add(b'changesets')
903 if not pushop.outgoing.missing:
903 if not pushop.outgoing.missing:
904 pushop.ui.status(_(b'no changes found\n'))
904 pushop.ui.status(_(b'no changes found\n'))
905 pushop.cgresult = 0
905 pushop.cgresult = 0
906 return
906 return
907
907
908 # This parameter tells the server that the following bundle is an
908 # This parameter tells the server that the following bundle is an
909 # infinitepush. This let's it switch the part processing to our infinitepush
909 # infinitepush. This let's it switch the part processing to our infinitepush
910 # code path.
910 # code path.
911 bundler.addparam(b"infinitepush", b"True")
911 bundler.addparam(b"infinitepush", b"True")
912
912
913 scratchparts = bundleparts.getscratchbranchparts(
913 scratchparts = bundleparts.getscratchbranchparts(
914 pushop.repo, pushop.remote, pushop.outgoing, pushop.ui, bookmark
914 pushop.repo, pushop.remote, pushop.outgoing, pushop.ui, bookmark
915 )
915 )
916
916
917 for scratchpart in scratchparts:
917 for scratchpart in scratchparts:
918 bundler.addpart(scratchpart)
918 bundler.addpart(scratchpart)
919
919
920 def handlereply(op):
920 def handlereply(op):
921 # server either succeeds or aborts; no code to read
921 # server either succeeds or aborts; no code to read
922 pushop.cgresult = 1
922 pushop.cgresult = 1
923
923
924 return handlereply
924 return handlereply
925
925
926
926
927 bundle2.capabilities[bundleparts.scratchbranchparttype] = ()
927 bundle2.capabilities[bundleparts.scratchbranchparttype] = ()
928
928
929
929
930 def _getrevs(bundle, oldnode, force, bookmark):
930 def _getrevs(bundle, oldnode, force, bookmark):
931 b'extracts and validates the revs to be imported'
931 b'extracts and validates the revs to be imported'
932 revs = [bundle[r] for r in bundle.revs(b'sort(bundle())')]
932 revs = [bundle[r] for r in bundle.revs(b'sort(bundle())')]
933
933
934 # new bookmark
934 # new bookmark
935 if oldnode is None:
935 if oldnode is None:
936 return revs
936 return revs
937
937
938 # Fast forward update
938 # Fast forward update
939 if oldnode in bundle and list(bundle.set(b'bundle() & %s::', oldnode)):
939 if oldnode in bundle and list(bundle.set(b'bundle() & %s::', oldnode)):
940 return revs
940 return revs
941
941
942 return revs
942 return revs
943
943
944
944
945 @contextlib.contextmanager
945 @contextlib.contextmanager
946 def logservicecall(logger, service, **kwargs):
946 def logservicecall(logger, service, **kwargs):
947 start = time.time()
947 start = time.time()
948 logger(service, eventtype=b'start', **kwargs)
948 logger(service, eventtype=b'start', **kwargs)
949 try:
949 try:
950 yield
950 yield
951 logger(
951 logger(
952 service,
952 service,
953 eventtype=b'success',
953 eventtype=b'success',
954 elapsedms=(time.time() - start) * 1000,
954 elapsedms=(time.time() - start) * 1000,
955 **kwargs
955 **kwargs
956 )
956 )
957 except Exception as e:
957 except Exception as e:
958 logger(
958 logger(
959 service,
959 service,
960 eventtype=b'failure',
960 eventtype=b'failure',
961 elapsedms=(time.time() - start) * 1000,
961 elapsedms=(time.time() - start) * 1000,
962 errormsg=str(e),
962 errormsg=stringutil.forcebytestr(e),
963 **kwargs
963 **kwargs
964 )
964 )
965 raise
965 raise
966
966
967
967
968 def _getorcreateinfinitepushlogger(op):
968 def _getorcreateinfinitepushlogger(op):
969 logger = op.records[b'infinitepushlogger']
969 logger = op.records[b'infinitepushlogger']
970 if not logger:
970 if not logger:
971 ui = op.repo.ui
971 ui = op.repo.ui
972 try:
972 try:
973 username = procutil.getuser()
973 username = procutil.getuser()
974 except Exception:
974 except Exception:
975 username = b'unknown'
975 username = b'unknown'
976 # Generate random request id to be able to find all logged entries
976 # Generate random request id to be able to find all logged entries
977 # for the same request. Since requestid is pseudo-generated it may
977 # for the same request. Since requestid is pseudo-generated it may
978 # not be unique, but we assume that (hostname, username, requestid)
978 # not be unique, but we assume that (hostname, username, requestid)
979 # is unique.
979 # is unique.
980 random.seed()
980 random.seed()
981 requestid = random.randint(0, 2000000000)
981 requestid = random.randint(0, 2000000000)
982 hostname = socket.gethostname()
982 hostname = socket.gethostname()
983 logger = functools.partial(
983 logger = functools.partial(
984 ui.log,
984 ui.log,
985 b'infinitepush',
985 b'infinitepush',
986 user=username,
986 user=username,
987 requestid=requestid,
987 requestid=requestid,
988 hostname=hostname,
988 hostname=hostname,
989 reponame=ui.config(b'infinitepush', b'reponame'),
989 reponame=ui.config(b'infinitepush', b'reponame'),
990 )
990 )
991 op.records.add(b'infinitepushlogger', logger)
991 op.records.add(b'infinitepushlogger', logger)
992 else:
992 else:
993 logger = logger[0]
993 logger = logger[0]
994 return logger
994 return logger
995
995
996
996
997 def storetobundlestore(orig, repo, op, unbundler):
997 def storetobundlestore(orig, repo, op, unbundler):
998 """stores the incoming bundle coming from push command to the bundlestore
998 """stores the incoming bundle coming from push command to the bundlestore
999 instead of applying on the revlogs"""
999 instead of applying on the revlogs"""
1000
1000
1001 repo.ui.status(_(b"storing changesets on the bundlestore\n"))
1001 repo.ui.status(_(b"storing changesets on the bundlestore\n"))
1002 bundler = bundle2.bundle20(repo.ui)
1002 bundler = bundle2.bundle20(repo.ui)
1003
1003
1004 # processing each part and storing it in bundler
1004 # processing each part and storing it in bundler
1005 with bundle2.partiterator(repo, op, unbundler) as parts:
1005 with bundle2.partiterator(repo, op, unbundler) as parts:
1006 for part in parts:
1006 for part in parts:
1007 bundlepart = None
1007 bundlepart = None
1008 if part.type == b'replycaps':
1008 if part.type == b'replycaps':
1009 # This configures the current operation to allow reply parts.
1009 # This configures the current operation to allow reply parts.
1010 bundle2._processpart(op, part)
1010 bundle2._processpart(op, part)
1011 else:
1011 else:
1012 bundlepart = bundle2.bundlepart(part.type, data=part.read())
1012 bundlepart = bundle2.bundlepart(part.type, data=part.read())
1013 for key, value in pycompat.iteritems(part.params):
1013 for key, value in pycompat.iteritems(part.params):
1014 bundlepart.addparam(key, value)
1014 bundlepart.addparam(key, value)
1015
1015
1016 # Certain parts require a response
1016 # Certain parts require a response
1017 if part.type in (b'pushkey', b'changegroup'):
1017 if part.type in (b'pushkey', b'changegroup'):
1018 if op.reply is not None:
1018 if op.reply is not None:
1019 rpart = op.reply.newpart(b'reply:%s' % part.type)
1019 rpart = op.reply.newpart(b'reply:%s' % part.type)
1020 rpart.addparam(
1020 rpart.addparam(
1021 b'in-reply-to', b'%d' % part.id, mandatory=False
1021 b'in-reply-to', b'%d' % part.id, mandatory=False
1022 )
1022 )
1023 rpart.addparam(b'return', b'1', mandatory=False)
1023 rpart.addparam(b'return', b'1', mandatory=False)
1024
1024
1025 op.records.add(part.type, {b'return': 1,})
1025 op.records.add(part.type, {b'return': 1,})
1026 if bundlepart:
1026 if bundlepart:
1027 bundler.addpart(bundlepart)
1027 bundler.addpart(bundlepart)
1028
1028
1029 # storing the bundle in the bundlestore
1029 # storing the bundle in the bundlestore
1030 buf = util.chunkbuffer(bundler.getchunks())
1030 buf = util.chunkbuffer(bundler.getchunks())
1031 fd, bundlefile = pycompat.mkstemp()
1031 fd, bundlefile = pycompat.mkstemp()
1032 try:
1032 try:
1033 try:
1033 try:
1034 fp = os.fdopen(fd, r'wb')
1034 fp = os.fdopen(fd, r'wb')
1035 fp.write(buf.read())
1035 fp.write(buf.read())
1036 finally:
1036 finally:
1037 fp.close()
1037 fp.close()
1038 storebundle(op, {}, bundlefile)
1038 storebundle(op, {}, bundlefile)
1039 finally:
1039 finally:
1040 try:
1040 try:
1041 os.unlink(bundlefile)
1041 os.unlink(bundlefile)
1042 except Exception:
1042 except Exception:
1043 # we would rather see the original exception
1043 # we would rather see the original exception
1044 pass
1044 pass
1045
1045
1046
1046
1047 def processparts(orig, repo, op, unbundler):
1047 def processparts(orig, repo, op, unbundler):
1048
1048
1049 # make sure we don't wrap processparts in case of `hg unbundle`
1049 # make sure we don't wrap processparts in case of `hg unbundle`
1050 if op.source == b'unbundle':
1050 if op.source == b'unbundle':
1051 return orig(repo, op, unbundler)
1051 return orig(repo, op, unbundler)
1052
1052
1053 # this server routes each push to bundle store
1053 # this server routes each push to bundle store
1054 if repo.ui.configbool(b'infinitepush', b'pushtobundlestore'):
1054 if repo.ui.configbool(b'infinitepush', b'pushtobundlestore'):
1055 return storetobundlestore(orig, repo, op, unbundler)
1055 return storetobundlestore(orig, repo, op, unbundler)
1056
1056
1057 if unbundler.params.get(b'infinitepush') != b'True':
1057 if unbundler.params.get(b'infinitepush') != b'True':
1058 return orig(repo, op, unbundler)
1058 return orig(repo, op, unbundler)
1059
1059
1060 handleallparts = repo.ui.configbool(b'infinitepush', b'storeallparts')
1060 handleallparts = repo.ui.configbool(b'infinitepush', b'storeallparts')
1061
1061
1062 bundler = bundle2.bundle20(repo.ui)
1062 bundler = bundle2.bundle20(repo.ui)
1063 cgparams = None
1063 cgparams = None
1064 with bundle2.partiterator(repo, op, unbundler) as parts:
1064 with bundle2.partiterator(repo, op, unbundler) as parts:
1065 for part in parts:
1065 for part in parts:
1066 bundlepart = None
1066 bundlepart = None
1067 if part.type == b'replycaps':
1067 if part.type == b'replycaps':
1068 # This configures the current operation to allow reply parts.
1068 # This configures the current operation to allow reply parts.
1069 bundle2._processpart(op, part)
1069 bundle2._processpart(op, part)
1070 elif part.type == bundleparts.scratchbranchparttype:
1070 elif part.type == bundleparts.scratchbranchparttype:
1071 # Scratch branch parts need to be converted to normal
1071 # Scratch branch parts need to be converted to normal
1072 # changegroup parts, and the extra parameters stored for later
1072 # changegroup parts, and the extra parameters stored for later
1073 # when we upload to the store. Eventually those parameters will
1073 # when we upload to the store. Eventually those parameters will
1074 # be put on the actual bundle instead of this part, then we can
1074 # be put on the actual bundle instead of this part, then we can
1075 # send a vanilla changegroup instead of the scratchbranch part.
1075 # send a vanilla changegroup instead of the scratchbranch part.
1076 cgversion = part.params.get(b'cgversion', b'01')
1076 cgversion = part.params.get(b'cgversion', b'01')
1077 bundlepart = bundle2.bundlepart(
1077 bundlepart = bundle2.bundlepart(
1078 b'changegroup', data=part.read()
1078 b'changegroup', data=part.read()
1079 )
1079 )
1080 bundlepart.addparam(b'version', cgversion)
1080 bundlepart.addparam(b'version', cgversion)
1081 cgparams = part.params
1081 cgparams = part.params
1082
1082
1083 # If we're not dumping all parts into the new bundle, we need to
1083 # If we're not dumping all parts into the new bundle, we need to
1084 # alert the future pushkey and phase-heads handler to skip
1084 # alert the future pushkey and phase-heads handler to skip
1085 # the part.
1085 # the part.
1086 if not handleallparts:
1086 if not handleallparts:
1087 op.records.add(
1087 op.records.add(
1088 scratchbranchparttype + b'_skippushkey', True
1088 scratchbranchparttype + b'_skippushkey', True
1089 )
1089 )
1090 op.records.add(
1090 op.records.add(
1091 scratchbranchparttype + b'_skipphaseheads', True
1091 scratchbranchparttype + b'_skipphaseheads', True
1092 )
1092 )
1093 else:
1093 else:
1094 if handleallparts:
1094 if handleallparts:
1095 # Ideally we would not process any parts, and instead just
1095 # Ideally we would not process any parts, and instead just
1096 # forward them to the bundle for storage, but since this
1096 # forward them to the bundle for storage, but since this
1097 # differs from previous behavior, we need to put it behind a
1097 # differs from previous behavior, we need to put it behind a
1098 # config flag for incremental rollout.
1098 # config flag for incremental rollout.
1099 bundlepart = bundle2.bundlepart(part.type, data=part.read())
1099 bundlepart = bundle2.bundlepart(part.type, data=part.read())
1100 for key, value in pycompat.iteritems(part.params):
1100 for key, value in pycompat.iteritems(part.params):
1101 bundlepart.addparam(key, value)
1101 bundlepart.addparam(key, value)
1102
1102
1103 # Certain parts require a response
1103 # Certain parts require a response
1104 if part.type == b'pushkey':
1104 if part.type == b'pushkey':
1105 if op.reply is not None:
1105 if op.reply is not None:
1106 rpart = op.reply.newpart(b'reply:pushkey')
1106 rpart = op.reply.newpart(b'reply:pushkey')
1107 rpart.addparam(
1107 rpart.addparam(
1108 b'in-reply-to', str(part.id), mandatory=False
1108 b'in-reply-to', str(part.id), mandatory=False
1109 )
1109 )
1110 rpart.addparam(b'return', b'1', mandatory=False)
1110 rpart.addparam(b'return', b'1', mandatory=False)
1111 else:
1111 else:
1112 bundle2._processpart(op, part)
1112 bundle2._processpart(op, part)
1113
1113
1114 if handleallparts:
1114 if handleallparts:
1115 op.records.add(part.type, {b'return': 1,})
1115 op.records.add(part.type, {b'return': 1,})
1116 if bundlepart:
1116 if bundlepart:
1117 bundler.addpart(bundlepart)
1117 bundler.addpart(bundlepart)
1118
1118
1119 # If commits were sent, store them
1119 # If commits were sent, store them
1120 if cgparams:
1120 if cgparams:
1121 buf = util.chunkbuffer(bundler.getchunks())
1121 buf = util.chunkbuffer(bundler.getchunks())
1122 fd, bundlefile = pycompat.mkstemp()
1122 fd, bundlefile = pycompat.mkstemp()
1123 try:
1123 try:
1124 try:
1124 try:
1125 fp = os.fdopen(fd, r'wb')
1125 fp = os.fdopen(fd, r'wb')
1126 fp.write(buf.read())
1126 fp.write(buf.read())
1127 finally:
1127 finally:
1128 fp.close()
1128 fp.close()
1129 storebundle(op, cgparams, bundlefile)
1129 storebundle(op, cgparams, bundlefile)
1130 finally:
1130 finally:
1131 try:
1131 try:
1132 os.unlink(bundlefile)
1132 os.unlink(bundlefile)
1133 except Exception:
1133 except Exception:
1134 # we would rather see the original exception
1134 # we would rather see the original exception
1135 pass
1135 pass
1136
1136
1137
1137
1138 def storebundle(op, params, bundlefile):
1138 def storebundle(op, params, bundlefile):
1139 log = _getorcreateinfinitepushlogger(op)
1139 log = _getorcreateinfinitepushlogger(op)
1140 parthandlerstart = time.time()
1140 parthandlerstart = time.time()
1141 log(scratchbranchparttype, eventtype=b'start')
1141 log(scratchbranchparttype, eventtype=b'start')
1142 index = op.repo.bundlestore.index
1142 index = op.repo.bundlestore.index
1143 store = op.repo.bundlestore.store
1143 store = op.repo.bundlestore.store
1144 op.records.add(scratchbranchparttype + b'_skippushkey', True)
1144 op.records.add(scratchbranchparttype + b'_skippushkey', True)
1145
1145
1146 bundle = None
1146 bundle = None
1147 try: # guards bundle
1147 try: # guards bundle
1148 bundlepath = b"bundle:%s+%s" % (op.repo.root, bundlefile)
1148 bundlepath = b"bundle:%s+%s" % (op.repo.root, bundlefile)
1149 bundle = hg.repository(op.repo.ui, bundlepath)
1149 bundle = hg.repository(op.repo.ui, bundlepath)
1150
1150
1151 bookmark = params.get(b'bookmark')
1151 bookmark = params.get(b'bookmark')
1152 bookprevnode = params.get(b'bookprevnode', b'')
1152 bookprevnode = params.get(b'bookprevnode', b'')
1153 force = params.get(b'force')
1153 force = params.get(b'force')
1154
1154
1155 if bookmark:
1155 if bookmark:
1156 oldnode = index.getnode(bookmark)
1156 oldnode = index.getnode(bookmark)
1157 else:
1157 else:
1158 oldnode = None
1158 oldnode = None
1159 bundleheads = bundle.revs(b'heads(bundle())')
1159 bundleheads = bundle.revs(b'heads(bundle())')
1160 if bookmark and len(bundleheads) > 1:
1160 if bookmark and len(bundleheads) > 1:
1161 raise error.Abort(
1161 raise error.Abort(
1162 _(b'cannot push more than one head to a scratch branch')
1162 _(b'cannot push more than one head to a scratch branch')
1163 )
1163 )
1164
1164
1165 revs = _getrevs(bundle, oldnode, force, bookmark)
1165 revs = _getrevs(bundle, oldnode, force, bookmark)
1166
1166
1167 # Notify the user of what is being pushed
1167 # Notify the user of what is being pushed
1168 plural = b's' if len(revs) > 1 else b''
1168 plural = b's' if len(revs) > 1 else b''
1169 op.repo.ui.warn(_(b"pushing %d commit%s:\n") % (len(revs), plural))
1169 op.repo.ui.warn(_(b"pushing %d commit%s:\n") % (len(revs), plural))
1170 maxoutput = 10
1170 maxoutput = 10
1171 for i in range(0, min(len(revs), maxoutput)):
1171 for i in range(0, min(len(revs), maxoutput)):
1172 firstline = bundle[revs[i]].description().split(b'\n')[0][:50]
1172 firstline = bundle[revs[i]].description().split(b'\n')[0][:50]
1173 op.repo.ui.warn(b" %s %s\n" % (revs[i], firstline))
1173 op.repo.ui.warn(b" %s %s\n" % (revs[i], firstline))
1174
1174
1175 if len(revs) > maxoutput + 1:
1175 if len(revs) > maxoutput + 1:
1176 op.repo.ui.warn(b" ...\n")
1176 op.repo.ui.warn(b" ...\n")
1177 firstline = bundle[revs[-1]].description().split(b'\n')[0][:50]
1177 firstline = bundle[revs[-1]].description().split(b'\n')[0][:50]
1178 op.repo.ui.warn(b" %s %s\n" % (revs[-1], firstline))
1178 op.repo.ui.warn(b" %s %s\n" % (revs[-1], firstline))
1179
1179
1180 nodesctx = [bundle[rev] for rev in revs]
1180 nodesctx = [bundle[rev] for rev in revs]
1181 inindex = lambda rev: bool(index.getbundle(bundle[rev].hex()))
1181 inindex = lambda rev: bool(index.getbundle(bundle[rev].hex()))
1182 if bundleheads:
1182 if bundleheads:
1183 newheadscount = sum(not inindex(rev) for rev in bundleheads)
1183 newheadscount = sum(not inindex(rev) for rev in bundleheads)
1184 else:
1184 else:
1185 newheadscount = 0
1185 newheadscount = 0
1186 # If there's a bookmark specified, there should be only one head,
1186 # If there's a bookmark specified, there should be only one head,
1187 # so we choose the last node, which will be that head.
1187 # so we choose the last node, which will be that head.
1188 # If a bug or malicious client allows there to be a bookmark
1188 # If a bug or malicious client allows there to be a bookmark
1189 # with multiple heads, we will place the bookmark on the last head.
1189 # with multiple heads, we will place the bookmark on the last head.
1190 bookmarknode = nodesctx[-1].hex() if nodesctx else None
1190 bookmarknode = nodesctx[-1].hex() if nodesctx else None
1191 key = None
1191 key = None
1192 if newheadscount:
1192 if newheadscount:
1193 with open(bundlefile, b'rb') as f:
1193 with open(bundlefile, b'rb') as f:
1194 bundledata = f.read()
1194 bundledata = f.read()
1195 with logservicecall(
1195 with logservicecall(
1196 log, b'bundlestore', bundlesize=len(bundledata)
1196 log, b'bundlestore', bundlesize=len(bundledata)
1197 ):
1197 ):
1198 bundlesizelimit = 100 * 1024 * 1024 # 100 MB
1198 bundlesizelimit = 100 * 1024 * 1024 # 100 MB
1199 if len(bundledata) > bundlesizelimit:
1199 if len(bundledata) > bundlesizelimit:
1200 error_msg = (
1200 error_msg = (
1201 b'bundle is too big: %d bytes. '
1201 b'bundle is too big: %d bytes. '
1202 + b'max allowed size is 100 MB'
1202 + b'max allowed size is 100 MB'
1203 )
1203 )
1204 raise error.Abort(error_msg % (len(bundledata),))
1204 raise error.Abort(error_msg % (len(bundledata),))
1205 key = store.write(bundledata)
1205 key = store.write(bundledata)
1206
1206
1207 with logservicecall(log, b'index', newheadscount=newheadscount), index:
1207 with logservicecall(log, b'index', newheadscount=newheadscount), index:
1208 if key:
1208 if key:
1209 index.addbundle(key, nodesctx)
1209 index.addbundle(key, nodesctx)
1210 if bookmark:
1210 if bookmark:
1211 index.addbookmark(bookmark, bookmarknode)
1211 index.addbookmark(bookmark, bookmarknode)
1212 _maybeaddpushbackpart(
1212 _maybeaddpushbackpart(
1213 op, bookmark, bookmarknode, bookprevnode, params
1213 op, bookmark, bookmarknode, bookprevnode, params
1214 )
1214 )
1215 log(
1215 log(
1216 scratchbranchparttype,
1216 scratchbranchparttype,
1217 eventtype=b'success',
1217 eventtype=b'success',
1218 elapsedms=(time.time() - parthandlerstart) * 1000,
1218 elapsedms=(time.time() - parthandlerstart) * 1000,
1219 )
1219 )
1220
1220
1221 except Exception as e:
1221 except Exception as e:
1222 log(
1222 log(
1223 scratchbranchparttype,
1223 scratchbranchparttype,
1224 eventtype=b'failure',
1224 eventtype=b'failure',
1225 elapsedms=(time.time() - parthandlerstart) * 1000,
1225 elapsedms=(time.time() - parthandlerstart) * 1000,
1226 errormsg=str(e),
1226 errormsg=stringutil.forcebytestr(e),
1227 )
1227 )
1228 raise
1228 raise
1229 finally:
1229 finally:
1230 if bundle:
1230 if bundle:
1231 bundle.close()
1231 bundle.close()
1232
1232
1233
1233
1234 @bundle2.parthandler(
1234 @bundle2.parthandler(
1235 scratchbranchparttype,
1235 scratchbranchparttype,
1236 (
1236 (
1237 b'bookmark',
1237 b'bookmark',
1238 b'bookprevnode',
1238 b'bookprevnode',
1239 b'force',
1239 b'force',
1240 b'pushbackbookmarks',
1240 b'pushbackbookmarks',
1241 b'cgversion',
1241 b'cgversion',
1242 ),
1242 ),
1243 )
1243 )
1244 def bundle2scratchbranch(op, part):
1244 def bundle2scratchbranch(op, part):
1245 '''unbundle a bundle2 part containing a changegroup to store'''
1245 '''unbundle a bundle2 part containing a changegroup to store'''
1246
1246
1247 bundler = bundle2.bundle20(op.repo.ui)
1247 bundler = bundle2.bundle20(op.repo.ui)
1248 cgversion = part.params.get(b'cgversion', b'01')
1248 cgversion = part.params.get(b'cgversion', b'01')
1249 cgpart = bundle2.bundlepart(b'changegroup', data=part.read())
1249 cgpart = bundle2.bundlepart(b'changegroup', data=part.read())
1250 cgpart.addparam(b'version', cgversion)
1250 cgpart.addparam(b'version', cgversion)
1251 bundler.addpart(cgpart)
1251 bundler.addpart(cgpart)
1252 buf = util.chunkbuffer(bundler.getchunks())
1252 buf = util.chunkbuffer(bundler.getchunks())
1253
1253
1254 fd, bundlefile = pycompat.mkstemp()
1254 fd, bundlefile = pycompat.mkstemp()
1255 try:
1255 try:
1256 try:
1256 try:
1257 fp = os.fdopen(fd, r'wb')
1257 fp = os.fdopen(fd, r'wb')
1258 fp.write(buf.read())
1258 fp.write(buf.read())
1259 finally:
1259 finally:
1260 fp.close()
1260 fp.close()
1261 storebundle(op, part.params, bundlefile)
1261 storebundle(op, part.params, bundlefile)
1262 finally:
1262 finally:
1263 try:
1263 try:
1264 os.unlink(bundlefile)
1264 os.unlink(bundlefile)
1265 except OSError as e:
1265 except OSError as e:
1266 if e.errno != errno.ENOENT:
1266 if e.errno != errno.ENOENT:
1267 raise
1267 raise
1268
1268
1269 return 1
1269 return 1
1270
1270
1271
1271
1272 def _maybeaddpushbackpart(op, bookmark, newnode, oldnode, params):
1272 def _maybeaddpushbackpart(op, bookmark, newnode, oldnode, params):
1273 if params.get(b'pushbackbookmarks'):
1273 if params.get(b'pushbackbookmarks'):
1274 if op.reply and b'pushback' in op.reply.capabilities:
1274 if op.reply and b'pushback' in op.reply.capabilities:
1275 params = {
1275 params = {
1276 b'namespace': b'bookmarks',
1276 b'namespace': b'bookmarks',
1277 b'key': bookmark,
1277 b'key': bookmark,
1278 b'new': newnode,
1278 b'new': newnode,
1279 b'old': oldnode,
1279 b'old': oldnode,
1280 }
1280 }
1281 op.reply.newpart(
1281 op.reply.newpart(
1282 b'pushkey', mandatoryparams=pycompat.iteritems(params)
1282 b'pushkey', mandatoryparams=pycompat.iteritems(params)
1283 )
1283 )
1284
1284
1285
1285
1286 def bundle2pushkey(orig, op, part):
1286 def bundle2pushkey(orig, op, part):
1287 '''Wrapper of bundle2.handlepushkey()
1287 '''Wrapper of bundle2.handlepushkey()
1288
1288
1289 The only goal is to skip calling the original function if flag is set.
1289 The only goal is to skip calling the original function if flag is set.
1290 It's set if infinitepush push is happening.
1290 It's set if infinitepush push is happening.
1291 '''
1291 '''
1292 if op.records[scratchbranchparttype + b'_skippushkey']:
1292 if op.records[scratchbranchparttype + b'_skippushkey']:
1293 if op.reply is not None:
1293 if op.reply is not None:
1294 rpart = op.reply.newpart(b'reply:pushkey')
1294 rpart = op.reply.newpart(b'reply:pushkey')
1295 rpart.addparam(b'in-reply-to', str(part.id), mandatory=False)
1295 rpart.addparam(b'in-reply-to', str(part.id), mandatory=False)
1296 rpart.addparam(b'return', b'1', mandatory=False)
1296 rpart.addparam(b'return', b'1', mandatory=False)
1297 return 1
1297 return 1
1298
1298
1299 return orig(op, part)
1299 return orig(op, part)
1300
1300
1301
1301
1302 def bundle2handlephases(orig, op, part):
1302 def bundle2handlephases(orig, op, part):
1303 '''Wrapper of bundle2.handlephases()
1303 '''Wrapper of bundle2.handlephases()
1304
1304
1305 The only goal is to skip calling the original function if flag is set.
1305 The only goal is to skip calling the original function if flag is set.
1306 It's set if infinitepush push is happening.
1306 It's set if infinitepush push is happening.
1307 '''
1307 '''
1308
1308
1309 if op.records[scratchbranchparttype + b'_skipphaseheads']:
1309 if op.records[scratchbranchparttype + b'_skipphaseheads']:
1310 return
1310 return
1311
1311
1312 return orig(op, part)
1312 return orig(op, part)
1313
1313
1314
1314
1315 def _asyncsavemetadata(root, nodes):
1315 def _asyncsavemetadata(root, nodes):
1316 '''starts a separate process that fills metadata for the nodes
1316 '''starts a separate process that fills metadata for the nodes
1317
1317
1318 This function creates a separate process and doesn't wait for it's
1318 This function creates a separate process and doesn't wait for it's
1319 completion. This was done to avoid slowing down pushes
1319 completion. This was done to avoid slowing down pushes
1320 '''
1320 '''
1321
1321
1322 maxnodes = 50
1322 maxnodes = 50
1323 if len(nodes) > maxnodes:
1323 if len(nodes) > maxnodes:
1324 return
1324 return
1325 nodesargs = []
1325 nodesargs = []
1326 for node in nodes:
1326 for node in nodes:
1327 nodesargs.append(b'--node')
1327 nodesargs.append(b'--node')
1328 nodesargs.append(node)
1328 nodesargs.append(node)
1329 with open(os.devnull, b'w+b') as devnull:
1329 with open(os.devnull, b'w+b') as devnull:
1330 cmdline = [
1330 cmdline = [
1331 util.hgexecutable(),
1331 util.hgexecutable(),
1332 b'debugfillinfinitepushmetadata',
1332 b'debugfillinfinitepushmetadata',
1333 b'-R',
1333 b'-R',
1334 root,
1334 root,
1335 ] + nodesargs
1335 ] + nodesargs
1336 # Process will run in background. We don't care about the return code
1336 # Process will run in background. We don't care about the return code
1337 subprocess.Popen(
1337 subprocess.Popen(
1338 pycompat.rapply(procutil.tonativestr, cmdline),
1338 pycompat.rapply(procutil.tonativestr, cmdline),
1339 close_fds=True,
1339 close_fds=True,
1340 shell=False,
1340 shell=False,
1341 stdin=devnull,
1341 stdin=devnull,
1342 stdout=devnull,
1342 stdout=devnull,
1343 stderr=devnull,
1343 stderr=devnull,
1344 )
1344 )
General Comments 0
You need to be logged in to leave comments. Login now