##// END OF EJS Templates
chore(deps): bumped celery to 5.3.6 and kombu to latest version
super-admin -
r1205:b35336c8 default
parent child Browse files
Show More
@@ -1,77 +1,77 b''
1 1 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
2 2
3 3 async-timeout==4.0.3
4 4 atomicwrites==1.4.1
5 celery==5.3.4
6 billiard==4.1.0
5 celery==5.3.6
6 billiard==4.2.0
7 7 click==8.1.3
8 8 click-didyoumean==0.3.0
9 9 click==8.1.3
10 10 click-plugins==1.1.1
11 11 click==8.1.3
12 12 click-repl==0.2.0
13 13 click==8.1.3
14 14 prompt-toolkit==3.0.38
15 15 wcwidth==0.2.6
16 16 six==1.16.0
17 kombu==5.3.2
18 amqp==5.1.1
17 kombu==5.3.5
18 amqp==5.2.0
19 19 vine==5.1.0
20 20 vine==5.1.0
21 21 python-dateutil==2.8.2
22 22 six==1.16.0
23 23 tzdata==2023.4
24 24 vine==5.1.0
25 25 contextlib2==21.6.0
26 26 cov-core==1.15.0
27 27 coverage==7.2.3
28 28 diskcache==5.6.3
29 29 dogpile.cache==1.3.0
30 30 decorator==5.1.1
31 31 stevedore==5.1.0
32 32 pbr==5.11.1
33 33 dulwich==0.21.6
34 34 urllib3==1.26.14
35 35 gunicorn==21.2.0
36 36 packaging==23.1
37 37 hg-evolve==11.0.2
38 38 importlib-metadata==6.0.0
39 39 zipp==3.15.0
40 40 mercurial==6.3.3
41 41 mock==5.0.2
42 42 more-itertools==9.1.0
43 43 msgpack==1.0.7
44 44 orjson==3.9.13
45 45 psutil==5.9.8
46 46 py==1.11.0
47 47 pygit2==1.13.3
48 48 cffi==1.16.0
49 49 pycparser==2.21
50 50 pygments==2.15.1
51 51 pyparsing==3.1.1
52 52 pyramid==2.0.2
53 53 hupper==1.12
54 54 plaster==1.1.2
55 55 plaster-pastedeploy==1.0.1
56 56 pastedeploy==3.1.0
57 57 plaster==1.1.2
58 58 translationstring==1.4
59 59 venusian==3.0.0
60 60 webob==1.8.7
61 61 zope.deprecation==5.0.0
62 62 zope.interface==6.1.0
63 63 redis==5.0.1
64 64 async-timeout==4.0.3
65 65 repoze.lru==0.7
66 66 scandir==1.10.0
67 67 setproctitle==1.3.3
68 68 subvertpy==0.11.0
69 69 waitress==3.0.0
70 70 wcwidth==0.2.6
71 71
72 72
73 73 ## test related requirements
74 74 #-r requirements_test.txt
75 75
76 76 ## uncomment to add the debug libraries
77 77 #-r requirements_debug.txt
@@ -1,796 +1,795 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import os
20 20 import sys
21 21 import logging
22 22 import collections
23 import importlib
24 23 import base64
25 24 import msgpack
26 25 import dataclasses
27 26 import pygit2
28 27
29 28 import http.client
30 29 from celery import Celery
31 30
32
33 31 import mercurial.scmutil
34 32 import mercurial.node
35 33
36 34 from vcsserver.lib.rc_json import json
37 35 from vcsserver import exceptions, subprocessio, settings
38 36 from vcsserver.str_utils import ascii_str, safe_str
39 37 from vcsserver.remote.git_remote import Repository
40 38
41 celery_app = Celery()
39 celery_app = Celery('__vcsserver__')
42 40 log = logging.getLogger(__name__)
43 41
44 42
45 43 class HooksHttpClient:
46 44 proto = 'msgpack.v1'
47 45 connection = None
48 46
49 47 def __init__(self, hooks_uri):
50 48 self.hooks_uri = hooks_uri
51 49
52 50 def __repr__(self):
53 51 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
54 52
55 53 def __call__(self, method, extras):
56 54 connection = http.client.HTTPConnection(self.hooks_uri)
57 55 # binary msgpack body
58 56 headers, body = self._serialize(method, extras)
59 57 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
60 58
61 59 try:
62 60 try:
63 61 connection.request('POST', '/', body, headers)
64 62 except Exception as error:
65 63 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
66 64 raise
67 65
68 66 response = connection.getresponse()
69 67 try:
70 68 return msgpack.load(response)
71 69 except Exception:
72 70 response_data = response.read()
73 71 log.exception('Failed to decode hook response json data. '
74 72 'response_code:%s, raw_data:%s',
75 73 response.status, response_data)
76 74 raise
77 75 finally:
78 76 connection.close()
79 77
80 78 @classmethod
81 79 def _serialize(cls, hook_name, extras):
82 80 data = {
83 81 'method': hook_name,
84 82 'extras': extras
85 83 }
86 84 headers = {
87 85 "rc-hooks-protocol": cls.proto,
88 86 "Connection": "keep-alive"
89 87 }
90 88 return headers, msgpack.packb(data)
91 89
92 90
93 91 class HooksCeleryClient:
94 92 TASK_TIMEOUT = 60 # time in seconds
95 93
96 94 def __init__(self, queue, backend):
97 celery_app.config_from_object({'broker_url': queue, 'result_backend': backend,
95 celery_app.config_from_object({
96 'broker_url': queue, 'result_backend': backend,
98 97 'broker_connection_retry_on_startup': True,
99 98 'task_serializer': 'msgpack',
100 99 'accept_content': ['json', 'msgpack'],
101 100 'result_serializer': 'msgpack',
102 101 'result_accept_content': ['json', 'msgpack']
103 102 })
104 103 self.celery_app = celery_app
105 104
106 105 def __call__(self, method, extras):
107 106 inquired_task = self.celery_app.signature(
108 107 f'rhodecode.lib.celerylib.tasks.{method}'
109 108 )
110 109 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
111 110
112 111
113 112 class HooksShadowRepoClient:
114 113
115 114 def __call__(self, hook_name, extras):
116 115 return {'output': '', 'status': 0}
117 116
118 117
119 118 class RemoteMessageWriter:
120 119 """Writer base class."""
121 120 def write(self, message):
122 121 raise NotImplementedError()
123 122
124 123
125 124 class HgMessageWriter(RemoteMessageWriter):
126 125 """Writer that knows how to send messages to mercurial clients."""
127 126
128 127 def __init__(self, ui):
129 128 self.ui = ui
130 129
131 130 def write(self, message: str):
132 131 # TODO: Check why the quiet flag is set by default.
133 132 old = self.ui.quiet
134 133 self.ui.quiet = False
135 134 self.ui.status(message.encode('utf-8'))
136 135 self.ui.quiet = old
137 136
138 137
139 138 class GitMessageWriter(RemoteMessageWriter):
140 139 """Writer that knows how to send messages to git clients."""
141 140
142 141 def __init__(self, stdout=None):
143 142 self.stdout = stdout or sys.stdout
144 143
145 144 def write(self, message: str):
146 145 self.stdout.write(message)
147 146
148 147
149 148 class SvnMessageWriter(RemoteMessageWriter):
150 149 """Writer that knows how to send messages to svn clients."""
151 150
152 151 def __init__(self, stderr=None):
153 152 # SVN needs data sent to stderr for back-to-client messaging
154 153 self.stderr = stderr or sys.stderr
155 154
156 155 def write(self, message):
157 156 self.stderr.write(message.encode('utf-8'))
158 157
159 158
160 159 def _handle_exception(result):
161 160 exception_class = result.get('exception')
162 161 exception_traceback = result.get('exception_traceback')
163 162 log.debug('Handling hook-call exception: %s', exception_class)
164 163
165 164 if exception_traceback:
166 165 log.error('Got traceback from remote call:%s', exception_traceback)
167 166
168 167 if exception_class == 'HTTPLockedRC':
169 168 raise exceptions.RepositoryLockedException()(*result['exception_args'])
170 169 elif exception_class == 'HTTPBranchProtected':
171 170 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
172 171 elif exception_class == 'RepositoryError':
173 172 raise exceptions.VcsException()(*result['exception_args'])
174 173 elif exception_class:
175 174 raise Exception(
176 175 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
177 176 )
178 177
179 178
180 179 def _get_hooks_client(extras):
181 180 hooks_uri = extras.get('hooks_uri')
182 181 task_queue = extras.get('task_queue')
183 182 task_backend = extras.get('task_backend')
184 183 is_shadow_repo = extras.get('is_shadow_repo')
185 184
186 185 if hooks_uri:
187 186 return HooksHttpClient(hooks_uri)
188 187 elif task_queue and task_backend:
189 188 return HooksCeleryClient(task_queue, task_backend)
190 189 elif is_shadow_repo:
191 190 return HooksShadowRepoClient()
192 191 else:
193 192 raise Exception("Hooks client not found!")
194 193
195 194
196 195 def _call_hook(hook_name, extras, writer):
197 196 hooks_client = _get_hooks_client(extras)
198 197 log.debug('Hooks, using client:%s', hooks_client)
199 198 result = hooks_client(hook_name, extras)
200 199 log.debug('Hooks got result: %s', result)
201 200 _handle_exception(result)
202 201 writer.write(result['output'])
203 202
204 203 return result['status']
205 204
206 205
207 206 def _extras_from_ui(ui):
208 207 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
209 208 if not hook_data:
210 209 # maybe it's inside environ ?
211 210 env_hook_data = os.environ.get('RC_SCM_DATA')
212 211 if env_hook_data:
213 212 hook_data = env_hook_data
214 213
215 214 extras = {}
216 215 if hook_data:
217 216 extras = json.loads(hook_data)
218 217 return extras
219 218
220 219
221 220 def _rev_range_hash(repo, node, check_heads=False):
222 221 from vcsserver.hgcompat import get_ctx
223 222
224 223 commits = []
225 224 revs = []
226 225 start = get_ctx(repo, node).rev()
227 226 end = len(repo)
228 227 for rev in range(start, end):
229 228 revs.append(rev)
230 229 ctx = get_ctx(repo, rev)
231 230 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
232 231 branch = safe_str(ctx.branch())
233 232 commits.append((commit_id, branch))
234 233
235 234 parent_heads = []
236 235 if check_heads:
237 236 parent_heads = _check_heads(repo, start, end, revs)
238 237 return commits, parent_heads
239 238
240 239
241 240 def _check_heads(repo, start, end, commits):
242 241 from vcsserver.hgcompat import get_ctx
243 242 changelog = repo.changelog
244 243 parents = set()
245 244
246 245 for new_rev in commits:
247 246 for p in changelog.parentrevs(new_rev):
248 247 if p == mercurial.node.nullrev:
249 248 continue
250 249 if p < start:
251 250 parents.add(p)
252 251
253 252 for p in parents:
254 253 branch = get_ctx(repo, p).branch()
255 254 # The heads descending from that parent, on the same branch
256 255 parent_heads = {p}
257 256 reachable = {p}
258 257 for x in range(p + 1, end):
259 258 if get_ctx(repo, x).branch() != branch:
260 259 continue
261 260 for pp in changelog.parentrevs(x):
262 261 if pp in reachable:
263 262 reachable.add(x)
264 263 parent_heads.discard(pp)
265 264 parent_heads.add(x)
266 265 # More than one head? Suggest merging
267 266 if len(parent_heads) > 1:
268 267 return list(parent_heads)
269 268
270 269 return []
271 270
272 271
273 272 def _get_git_env():
274 273 env = {}
275 274 for k, v in os.environ.items():
276 275 if k.startswith('GIT'):
277 276 env[k] = v
278 277
279 278 # serialized version
280 279 return [(k, v) for k, v in env.items()]
281 280
282 281
283 282 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
284 283 env = {}
285 284 for k, v in os.environ.items():
286 285 if k.startswith('HG'):
287 286 env[k] = v
288 287
289 288 env['HG_NODE'] = old_rev
290 289 env['HG_NODE_LAST'] = new_rev
291 290 env['HG_TXNID'] = txnid
292 291 env['HG_PENDING'] = repo_path
293 292
294 293 return [(k, v) for k, v in env.items()]
295 294
296 295
297 296 def repo_size(ui, repo, **kwargs):
298 297 extras = _extras_from_ui(ui)
299 298 return _call_hook('repo_size', extras, HgMessageWriter(ui))
300 299
301 300
302 301 def pre_pull(ui, repo, **kwargs):
303 302 extras = _extras_from_ui(ui)
304 303 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
305 304
306 305
307 306 def pre_pull_ssh(ui, repo, **kwargs):
308 307 extras = _extras_from_ui(ui)
309 308 if extras and extras.get('SSH'):
310 309 return pre_pull(ui, repo, **kwargs)
311 310 return 0
312 311
313 312
314 313 def post_pull(ui, repo, **kwargs):
315 314 extras = _extras_from_ui(ui)
316 315 return _call_hook('post_pull', extras, HgMessageWriter(ui))
317 316
318 317
319 318 def post_pull_ssh(ui, repo, **kwargs):
320 319 extras = _extras_from_ui(ui)
321 320 if extras and extras.get('SSH'):
322 321 return post_pull(ui, repo, **kwargs)
323 322 return 0
324 323
325 324
326 325 def pre_push(ui, repo, node=None, **kwargs):
327 326 """
328 327 Mercurial pre_push hook
329 328 """
330 329 extras = _extras_from_ui(ui)
331 330 detect_force_push = extras.get('detect_force_push')
332 331
333 332 rev_data = []
334 333 hook_type: str = safe_str(kwargs.get('hooktype'))
335 334
336 335 if node and hook_type == 'pretxnchangegroup':
337 336 branches = collections.defaultdict(list)
338 337 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
339 338 for commit_id, branch in commits:
340 339 branches[branch].append(commit_id)
341 340
342 341 for branch, commits in branches.items():
343 342 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
344 343 rev_data.append({
345 344 'total_commits': len(commits),
346 345 'old_rev': old_rev,
347 346 'new_rev': commits[-1],
348 347 'ref': '',
349 348 'type': 'branch',
350 349 'name': branch,
351 350 })
352 351
353 352 for push_ref in rev_data:
354 353 push_ref['multiple_heads'] = _heads
355 354
356 355 repo_path = os.path.join(
357 356 extras.get('repo_store', ''), extras.get('repository', ''))
358 357 push_ref['hg_env'] = _get_hg_env(
359 358 old_rev=push_ref['old_rev'],
360 359 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
361 360 repo_path=repo_path)
362 361
363 362 extras['hook_type'] = hook_type or 'pre_push'
364 363 extras['commit_ids'] = rev_data
365 364
366 365 return _call_hook('pre_push', extras, HgMessageWriter(ui))
367 366
368 367
369 368 def pre_push_ssh(ui, repo, node=None, **kwargs):
370 369 extras = _extras_from_ui(ui)
371 370 if extras.get('SSH'):
372 371 return pre_push(ui, repo, node, **kwargs)
373 372
374 373 return 0
375 374
376 375
377 376 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
378 377 """
379 378 Mercurial pre_push hook for SSH
380 379 """
381 380 extras = _extras_from_ui(ui)
382 381 if extras.get('SSH'):
383 382 permission = extras['SSH_PERMISSIONS']
384 383
385 384 if 'repository.write' == permission or 'repository.admin' == permission:
386 385 return 0
387 386
388 387 # non-zero ret code
389 388 return 1
390 389
391 390 return 0
392 391
393 392
394 393 def post_push(ui, repo, node, **kwargs):
395 394 """
396 395 Mercurial post_push hook
397 396 """
398 397 extras = _extras_from_ui(ui)
399 398
400 399 commit_ids = []
401 400 branches = []
402 401 bookmarks = []
403 402 tags = []
404 403 hook_type: str = safe_str(kwargs.get('hooktype'))
405 404
406 405 commits, _heads = _rev_range_hash(repo, node)
407 406 for commit_id, branch in commits:
408 407 commit_ids.append(commit_id)
409 408 if branch not in branches:
410 409 branches.append(branch)
411 410
412 411 if hasattr(ui, '_rc_pushkey_bookmarks'):
413 412 bookmarks = ui._rc_pushkey_bookmarks
414 413
415 414 extras['hook_type'] = hook_type or 'post_push'
416 415 extras['commit_ids'] = commit_ids
417 416
418 417 extras['new_refs'] = {
419 418 'branches': branches,
420 419 'bookmarks': bookmarks,
421 420 'tags': tags
422 421 }
423 422
424 423 return _call_hook('post_push', extras, HgMessageWriter(ui))
425 424
426 425
427 426 def post_push_ssh(ui, repo, node, **kwargs):
428 427 """
429 428 Mercurial post_push hook for SSH
430 429 """
431 430 if _extras_from_ui(ui).get('SSH'):
432 431 return post_push(ui, repo, node, **kwargs)
433 432 return 0
434 433
435 434
436 435 def key_push(ui, repo, **kwargs):
437 436 from vcsserver.hgcompat import get_ctx
438 437
439 438 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
440 439 # store new bookmarks in our UI object propagated later to post_push
441 440 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
442 441 return
443 442
444 443
445 444 # backward compat
446 445 log_pull_action = post_pull
447 446
448 447 # backward compat
449 448 log_push_action = post_push
450 449
451 450
452 451 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
453 452 """
454 453 Old hook name: keep here for backward compatibility.
455 454
456 455 This is only required when the installed git hooks are not upgraded.
457 456 """
458 457 pass
459 458
460 459
461 460 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
462 461 """
463 462 Old hook name: keep here for backward compatibility.
464 463
465 464 This is only required when the installed git hooks are not upgraded.
466 465 """
467 466 pass
468 467
469 468
470 469 @dataclasses.dataclass
471 470 class HookResponse:
472 471 status: int
473 472 output: str
474 473
475 474
476 475 def git_pre_pull(extras) -> HookResponse:
477 476 """
478 477 Pre pull hook.
479 478
480 479 :param extras: dictionary containing the keys defined in simplevcs
481 480 :type extras: dict
482 481
483 482 :return: status code of the hook. 0 for success.
484 483 :rtype: int
485 484 """
486 485
487 486 if 'pull' not in extras['hooks']:
488 487 return HookResponse(0, '')
489 488
490 489 stdout = io.StringIO()
491 490 try:
492 491 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
493 492
494 493 except Exception as error:
495 494 log.exception('Failed to call pre_pull hook')
496 495 status_code = 128
497 496 stdout.write(f'ERROR: {error}\n')
498 497
499 498 return HookResponse(status_code, stdout.getvalue())
500 499
501 500
502 501 def git_post_pull(extras) -> HookResponse:
503 502 """
504 503 Post pull hook.
505 504
506 505 :param extras: dictionary containing the keys defined in simplevcs
507 506 :type extras: dict
508 507
509 508 :return: status code of the hook. 0 for success.
510 509 :rtype: int
511 510 """
512 511 if 'pull' not in extras['hooks']:
513 512 return HookResponse(0, '')
514 513
515 514 stdout = io.StringIO()
516 515 try:
517 516 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
518 517 except Exception as error:
519 518 status = 128
520 519 stdout.write(f'ERROR: {error}\n')
521 520
522 521 return HookResponse(status, stdout.getvalue())
523 522
524 523
525 524 def _parse_git_ref_lines(revision_lines):
526 525 rev_data = []
527 526 for revision_line in revision_lines or []:
528 527 old_rev, new_rev, ref = revision_line.strip().split(' ')
529 528 ref_data = ref.split('/', 2)
530 529 if ref_data[1] in ('tags', 'heads'):
531 530 rev_data.append({
532 531 # NOTE(marcink):
533 532 # we're unable to tell total_commits for git at this point
534 533 # but we set the variable for consistency with GIT
535 534 'total_commits': -1,
536 535 'old_rev': old_rev,
537 536 'new_rev': new_rev,
538 537 'ref': ref,
539 538 'type': ref_data[1],
540 539 'name': ref_data[2],
541 540 })
542 541 return rev_data
543 542
544 543
545 544 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
546 545 """
547 546 Pre push hook.
548 547
549 548 :return: status code of the hook. 0 for success.
550 549 """
551 550 extras = json.loads(env['RC_SCM_DATA'])
552 551 rev_data = _parse_git_ref_lines(revision_lines)
553 552 if 'push' not in extras['hooks']:
554 553 return 0
555 554 empty_commit_id = '0' * 40
556 555
557 556 detect_force_push = extras.get('detect_force_push')
558 557
559 558 for push_ref in rev_data:
560 559 # store our git-env which holds the temp store
561 560 push_ref['git_env'] = _get_git_env()
562 561 push_ref['pruned_sha'] = ''
563 562 if not detect_force_push:
564 563 # don't check for forced-push when we don't need to
565 564 continue
566 565
567 566 type_ = push_ref['type']
568 567 new_branch = push_ref['old_rev'] == empty_commit_id
569 568 delete_branch = push_ref['new_rev'] == empty_commit_id
570 569 if type_ == 'heads' and not (new_branch or delete_branch):
571 570 old_rev = push_ref['old_rev']
572 571 new_rev = push_ref['new_rev']
573 572 cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, f'^{new_rev}']
574 573 stdout, stderr = subprocessio.run_command(
575 574 cmd, env=os.environ.copy())
576 575 # means we're having some non-reachable objects, this forced push was used
577 576 if stdout:
578 577 push_ref['pruned_sha'] = stdout.splitlines()
579 578
580 579 extras['hook_type'] = 'pre_receive'
581 580 extras['commit_ids'] = rev_data
582 581
583 582 stdout = sys.stdout
584 583 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
585 584
586 585 return status_code
587 586
588 587
589 588 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
590 589 """
591 590 Post push hook.
592 591
593 592 :return: status code of the hook. 0 for success.
594 593 """
595 594 extras = json.loads(env['RC_SCM_DATA'])
596 595 if 'push' not in extras['hooks']:
597 596 return 0
598 597
599 598 rev_data = _parse_git_ref_lines(revision_lines)
600 599
601 600 git_revs = []
602 601
603 602 # N.B.(skreft): it is ok to just call git, as git before calling a
604 603 # subcommand sets the PATH environment variable so that it point to the
605 604 # correct version of the git executable.
606 605 empty_commit_id = '0' * 40
607 606 branches = []
608 607 tags = []
609 608 for push_ref in rev_data:
610 609 type_ = push_ref['type']
611 610
612 611 if type_ == 'heads':
613 612 # starting new branch case
614 613 if push_ref['old_rev'] == empty_commit_id:
615 614 push_ref_name = push_ref['name']
616 615
617 616 if push_ref_name not in branches:
618 617 branches.append(push_ref_name)
619 618
620 619 need_head_set = ''
621 620 with Repository(os.getcwd()) as repo:
622 621 try:
623 622 repo.head
624 623 except pygit2.GitError:
625 624 need_head_set = f'refs/heads/{push_ref_name}'
626 625
627 626 if need_head_set:
628 627 repo.set_head(need_head_set)
629 628 print(f"Setting default branch to {push_ref_name}")
630 629
631 630 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
632 631 stdout, stderr = subprocessio.run_command(
633 632 cmd, env=os.environ.copy())
634 633 heads = safe_str(stdout)
635 634 heads = heads.replace(push_ref['ref'], '')
636 635 heads = ' '.join(head for head
637 636 in heads.splitlines() if head) or '.'
638 637 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
639 638 '--pretty=format:%H', '--', push_ref['new_rev'],
640 639 '--not', heads]
641 640 stdout, stderr = subprocessio.run_command(
642 641 cmd, env=os.environ.copy())
643 642 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
644 643
645 644 # delete branch case
646 645 elif push_ref['new_rev'] == empty_commit_id:
647 646 git_revs.append(f'delete_branch=>{push_ref["name"]}')
648 647 else:
649 648 if push_ref['name'] not in branches:
650 649 branches.append(push_ref['name'])
651 650
652 651 cmd = [settings.GIT_EXECUTABLE, 'log',
653 652 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
654 653 '--reverse', '--pretty=format:%H']
655 654 stdout, stderr = subprocessio.run_command(
656 655 cmd, env=os.environ.copy())
657 656 # we get bytes from stdout, we need str to be consistent
658 657 log_revs = list(map(ascii_str, stdout.splitlines()))
659 658 git_revs.extend(log_revs)
660 659
661 660 # Pure pygit2 impl. but still 2-3x slower :/
662 661 # results = []
663 662 #
664 663 # with Repository(os.getcwd()) as repo:
665 664 # repo_new_rev = repo[push_ref['new_rev']]
666 665 # repo_old_rev = repo[push_ref['old_rev']]
667 666 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
668 667 #
669 668 # for commit in walker:
670 669 # if commit.id == repo_old_rev.id:
671 670 # break
672 671 # results.append(commit.id.hex)
673 672 # # reverse the order, can't use GIT_SORT_REVERSE
674 673 # log_revs = results[::-1]
675 674
676 675 elif type_ == 'tags':
677 676 if push_ref['name'] not in tags:
678 677 tags.append(push_ref['name'])
679 678 git_revs.append(f'tag=>{push_ref["name"]}')
680 679
681 680 extras['hook_type'] = 'post_receive'
682 681 extras['commit_ids'] = git_revs
683 682 extras['new_refs'] = {
684 683 'branches': branches,
685 684 'bookmarks': [],
686 685 'tags': tags,
687 686 }
688 687
689 688 stdout = sys.stdout
690 689
691 690 if 'repo_size' in extras['hooks']:
692 691 try:
693 692 _call_hook('repo_size', extras, GitMessageWriter(stdout))
694 693 except Exception:
695 694 pass
696 695
697 696 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
698 697 return status_code
699 698
700 699
701 700 def _get_extras_from_txn_id(path, txn_id):
702 701 extras = {}
703 702 try:
704 703 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
705 704 '-t', txn_id,
706 705 '--revprop', path, 'rc-scm-extras']
707 706 stdout, stderr = subprocessio.run_command(
708 707 cmd, env=os.environ.copy())
709 708 extras = json.loads(base64.urlsafe_b64decode(stdout))
710 709 except Exception:
711 710 log.exception('Failed to extract extras info from txn_id')
712 711
713 712 return extras
714 713
715 714
716 715 def _get_extras_from_commit_id(commit_id, path):
717 716 extras = {}
718 717 try:
719 718 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
720 719 '-r', commit_id,
721 720 '--revprop', path, 'rc-scm-extras']
722 721 stdout, stderr = subprocessio.run_command(
723 722 cmd, env=os.environ.copy())
724 723 extras = json.loads(base64.urlsafe_b64decode(stdout))
725 724 except Exception:
726 725 log.exception('Failed to extract extras info from commit_id')
727 726
728 727 return extras
729 728
730 729
731 730 def svn_pre_commit(repo_path, commit_data, env):
732 731 path, txn_id = commit_data
733 732 branches = []
734 733 tags = []
735 734
736 735 if env.get('RC_SCM_DATA'):
737 736 extras = json.loads(env['RC_SCM_DATA'])
738 737 else:
739 738 # fallback method to read from TXN-ID stored data
740 739 extras = _get_extras_from_txn_id(path, txn_id)
741 740 if not extras:
742 741 return 0
743 742
744 743 extras['hook_type'] = 'pre_commit'
745 744 extras['commit_ids'] = [txn_id]
746 745 extras['txn_id'] = txn_id
747 746 extras['new_refs'] = {
748 747 'total_commits': 1,
749 748 'branches': branches,
750 749 'bookmarks': [],
751 750 'tags': tags,
752 751 }
753 752
754 753 return _call_hook('pre_push', extras, SvnMessageWriter())
755 754
756 755
757 756 def svn_post_commit(repo_path, commit_data, env):
758 757 """
759 758 commit_data is path, rev, txn_id
760 759 """
761 760 if len(commit_data) == 3:
762 761 path, commit_id, txn_id = commit_data
763 762 elif len(commit_data) == 2:
764 763 log.error('Failed to extract txn_id from commit_data using legacy method. '
765 764 'Some functionality might be limited')
766 765 path, commit_id = commit_data
767 766 txn_id = None
768 767
769 768 branches = []
770 769 tags = []
771 770
772 771 if env.get('RC_SCM_DATA'):
773 772 extras = json.loads(env['RC_SCM_DATA'])
774 773 else:
775 774 # fallback method to read from TXN-ID stored data
776 775 extras = _get_extras_from_commit_id(commit_id, path)
777 776 if not extras:
778 777 return 0
779 778
780 779 extras['hook_type'] = 'post_commit'
781 780 extras['commit_ids'] = [commit_id]
782 781 extras['txn_id'] = txn_id
783 782 extras['new_refs'] = {
784 783 'branches': branches,
785 784 'bookmarks': [],
786 785 'tags': tags,
787 786 'total_commits': 1,
788 787 }
789 788
790 789 if 'repo_size' in extras['hooks']:
791 790 try:
792 791 _call_hook('repo_size', extras, SvnMessageWriter())
793 792 except Exception:
794 793 pass
795 794
796 795 return _call_hook('post_push', extras, SvnMessageWriter())
General Comments 0
You need to be logged in to leave comments. Login now