##// END OF EJS Templates
fix(svn): svn events fixes and change the way how we handle the events
super-admin -
r5459:7f730862 default
parent child Browse files
Show More
@@ -0,0 +1,132 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import logging
20 import redis
21
22 from ..lib import rc_cache
23 from ..lib.ext_json import json
24
25
26 log = logging.getLogger(__name__)
27
28 redis_client = None
29
30
31 class RedisTxnClient:
32
33 def __init__(self, url):
34 self.url = url
35 self._create_client(url)
36
37 def _create_client(self, url):
38 connection_pool = redis.ConnectionPool.from_url(url)
39 self.writer_client = redis.StrictRedis(
40 connection_pool=connection_pool
41 )
42 self.reader_client = self.writer_client
43
44 def set(self, key, value, expire=24 * 60000):
45 self.writer_client.set(key, value, ex=expire)
46
47 def get(self, key):
48 return self.reader_client.get(key)
49
50 def delete(self, key):
51 self.writer_client.delete(key)
52
53
54 def get_redis_client(url=''):
55
56 global redis_client
57 if redis_client is not None:
58 return redis_client
59 if not url:
60 from rhodecode import CONFIG
61 url = CONFIG['vcs.svn.redis_conn']
62 redis_client = RedisTxnClient(url)
63 return redis_client
64
65
66 def extract_svn_txn_id(data: bytes):
67 """
68 Helper method for extraction of svn txn_id from submitted XML data during
69 POST operations
70 """
71 import re
72 from lxml import etree
73
74 try:
75 root = etree.fromstring(data)
76 pat = re.compile(r'/txn/(?P<txn_id>.*)')
77 for el in root:
78 if el.tag == '{DAV:}source':
79 for sub_el in el:
80 if sub_el.tag == '{DAV:}href':
81 match = pat.search(sub_el.text)
82 if match:
83 svn_tx_id = match.groupdict()['txn_id']
84 return svn_tx_id
85 except Exception:
86 log.exception('Failed to extract txn_id')
87
88
89 def get_txn_id_data_key(repo_path, svn_txn_id):
90 log.debug('svn-txn-id: %s, obtaining data path', svn_txn_id)
91 repo_key = rc_cache.utils.compute_key_from_params(repo_path)
92 final_key = f'{repo_key}.{svn_txn_id}.svn_txn_id'
93 log.debug('computed final key: %s', final_key)
94
95 return final_key
96
97
98 def store_txn_id_data(repo_path, svn_txn_id, data_dict):
99 log.debug('svn-txn-id: %s, storing data', svn_txn_id)
100
101 if not svn_txn_id:
102 log.warning('Cannot store txn_id because it is empty')
103 return
104
105 redis_conn = get_redis_client()
106
107 store_key = get_txn_id_data_key(repo_path, svn_txn_id)
108 store_data = json.dumps(data_dict)
109 redis_conn.set(store_key, store_data)
110
111
112 def get_txn_id_from_store(repo_path, svn_txn_id, rm_on_read=False):
113 """
114 Reads txn_id from store and if present returns the data for callback manager
115 """
116 log.debug('svn-txn-id: %s, retrieving data', svn_txn_id)
117 redis_conn = get_redis_client()
118
119 store_key = get_txn_id_data_key(repo_path, svn_txn_id)
120 data = {}
121 redis_conn.get(store_key)
122 try:
123 raw_data = redis_conn.get(store_key)
124 data = json.loads(raw_data)
125 except Exception:
126 log.exception('Failed to get txn_id metadata')
127
128 if rm_on_read:
129 log.debug('Cleaning up txn_id at %s', store_key)
130 redis_conn.delete(store_key)
131
132 return data
@@ -0,0 +1,226 b''
1
2 # Copyright (C) 2010-2023 RhodeCode GmbH
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License, version 3
6 # (only), as published by the Free Software Foundation.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 #
16 # This program is dual-licensed. If you wish to learn more about the
17 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
19
20 """
21 Test suite for making push/pull operations, on specially modified INI files
22
23 .. important::
24
25 You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
26 to redirect things to stderr instead of stdout.
27 """
28
29
30 import time
31
32 import pytest
33
34 from rhodecode.lib import rc_cache
35 from rhodecode.model.db import Repository, UserIpMap, CacheKey
36 from rhodecode.model.meta import Session
37 from rhodecode.model.repo import RepoModel
38 from rhodecode.model.user import UserModel
39 from rhodecode.tests import (GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN)
40
41 from rhodecode.tests.vcs_operations import (
42 Command, _check_proper_clone, _add_files_and_push, HG_REPO_WITH_GROUP)
43
44
45 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
46 class TestVCSOperations(object):
47
48 def test_clone_hg_repo_by_admin(self, rc_web_server, tmpdir):
49 clone_url = rc_web_server.repo_clone_url(HG_REPO)
50 stdout, stderr = Command('/tmp').execute(
51 'hg clone', clone_url, tmpdir.strpath)
52 _check_proper_clone(stdout, stderr, 'hg')
53
54 def test_clone_hg_repo_by_admin_pull_protocol(self, rc_web_server, tmpdir):
55 clone_url = rc_web_server.repo_clone_url(HG_REPO)
56 stdout, stderr = Command('/tmp').execute(
57 'hg clone --pull', clone_url, tmpdir.strpath)
58 _check_proper_clone(stdout, stderr, 'hg')
59
60 def test_clone_hg_repo_by_admin_pull_stream_protocol(self, rc_web_server, tmpdir):
61 clone_url = rc_web_server.repo_clone_url(HG_REPO)
62 stdout, stderr = Command('/tmp').execute(
63 'hg clone --pull --stream', clone_url, tmpdir.strpath)
64 assert 'files to transfer,' in stdout
65 assert 'transferred 1.' in stdout
66 assert '114 files updated,' in stdout
67
68 def test_clone_hg_repo_by_id_by_admin(self, rc_web_server, tmpdir):
69 repo_id = Repository.get_by_repo_name(HG_REPO).repo_id
70 clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
71 stdout, stderr = Command('/tmp').execute(
72 'hg clone', clone_url, tmpdir.strpath)
73 _check_proper_clone(stdout, stderr, 'hg')
74
75 def test_clone_hg_repo_with_group_by_admin(self, rc_web_server, tmpdir):
76 clone_url = rc_web_server.repo_clone_url(HG_REPO_WITH_GROUP)
77 stdout, stderr = Command('/tmp').execute(
78 'hg clone', clone_url, tmpdir.strpath)
79 _check_proper_clone(stdout, stderr, 'hg')
80
81 def test_clone_wrong_credentials_hg(self, rc_web_server, tmpdir):
82 clone_url = rc_web_server.repo_clone_url(HG_REPO, passwd='bad!')
83 stdout, stderr = Command('/tmp').execute(
84 'hg clone', clone_url, tmpdir.strpath)
85 assert 'abort: authorization failed' in stderr
86
87 def test_clone_git_dir_as_hg(self, rc_web_server, tmpdir):
88 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
89 stdout, stderr = Command('/tmp').execute(
90 'hg clone', clone_url, tmpdir.strpath)
91 assert 'HTTP Error 404: Not Found' in stderr
92
93 def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
94 clone_url = rc_web_server.repo_clone_url('trololo')
95 stdout, stderr = Command('/tmp').execute(
96 'hg clone', clone_url, tmpdir.strpath)
97 assert 'HTTP Error 404: Not Found' in stderr
98
99 def test_clone_hg_with_slashes(self, rc_web_server, tmpdir):
100 clone_url = rc_web_server.repo_clone_url('//' + HG_REPO)
101 stdout, stderr = Command('/tmp').execute('hg clone', clone_url, tmpdir.strpath)
102 assert 'HTTP Error 404: Not Found' in stderr
103
104 def test_clone_existing_path_hg_not_in_database(
105 self, rc_web_server, tmpdir, fs_repo_only):
106
107 db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
108 clone_url = rc_web_server.repo_clone_url(db_name)
109 stdout, stderr = Command('/tmp').execute(
110 'hg clone', clone_url, tmpdir.strpath)
111 assert 'HTTP Error 404: Not Found' in stderr
112
113 def test_clone_existing_path_hg_not_in_database_different_scm(
114 self, rc_web_server, tmpdir, fs_repo_only):
115 db_name = fs_repo_only('not-in-db-git', repo_type='git')
116 clone_url = rc_web_server.repo_clone_url(db_name)
117 stdout, stderr = Command('/tmp').execute(
118 'hg clone', clone_url, tmpdir.strpath)
119 assert 'HTTP Error 404: Not Found' in stderr
120
121 def test_clone_non_existing_store_path_hg(self, rc_web_server, tmpdir, user_util):
122 repo = user_util.create_repo()
123 clone_url = rc_web_server.repo_clone_url(repo.repo_name)
124
125 # Damage repo by removing it's folder
126 RepoModel()._delete_filesystem_repo(repo)
127
128 stdout, stderr = Command('/tmp').execute(
129 'hg clone', clone_url, tmpdir.strpath)
130 assert 'HTTP Error 404: Not Found' in stderr
131
132 def test_push_new_file_hg(self, rc_web_server, tmpdir):
133 clone_url = rc_web_server.repo_clone_url(HG_REPO)
134 stdout, stderr = Command('/tmp').execute(
135 'hg clone', clone_url, tmpdir.strpath)
136
137 stdout, stderr = _add_files_and_push(
138 'hg', tmpdir.strpath, clone_url=clone_url)
139
140 assert 'pushing to' in stdout
141 assert 'size summary' in stdout
142
143 def test_push_invalidates_cache(self, rc_web_server, tmpdir):
144 hg_repo = Repository.get_by_repo_name(HG_REPO)
145
146 # init cache objects
147 CacheKey.delete_all_cache()
148
149 repo_namespace_key = CacheKey.REPO_INVALIDATION_NAMESPACE.format(repo_id=hg_repo.repo_id)
150
151 inv_context_manager = rc_cache.InvalidationContext(key=repo_namespace_key)
152
153 with inv_context_manager as invalidation_context:
154 # __enter__ will create and register cache objects
155 pass
156
157 cache_keys = hg_repo.cache_keys
158 assert cache_keys != []
159 old_ids = [x.cache_state_uid for x in cache_keys]
160
161 # clone to init cache
162 clone_url = rc_web_server.repo_clone_url(hg_repo.repo_name)
163 stdout, stderr = Command('/tmp').execute(
164 'hg clone', clone_url, tmpdir.strpath)
165
166 cache_keys = hg_repo.cache_keys
167 assert cache_keys != []
168 for key in cache_keys:
169 assert key.cache_active is True
170
171 # PUSH that should trigger invalidation cache
172 stdout, stderr = _add_files_and_push(
173 'hg', tmpdir.strpath, clone_url=clone_url, files_no=1)
174
175 # flush...
176 Session().commit()
177 hg_repo = Repository.get_by_repo_name(HG_REPO)
178 cache_keys = hg_repo.cache_keys
179 assert cache_keys != []
180 new_ids = [x.cache_state_uid for x in cache_keys]
181 assert new_ids != old_ids
182
183 def test_push_wrong_credentials_hg(self, rc_web_server, tmpdir):
184 clone_url = rc_web_server.repo_clone_url(HG_REPO)
185 stdout, stderr = Command('/tmp').execute(
186 'hg clone', clone_url, tmpdir.strpath)
187
188 push_url = rc_web_server.repo_clone_url(
189 HG_REPO, user='bad', passwd='name')
190 stdout, stderr = _add_files_and_push(
191 'hg', tmpdir.strpath, clone_url=push_url)
192
193 assert 'abort: authorization failed' in stderr
194
195 def test_push_back_to_wrong_url_hg(self, rc_web_server, tmpdir):
196 clone_url = rc_web_server.repo_clone_url(HG_REPO)
197 stdout, stderr = Command('/tmp').execute(
198 'hg clone', clone_url, tmpdir.strpath)
199
200 stdout, stderr = _add_files_and_push(
201 'hg', tmpdir.strpath,
202 clone_url=rc_web_server.repo_clone_url('not-existing'))
203
204 assert 'HTTP Error 404: Not Found' in stderr
205
206 def test_ip_restriction_hg(self, rc_web_server, tmpdir):
207 user_model = UserModel()
208 try:
209 user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
210 Session().commit()
211 time.sleep(2)
212 clone_url = rc_web_server.repo_clone_url(HG_REPO)
213 stdout, stderr = Command('/tmp').execute(
214 'hg clone', clone_url, tmpdir.strpath)
215 assert 'abort: HTTP Error 403: Forbidden' in stderr
216 finally:
217 # release IP restrictions
218 for ip in UserIpMap.getAll():
219 UserIpMap.delete(ip.ip_id)
220 Session().commit()
221
222 time.sleep(2)
223
224 stdout, stderr = Command('/tmp').execute(
225 'hg clone', clone_url, tmpdir.strpath)
226 _check_proper_clone(stdout, stderr, 'hg')
@@ -0,0 +1,197 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 """
20 Test suite for making push/pull operations, on specially modified INI files
21
22 .. important::
23
24 You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
25 to redirect things to stderr instead of stdout.
26 """
27
28
29 import time
30 import pytest
31
32 from rhodecode.model.db import Repository, UserIpMap
33 from rhodecode.model.meta import Session
34 from rhodecode.model.repo import RepoModel
35 from rhodecode.model.user import UserModel
36 from rhodecode.tests import (SVN_REPO, TEST_USER_ADMIN_LOGIN)
37
38
39 from rhodecode.tests.vcs_operations import (
40 Command, _check_proper_clone, _check_proper_svn_push,
41 _add_files_and_push, SVN_REPO_WITH_GROUP)
42
43
44 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
45 class TestVCSOperations(object):
46
47 def test_clone_svn_repo_by_admin(self, rc_web_server, tmpdir):
48 clone_url = rc_web_server.repo_clone_url(SVN_REPO)
49 username, password = rc_web_server.repo_clone_credentials()
50
51 cmd = Command('/tmp')
52
53 auth = f'--non-interactive --username={username} --password={password}'
54 stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
55 _check_proper_clone(stdout, stderr, 'svn')
56 cmd.assert_returncode_success()
57
58 def test_clone_svn_repo_by_id_by_admin(self, rc_web_server, tmpdir):
59 repo_id = Repository.get_by_repo_name(SVN_REPO).repo_id
60 username, password = rc_web_server.repo_clone_credentials()
61
62 clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
63 cmd = Command('/tmp')
64 auth = f'--non-interactive --username={username} --password={password}'
65 stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
66 _check_proper_clone(stdout, stderr, 'svn')
67 cmd.assert_returncode_success()
68
69 def test_clone_svn_repo_with_group_by_admin(self, rc_web_server, tmpdir):
70 clone_url = rc_web_server.repo_clone_url(SVN_REPO_WITH_GROUP)
71 username, password = rc_web_server.repo_clone_credentials()
72
73 cmd = Command('/tmp')
74 auth = f'--non-interactive --username={username} --password={password}'
75 stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
76 _check_proper_clone(stdout, stderr, 'svn')
77 cmd.assert_returncode_success()
78
79 def test_clone_wrong_credentials_svn(self, rc_web_server, tmpdir):
80 clone_url = rc_web_server.repo_clone_url(SVN_REPO)
81 username, password = rc_web_server.repo_clone_credentials()
82 password = 'bad-password'
83
84 auth = f'--non-interactive --username={username} --password={password}'
85 stdout, stderr = Command('/tmp').execute(
86 f'svn checkout {auth}', clone_url, tmpdir.strpath)
87 assert 'fatal: Authentication failed' in stderr
88
89 def test_clone_svn_with_slashes(self, rc_web_server, tmpdir):
90 clone_url = rc_web_server.repo_clone_url('//' + SVN_REPO)
91 stdout, stderr = Command('/tmp').execute('svn checkout', clone_url)
92 assert 'not found' in stderr
93
94 def test_clone_existing_path_svn_not_in_database(
95 self, rc_web_server, tmpdir, fs_repo_only):
96 db_name = fs_repo_only('not-in-db-git', repo_type='git')
97 clone_url = rc_web_server.repo_clone_url(db_name)
98 username, password = '', ''
99 auth = f'--non-interactive --username={username} --password={password}'
100
101 stdout, stderr = Command('/tmp').execute(
102 f'svn checkout {auth}', clone_url, tmpdir.strpath)
103 assert 'not found' in stderr
104
105 def test_clone_existing_path_svn_not_in_database_different_scm(
106 self, rc_web_server, tmpdir, fs_repo_only):
107 db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
108 clone_url = rc_web_server.repo_clone_url(db_name)
109
110 username, password = '', ''
111 auth = f'--non-interactive --username={username} --password={password}'
112 stdout, stderr = Command('/tmp').execute(
113 f'svn checkout {auth}', clone_url, tmpdir.strpath)
114 assert 'not found' in stderr
115
116 def test_clone_non_existing_store_path_svn(self, rc_web_server, tmpdir, user_util):
117 repo = user_util.create_repo(repo_type='git')
118 clone_url = rc_web_server.repo_clone_url(repo.repo_name)
119
120 # Damage repo by removing it's folder
121 RepoModel()._delete_filesystem_repo(repo)
122
123 username, password = '', ''
124 auth = f'--non-interactive --username={username} --password={password}'
125 stdout, stderr = Command('/tmp').execute(
126 f'svn checkout {auth}', clone_url, tmpdir.strpath)
127 assert 'not found' in stderr
128
129 def test_push_new_file_svn(self, rc_web_server, tmpdir):
130 clone_url = rc_web_server.repo_clone_url(SVN_REPO)
131 username, password = '', ''
132 auth = f'--non-interactive --username={username} --password={password}'
133
134 stdout, stderr = Command('/tmp').execute(
135 f'svn checkout {auth}', clone_url, tmpdir.strpath)
136
137 # commit some stuff into this repo
138 stdout, stderr = _add_files_and_push(
139 'svn', tmpdir.strpath, clone_url=clone_url)
140
141 _check_proper_svn_push(stdout, stderr)
142
143 def test_push_wrong_credentials_svn(self, rc_web_server, tmpdir):
144 clone_url = rc_web_server.repo_clone_url(SVN_REPO)
145
146 username, password = '', ''
147 auth = f'--non-interactive --username={username} --password={password}'
148 stdout, stderr = Command('/tmp').execute(
149 f'svn checkout {auth}', clone_url, tmpdir.strpath)
150
151 push_url = rc_web_server.repo_clone_url(
152 SVN_REPO, user='bad', passwd='name')
153 stdout, stderr = _add_files_and_push(
154 'svn', tmpdir.strpath, clone_url=push_url)
155
156 assert 'fatal: Authentication failed' in stderr
157
158 def test_push_back_to_wrong_url_svn(self, rc_web_server, tmpdir):
159 clone_url = rc_web_server.repo_clone_url(SVN_REPO)
160 username, password = '', ''
161 auth = f'--non-interactive --username={username} --password={password}'
162 Command('/tmp').execute(
163 f'svn checkout {auth}', clone_url, tmpdir.strpath)
164
165 stdout, stderr = _add_files_and_push(
166 'svn', tmpdir.strpath,
167 clone_url=rc_web_server.repo_clone_url('not-existing'))
168
169 assert 'not found' in stderr
170
171 def test_ip_restriction_svn(self, rc_web_server, tmpdir):
172 user_model = UserModel()
173 username, password = '', ''
174 auth = f'--non-interactive --username={username} --password={password}'
175
176 try:
177 user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
178 Session().commit()
179 time.sleep(2)
180 clone_url = rc_web_server.repo_clone_url(SVN_REPO)
181
182 stdout, stderr = Command('/tmp').execute(
183 f'svn checkout {auth}', clone_url, tmpdir.strpath)
184 msg = "The requested URL returned error: 403"
185 assert msg in stderr
186 finally:
187 # release IP restrictions
188 for ip in UserIpMap.getAll():
189 UserIpMap.delete(ip.ip_id)
190 Session().commit()
191
192 time.sleep(2)
193
194 cmd = Command('/tmp')
195 stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
196 cmd.assert_returncode_success()
197 _check_proper_clone(stdout, stderr, 'svn')
@@ -657,6 +657,10 b' vcs.methods.cache = true'
657 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
657 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
658 #vcs.svn.compatible_version = 1.8
658 #vcs.svn.compatible_version = 1.8
659
659
660 ; Redis connection settings for svn integrations logic
661 ; This connection string needs to be the same on ce and vcsserver
662 vcs.svn.redis_conn = redis://redis:6379/0
663
660 ; Enable SVN proxy of requests over HTTP
664 ; Enable SVN proxy of requests over HTTP
661 vcs.svn.proxy.enabled = true
665 vcs.svn.proxy.enabled = true
662
666
@@ -625,6 +625,10 b' vcs.methods.cache = true'
625 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
625 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
626 #vcs.svn.compatible_version = 1.8
626 #vcs.svn.compatible_version = 1.8
627
627
628 ; Redis connection settings for svn integrations logic
629 ; This connection string needs to be the same on ce and vcsserver
630 vcs.svn.redis_conn = redis://redis:6379/0
631
628 ; Enable SVN proxy of requests over HTTP
632 ; Enable SVN proxy of requests over HTTP
629 vcs.svn.proxy.enabled = true
633 vcs.svn.proxy.enabled = true
630
634
@@ -103,6 +103,7 b' def sanitize_settings_and_apply_defaults'
103 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
103 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
104
104
105 settings_maker.make_setting('vcs.svn.compatible_version', '')
105 settings_maker.make_setting('vcs.svn.compatible_version', '')
106 settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
106 settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
107 settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
107 settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
108 settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
108 settings_maker.make_setting('vcs.hooks.protocol', 'http')
109 settings_maker.make_setting('vcs.hooks.protocol', 'http')
@@ -258,8 +258,7 b' class ActionParser(object):'
258 commit = repo.get_commit(commit_id=commit_id)
258 commit = repo.get_commit(commit_id=commit_id)
259 commits.append(commit)
259 commits.append(commit)
260 except CommitDoesNotExistError:
260 except CommitDoesNotExistError:
261 log.error(
261 log.error('cannot find commit id %s in this repository',
262 'cannot find commit id %s in this repository',
263 commit_id)
262 commit_id)
264 commits.append(commit_id)
263 commits.append(commit_id)
265 continue
264 continue
@@ -15,13 +15,14 b''
15 # This program is dual-licensed. If you wish to learn more about the
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
18 import os
19 import os
19 import time
20 import time
20 import logging
21 import logging
21 import tempfile
22
22
23 from rhodecode.lib.config_utils import get_config
23 from rhodecode.lib.config_utils import get_config
24 from rhodecode.lib.ext_json import json
24
25 from rhodecode.lib.svn_txn_utils import get_txn_id_from_store
25
26
26 log = logging.getLogger(__name__)
27 log = logging.getLogger(__name__)
27
28
@@ -47,49 +48,22 b' class HooksModuleCallbackDaemon(BaseHook'
47 super().__init__()
48 super().__init__()
48 self.hooks_module = module
49 self.hooks_module = module
49
50
50
51 def __repr__(self):
51 def get_txn_id_data_path(txn_id):
52 return f'HooksModuleCallbackDaemon(hooks_module={self.hooks_module})'
52 import rhodecode
53
54 root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
55 final_dir = os.path.join(root, 'svn_txn_id')
56
57 if not os.path.isdir(final_dir):
58 os.makedirs(final_dir)
59 return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
60
61
62 def store_txn_id_data(txn_id, data_dict):
63 if not txn_id:
64 log.warning('Cannot store txn_id because it is empty')
65 return
66
67 path = get_txn_id_data_path(txn_id)
68 try:
69 with open(path, 'wb') as f:
70 f.write(json.dumps(data_dict))
71 except Exception:
72 log.exception('Failed to write txn_id metadata')
73
74
75 def get_txn_id_from_store(txn_id):
76 """
77 Reads txn_id from store and if present returns the data for callback manager
78 """
79 path = get_txn_id_data_path(txn_id)
80 try:
81 with open(path, 'rb') as f:
82 return json.loads(f.read())
83 except Exception:
84 return {}
85
53
86
54
87 def prepare_callback_daemon(extras, protocol, host, txn_id=None):
55 def prepare_callback_daemon(extras, protocol, host, txn_id=None):
88 txn_details = get_txn_id_from_store(txn_id)
56
89 port = txn_details.get('port', 0)
90 match protocol:
57 match protocol:
91 case 'http':
58 case 'http':
92 from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
59 from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
60 port = 0
61 if txn_id:
62 # read txn-id to re-use the PORT for callback daemon
63 repo_path = os.path.join(extras['repo_store'], extras['repository'])
64 txn_details = get_txn_id_from_store(repo_path, txn_id)
65 port = txn_details.get('port', 0)
66
93 callback_daemon = HttpHooksCallbackDaemon(
67 callback_daemon = HttpHooksCallbackDaemon(
94 txn_id=txn_id, host=host, port=port)
68 txn_id=txn_id, host=host, port=port)
95 case 'celery':
69 case 'celery':
@@ -28,3 +28,6 b' class CeleryHooksCallbackDaemon(BaseHook'
28 # TODO: replace this with settings bootstrapped...
28 # TODO: replace this with settings bootstrapped...
29 self.task_queue = config.get('app:main', 'celery.broker_url')
29 self.task_queue = config.get('app:main', 'celery.broker_url')
30 self.task_backend = config.get('app:main', 'celery.result_backend')
30 self.task_backend = config.get('app:main', 'celery.result_backend')
31
32 def __repr__(self):
33 return f'CeleryHooksCallbackDaemon(task_queue={self.task_queue}, task_backend={self.task_backend})'
@@ -30,7 +30,7 b' from socketserver import TCPServer'
30 from rhodecode.model import meta
30 from rhodecode.model import meta
31 from rhodecode.lib.ext_json import json
31 from rhodecode.lib.ext_json import json
32 from rhodecode.lib import rc_cache
32 from rhodecode.lib import rc_cache
33 from rhodecode.lib.hook_daemon.base import get_txn_id_data_path
33 from rhodecode.lib.svn_txn_utils import get_txn_id_data_key
34 from rhodecode.lib.hook_daemon.hook_module import Hooks
34 from rhodecode.lib.hook_daemon.hook_module import Hooks
35
35
36 log = logging.getLogger(__name__)
36 log = logging.getLogger(__name__)
@@ -185,9 +185,12 b' class HttpHooksCallbackDaemon(ThreadedHo'
185
185
186 use_gevent = False
186 use_gevent = False
187
187
188 def __repr__(self):
189 return f'HttpHooksCallbackDaemon(hooks_uri={self.hooks_uri})'
190
188 @property
191 @property
189 def _hook_prefix(self):
192 def _hook_prefix(self):
190 return 'HOOKS: {} '.format(self.hooks_uri)
193 return f'HOOKS: {self.hooks_uri} '
191
194
192 def get_hostname(self):
195 def get_hostname(self):
193 return socket.gethostname() or '127.0.0.1'
196 return socket.gethostname() or '127.0.0.1'
@@ -205,7 +208,7 b' class HttpHooksCallbackDaemon(ThreadedHo'
205 port = self.get_available_port()
208 port = self.get_available_port()
206
209
207 server_address = (host, port)
210 server_address = (host, port)
208 self.hooks_uri = '{}:{}'.format(host, port)
211 self.hooks_uri = f'{host}:{port}'
209 self.txn_id = txn_id
212 self.txn_id = txn_id
210 self._done = False
213 self._done = False
211
214
@@ -249,7 +252,9 b' class HttpHooksCallbackDaemon(ThreadedHo'
249 self._daemon = None
252 self._daemon = None
250 self._callback_thread = None
253 self._callback_thread = None
251 if self.txn_id:
254 if self.txn_id:
252 txn_id_file = get_txn_id_data_path(self.txn_id)
255 #TODO: figure out the repo_path...
256 repo_path = ''
257 txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
253 log.debug('Cleaning up TXN ID %s', txn_id_file)
258 log.debug('Cleaning up TXN ID %s', txn_id_file)
254 if os.path.isfile(txn_id_file):
259 if os.path.isfile(txn_id_file):
255 os.remove(txn_id_file)
260 os.remove(txn_id_file)
@@ -272,7 +277,9 b' class HttpHooksCallbackDaemon(ThreadedHo'
272 self._callback_greenlet = None
277 self._callback_greenlet = None
273
278
274 if self.txn_id:
279 if self.txn_id:
275 txn_id_file = get_txn_id_data_path(self.txn_id)
280 #TODO: figure out the repo_path...
281 repo_path = ''
282 txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
276 log.debug('Cleaning up TXN ID %s', txn_id_file)
283 log.debug('Cleaning up TXN ID %s', txn_id_file)
277 if os.path.isfile(txn_id_file):
284 if os.path.isfile(txn_id_file):
278 os.remove(txn_id_file)
285 os.remove(txn_id_file)
@@ -17,7 +17,8 b''
17 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
19
19
20 import base64
20 import re
21 import os
21 import logging
22 import logging
22 import urllib.request
23 import urllib.request
23 import urllib.parse
24 import urllib.parse
@@ -28,14 +29,10 b' import requests'
28 from pyramid.httpexceptions import HTTPNotAcceptable
29 from pyramid.httpexceptions import HTTPNotAcceptable
29
30
30 from rhodecode import ConfigGet
31 from rhodecode import ConfigGet
31 from rhodecode.lib import rc_cache
32 from rhodecode.lib.middleware import simplevcs
32 from rhodecode.lib.middleware import simplevcs
33 from rhodecode.lib.middleware.utils import get_path_info
33 from rhodecode.lib.middleware.utils import get_path_info
34 from rhodecode.lib.utils import is_valid_repo
34 from rhodecode.lib.utils import is_valid_repo
35 from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
35 from rhodecode.lib.str_utils import safe_str
36 from rhodecode.lib.type_utils import str2bool
37 from rhodecode.lib.ext_json import json
38 from rhodecode.lib.hook_daemon.base import store_txn_id_data
39
36
40 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
41
38
@@ -63,28 +60,11 b' class SimpleSvnApp(object):'
63
60
64 # stream control flag, based on request and content type...
61 # stream control flag, based on request and content type...
65 stream = False
62 stream = False
66
67 if req_method in ['MKCOL'] or has_content_length:
63 if req_method in ['MKCOL'] or has_content_length:
68 data_processed = False
69 # read chunk to check if we have txn-with-props
70 initial_data: bytes = data_io.read(1024)
71 if initial_data.startswith(b'(create-txn-with-props'):
72 data_io = initial_data + data_io.read()
73 # store on-the-fly our rc_extra using svn revision properties
74 # those can be read later on in hooks executed so we have a way
75 # to pass in the data into svn hooks
76 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
77 rc_data_len = str(len(rc_data))
78 # header defines data length, and serialized data
79 skel = b' rc-scm-extras %b %b' % (safe_bytes(rc_data_len), safe_bytes(rc_data))
80 data_io = data_io[:-2] + skel + b'))'
81 data_processed = True
82
83 if not data_processed:
84 # NOTE(johbo): Avoid that we end up with sending the request in chunked
64 # NOTE(johbo): Avoid that we end up with sending the request in chunked
85 # transfer encoding (mainly on Gunicorn). If we know the content
65 # transfer encoding (mainly on Gunicorn). If we know the content
86 # length, then we should transfer the payload in one request.
66 # length, then we should transfer the payload in one request.
87 data_io = initial_data + data_io.read()
67 data_io = data_io.read()
88
68
89 if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
69 if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
90 # NOTE(marcink): when getting/uploading files, we want to STREAM content
70 # NOTE(marcink): when getting/uploading files, we want to STREAM content
@@ -101,6 +81,7 b' class SimpleSvnApp(object):'
101 stream=stream
81 stream=stream
102 )
82 )
103 if req_method in ['HEAD', 'DELETE']:
83 if req_method in ['HEAD', 'DELETE']:
84 # NOTE(marcink): HEAD might be deprecated for SVN 1.14+ protocol
104 del call_kwargs['data']
85 del call_kwargs['data']
105
86
106 try:
87 try:
@@ -120,14 +101,6 b' class SimpleSvnApp(object):'
120 log.debug('got response code: %s', response.status_code)
101 log.debug('got response code: %s', response.status_code)
121
102
122 response_headers = self._get_response_headers(response.headers)
103 response_headers = self._get_response_headers(response.headers)
123
124 if response.headers.get('SVN-Txn-name'):
125 svn_tx_id = response.headers.get('SVN-Txn-name')
126 txn_id = rc_cache.utils.compute_key_from_params(
127 self.config['repository'], svn_tx_id)
128 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
129 store_txn_id_data(txn_id, {'port': port})
130
131 start_response(f'{response.status_code} {response.reason}', response_headers)
104 start_response(f'{response.status_code} {response.reason}', response_headers)
132 return response.iter_content(chunk_size=1024)
105 return response.iter_content(chunk_size=1024)
133
106
@@ -137,6 +110,20 b' class SimpleSvnApp(object):'
137 url_path = urllib.parse.quote(url_path, safe="/:=~+!$,;'")
110 url_path = urllib.parse.quote(url_path, safe="/:=~+!$,;'")
138 return url_path
111 return url_path
139
112
113 def _get_txn_id(self, environ):
114 url = environ['RAW_URI']
115
116 # Define the regex pattern
117 pattern = r'/txr/([^/]+)/'
118
119 # Search for the pattern in the URL
120 match = re.search(pattern, url)
121
122 # Check if a match is found and extract the captured group
123 if match:
124 txn_id = match.group(1)
125 return txn_id
126
140 def _get_request_headers(self, environ):
127 def _get_request_headers(self, environ):
141 headers = {}
128 headers = {}
142 whitelist = {
129 whitelist = {
@@ -182,10 +169,39 b' class DisabledSimpleSvnApp(object):'
182
169
183
170
184 class SimpleSvn(simplevcs.SimpleVCS):
171 class SimpleSvn(simplevcs.SimpleVCS):
172 """
173 details: https://svn.apache.org/repos/asf/subversion/trunk/notes/http-and-webdav/webdav-protocol
174
175 Read Commands : (OPTIONS, PROPFIND, GET, REPORT)
176
177 GET: fetch info about resources
178 PROPFIND: Used to retrieve properties of resources.
179 REPORT: Used for specialized queries to the repository. E.g History etc...
180 OPTIONS: request is sent to an SVN server, the server responds with information about the available HTTP
181 methods and other server capabilities.
182
183 Write Commands : (MKACTIVITY, PROPPATCH, PUT, CHECKOUT, MKCOL, MOVE,
184 -------------- COPY, DELETE, LOCK, UNLOCK, MERGE)
185
186 With the exception of LOCK/UNLOCK, every write command performs some
187 sort of DeltaV commit operation. In DeltaV, a commit always starts
188 by creating a transaction (MKACTIVITY), applies a log message
189 (PROPPATCH), does some other write methods, and then ends by
190 committing the transaction (MERGE). If the MERGE fails, the client
191 may try to remove the transaction with a DELETE.
192
193 PROPPATCH: Used to set and/or remove properties on resources.
194 MKCOL: Creates a new collection (directory).
195 DELETE: Removes a resource.
196 COPY and MOVE: Used for copying and moving resources.
197 MERGE: Used to merge changes from different branches.
198 CHECKOUT, CHECKIN, UNCHECKOUT: DeltaV methods for managing working resources and versions.
199 """
185
200
186 SCM = 'svn'
201 SCM = 'svn'
187 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
202 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
188 DEFAULT_HTTP_SERVER = 'http://localhost:8090'
203 WRITE_COMMANDS = ('MERGE', 'POST', 'PUT', 'COPY', 'MOVE', 'DELETE', 'MKCOL')
204 DEFAULT_HTTP_SERVER = 'http://svn:8090'
189
205
190 def _get_repository_name(self, environ):
206 def _get_repository_name(self, environ):
191 """
207 """
@@ -218,10 +234,10 b' class SimpleSvn(simplevcs.SimpleVCS):'
218 else 'push')
234 else 'push')
219
235
220 def _should_use_callback_daemon(self, extras, environ, action):
236 def _should_use_callback_daemon(self, extras, environ, action):
221 # only MERGE command triggers hooks, so we don't want to start
237 # only PUT & MERGE command triggers hooks, so we don't want to start
222 # hooks server too many times. POST however starts the svn transaction
238 # hooks server too many times. POST however starts the svn transaction
223 # so we also need to run the init of callback daemon of POST
239 # so we also need to run the init of callback daemon of POST
224 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
240 if environ['REQUEST_METHOD'] not in self.READ_ONLY_COMMANDS:
225 return True
241 return True
226 return False
242 return False
227
243
@@ -25,11 +25,9 b" It's implemented with basic auth functio"
25
25
26 import os
26 import os
27 import re
27 import re
28 import io
29 import logging
28 import logging
30 import importlib
29 import importlib
31 from functools import wraps
30 from functools import wraps
32 from lxml import etree
33
31
34 import time
32 import time
35 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
33 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
@@ -41,6 +39,7 b' from zope.cachedescriptors.property impo'
41 import rhodecode
39 import rhodecode
42 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
40 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
43 from rhodecode.lib import rc_cache
41 from rhodecode.lib import rc_cache
42 from rhodecode.lib.svn_txn_utils import store_txn_id_data
44 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
43 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
45 from rhodecode.lib.base import (
44 from rhodecode.lib.base import (
46 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
45 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -48,7 +47,7 b' from rhodecode.lib.exceptions import (Us'
48 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
47 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
49 from rhodecode.lib.middleware import appenlight
48 from rhodecode.lib.middleware import appenlight
50 from rhodecode.lib.middleware.utils import scm_app_http
49 from rhodecode.lib.middleware.utils import scm_app_http
51 from rhodecode.lib.str_utils import safe_bytes
50 from rhodecode.lib.str_utils import safe_bytes, safe_int
52 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
51 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
53 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
52 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
54 from rhodecode.lib.vcs.conf import settings as vcs_settings
53 from rhodecode.lib.vcs.conf import settings as vcs_settings
@@ -63,29 +62,6 b' from rhodecode.model.settings import Set'
63 log = logging.getLogger(__name__)
62 log = logging.getLogger(__name__)
64
63
65
64
66 def extract_svn_txn_id(acl_repo_name, data: bytes):
67 """
68 Helper method for extraction of svn txn_id from submitted XML data during
69 POST operations
70 """
71
72 try:
73 root = etree.fromstring(data)
74 pat = re.compile(r'/txn/(?P<txn_id>.*)')
75 for el in root:
76 if el.tag == '{DAV:}source':
77 for sub_el in el:
78 if sub_el.tag == '{DAV:}href':
79 match = pat.search(sub_el.text)
80 if match:
81 svn_tx_id = match.groupdict()['txn_id']
82 txn_id = rc_cache.utils.compute_key_from_params(
83 acl_repo_name, svn_tx_id)
84 return txn_id
85 except Exception:
86 log.exception('Failed to extract txn_id')
87
88
89 def initialize_generator(factory):
65 def initialize_generator(factory):
90 """
66 """
91 Initializes the returned generator by draining its first element.
67 Initializes the returned generator by draining its first element.
@@ -468,7 +444,6 b' class SimpleVCS(object):'
468 log.debug('Not enough credentials to access repo: `%s` '
444 log.debug('Not enough credentials to access repo: `%s` '
469 'repository as anonymous user', self.acl_repo_name)
445 'repository as anonymous user', self.acl_repo_name)
470
446
471
472 username = None
447 username = None
473 # ==============================================================
448 # ==============================================================
474 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
449 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
@@ -582,6 +557,24 b' class SimpleVCS(object):'
582 return self._generate_vcs_response(
557 return self._generate_vcs_response(
583 environ, start_response, repo_path, extras, action)
558 environ, start_response, repo_path, extras, action)
584
559
560 def _get_txn_id(self, environ):
561
562 for k in ['RAW_URI', 'HTTP_DESTINATION']:
563 url = environ.get(k)
564 if not url:
565 continue
566
567 # regex to search for svn-txn-id
568 pattern = r'/!svn/txr/([^/]+)/'
569
570 # Search for the pattern in the URL
571 match = re.search(pattern, url)
572
573 # Check if a match is found and extract the captured group
574 if match:
575 txn_id = match.group(1)
576 return txn_id
577
585 @initialize_generator
578 @initialize_generator
586 def _generate_vcs_response(
579 def _generate_vcs_response(
587 self, environ, start_response, repo_path, extras, action):
580 self, environ, start_response, repo_path, extras, action):
@@ -593,28 +586,23 b' class SimpleVCS(object):'
593 also handles the locking exceptions which will be triggered when
586 also handles the locking exceptions which will be triggered when
594 the first chunk is produced by the underlying WSGI application.
587 the first chunk is produced by the underlying WSGI application.
595 """
588 """
596
589 svn_txn_id = ''
597 txn_id = ''
590 if action == 'push':
598 if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
591 svn_txn_id = self._get_txn_id(environ)
599 # case for SVN, we want to re-use the callback daemon port
600 # so we use the txn_id, for this we peek the body, and still save
601 # it as wsgi.input
602
603 stream = environ['wsgi.input']
604
605 if isinstance(stream, io.BytesIO):
606 data: bytes = stream.getvalue()
607 elif hasattr(stream, 'buf'): # most likely gunicorn.http.body.Body
608 data: bytes = stream.buf.getvalue()
609 else:
610 # fallback to the crudest way, copy the iterator
611 data = safe_bytes(stream.read())
612 environ['wsgi.input'] = io.BytesIO(data)
613
614 txn_id = extract_svn_txn_id(self.acl_repo_name, data)
615
592
616 callback_daemon, extras = self._prepare_callback_daemon(
593 callback_daemon, extras = self._prepare_callback_daemon(
617 extras, environ, action, txn_id=txn_id)
594 extras, environ, action, txn_id=svn_txn_id)
595
596 if svn_txn_id:
597
598 port = safe_int(extras['hooks_uri'].split(':')[-1])
599 txn_id_data = extras.copy()
600 txn_id_data.update({'port': port})
601 txn_id_data.update({'req_method': environ['REQUEST_METHOD']})
602
603 full_repo_path = repo_path
604 store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
605
618 log.debug('HOOKS extras is %s', extras)
606 log.debug('HOOKS extras is %s', extras)
619
607
620 http_scheme = self._get_http_scheme(environ)
608 http_scheme = self._get_http_scheme(environ)
@@ -677,6 +665,7 b' class SimpleVCS(object):'
677
665
678 def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
666 def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
679 protocol = vcs_settings.HOOKS_PROTOCOL
667 protocol = vcs_settings.HOOKS_PROTOCOL
668
680 if not self._should_use_callback_daemon(extras, environ, action):
669 if not self._should_use_callback_daemon(extras, environ, action):
681 # disable callback daemon for actions that don't require it
670 # disable callback daemon for actions that don't require it
682 protocol = 'local'
671 protocol = 'local'
@@ -12,16 +12,14 b''
12 ******************************************************************************/
12 ******************************************************************************/
13 function registerRCRoutes() {
13 function registerRCRoutes() {
14 // routes registration
14 // routes registration
15 pyroutes.register('admin_artifacts', '/_admin/artifacts', []);
15 pyroutes.register('admin_artifacts', '/_admin/_admin/artifacts', []);
16 pyroutes.register('admin_artifacts_data', '/_admin/artifacts-data', []);
16 pyroutes.register('admin_artifacts_delete', '/_admin/_admin/artifacts/%(uid)s/delete', ['uid']);
17 pyroutes.register('admin_artifacts_delete', '/_admin/artifacts/%(uid)s/delete', ['uid']);
17 pyroutes.register('admin_artifacts_show_all', '/_admin/_admin/artifacts', []);
18 pyroutes.register('admin_artifacts_show_all', '/_admin/artifacts', []);
18 pyroutes.register('admin_artifacts_show_info', '/_admin/_admin/artifacts/%(uid)s', ['uid']);
19 pyroutes.register('admin_artifacts_show_info', '/_admin/artifacts/%(uid)s', ['uid']);
19 pyroutes.register('admin_artifacts_update', '/_admin/_admin/artifacts/%(uid)s/update', ['uid']);
20 pyroutes.register('admin_artifacts_update', '/_admin/artifacts/%(uid)s/update', ['uid']);
21 pyroutes.register('admin_audit_log_entry', '/_admin/audit_logs/%(audit_log_id)s', ['audit_log_id']);
20 pyroutes.register('admin_audit_log_entry', '/_admin/audit_logs/%(audit_log_id)s', ['audit_log_id']);
22 pyroutes.register('admin_audit_logs', '/_admin/audit_logs', []);
21 pyroutes.register('admin_audit_logs', '/_admin/audit_logs', []);
23 pyroutes.register('admin_automation', '/_admin/automation', []);
22 pyroutes.register('admin_automation', '/_admin/_admin/automation', []);
24 pyroutes.register('admin_automation_update', '/_admin/automation/%(entry_id)s/update', ['entry_id']);
25 pyroutes.register('admin_defaults_repositories', '/_admin/defaults/repositories', []);
23 pyroutes.register('admin_defaults_repositories', '/_admin/defaults/repositories', []);
26 pyroutes.register('admin_defaults_repositories_update', '/_admin/defaults/repositories/update', []);
24 pyroutes.register('admin_defaults_repositories_update', '/_admin/defaults/repositories/update', []);
27 pyroutes.register('admin_home', '/_admin', []);
25 pyroutes.register('admin_home', '/_admin', []);
@@ -29,7 +27,6 b' function registerRCRoutes() {'
29 pyroutes.register('admin_permissions_application_update', '/_admin/permissions/application/update', []);
27 pyroutes.register('admin_permissions_application_update', '/_admin/permissions/application/update', []);
30 pyroutes.register('admin_permissions_auth_token_access', '/_admin/permissions/auth_token_access', []);
28 pyroutes.register('admin_permissions_auth_token_access', '/_admin/permissions/auth_token_access', []);
31 pyroutes.register('admin_permissions_branch', '/_admin/permissions/branch', []);
29 pyroutes.register('admin_permissions_branch', '/_admin/permissions/branch', []);
32 pyroutes.register('admin_permissions_branch_update', '/_admin/permissions/branch/update', []);
33 pyroutes.register('admin_permissions_global', '/_admin/permissions/global', []);
30 pyroutes.register('admin_permissions_global', '/_admin/permissions/global', []);
34 pyroutes.register('admin_permissions_global_update', '/_admin/permissions/global/update', []);
31 pyroutes.register('admin_permissions_global_update', '/_admin/permissions/global/update', []);
35 pyroutes.register('admin_permissions_ips', '/_admin/permissions/ips', []);
32 pyroutes.register('admin_permissions_ips', '/_admin/permissions/ips', []);
@@ -39,8 +36,7 b' function registerRCRoutes() {'
39 pyroutes.register('admin_permissions_ssh_keys', '/_admin/permissions/ssh_keys', []);
36 pyroutes.register('admin_permissions_ssh_keys', '/_admin/permissions/ssh_keys', []);
40 pyroutes.register('admin_permissions_ssh_keys_data', '/_admin/permissions/ssh_keys/data', []);
37 pyroutes.register('admin_permissions_ssh_keys_data', '/_admin/permissions/ssh_keys/data', []);
41 pyroutes.register('admin_permissions_ssh_keys_update', '/_admin/permissions/ssh_keys/update', []);
38 pyroutes.register('admin_permissions_ssh_keys_update', '/_admin/permissions/ssh_keys/update', []);
42 pyroutes.register('admin_scheduler', '/_admin/scheduler', []);
39 pyroutes.register('admin_scheduler', '/_admin/_admin/scheduler', []);
43 pyroutes.register('admin_scheduler_show_tasks', '/_admin/scheduler/_tasks', []);
44 pyroutes.register('admin_settings', '/_admin/settings', []);
40 pyroutes.register('admin_settings', '/_admin/settings', []);
45 pyroutes.register('admin_settings_email', '/_admin/settings/email', []);
41 pyroutes.register('admin_settings_email', '/_admin/settings/email', []);
46 pyroutes.register('admin_settings_email_update', '/_admin/settings/email/update', []);
42 pyroutes.register('admin_settings_email_update', '/_admin/settings/email/update', []);
@@ -59,8 +55,6 b' function registerRCRoutes() {'
59 pyroutes.register('admin_settings_issuetracker_update', '/_admin/settings/issue-tracker/update', []);
55 pyroutes.register('admin_settings_issuetracker_update', '/_admin/settings/issue-tracker/update', []);
60 pyroutes.register('admin_settings_labs', '/_admin/settings/labs', []);
56 pyroutes.register('admin_settings_labs', '/_admin/settings/labs', []);
61 pyroutes.register('admin_settings_labs_update', '/_admin/settings/labs/update', []);
57 pyroutes.register('admin_settings_labs_update', '/_admin/settings/labs/update', []);
62 pyroutes.register('admin_settings_license', '/_admin/settings/license', []);
63 pyroutes.register('admin_settings_license_unlock', '/_admin/settings/license_unlock', []);
64 pyroutes.register('admin_settings_mapping', '/_admin/settings/mapping', []);
58 pyroutes.register('admin_settings_mapping', '/_admin/settings/mapping', []);
65 pyroutes.register('admin_settings_mapping_update', '/_admin/settings/mapping/update', []);
59 pyroutes.register('admin_settings_mapping_update', '/_admin/settings/mapping/update', []);
66 pyroutes.register('admin_settings_open_source', '/_admin/settings/open_source', []);
60 pyroutes.register('admin_settings_open_source', '/_admin/settings/open_source', []);
@@ -68,12 +62,6 b' function registerRCRoutes() {'
68 pyroutes.register('admin_settings_process_management_data', '/_admin/settings/process_management/data', []);
62 pyroutes.register('admin_settings_process_management_data', '/_admin/settings/process_management/data', []);
69 pyroutes.register('admin_settings_process_management_master_signal', '/_admin/settings/process_management/master_signal', []);
63 pyroutes.register('admin_settings_process_management_master_signal', '/_admin/settings/process_management/master_signal', []);
70 pyroutes.register('admin_settings_process_management_signal', '/_admin/settings/process_management/signal', []);
64 pyroutes.register('admin_settings_process_management_signal', '/_admin/settings/process_management/signal', []);
71 pyroutes.register('admin_settings_scheduler_create', '/_admin/scheduler/create', []);
72 pyroutes.register('admin_settings_scheduler_delete', '/_admin/scheduler/%(schedule_id)s/delete', ['schedule_id']);
73 pyroutes.register('admin_settings_scheduler_edit', '/_admin/scheduler/%(schedule_id)s', ['schedule_id']);
74 pyroutes.register('admin_settings_scheduler_execute', '/_admin/scheduler/%(schedule_id)s/execute', ['schedule_id']);
75 pyroutes.register('admin_settings_scheduler_new', '/_admin/scheduler/new', []);
76 pyroutes.register('admin_settings_scheduler_update', '/_admin/scheduler/%(schedule_id)s/update', ['schedule_id']);
77 pyroutes.register('admin_settings_search', '/_admin/settings/search', []);
65 pyroutes.register('admin_settings_search', '/_admin/settings/search', []);
78 pyroutes.register('admin_settings_sessions', '/_admin/settings/sessions', []);
66 pyroutes.register('admin_settings_sessions', '/_admin/settings/sessions', []);
79 pyroutes.register('admin_settings_sessions_cleanup', '/_admin/settings/sessions/cleanup', []);
67 pyroutes.register('admin_settings_sessions_cleanup', '/_admin/settings/sessions/cleanup', []);
@@ -97,7 +85,6 b' function registerRCRoutes() {'
97 pyroutes.register('channelstream_proxy', '/_channelstream', []);
85 pyroutes.register('channelstream_proxy', '/_channelstream', []);
98 pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
86 pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
99 pyroutes.register('check_2fa', '/_admin/check_2fa', []);
87 pyroutes.register('check_2fa', '/_admin/check_2fa', []);
100 pyroutes.register('commit_draft_comments_submit', '/%(repo_name)s/changeset/%(commit_id)s/draft_comments_submit', ['repo_name', 'commit_id']);
101 pyroutes.register('debug_style_email', '/_admin/debug_style/email/%(email_id)s', ['email_id']);
88 pyroutes.register('debug_style_email', '/_admin/debug_style/email/%(email_id)s', ['email_id']);
102 pyroutes.register('debug_style_email_plain_rendered', '/_admin/debug_style/email-rendered/%(email_id)s', ['email_id']);
89 pyroutes.register('debug_style_email_plain_rendered', '/_admin/debug_style/email-rendered/%(email_id)s', ['email_id']);
103 pyroutes.register('debug_style_home', '/_admin/debug_style', []);
90 pyroutes.register('debug_style_home', '/_admin/debug_style', []);
@@ -222,8 +209,6 b' function registerRCRoutes() {'
222 pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
209 pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
223 pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
210 pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
224 pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
211 pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
225 pyroutes.register('my_account_external_identity', '/_admin/my_account/external-identity', []);
226 pyroutes.register('my_account_external_identity_delete', '/_admin/my_account/external-identity/delete', []);
227 pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
212 pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
228 pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
213 pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
229 pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
214 pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
@@ -254,7 +239,6 b' function registerRCRoutes() {'
254 pyroutes.register('ops_healthcheck', '/_admin/ops/status', []);
239 pyroutes.register('ops_healthcheck', '/_admin/ops/status', []);
255 pyroutes.register('ops_ping', '/_admin/ops/ping', []);
240 pyroutes.register('ops_ping', '/_admin/ops/ping', []);
256 pyroutes.register('ops_redirect_test', '/_admin/ops/redirect', []);
241 pyroutes.register('ops_redirect_test', '/_admin/ops/redirect', []);
257 pyroutes.register('plugin_admin_chat', '/_admin/plugin_admin_chat/%(action)s', ['action']);
258 pyroutes.register('pull_requests_global', '/_admin/pull-request/%(pull_request_id)s', ['pull_request_id']);
242 pyroutes.register('pull_requests_global', '/_admin/pull-request/%(pull_request_id)s', ['pull_request_id']);
259 pyroutes.register('pull_requests_global_0', '/_admin/pull_requests/%(pull_request_id)s', ['pull_request_id']);
243 pyroutes.register('pull_requests_global_0', '/_admin/pull_requests/%(pull_request_id)s', ['pull_request_id']);
260 pyroutes.register('pull_requests_global_1', '/_admin/pull-requests/%(pull_request_id)s', ['pull_request_id']);
244 pyroutes.register('pull_requests_global_1', '/_admin/pull-requests/%(pull_request_id)s', ['pull_request_id']);
@@ -264,7 +248,6 b' function registerRCRoutes() {'
264 pyroutes.register('pullrequest_comments', '/%(repo_name)s/pull-request/%(pull_request_id)s/comments', ['repo_name', 'pull_request_id']);
248 pyroutes.register('pullrequest_comments', '/%(repo_name)s/pull-request/%(pull_request_id)s/comments', ['repo_name', 'pull_request_id']);
265 pyroutes.register('pullrequest_create', '/%(repo_name)s/pull-request/create', ['repo_name']);
249 pyroutes.register('pullrequest_create', '/%(repo_name)s/pull-request/create', ['repo_name']);
266 pyroutes.register('pullrequest_delete', '/%(repo_name)s/pull-request/%(pull_request_id)s/delete', ['repo_name', 'pull_request_id']);
250 pyroutes.register('pullrequest_delete', '/%(repo_name)s/pull-request/%(pull_request_id)s/delete', ['repo_name', 'pull_request_id']);
267 pyroutes.register('pullrequest_draft_comments_submit', '/%(repo_name)s/pull-request/%(pull_request_id)s/draft_comments_submit', ['repo_name', 'pull_request_id']);
268 pyroutes.register('pullrequest_drafts', '/%(repo_name)s/pull-request/%(pull_request_id)s/drafts', ['repo_name', 'pull_request_id']);
251 pyroutes.register('pullrequest_drafts', '/%(repo_name)s/pull-request/%(pull_request_id)s/drafts', ['repo_name', 'pull_request_id']);
269 pyroutes.register('pullrequest_merge', '/%(repo_name)s/pull-request/%(pull_request_id)s/merge', ['repo_name', 'pull_request_id']);
252 pyroutes.register('pullrequest_merge', '/%(repo_name)s/pull-request/%(pull_request_id)s/merge', ['repo_name', 'pull_request_id']);
270 pyroutes.register('pullrequest_new', '/%(repo_name)s/pull-request/new', ['repo_name']);
253 pyroutes.register('pullrequest_new', '/%(repo_name)s/pull-request/new', ['repo_name']);
@@ -277,18 +260,8 b' function registerRCRoutes() {'
277 pyroutes.register('pullrequest_update', '/%(repo_name)s/pull-request/%(pull_request_id)s/update', ['repo_name', 'pull_request_id']);
260 pyroutes.register('pullrequest_update', '/%(repo_name)s/pull-request/%(pull_request_id)s/update', ['repo_name', 'pull_request_id']);
278 pyroutes.register('register', '/_admin/register', []);
261 pyroutes.register('register', '/_admin/register', []);
279 pyroutes.register('repo_archivefile', '/%(repo_name)s/archive/%(fname)s', ['repo_name', 'fname']);
262 pyroutes.register('repo_archivefile', '/%(repo_name)s/archive/%(fname)s', ['repo_name', 'fname']);
280 pyroutes.register('repo_artifacts_data', '/%(repo_name)s/artifacts_data', ['repo_name']);
281 pyroutes.register('repo_artifacts_delete', '/%(repo_name)s/artifacts/delete/%(uid)s', ['repo_name', 'uid']);
282 pyroutes.register('repo_artifacts_get', '/%(repo_name)s/artifacts/download/%(uid)s', ['repo_name', 'uid']);
283 pyroutes.register('repo_artifacts_info', '/%(repo_name)s/artifacts/info/%(uid)s', ['repo_name', 'uid']);
284 pyroutes.register('repo_artifacts_list', '/%(repo_name)s/artifacts', ['repo_name']);
263 pyroutes.register('repo_artifacts_list', '/%(repo_name)s/artifacts', ['repo_name']);
285 pyroutes.register('repo_artifacts_new', '/%(repo_name)s/artifacts/new', ['repo_name']);
286 pyroutes.register('repo_artifacts_store', '/%(repo_name)s/artifacts/store', ['repo_name']);
287 pyroutes.register('repo_artifacts_stream_script', '/_file_store/stream-upload-script', []);
288 pyroutes.register('repo_artifacts_stream_store', '/_file_store/stream-upload', []);
289 pyroutes.register('repo_artifacts_update', '/%(repo_name)s/artifacts/update/%(uid)s', ['repo_name', 'uid']);
290 pyroutes.register('repo_automation', '/%(repo_name)s/settings/automation', ['repo_name']);
264 pyroutes.register('repo_automation', '/%(repo_name)s/settings/automation', ['repo_name']);
291 pyroutes.register('repo_automation_update', '/%(repo_name)s/settings/automation/%(entry_id)s/update', ['repo_name', 'entry_id']);
292 pyroutes.register('repo_changelog', '/%(repo_name)s/changelog', ['repo_name']);
265 pyroutes.register('repo_changelog', '/%(repo_name)s/changelog', ['repo_name']);
293 pyroutes.register('repo_changelog_file', '/%(repo_name)s/changelog/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
266 pyroutes.register('repo_changelog_file', '/%(repo_name)s/changelog/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
294 pyroutes.register('repo_commit', '/%(repo_name)s/changeset/%(commit_id)s', ['repo_name', 'commit_id']);
267 pyroutes.register('repo_commit', '/%(repo_name)s/changeset/%(commit_id)s', ['repo_name', 'commit_id']);
@@ -366,9 +339,6 b' function registerRCRoutes() {'
366 pyroutes.register('repo_refs_changelog_data', '/%(repo_name)s/refs-data-changelog', ['repo_name']);
339 pyroutes.register('repo_refs_changelog_data', '/%(repo_name)s/refs-data-changelog', ['repo_name']);
367 pyroutes.register('repo_refs_data', '/%(repo_name)s/refs-data', ['repo_name']);
340 pyroutes.register('repo_refs_data', '/%(repo_name)s/refs-data', ['repo_name']);
368 pyroutes.register('repo_reviewers', '/%(repo_name)s/settings/review/rules', ['repo_name']);
341 pyroutes.register('repo_reviewers', '/%(repo_name)s/settings/review/rules', ['repo_name']);
369 pyroutes.register('repo_reviewers_review_rule_delete', '/%(repo_name)s/settings/review/rules/%(rule_id)s/delete', ['repo_name', 'rule_id']);
370 pyroutes.register('repo_reviewers_review_rule_edit', '/%(repo_name)s/settings/review/rules/%(rule_id)s', ['repo_name', 'rule_id']);
371 pyroutes.register('repo_reviewers_review_rule_new', '/%(repo_name)s/settings/review/rules/new', ['repo_name']);
372 pyroutes.register('repo_settings_quick_actions', '/%(repo_name)s/settings/quick-action', ['repo_name']);
342 pyroutes.register('repo_settings_quick_actions', '/%(repo_name)s/settings/quick-action', ['repo_name']);
373 pyroutes.register('repo_stats', '/%(repo_name)s/repo_stats/%(commit_id)s', ['repo_name', 'commit_id']);
343 pyroutes.register('repo_stats', '/%(repo_name)s/repo_stats/%(commit_id)s', ['repo_name', 'commit_id']);
374 pyroutes.register('repo_summary', '/%(repo_name)s', ['repo_name']);
344 pyroutes.register('repo_summary', '/%(repo_name)s', ['repo_name']);
@@ -17,7 +17,7 b' examples = ['
17
17
18 (
18 (
19 'Tickets with #123 (Redmine etc)',
19 'Tickets with #123 (Redmine etc)',
20 '(?<![a-zA-Z0-9_/]{1,10}-?)(#)(?P<issue_id>\d+)',
20 '(?<![a-zA-Z0-9_/]{1,10}-?)(#)(?P<issue_id>[0-9]+)',
21 'https://myissueserver.com/${repo}/issue/${issue_id}',
21 'https://myissueserver.com/${repo}/issue/${issue_id}',
22 ''
22 ''
23 ),
23 ),
@@ -60,7 +60,7 b' examples = ['
60
60
61 (
61 (
62 'Pivotal Tracker',
62 'Pivotal Tracker',
63 '(?:pivot-)(?P<project_id>\d+)-(?P<story>\d+)',
63 '(?:pivot-)(?P<project_id>\d+)-(?P<story>[0-9]+)',
64 'https://www.pivotaltracker.com/s/projects/${project_id}/stories/${story}',
64 'https://www.pivotaltracker.com/s/projects/${project_id}/stories/${story}',
65 'PIV-',
65 'PIV-',
66 ),
66 ),
@@ -332,7 +332,6 b''
332 POST request to trigger the (re)generation of the mod_dav_svn config. */
332 POST request to trigger the (re)generation of the mod_dav_svn config. */
333 $('#vcs_svn_generate_cfg').on('click', function(event) {
333 $('#vcs_svn_generate_cfg').on('click', function(event) {
334 event.preventDefault();
334 event.preventDefault();
335 alert('i cliked it !!')
336 var url = "${h.route_path('admin_settings_vcs_svn_generate_cfg')}";
335 var url = "${h.route_path('admin_settings_vcs_svn_generate_cfg')}";
337 var jqxhr = $.post(url, {'csrf_token': CSRF_TOKEN});
336 var jqxhr = $.post(url, {'csrf_token': CSRF_TOKEN});
338 jqxhr.done(function(data) {
337 jqxhr.done(function(data) {
@@ -161,7 +161,7 b' def vcsserver_port(request):'
161
161
162
162
163 @pytest.fixture(scope='session')
163 @pytest.fixture(scope='session')
164 def available_port_factory():
164 def available_port_factory() -> get_available_port:
165 """
165 """
166 Returns a callable which returns free port numbers.
166 Returns a callable which returns free port numbers.
167 """
167 """
@@ -304,7 +304,8 b' class TestPrepareHooksDaemon(object):'
304 'txn_id': 'txnid2',
304 'txn_id': 'txnid2',
305 'hooks_protocol': protocol.lower(),
305 'hooks_protocol': protocol.lower(),
306 'task_backend': '',
306 'task_backend': '',
307 'task_queue': ''
307 'task_queue': '',
308 'repo_store': '/var/opt/rhodecode_repo_store'
308 }
309 }
309 callback, extras = hook_base.prepare_callback_daemon(
310 callback, extras = hook_base.prepare_callback_daemon(
310 expected_extras.copy(), protocol=protocol, host='127.0.0.1',
311 expected_extras.copy(), protocol=protocol, host='127.0.0.1',
@@ -148,7 +148,7 b' class RcVCSServer(ServerBase):'
148 self._args = [
148 self._args = [
149 'gunicorn',
149 'gunicorn',
150 '--bind', self.bind_addr,
150 '--bind', self.bind_addr,
151 '--worker-class', 'gevent',
151 '--worker-class', 'gthread',
152 '--backlog', '16',
152 '--backlog', '16',
153 '--timeout', '300',
153 '--timeout', '300',
154 '--workers', workers,
154 '--workers', workers,
@@ -185,7 +185,7 b' class RcWebServer(ServerBase):'
185 self._args = [
185 self._args = [
186 'gunicorn',
186 'gunicorn',
187 '--bind', self.bind_addr,
187 '--bind', self.bind_addr,
188 '--worker-class', 'gevent',
188 '--worker-class', 'gthread',
189 '--backlog', '16',
189 '--backlog', '16',
190 '--timeout', '300',
190 '--timeout', '300',
191 '--workers', workers,
191 '--workers', workers,
@@ -219,3 +219,11 b' class RcWebServer(ServerBase):'
219 params.update(**kwargs)
219 params.update(**kwargs)
220 _url = f"http://{params['user']}:{params['passwd']}@{params['host']}/{params['cloned_repo']}"
220 _url = f"http://{params['user']}:{params['passwd']}@{params['host']}/{params['cloned_repo']}"
221 return _url
221 return _url
222
223 def repo_clone_credentials(self, **kwargs):
224 params = {
225 'user': TEST_USER_ADMIN_LOGIN,
226 'passwd': TEST_USER_ADMIN_PASS,
227 }
228 params.update(**kwargs)
229 return params['user'], params['passwd']
@@ -26,20 +26,21 b' Base for test suite for making push/pull'
26 to redirect things to stderr instead of stdout.
26 to redirect things to stderr instead of stdout.
27 """
27 """
28
28
29 from os.path import join as jn
29
30 from subprocess import Popen, PIPE
31 import logging
30 import logging
32 import os
31 import os
33 import tempfile
32 import tempfile
33 import subprocess
34
34
35 from rhodecode.lib.str_utils import safe_str
35 from rhodecode.lib.str_utils import safe_str
36 from rhodecode.tests import GIT_REPO, HG_REPO
36 from rhodecode.tests import GIT_REPO, HG_REPO, SVN_REPO
37
37
38 DEBUG = True
38 DEBUG = True
39 RC_LOG = os.path.join(tempfile.gettempdir(), 'rc.log')
39 RC_LOG = os.path.join(tempfile.gettempdir(), 'rc.log')
40 REPO_GROUP = 'a_repo_group'
40 REPO_GROUP = 'a_repo_group'
41 HG_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, HG_REPO)
41 HG_REPO_WITH_GROUP = f'{REPO_GROUP}/{HG_REPO}'
42 GIT_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, GIT_REPO)
42 GIT_REPO_WITH_GROUP = f'{REPO_GROUP}/{GIT_REPO}'
43 SVN_REPO_WITH_GROUP = f'{REPO_GROUP}/{SVN_REPO}'
43
44
44 log = logging.getLogger(__name__)
45 log = logging.getLogger(__name__)
45
46
@@ -65,7 +66,8 b' class Command(object):'
65 if key.startswith('COV_CORE_'):
66 if key.startswith('COV_CORE_'):
66 del env[key]
67 del env[key]
67
68
68 self.process = Popen(command, shell=True, stdout=PIPE, stderr=PIPE,
69 self.process = subprocess.Popen(
70 command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
69 cwd=self.cwd, env=env)
71 cwd=self.cwd, env=env)
70 stdout, stderr = self.process.communicate()
72 stdout, stderr = self.process.communicate()
71
73
@@ -85,12 +87,14 b' def _add_files(vcs, dest, clone_url=None'
85 full_name = 'Marcin KuΕΊminski'
87 full_name = 'Marcin KuΕΊminski'
86 email = 'me@email.com'
88 email = 'me@email.com'
87 git_ident = f"git config user.name {full_name} && git config user.email {email}"
89 git_ident = f"git config user.name {full_name} && git config user.email {email}"
88 cwd = path = jn(dest)
90 cwd = path = os.path.join(dest)
89
91
90 tags = tags or []
92 tags = tags or []
91 added_file = jn(path, '{}_setup.py'.format(next(tempfile._RandomNameSequence())))
93 name_sequence = next(tempfile._RandomNameSequence())
92 Command(cwd).execute('touch %s' % added_file)
94 added_file = os.path.join(path, f'{name_sequence}_setup.py')
93 Command(cwd).execute('%s add %s' % (vcs, added_file))
95
96 Command(cwd).execute(f'touch {added_file}')
97 Command(cwd).execute(f'{vcs} add {added_file}')
94 author_str = 'Marcin KuΕΊminski <me@email.com>'
98 author_str = 'Marcin KuΕΊminski <me@email.com>'
95
99
96 for i in range(kwargs.get('files_no', 3)):
100 for i in range(kwargs.get('files_no', 3)):
@@ -128,7 +132,7 b' def _add_files_and_push(vcs, dest, clone'
128 vcs is git or hg and defines what VCS we want to make those files for
132 vcs is git or hg and defines what VCS we want to make those files for
129 """
133 """
130 git_ident = "git config user.name Marcin KuΕΊminski && git config user.email me@email.com"
134 git_ident = "git config user.name Marcin KuΕΊminski && git config user.email me@email.com"
131 cwd = jn(dest)
135 cwd = os.path.join(dest)
132
136
133 # commit some stuff into this repo
137 # commit some stuff into this repo
134 _add_files(vcs, dest, clone_url, tags, target_branch, new_branch, **kwargs)
138 _add_files(vcs, dest, clone_url, tags, target_branch, new_branch, **kwargs)
@@ -147,12 +151,15 b' def _add_files_and_push(vcs, dest, clone'
147 if new_branch:
151 if new_branch:
148 maybe_new_branch = '--new-branch'
152 maybe_new_branch = '--new-branch'
149 stdout, stderr = Command(cwd).execute(
153 stdout, stderr = Command(cwd).execute(
150 'hg push --traceback --verbose {} -r {} {}'.format(maybe_new_branch, target_branch, clone_url)
154 f'hg push --traceback --verbose {maybe_new_branch} -r {target_branch} {clone_url}'
151 )
155 )
152 elif vcs == 'git':
156 elif vcs == 'git':
153 stdout, stderr = Command(cwd).execute(
157 stdout, stderr = Command(cwd).execute(
154 """{} &&
158 f'{git_ident} && git push --verbose --tags {clone_url} {target_branch}'
155 git push --verbose --tags {} {}""".format(git_ident, clone_url, target_branch)
159 )
160 elif vcs == 'svn':
161 stdout, stderr = Command(cwd).execute(
162 f'svn ci -m "pushing to {target_branch}"'
156 )
163 )
157
164
158 return stdout, stderr
165 return stdout, stderr
@@ -179,6 +186,13 b' def _check_proper_hg_push(stdout, stderr'
179 assert 'abort:' not in stderr
186 assert 'abort:' not in stderr
180
187
181
188
189 def _check_proper_svn_push(stdout, stderr):
190 assert 'pushing to' in stdout
191 assert 'searching for changes' in stdout
192
193 assert 'abort:' not in stderr
194
195
182 def _check_proper_clone(stdout, stderr, vcs):
196 def _check_proper_clone(stdout, stderr, vcs):
183 if vcs == 'hg':
197 if vcs == 'hg':
184 assert 'requesting all changes' in stdout
198 assert 'requesting all changes' in stdout
@@ -193,3 +207,8 b' def _check_proper_clone(stdout, stderr, '
193 assert 'Cloning into' in stderr
207 assert 'Cloning into' in stderr
194 assert 'abort:' not in stderr
208 assert 'abort:' not in stderr
195 assert 'fatal:' not in stderr
209 assert 'fatal:' not in stderr
210
211 if vcs == 'svn':
212 assert 'dupa' in stdout
213
214
@@ -42,7 +42,7 b' from rhodecode.model.db import Repositor'
42 from rhodecode.model.meta import Session
42 from rhodecode.model.meta import Session
43 from rhodecode.integrations.types.webhook import WebhookIntegrationType
43 from rhodecode.integrations.types.webhook import WebhookIntegrationType
44
44
45 from rhodecode.tests import GIT_REPO, HG_REPO
45 from rhodecode.tests import GIT_REPO, HG_REPO, SVN_REPO
46 from rhodecode.tests.conftest import HTTPBIN_DOMAIN, HTTPBIN_POST
46 from rhodecode.tests.conftest import HTTPBIN_DOMAIN, HTTPBIN_POST
47 from rhodecode.tests.fixture import Fixture
47 from rhodecode.tests.fixture import Fixture
48 from rhodecode.tests.server_utils import RcWebServer
48 from rhodecode.tests.server_utils import RcWebServer
@@ -51,13 +51,15 b' from rhodecode.tests.server_utils import'
51 REPO_GROUP = 'a_repo_group'
51 REPO_GROUP = 'a_repo_group'
52 HG_REPO_WITH_GROUP = f'{REPO_GROUP}/{HG_REPO}'
52 HG_REPO_WITH_GROUP = f'{REPO_GROUP}/{HG_REPO}'
53 GIT_REPO_WITH_GROUP = f'{REPO_GROUP}/{GIT_REPO}'
53 GIT_REPO_WITH_GROUP = f'{REPO_GROUP}/{GIT_REPO}'
54 SVN_REPO_WITH_GROUP = f'{REPO_GROUP}/{SVN_REPO}'
54
55
55 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
56
57
57
58
58 def check_httpbin_connection():
59 def check_httpbin_connection():
60 log.debug('Checking if HTTPBIN_DOMAIN: %s is available', HTTPBIN_DOMAIN)
59 try:
61 try:
60 response = requests.get(HTTPBIN_DOMAIN)
62 response = requests.get(HTTPBIN_DOMAIN, timeout=5)
61 return response.status_code == 200
63 return response.status_code == 200
62 except Exception as e:
64 except Exception as e:
63 print(e)
65 print(e)
@@ -102,11 +104,15 b' def repos(request, db_connection):'
102 fixture.create_fork(GIT_REPO, GIT_REPO,
104 fixture.create_fork(GIT_REPO, GIT_REPO,
103 repo_name_full=GIT_REPO_WITH_GROUP,
105 repo_name_full=GIT_REPO_WITH_GROUP,
104 repo_group=repo_group_id)
106 repo_group=repo_group_id)
107 fixture.create_fork(SVN_REPO, SVN_REPO,
108 repo_name_full=SVN_REPO_WITH_GROUP,
109 repo_group=repo_group_id)
105
110
106 @request.addfinalizer
111 @request.addfinalizer
107 def cleanup():
112 def cleanup():
108 fixture.destroy_repo(HG_REPO_WITH_GROUP)
113 fixture.destroy_repo(HG_REPO_WITH_GROUP)
109 fixture.destroy_repo(GIT_REPO_WITH_GROUP)
114 fixture.destroy_repo(GIT_REPO_WITH_GROUP)
115 fixture.destroy_repo(SVN_REPO_WITH_GROUP)
110 fixture.destroy_repo_group(repo_group_id)
116 fixture.destroy_repo_group(repo_group_id)
111
117
112
118
@@ -139,11 +145,11 b' def rc_web_server('
139 """
145 """
140 Run the web server as a subprocess. with its own instance of vcsserver
146 Run the web server as a subprocess. with its own instance of vcsserver
141 """
147 """
142 rcweb_port = available_port_factory()
148 rcweb_port: int = available_port_factory()
143 log.info('Using rcweb ops test port {}'.format(rcweb_port))
149 log.info('Using rcweb ops test port %s', rcweb_port)
144
150
145 vcsserver_port = available_port_factory()
151 vcsserver_port: int = available_port_factory()
146 log.info('Using vcsserver ops test port {}'.format(vcsserver_port))
152 log.info('Using vcsserver ops test port %s', vcsserver_port)
147
153
148 vcs_log = os.path.join(tempfile.gettempdir(), 'rc_op_vcs.log')
154 vcs_log = os.path.join(tempfile.gettempdir(), 'rc_op_vcs.log')
149 vcsserver_factory(
155 vcsserver_factory(
@@ -303,5 +309,3 b' def branch_permission_setter(request):'
303 Session().commit()
309 Session().commit()
304
310
305 return _branch_permissions_setter
311 return _branch_permissions_setter
306
307
@@ -32,7 +32,7 b' from rhodecode.lib.vcs.backends.git.repo'
32 from rhodecode.lib.vcs.nodes import FileNode
32 from rhodecode.lib.vcs.nodes import FileNode
33 from rhodecode.tests import GIT_REPO
33 from rhodecode.tests import GIT_REPO
34 from rhodecode.tests.vcs_operations import Command
34 from rhodecode.tests.vcs_operations import Command
35 from .test_vcs_operations import _check_proper_clone, _check_proper_git_push
35 from .test_vcs_operations_git import _check_proper_clone, _check_proper_git_push
36
36
37
37
38 def test_git_clone_with_small_push_buffer(backend_git, rc_web_server, tmpdir):
38 def test_git_clone_with_small_push_buffer(backend_git, rc_web_server, tmpdir):
@@ -28,47 +28,23 b' Test suite for making push/pull operatio'
28
28
29
29
30 import time
30 import time
31 import logging
32
33 import pytest
31 import pytest
34
32
35 from rhodecode.lib import rc_cache
33 from rhodecode.model.db import Repository, UserIpMap
36 from rhodecode.model.auth_token import AuthTokenModel
37 from rhodecode.model.db import Repository, UserIpMap, CacheKey
38 from rhodecode.model.meta import Session
34 from rhodecode.model.meta import Session
39 from rhodecode.model.repo import RepoModel
35 from rhodecode.model.repo import RepoModel
40 from rhodecode.model.user import UserModel
36 from rhodecode.model.user import UserModel
41 from rhodecode.tests import (GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN)
37 from rhodecode.tests import (GIT_REPO, TEST_USER_ADMIN_LOGIN)
42 from rhodecode.tests.utils import assert_message_in_log
38
43
39
44 from rhodecode.tests.vcs_operations import (
40 from rhodecode.tests.vcs_operations import (
45 Command, _check_proper_clone, _check_proper_git_push,
41 Command, _check_proper_clone, _check_proper_git_push,
46 _add_files_and_push, HG_REPO_WITH_GROUP, GIT_REPO_WITH_GROUP)
42 _add_files_and_push, GIT_REPO_WITH_GROUP)
47
43
48
44
49 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
45 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
50 class TestVCSOperations(object):
46 class TestVCSOperations(object):
51
47
52 def test_clone_hg_repo_by_admin(self, rc_web_server, tmpdir):
53 clone_url = rc_web_server.repo_clone_url(HG_REPO)
54 stdout, stderr = Command('/tmp').execute(
55 'hg clone', clone_url, tmpdir.strpath)
56 _check_proper_clone(stdout, stderr, 'hg')
57
58 def test_clone_hg_repo_by_admin_pull_protocol(self, rc_web_server, tmpdir):
59 clone_url = rc_web_server.repo_clone_url(HG_REPO)
60 stdout, stderr = Command('/tmp').execute(
61 'hg clone --pull', clone_url, tmpdir.strpath)
62 _check_proper_clone(stdout, stderr, 'hg')
63
64 def test_clone_hg_repo_by_admin_pull_stream_protocol(self, rc_web_server, tmpdir):
65 clone_url = rc_web_server.repo_clone_url(HG_REPO)
66 stdout, stderr = Command('/tmp').execute(
67 'hg clone --pull --stream', clone_url, tmpdir.strpath)
68 assert 'files to transfer,' in stdout
69 assert 'transferred 1.' in stdout
70 assert '114 files updated,' in stdout
71
72 def test_clone_git_repo_by_admin(self, rc_web_server, tmpdir):
48 def test_clone_git_repo_by_admin(self, rc_web_server, tmpdir):
73 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
49 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
74 cmd = Command('/tmp')
50 cmd = Command('/tmp')
@@ -83,13 +59,6 b' class TestVCSOperations(object):'
83 _check_proper_clone(stdout, stderr, 'git')
59 _check_proper_clone(stdout, stderr, 'git')
84 cmd.assert_returncode_success()
60 cmd.assert_returncode_success()
85
61
86 def test_clone_hg_repo_by_id_by_admin(self, rc_web_server, tmpdir):
87 repo_id = Repository.get_by_repo_name(HG_REPO).repo_id
88 clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
89 stdout, stderr = Command('/tmp').execute(
90 'hg clone', clone_url, tmpdir.strpath)
91 _check_proper_clone(stdout, stderr, 'hg')
92
93 def test_clone_git_repo_by_id_by_admin(self, rc_web_server, tmpdir):
62 def test_clone_git_repo_by_id_by_admin(self, rc_web_server, tmpdir):
94 repo_id = Repository.get_by_repo_name(GIT_REPO).repo_id
63 repo_id = Repository.get_by_repo_name(GIT_REPO).repo_id
95 clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
64 clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
@@ -98,12 +67,6 b' class TestVCSOperations(object):'
98 _check_proper_clone(stdout, stderr, 'git')
67 _check_proper_clone(stdout, stderr, 'git')
99 cmd.assert_returncode_success()
68 cmd.assert_returncode_success()
100
69
101 def test_clone_hg_repo_with_group_by_admin(self, rc_web_server, tmpdir):
102 clone_url = rc_web_server.repo_clone_url(HG_REPO_WITH_GROUP)
103 stdout, stderr = Command('/tmp').execute(
104 'hg clone', clone_url, tmpdir.strpath)
105 _check_proper_clone(stdout, stderr, 'hg')
106
107 def test_clone_git_repo_with_group_by_admin(self, rc_web_server, tmpdir):
70 def test_clone_git_repo_with_group_by_admin(self, rc_web_server, tmpdir):
108 clone_url = rc_web_server.repo_clone_url(GIT_REPO_WITH_GROUP)
71 clone_url = rc_web_server.repo_clone_url(GIT_REPO_WITH_GROUP)
109 cmd = Command('/tmp')
72 cmd = Command('/tmp')
@@ -121,11 +84,6 b' class TestVCSOperations(object):'
121 assert 'Cloning into' in stderr
84 assert 'Cloning into' in stderr
122 cmd.assert_returncode_success()
85 cmd.assert_returncode_success()
123
86
124 def test_clone_wrong_credentials_hg(self, rc_web_server, tmpdir):
125 clone_url = rc_web_server.repo_clone_url(HG_REPO, passwd='bad!')
126 stdout, stderr = Command('/tmp').execute(
127 'hg clone', clone_url, tmpdir.strpath)
128 assert 'abort: authorization failed' in stderr
129
87
130 def test_clone_wrong_credentials_git(self, rc_web_server, tmpdir):
88 def test_clone_wrong_credentials_git(self, rc_web_server, tmpdir):
131 clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!')
89 clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!')
@@ -139,12 +97,6 b' class TestVCSOperations(object):'
139 'hg clone', clone_url, tmpdir.strpath)
97 'hg clone', clone_url, tmpdir.strpath)
140 assert 'HTTP Error 404: Not Found' in stderr
98 assert 'HTTP Error 404: Not Found' in stderr
141
99
142 def test_clone_hg_repo_as_git(self, rc_web_server, tmpdir):
143 clone_url = rc_web_server.repo_clone_url(HG_REPO)
144 stdout, stderr = Command('/tmp').execute(
145 'git clone', clone_url, tmpdir.strpath)
146 assert 'not found' in stderr
147
148 def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
100 def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
149 clone_url = rc_web_server.repo_clone_url('trololo')
101 clone_url = rc_web_server.repo_clone_url('trololo')
150 stdout, stderr = Command('/tmp').execute(
102 stdout, stderr = Command('/tmp').execute(
@@ -156,25 +108,11 b' class TestVCSOperations(object):'
156 stdout, stderr = Command('/tmp').execute('git clone', clone_url)
108 stdout, stderr = Command('/tmp').execute('git clone', clone_url)
157 assert 'not found' in stderr
109 assert 'not found' in stderr
158
110
159 def test_clone_hg_with_slashes(self, rc_web_server, tmpdir):
160 clone_url = rc_web_server.repo_clone_url('//' + HG_REPO)
161 stdout, stderr = Command('/tmp').execute('hg clone', clone_url, tmpdir.strpath)
162 assert 'HTTP Error 404: Not Found' in stderr
163
164 def test_clone_git_with_slashes(self, rc_web_server, tmpdir):
111 def test_clone_git_with_slashes(self, rc_web_server, tmpdir):
165 clone_url = rc_web_server.repo_clone_url('//' + GIT_REPO)
112 clone_url = rc_web_server.repo_clone_url('//' + GIT_REPO)
166 stdout, stderr = Command('/tmp').execute('git clone', clone_url)
113 stdout, stderr = Command('/tmp').execute('git clone', clone_url)
167 assert 'not found' in stderr
114 assert 'not found' in stderr
168
115
169 def test_clone_existing_path_hg_not_in_database(
170 self, rc_web_server, tmpdir, fs_repo_only):
171
172 db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
173 clone_url = rc_web_server.repo_clone_url(db_name)
174 stdout, stderr = Command('/tmp').execute(
175 'hg clone', clone_url, tmpdir.strpath)
176 assert 'HTTP Error 404: Not Found' in stderr
177
178 def test_clone_existing_path_git_not_in_database(
116 def test_clone_existing_path_git_not_in_database(
179 self, rc_web_server, tmpdir, fs_repo_only):
117 self, rc_web_server, tmpdir, fs_repo_only):
180 db_name = fs_repo_only('not-in-db-git', repo_type='git')
118 db_name = fs_repo_only('not-in-db-git', repo_type='git')
@@ -183,14 +121,6 b' class TestVCSOperations(object):'
183 'git clone', clone_url, tmpdir.strpath)
121 'git clone', clone_url, tmpdir.strpath)
184 assert 'not found' in stderr
122 assert 'not found' in stderr
185
123
186 def test_clone_existing_path_hg_not_in_database_different_scm(
187 self, rc_web_server, tmpdir, fs_repo_only):
188 db_name = fs_repo_only('not-in-db-git', repo_type='git')
189 clone_url = rc_web_server.repo_clone_url(db_name)
190 stdout, stderr = Command('/tmp').execute(
191 'hg clone', clone_url, tmpdir.strpath)
192 assert 'HTTP Error 404: Not Found' in stderr
193
194 def test_clone_existing_path_git_not_in_database_different_scm(
124 def test_clone_existing_path_git_not_in_database_different_scm(
195 self, rc_web_server, tmpdir, fs_repo_only):
125 self, rc_web_server, tmpdir, fs_repo_only):
196 db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
126 db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
@@ -199,17 +129,6 b' class TestVCSOperations(object):'
199 'git clone', clone_url, tmpdir.strpath)
129 'git clone', clone_url, tmpdir.strpath)
200 assert 'not found' in stderr
130 assert 'not found' in stderr
201
131
202 def test_clone_non_existing_store_path_hg(self, rc_web_server, tmpdir, user_util):
203 repo = user_util.create_repo()
204 clone_url = rc_web_server.repo_clone_url(repo.repo_name)
205
206 # Damage repo by removing it's folder
207 RepoModel()._delete_filesystem_repo(repo)
208
209 stdout, stderr = Command('/tmp').execute(
210 'hg clone', clone_url, tmpdir.strpath)
211 assert 'HTTP Error 404: Not Found' in stderr
212
213 def test_clone_non_existing_store_path_git(self, rc_web_server, tmpdir, user_util):
132 def test_clone_non_existing_store_path_git(self, rc_web_server, tmpdir, user_util):
214 repo = user_util.create_repo(repo_type='git')
133 repo = user_util.create_repo(repo_type='git')
215 clone_url = rc_web_server.repo_clone_url(repo.repo_name)
134 clone_url = rc_web_server.repo_clone_url(repo.repo_name)
@@ -221,17 +140,6 b' class TestVCSOperations(object):'
221 'git clone', clone_url, tmpdir.strpath)
140 'git clone', clone_url, tmpdir.strpath)
222 assert 'not found' in stderr
141 assert 'not found' in stderr
223
142
224 def test_push_new_file_hg(self, rc_web_server, tmpdir):
225 clone_url = rc_web_server.repo_clone_url(HG_REPO)
226 stdout, stderr = Command('/tmp').execute(
227 'hg clone', clone_url, tmpdir.strpath)
228
229 stdout, stderr = _add_files_and_push(
230 'hg', tmpdir.strpath, clone_url=clone_url)
231
232 assert 'pushing to' in stdout
233 assert 'size summary' in stdout
234
235 def test_push_new_file_git(self, rc_web_server, tmpdir):
143 def test_push_new_file_git(self, rc_web_server, tmpdir):
236 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
144 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
237 stdout, stderr = Command('/tmp').execute(
145 stdout, stderr = Command('/tmp').execute(
@@ -243,58 +151,6 b' class TestVCSOperations(object):'
243
151
244 _check_proper_git_push(stdout, stderr)
152 _check_proper_git_push(stdout, stderr)
245
153
246 def test_push_invalidates_cache(self, rc_web_server, tmpdir):
247 hg_repo = Repository.get_by_repo_name(HG_REPO)
248
249 # init cache objects
250 CacheKey.delete_all_cache()
251
252 repo_namespace_key = CacheKey.REPO_INVALIDATION_NAMESPACE.format(repo_id=hg_repo.repo_id)
253
254 inv_context_manager = rc_cache.InvalidationContext(key=repo_namespace_key)
255
256 with inv_context_manager as invalidation_context:
257 # __enter__ will create and register cache objects
258 pass
259
260 cache_keys = hg_repo.cache_keys
261 assert cache_keys != []
262 old_ids = [x.cache_state_uid for x in cache_keys]
263
264 # clone to init cache
265 clone_url = rc_web_server.repo_clone_url(hg_repo.repo_name)
266 stdout, stderr = Command('/tmp').execute(
267 'hg clone', clone_url, tmpdir.strpath)
268
269 cache_keys = hg_repo.cache_keys
270 assert cache_keys != []
271 for key in cache_keys:
272 assert key.cache_active is True
273
274 # PUSH that should trigger invalidation cache
275 stdout, stderr = _add_files_and_push(
276 'hg', tmpdir.strpath, clone_url=clone_url, files_no=1)
277
278 # flush...
279 Session().commit()
280 hg_repo = Repository.get_by_repo_name(HG_REPO)
281 cache_keys = hg_repo.cache_keys
282 assert cache_keys != []
283 new_ids = [x.cache_state_uid for x in cache_keys]
284 assert new_ids != old_ids
285
286 def test_push_wrong_credentials_hg(self, rc_web_server, tmpdir):
287 clone_url = rc_web_server.repo_clone_url(HG_REPO)
288 stdout, stderr = Command('/tmp').execute(
289 'hg clone', clone_url, tmpdir.strpath)
290
291 push_url = rc_web_server.repo_clone_url(
292 HG_REPO, user='bad', passwd='name')
293 stdout, stderr = _add_files_and_push(
294 'hg', tmpdir.strpath, clone_url=push_url)
295
296 assert 'abort: authorization failed' in stderr
297
298 def test_push_wrong_credentials_git(self, rc_web_server, tmpdir):
154 def test_push_wrong_credentials_git(self, rc_web_server, tmpdir):
299 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
155 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
300 stdout, stderr = Command('/tmp').execute(
156 stdout, stderr = Command('/tmp').execute(
@@ -307,17 +163,6 b' class TestVCSOperations(object):'
307
163
308 assert 'fatal: Authentication failed' in stderr
164 assert 'fatal: Authentication failed' in stderr
309
165
310 def test_push_back_to_wrong_url_hg(self, rc_web_server, tmpdir):
311 clone_url = rc_web_server.repo_clone_url(HG_REPO)
312 stdout, stderr = Command('/tmp').execute(
313 'hg clone', clone_url, tmpdir.strpath)
314
315 stdout, stderr = _add_files_and_push(
316 'hg', tmpdir.strpath,
317 clone_url=rc_web_server.repo_clone_url('not-existing'))
318
319 assert 'HTTP Error 404: Not Found' in stderr
320
321 def test_push_back_to_wrong_url_git(self, rc_web_server, tmpdir):
166 def test_push_back_to_wrong_url_git(self, rc_web_server, tmpdir):
322 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
167 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
323 stdout, stderr = Command('/tmp').execute(
168 stdout, stderr = Command('/tmp').execute(
@@ -329,28 +174,6 b' class TestVCSOperations(object):'
329
174
330 assert 'not found' in stderr
175 assert 'not found' in stderr
331
176
332 def test_ip_restriction_hg(self, rc_web_server, tmpdir):
333 user_model = UserModel()
334 try:
335 user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
336 Session().commit()
337 time.sleep(2)
338 clone_url = rc_web_server.repo_clone_url(HG_REPO)
339 stdout, stderr = Command('/tmp').execute(
340 'hg clone', clone_url, tmpdir.strpath)
341 assert 'abort: HTTP Error 403: Forbidden' in stderr
342 finally:
343 # release IP restrictions
344 for ip in UserIpMap.getAll():
345 UserIpMap.delete(ip.ip_id)
346 Session().commit()
347
348 time.sleep(2)
349
350 stdout, stderr = Command('/tmp').execute(
351 'hg clone', clone_url, tmpdir.strpath)
352 _check_proper_clone(stdout, stderr, 'hg')
353
354 def test_ip_restriction_git(self, rc_web_server, tmpdir):
177 def test_ip_restriction_git(self, rc_web_server, tmpdir):
355 user_model = UserModel()
178 user_model = UserModel()
356 try:
179 try:
@@ -42,6 +42,7 b' connection_available = pytest.mark.skipi'
42 "enable_webhook_push_integration")
42 "enable_webhook_push_integration")
43 class TestVCSOperationsOnCustomIniConfig(object):
43 class TestVCSOperationsOnCustomIniConfig(object):
44
44
45 @connection_available
45 def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir):
46 def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir):
46 clone_url = rc_web_server.repo_clone_url(HG_REPO)
47 clone_url = rc_web_server.repo_clone_url(HG_REPO)
47 stdout, stderr = Command('/tmp').execute(
48 stdout, stderr = Command('/tmp').execute(
@@ -56,6 +57,7 b' class TestVCSOperationsOnCustomIniConfig'
56 assert 'ERROR' not in rc_log
57 assert 'ERROR' not in rc_log
57 assert "{'name': 'v1.0.0'," in rc_log
58 assert "{'name': 'v1.0.0'," in rc_log
58
59
60 @connection_available
59 def test_push_tag_with_commit_git(
61 def test_push_tag_with_commit_git(
60 self, rc_web_server, tmpdir):
62 self, rc_web_server, tmpdir):
61 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
63 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
@@ -71,6 +73,7 b' class TestVCSOperationsOnCustomIniConfig'
71 assert 'ERROR' not in rc_log
73 assert 'ERROR' not in rc_log
72 assert "{'name': 'v1.0.0'," in rc_log
74 assert "{'name': 'v1.0.0'," in rc_log
73
75
76 @connection_available
74 def test_push_tag_with_no_commit_git(
77 def test_push_tag_with_no_commit_git(
75 self, rc_web_server, tmpdir):
78 self, rc_web_server, tmpdir):
76 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
79 clone_url = rc_web_server.repo_clone_url(GIT_REPO)
@@ -7,7 +7,7 b''
7 [server:main]
7 [server:main]
8 ; COMMON HOST/IP CONFIG
8 ; COMMON HOST/IP CONFIG
9 host = 127.0.0.1
9 host = 127.0.0.1
10 port = 9900
10 port = 10010
11
11
12
12
13 ; ###########################
13 ; ###########################
@@ -22,6 +22,17 b' use = egg:gunicorn#main'
22 [app:main]
22 [app:main]
23 ; The %(here)s variable will be replaced with the absolute path of parent directory
23 ; The %(here)s variable will be replaced with the absolute path of parent directory
24 ; of this file
24 ; of this file
25 ; Each option in the app:main can be override by an environmental variable
26 ;
27 ;To override an option:
28 ;
29 ;RC_<KeyName>
30 ;Everything should be uppercase, . and - should be replaced by _.
31 ;For example, if you have these configuration settings:
32 ;rc_cache.repo_object.backend = foo
33 ;can be overridden by
34 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
35
25 use = egg:rhodecode-vcsserver
36 use = egg:rhodecode-vcsserver
26
37
27 ; Pyramid default locales, we need this to be set
38 ; Pyramid default locales, we need this to be set
@@ -30,11 +41,15 b' pyramid.default_locale_name = en'
30 ; default locale used by VCS systems
41 ; default locale used by VCS systems
31 locale = en_US.UTF-8
42 locale = en_US.UTF-8
32
43
33 ; path to binaries for vcsserver, it should be set by the installer
44 ; path to binaries (hg,git,svn) for vcsserver, it should be set by the installer
34 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
45 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
35 ; it can also be a path to nix-build output in case of development
46 ; or /usr/local/bin/rhodecode_bin/vcs_bin
36 core.binary_dir =
47 core.binary_dir =
37
48
49 ; Redis connection settings for svn integrations logic
50 ; This connection string needs to be the same on ce and vcsserver
51 vcs.svn.redis_conn = redis://redis:6379/0
52
38 ; Custom exception store path, defaults to TMPDIR
53 ; Custom exception store path, defaults to TMPDIR
39 ; This is used to store exception from RhodeCode in shared directory
54 ; This is used to store exception from RhodeCode in shared directory
40 #exception_tracker.store_path =
55 #exception_tracker.store_path =
@@ -52,14 +67,14 b' cache_dir = %(here)s/data'
52 ; ***************************************
67 ; ***************************************
53
68
54 ; `repo_object` cache settings for vcs methods for repositories
69 ; `repo_object` cache settings for vcs methods for repositories
55 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
70 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
56
71
57 ; cache auto-expires after N seconds
72 ; cache auto-expires after N seconds
58 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
73 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
59 rc_cache.repo_object.expiration_time = 2592000
74 #rc_cache.repo_object.expiration_time = 2592000
60
75
61 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
76 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
62 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
77 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
63
78
64 ; ***********************************************************
79 ; ***********************************************************
65 ; `repo_object` cache with redis backend
80 ; `repo_object` cache with redis backend
@@ -83,19 +98,32 b' rc_cache.repo_object.expiration_time = 2'
83 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
98 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
84 #rc_cache.repo_object.arguments.distributed_lock = true
99 #rc_cache.repo_object.arguments.distributed_lock = true
85
100
86 # legacy cache regions, please don't change
101 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
87 beaker.cache.regions = repo_object
102 #rc_cache.repo_object.arguments.lock_auto_renewal = true
88 beaker.cache.repo_object.type = memorylru
103
89 beaker.cache.repo_object.max_items = 100
104 ; Statsd client config, this is used to send metrics to statsd
90 # cache auto-expires after N seconds
105 ; We recommend setting statsd_exported and scrape them using Promethues
91 beaker.cache.repo_object.expire = 300
106 #statsd.enabled = false
92 beaker.cache.repo_object.enabled = true
107 #statsd.statsd_host = 0.0.0.0
108 #statsd.statsd_port = 8125
109 #statsd.statsd_prefix =
110 #statsd.statsd_ipv6 = false
93
111
112 ; configure logging automatically at server startup set to false
113 ; to use the below custom logging config.
114 ; RC_LOGGING_FORMATTER
115 ; RC_LOGGING_LEVEL
116 ; env variables can control the settings for logging in case of autoconfigure
94
117
118 #logging.autoconfigure = true
119
120 ; specify your own custom logging config file to configure logging
121 #logging.logging_conf_file = /path/to/custom_logging.ini
95
122
96 ; #####################
123 ; #####################
97 ; LOGGING CONFIGURATION
124 ; LOGGING CONFIGURATION
98 ; #####################
125 ; #####################
126
99 [loggers]
127 [loggers]
100 keys = root, vcsserver
128 keys = root, vcsserver
101
129
@@ -103,7 +131,7 b' keys = root, vcsserver'
103 keys = console
131 keys = console
104
132
105 [formatters]
133 [formatters]
106 keys = generic
134 keys = generic, json
107
135
108 ; #######
136 ; #######
109 ; LOGGERS
137 ; LOGGERS
@@ -113,12 +141,11 b' level = NOTSET'
113 handlers = console
141 handlers = console
114
142
115 [logger_vcsserver]
143 [logger_vcsserver]
116 level = DEBUG
144 level = INFO
117 handlers =
145 handlers =
118 qualname = vcsserver
146 qualname = vcsserver
119 propagate = 1
147 propagate = 1
120
148
121
122 ; ########
149 ; ########
123 ; HANDLERS
150 ; HANDLERS
124 ; ########
151 ; ########
@@ -127,6 +154,8 b' propagate = 1'
127 class = StreamHandler
154 class = StreamHandler
128 args = (sys.stderr, )
155 args = (sys.stderr, )
129 level = DEBUG
156 level = DEBUG
157 ; To enable JSON formatted logs replace 'generic' with 'json'
158 ; This allows sending properly formatted logs to grafana loki or elasticsearch
130 formatter = generic
159 formatter = generic
131
160
132 ; ##########
161 ; ##########
@@ -136,3 +165,7 b' formatter = generic'
136 [formatter_generic]
165 [formatter_generic]
137 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
166 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
138 datefmt = %Y-%m-%d %H:%M:%S
167 datefmt = %Y-%m-%d %H:%M:%S
168
169 [formatter_json]
170 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
171 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
General Comments 0
You need to be logged in to leave comments. Login now