##// END OF EJS Templates
merge: Resolved conflicts
andverb -
r1266:6139e442 merge v5.1.0 stable
parent child Browse files
Show More
@@ -0,0 +1,79 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import logging
20
21 from .backends.fanout_cache import FileSystemFanoutCache
22 from .backends.objectstore_cache import ObjectStoreCache
23
24 from .utils import archive_iterator # noqa
25 from .lock import ArchiveCacheGenerationLock # noqa
26
27 log = logging.getLogger(__name__)
28
29
30 cache_meta = None
31
32
33 def includeme(config):
34 return # vcsserver gets its config from rhodecode on a remote call
35 # init our cache at start
36 settings = config.get_settings()
37 get_archival_cache_store(settings)
38
39
40 def get_archival_config(config):
41
42 final_config = {
43
44 }
45
46 for k, v in config.items():
47 if k.startswith('archive_cache'):
48 final_config[k] = v
49
50 return final_config
51
52
53 def get_archival_cache_store(config, always_init=False):
54
55 global cache_meta
56 if cache_meta is not None and not always_init:
57 return cache_meta
58
59 config = get_archival_config(config)
60 backend = config['archive_cache.backend.type']
61
62 archive_cache_locking_url = config['archive_cache.locking.url']
63
64 match backend:
65 case 'filesystem':
66 d_cache = FileSystemFanoutCache(
67 locking_url=archive_cache_locking_url,
68 **config
69 )
70 case 'objectstore':
71 d_cache = ObjectStoreCache(
72 locking_url=archive_cache_locking_url,
73 **config
74 )
75 case _:
76 raise ValueError(f'archive_cache.backend.type only supports "filesystem" or "objectstore" got {backend} ')
77
78 cache_meta = d_cache
79 return cache_meta
@@ -0,0 +1,17 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
@@ -0,0 +1,372 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import os
20 import functools
21 import logging
22 import typing
23 import time
24 import zlib
25
26 from ...ext_json import json
27 from ..utils import StatsDB, NOT_GIVEN, ShardFileReader, EVICTION_POLICY, format_size
28 from ..lock import GenerationLock
29
30 log = logging.getLogger(__name__)
31
32
33 class BaseShard:
34 storage_type: str = ''
35 fs = None
36
37 @classmethod
38 def hash(cls, key):
39 """Compute portable hash for `key`.
40
41 :param key: key to hash
42 :return: hash value
43
44 """
45 mask = 0xFFFFFFFF
46 return zlib.adler32(key.encode('utf-8')) & mask # noqa
47
48 def _write_file(self, full_path, read_iterator, mode):
49 raise NotImplementedError
50
51 def _get_keyfile(self, key):
52 raise NotImplementedError
53
54 def random_filename(self):
55 raise NotImplementedError
56
57 def store(self, *args, **kwargs):
58 raise NotImplementedError
59
60 def _store(self, key, value_reader, metadata, mode):
61 (filename, # hash-name
62 full_path # full-path/hash-name
63 ) = self.random_filename()
64
65 key_file, key_file_path = self._get_keyfile(key)
66
67 # STORE METADATA
68 _metadata = {
69 "version": "v1",
70
71 "key_file": key_file, # this is the .key.json file storing meta
72 "key_file_path": key_file_path, # full path to key_file
73 "archive_key": key, # original name we stored archive under, e.g my-archive.zip
74 "archive_filename": filename, # the actual filename we stored that file under
75 "archive_full_path": full_path,
76
77 "store_time": time.time(),
78 "access_count": 0,
79 "access_time": 0,
80
81 "size": 0
82 }
83 if metadata:
84 _metadata.update(metadata)
85
86 read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
87 size, sha256 = self._write_file(full_path, read_iterator, mode)
88 _metadata['size'] = size
89 _metadata['sha256'] = sha256
90
91 # after archive is finished, we create a key to save the presence of the binary file
92 with self.fs.open(key_file_path, 'wb') as f:
93 f.write(json.dumps(_metadata))
94
95 return key, filename, size, _metadata
96
97 def fetch(self, *args, **kwargs):
98 raise NotImplementedError
99
100 def _fetch(self, key, retry, retry_attempts, retry_backoff,
101 presigned_url_expires: int = 0) -> tuple[ShardFileReader, dict]:
102 if retry is NOT_GIVEN:
103 retry = False
104 if retry_attempts is NOT_GIVEN:
105 retry_attempts = 0
106
107 if retry and retry_attempts > 0:
108 for attempt in range(1, retry_attempts + 1):
109 if key in self:
110 break
111 # we didn't find the key, wait retry_backoff N seconds, and re-check
112 time.sleep(retry_backoff)
113
114 if key not in self:
115 log.exception(f'requested key={key} not found in {self} retry={retry}, attempts={retry_attempts}')
116 raise KeyError(key)
117
118 key_file, key_file_path = self._get_keyfile(key)
119 with self.fs.open(key_file_path, 'rb') as f:
120 metadata = json.loads(f.read())
121
122 archive_path = metadata['archive_full_path']
123 if presigned_url_expires and presigned_url_expires > 0:
124 metadata['url'] = self.fs.url(archive_path, expires=presigned_url_expires)
125
126 try:
127 return ShardFileReader(self.fs.open(archive_path, 'rb')), metadata
128 finally:
129 # update usage stats, count and accessed
130 metadata["access_count"] = metadata.get("access_count", 0) + 1
131 metadata["access_time"] = time.time()
132 log.debug('Updated %s with access snapshot, access_count=%s access_time=%s',
133 key_file, metadata['access_count'], metadata['access_time'])
134 with self.fs.open(key_file_path, 'wb') as f:
135 f.write(json.dumps(metadata))
136
137 def remove(self, *args, **kwargs):
138 raise NotImplementedError
139
140 def _remove(self, key):
141 if key not in self:
142 log.exception(f'requested key={key} not found in {self}')
143 raise KeyError(key)
144
145 key_file, key_file_path = self._get_keyfile(key)
146 with self.fs.open(key_file_path, 'rb') as f:
147 metadata = json.loads(f.read())
148
149 archive_path = metadata['archive_full_path']
150 self.fs.rm(archive_path)
151 self.fs.rm(key_file_path)
152 return 1
153
154 @property
155 def storage_medium(self):
156 return getattr(self, self.storage_type)
157
158 @property
159 def key_suffix(self):
160 return 'key.json'
161
162 def __contains__(self, key):
163 """Return `True` if `key` matching item is found in cache.
164
165 :param key: key matching item
166 :return: True if key matching item
167
168 """
169 key_file, key_file_path = self._get_keyfile(key)
170 return self.fs.exists(key_file_path)
171
172
173 class BaseCache:
174 _locking_url: str = ''
175 _storage_path: str = ''
176 _config: dict = {}
177 retry = False
178 retry_attempts: int = 0
179 retry_backoff: int | float = 1
180 _shards = tuple()
181 shard_cls = BaseShard
182 # define the presigned url expiration, 0 == disabled
183 presigned_url_expires: int = 0
184
185 def __contains__(self, key):
186 """Return `True` if `key` matching item is found in cache.
187
188 :param key: key matching item
189 :return: True if key matching item
190
191 """
192 return self.has_key(key)
193
194 def __repr__(self):
195 return f'<{self.__class__.__name__}(storage={self._storage_path})>'
196
197 @classmethod
198 def gb_to_bytes(cls, gb):
199 return gb * (1024 ** 3)
200
201 @property
202 def storage_path(self):
203 return self._storage_path
204
205 @classmethod
206 def get_stats_db(cls):
207 return StatsDB()
208
209 def get_conf(self, key, pop=False):
210 if key not in self._config:
211 raise ValueError(f"No configuration key '{key}', please make sure it exists in archive_cache config")
212 val = self._config[key]
213 if pop:
214 del self._config[key]
215 return val
216
217 def _get_shard(self, key) -> shard_cls:
218 index = self._hash(key) % self._shard_count
219 shard = self._shards[index]
220 return shard
221
222 def _get_size(self, shard, archive_path):
223 raise NotImplementedError
224
225 def store(self, key, value_reader, metadata=None):
226 shard = self._get_shard(key)
227 return shard.store(key, value_reader, metadata)
228
229 def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN) -> tuple[typing.BinaryIO, dict]:
230 """
231 Return file handle corresponding to `key` from specific shard cache.
232 """
233 if retry is NOT_GIVEN:
234 retry = self.retry
235 if retry_attempts is NOT_GIVEN:
236 retry_attempts = self.retry_attempts
237 retry_backoff = self.retry_backoff
238 presigned_url_expires = self.presigned_url_expires
239
240 shard = self._get_shard(key)
241 return shard.fetch(key, retry=retry,
242 retry_attempts=retry_attempts,
243 retry_backoff=retry_backoff,
244 presigned_url_expires=presigned_url_expires)
245
246 def remove(self, key):
247 shard = self._get_shard(key)
248 return shard.remove(key)
249
250 def has_key(self, archive_key):
251 """Return `True` if `key` matching item is found in cache.
252
253 :param archive_key: key for item, this is a unique archive name we want to store data under. e.g my-archive-svn.zip
254 :return: True if key is found
255
256 """
257 shard = self._get_shard(archive_key)
258 return archive_key in shard
259
260 def iter_keys(self):
261 for shard in self._shards:
262 if shard.fs.exists(shard.storage_medium):
263 for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
264 for key_file_path in _files:
265 if key_file_path.endswith(shard.key_suffix):
266 yield shard, key_file_path
267
268 def get_lock(self, lock_key):
269 return GenerationLock(lock_key, self._locking_url)
270
271 def evict(self, policy=None, size_limit=None) -> dict:
272 """
273 Remove old items based on the conditions
274
275
276 explanation of this algo:
277 iterate over each shard, then for each shard iterate over the .key files
278 read the key files metadata stored. This gives us a full list of keys, cached_archived, their size and
279 access data, time creation, and access counts.
280
281 Store that into a memory DB in order we can run different sorting strategies easily.
282 Summing the size is a sum sql query.
283
284 Then we run a sorting strategy based on eviction policy.
285 We iterate over sorted keys, and remove each checking if we hit the overall limit.
286 """
287 removal_info = {
288 "removed_items": 0,
289 "removed_size": 0
290 }
291 policy = policy or self._eviction_policy
292 size_limit = size_limit or self._cache_size_limit
293
294 select_policy = EVICTION_POLICY[policy]['evict']
295
296 log.debug('Running eviction policy \'%s\', and checking for size limit: %s',
297 policy, format_size(size_limit))
298
299 if select_policy is None:
300 return removal_info
301
302 db = self.get_stats_db()
303
304 data = []
305 cnt = 1
306
307 for shard, key_file in self.iter_keys():
308 with shard.fs.open(os.path.join(shard.storage_medium, key_file), 'rb') as f:
309 metadata = json.loads(f.read())
310
311 key_file_path = os.path.join(shard.storage_medium, key_file)
312
313 archive_key = metadata['archive_key']
314 archive_path = metadata['archive_full_path']
315
316 size = metadata.get('size')
317 if not size:
318 # in case we don't have size re-calc it...
319 size = self._get_size(shard, archive_path)
320
321 data.append([
322 cnt,
323 key_file,
324 key_file_path,
325 archive_key,
326 archive_path,
327 metadata.get('store_time', 0),
328 metadata.get('access_time', 0),
329 metadata.get('access_count', 0),
330 size,
331 ])
332 cnt += 1
333
334 # Insert bulk data using executemany
335 db.bulk_insert(data)
336
337 total_size = db.get_total_size()
338 log.debug('Analyzed %s keys, occupying: %s, running eviction to match %s',
339 len(data), format_size(total_size), format_size(size_limit))
340
341 removed_items = 0
342 removed_size = 0
343 for key_file, archive_key, size in db.get_sorted_keys(select_policy):
344 # simulate removal impact BEFORE removal
345 total_size -= size
346
347 if total_size <= size_limit:
348 # we obtained what we wanted...
349 break
350
351 self.remove(archive_key)
352 removed_items += 1
353 removed_size += size
354 removal_info['removed_items'] = removed_items
355 removal_info['removed_size'] = removed_size
356 log.debug('Removed %s cache archives, and reduced size by: %s',
357 removed_items, format_size(removed_size))
358 return removal_info
359
360 def get_statistics(self):
361 total_files = 0
362 total_size = 0
363 meta = {}
364
365 for shard, key_file in self.iter_keys():
366 json_key = f"{shard.storage_medium}/{key_file}"
367 with shard.fs.open(json_key, 'rb') as f:
368 total_files += 1
369 metadata = json.loads(f.read())
370 total_size += metadata['size']
371
372 return total_files, total_size, meta
@@ -0,0 +1,177 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import codecs
20 import hashlib
21 import logging
22 import os
23 import typing
24
25 import fsspec
26
27 from .base import BaseCache, BaseShard
28 from ..utils import ShardFileReader, NOT_GIVEN
29 from ...type_utils import str2bool
30
31 log = logging.getLogger(__name__)
32
33
34 class FileSystemShard(BaseShard):
35
36 def __init__(self, index, directory, directory_folder, fs, **settings):
37 self._index: int = index
38 self._directory: str = directory
39 self._directory_folder: str = directory_folder
40 self.storage_type: str = 'directory'
41
42 self.fs = fs
43
44 @property
45 def directory(self) -> str:
46 """Cache directory final path."""
47 return os.path.join(self._directory, self._directory_folder)
48
49 def _get_keyfile(self, archive_key) -> tuple[str, str]:
50 key_file: str = f'{archive_key}.{self.key_suffix}'
51 return key_file, os.path.join(self.directory, key_file)
52
53 def _get_writer(self, path, mode):
54 for count in range(1, 11):
55 try:
56 # Another cache may have deleted the directory before
57 # the file could be opened.
58 return self.fs.open(path, mode)
59 except OSError:
60 if count == 10:
61 # Give up after 10 tries to open the file.
62 raise
63 continue
64
65 def _write_file(self, full_path, iterator, mode):
66
67 # ensure dir exists
68 destination, _ = os.path.split(full_path)
69 if not self.fs.exists(destination):
70 self.fs.makedirs(destination)
71
72 writer = self._get_writer(full_path, mode)
73
74 digest = hashlib.sha256()
75 with writer:
76 size = 0
77 for chunk in iterator:
78 size += len(chunk)
79 digest.update(chunk)
80 writer.write(chunk)
81 writer.flush()
82 # Get the file descriptor
83 fd = writer.fileno()
84
85 # Sync the file descriptor to disk, helps with NFS cases...
86 os.fsync(fd)
87 sha256 = digest.hexdigest()
88 log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
89 return size, sha256
90
91 def store(self, key, value_reader, metadata: dict | None = None):
92 return self._store(key, value_reader, metadata, mode='xb')
93
94 def fetch(self, key, retry=NOT_GIVEN,
95 retry_attempts=NOT_GIVEN, retry_backoff=1, **kwargs) -> tuple[ShardFileReader, dict]:
96 return self._fetch(key, retry, retry_attempts, retry_backoff)
97
98 def remove(self, key):
99 return self._remove(key)
100
101 def random_filename(self):
102 """Return filename and full-path tuple for file storage.
103
104 Filename will be a randomly generated 28 character hexadecimal string
105 with ".archive_cache" suffixed. Two levels of sub-directories will be used to
106 reduce the size of directories. On older filesystems, lookups in
107 directories with many files may be slow.
108 """
109
110 hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
111
112 archive_name = hex_name[4:] + '.archive_cache'
113 filename = f"{hex_name[:2]}/{hex_name[2:4]}/{archive_name}"
114
115 full_path = os.path.join(self.directory, filename)
116 return archive_name, full_path
117
118 def __repr__(self):
119 return f'{self.__class__.__name__}(index={self._index}, dir={self.directory})'
120
121
122 class FileSystemFanoutCache(BaseCache):
123 shard_name: str = 'shard_{:03d}'
124 shard_cls = FileSystemShard
125
126 def __init__(self, locking_url, **settings):
127 """
128 Initialize file system cache instance.
129
130 :param str locking_url: redis url for a lock
131 :param settings: settings dict
132
133 """
134 self._locking_url = locking_url
135 self._config = settings
136 cache_dir = self.get_conf('archive_cache.filesystem.store_dir')
137 directory = str(cache_dir)
138 directory = os.path.expanduser(directory)
139 directory = os.path.expandvars(directory)
140 self._directory = directory
141 self._storage_path = directory # common path for all from BaseCache
142
143 self._shard_count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
144 if self._shard_count < 1:
145 raise ValueError('cache_shards must be 1 or more')
146
147 self._eviction_policy = self.get_conf('archive_cache.filesystem.eviction_policy', pop=True)
148 self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.filesystem.cache_size_gb')))
149
150 self.retry = str2bool(self.get_conf('archive_cache.filesystem.retry', pop=True))
151 self.retry_attempts = int(self.get_conf('archive_cache.filesystem.retry_attempts', pop=True))
152 self.retry_backoff = int(self.get_conf('archive_cache.filesystem.retry_backoff', pop=True))
153
154 log.debug('Initializing %s archival cache instance', self)
155 fs = fsspec.filesystem('file')
156 # check if it's ok to write, and re-create the archive cache main dir
157 # A directory is the virtual equivalent of a physical file cabinet.
158 # In other words, it's a container for organizing digital data.
159 # Unlike a folder, which can only store files, a directory can store files,
160 # subdirectories, and other directories.
161 if not fs.exists(self._directory):
162 fs.makedirs(self._directory, exist_ok=True)
163
164 self._shards = tuple(
165 self.shard_cls(
166 index=num,
167 directory=directory,
168 directory_folder=self.shard_name.format(num),
169 fs=fs,
170 **settings,
171 )
172 for num in range(self._shard_count)
173 )
174 self._hash = self._shards[0].hash
175
176 def _get_size(self, shard, archive_path):
177 return os.stat(archive_path).st_size
@@ -0,0 +1,173 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import codecs
20 import hashlib
21 import logging
22 import os
23 import typing
24
25 import fsspec
26
27 from .base import BaseCache, BaseShard
28 from ..utils import ShardFileReader, NOT_GIVEN
29 from ...type_utils import str2bool
30
31 log = logging.getLogger(__name__)
32
33
34 class S3Shard(BaseShard):
35
36 def __init__(self, index, bucket, bucket_folder, fs, **settings):
37 self._index: int = index
38 self._bucket_folder: str = bucket_folder
39 self.storage_type: str = 'bucket'
40 self._bucket_main: str = bucket
41
42 self.fs = fs
43
44 @property
45 def bucket(self) -> str:
46 """Cache bucket final path."""
47 return os.path.join(self._bucket_main, self._bucket_folder)
48
49 def _get_keyfile(self, archive_key) -> tuple[str, str]:
50 key_file: str = f'{archive_key}-{self.key_suffix}'
51 return key_file, os.path.join(self.bucket, key_file)
52
53 def _get_writer(self, path, mode):
54 return self.fs.open(path, 'wb')
55
56 def _write_file(self, full_path, iterator, mode):
57
58 # ensure folder in bucket exists
59 destination = self.bucket
60 if not self.fs.exists(destination):
61 self.fs.mkdir(destination, s3_additional_kwargs={})
62
63 writer = self._get_writer(full_path, mode)
64
65 digest = hashlib.sha256()
66 with writer:
67 size = 0
68 for chunk in iterator:
69 size += len(chunk)
70 digest.update(chunk)
71 writer.write(chunk)
72
73 sha256 = digest.hexdigest()
74 log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
75 return size, sha256
76
77 def store(self, key, value_reader, metadata: dict | None = None):
78 return self._store(key, value_reader, metadata, mode='wb')
79
80 def fetch(self, key, retry=NOT_GIVEN,
81 retry_attempts=NOT_GIVEN, retry_backoff=1,
82 presigned_url_expires: int = 0) -> tuple[ShardFileReader, dict]:
83 return self._fetch(key, retry, retry_attempts, retry_backoff, presigned_url_expires=presigned_url_expires)
84
85 def remove(self, key):
86 return self._remove(key)
87
88 def random_filename(self):
89 """Return filename and full-path tuple for file storage.
90
91 Filename will be a randomly generated 28 character hexadecimal string
92 with ".archive_cache" suffixed. Two levels of sub-directories will be used to
93 reduce the size of directories. On older filesystems, lookups in
94 directories with many files may be slow.
95 """
96
97 hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
98
99 archive_name = hex_name[4:] + '.archive_cache'
100 filename = f"{hex_name[:2]}-{hex_name[2:4]}-{archive_name}"
101
102 full_path = os.path.join(self.bucket, filename)
103 return archive_name, full_path
104
105 def __repr__(self):
106 return f'{self.__class__.__name__}(index={self._index}, bucket={self.bucket})'
107
108
109 class ObjectStoreCache(BaseCache):
110 shard_name: str = 'shard-{:03d}'
111 shard_cls = S3Shard
112
113 def __init__(self, locking_url, **settings):
114 """
115 Initialize objectstore cache instance.
116
117 :param str locking_url: redis url for a lock
118 :param settings: settings dict
119
120 """
121 self._locking_url = locking_url
122 self._config = settings
123
124 objectstore_url = self.get_conf('archive_cache.objectstore.url')
125 self._storage_path = objectstore_url # common path for all from BaseCache
126
127 self._shard_count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
128 if self._shard_count < 1:
129 raise ValueError('cache_shards must be 1 or more')
130
131 self._bucket = settings.pop('archive_cache.objectstore.bucket')
132 if not self._bucket:
133 raise ValueError('archive_cache.objectstore.bucket needs to have a value')
134
135 self._eviction_policy = self.get_conf('archive_cache.objectstore.eviction_policy', pop=True)
136 self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.objectstore.cache_size_gb')))
137
138 self.retry = str2bool(self.get_conf('archive_cache.objectstore.retry', pop=True))
139 self.retry_attempts = int(self.get_conf('archive_cache.objectstore.retry_attempts', pop=True))
140 self.retry_backoff = int(self.get_conf('archive_cache.objectstore.retry_backoff', pop=True))
141
142 endpoint_url = settings.pop('archive_cache.objectstore.url')
143 key = settings.pop('archive_cache.objectstore.key')
144 secret = settings.pop('archive_cache.objectstore.secret')
145 region = settings.pop('archive_cache.objectstore.region')
146
147 log.debug('Initializing %s archival cache instance', self)
148
149 fs = fsspec.filesystem(
150 's3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret, client_kwargs={'region_name': region}
151 )
152
153 # init main bucket
154 if not fs.exists(self._bucket):
155 fs.mkdir(self._bucket)
156
157 self._shards = tuple(
158 self.shard_cls(
159 index=num,
160 bucket=self._bucket,
161 bucket_folder=self.shard_name.format(num),
162 fs=fs,
163 **settings,
164 )
165 for num in range(self._shard_count)
166 )
167 self._hash = self._shards[0].hash
168
169 def _get_size(self, shard, archive_path):
170 return shard.fs.info(archive_path)['size']
171
172 def set_presigned_url_expiry(self, val: int) -> None:
173 self.presigned_url_expires = val
@@ -0,0 +1,62 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import redis
20 from .._vendor import redis_lock
21
22
23 class ArchiveCacheGenerationLock(Exception):
24 pass
25
26
27 class GenerationLock:
28 """
29 Locking mechanism that detects if a lock is acquired
30
31 with GenerationLock(lock_key):
32 compute_archive()
33 """
34 lock_timeout = 7200
35
36 def __init__(self, lock_key, url):
37 self.lock_key = lock_key
38 self._create_client(url)
39 self.lock = self.get_lock()
40
41 def _create_client(self, url):
42 connection_pool = redis.ConnectionPool.from_url(url)
43 self.writer_client = redis.StrictRedis(
44 connection_pool=connection_pool
45 )
46 self.reader_client = self.writer_client
47
48 def get_lock(self):
49 return redis_lock.Lock(
50 redis_client=self.writer_client,
51 name=self.lock_key,
52 expire=self.lock_timeout,
53 strict=True
54 )
55
56 def __enter__(self):
57 acquired = self.lock.acquire(blocking=False)
58 if not acquired:
59 raise ArchiveCacheGenerationLock('Failed to create a lock')
60
61 def __exit__(self, exc_type, exc_val, exc_tb):
62 self.lock.release()
@@ -0,0 +1,134 b''
1 # Copyright (C) 2015-2024 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import sqlite3
20 import s3fs.core
21
22 NOT_GIVEN = -917
23
24
25 EVICTION_POLICY = {
26 'none': {
27 'evict': None,
28 },
29 'least-recently-stored': {
30 'evict': 'SELECT {fields} FROM archive_cache ORDER BY store_time',
31 },
32 'least-recently-used': {
33 'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_time',
34 },
35 'least-frequently-used': {
36 'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_count',
37 },
38 }
39
40
41 def archive_iterator(_reader, block_size: int = 4096 * 512):
42 # 4096 * 64 = 64KB
43 while 1:
44 data = _reader.read(block_size)
45 if not data:
46 break
47 yield data
48
49
50 def format_size(size):
51 # Convert size in bytes to a human-readable format (e.g., KB, MB, GB)
52 for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
53 if size < 1024:
54 return f"{size:.2f} {unit}"
55 size /= 1024
56
57
58 class StatsDB:
59
60 def __init__(self):
61 self.connection = sqlite3.connect(':memory:')
62 self._init_db()
63
64 def _init_db(self):
65 qry = '''
66 CREATE TABLE IF NOT EXISTS archive_cache (
67 rowid INTEGER PRIMARY KEY,
68 key_file TEXT,
69 key_file_path TEXT,
70 archive_key TEXT,
71 archive_path TEXT,
72 store_time REAL,
73 access_time REAL,
74 access_count INTEGER DEFAULT 0,
75 size INTEGER DEFAULT 0
76 )
77 '''
78
79 self.sql(qry)
80 self.connection.commit()
81
82 @property
83 def sql(self):
84 return self.connection.execute
85
86 def bulk_insert(self, rows):
87 qry = '''
88 INSERT INTO archive_cache (
89 rowid,
90 key_file,
91 key_file_path,
92 archive_key,
93 archive_path,
94 store_time,
95 access_time,
96 access_count,
97 size
98 )
99 VALUES (
100 ?, ?, ?, ?, ?, ?, ?, ?, ?
101 )
102 '''
103 cursor = self.connection.cursor()
104 cursor.executemany(qry, rows)
105 self.connection.commit()
106
107 def get_total_size(self):
108 qry = 'SELECT COALESCE(SUM(size), 0) FROM archive_cache'
109 ((total_size,),) = self.sql(qry).fetchall()
110 return total_size
111
112 def get_sorted_keys(self, select_policy):
113 select_policy_qry = select_policy.format(fields='key_file, archive_key, size')
114 return self.sql(select_policy_qry).fetchall()
115
116
117 class ShardFileReader:
118
119 def __init__(self, file_like_reader):
120 self._file_like_reader = file_like_reader
121
122 def __getattr__(self, item):
123 if isinstance(self._file_like_reader, s3fs.core.S3File):
124 match item:
125 case 'name':
126 # S3 FileWrapper doesn't support name attribute, and we use it
127 return self._file_like_reader.full_name
128 case _:
129 return getattr(self._file_like_reader, item)
130 else:
131 return getattr(self._file_like_reader, item)
132
133 def __repr__(self):
134 return f'<{self.__class__.__name__}={self._file_like_reader}>'
@@ -0,0 +1,111 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import logging
19 import redis
20
21 from ..lib import rc_cache
22 from ..lib.ext_json import json
23
24
25 log = logging.getLogger(__name__)
26
27 redis_client = None
28
29
30 class RedisTxnClient:
31
32 def __init__(self, url):
33 self.url = url
34 self._create_client(url)
35
36 def _create_client(self, url):
37 connection_pool = redis.ConnectionPool.from_url(url)
38 self.writer_client = redis.StrictRedis(
39 connection_pool=connection_pool
40 )
41 self.reader_client = self.writer_client
42
43 def set(self, key, value):
44 self.writer_client.set(key, value)
45
46 def get(self, key):
47 return self.reader_client.get(key)
48
49 def delete(self, key):
50 self.writer_client.delete(key)
51
52
53 def get_redis_client(url=''):
54
55 global redis_client
56 if redis_client is not None:
57 return redis_client
58 if not url:
59 from vcsserver import CONFIG
60 url = CONFIG['vcs.svn.redis_conn']
61 redis_client = RedisTxnClient(url)
62 return redis_client
63
64
65 def get_txn_id_data_key(repo_path, svn_txn_id):
66 log.debug('svn-txn-id: %s, obtaining data path', svn_txn_id)
67 repo_key = rc_cache.utils.compute_key_from_params(repo_path)
68 final_key = f'{repo_key}.{svn_txn_id}.svn_txn_id'
69 log.debug('computed final key: %s', final_key)
70
71 return final_key
72
73
74 def store_txn_id_data(repo_path, svn_txn_id, data_dict):
75 log.debug('svn-txn-id: %s, storing data', svn_txn_id)
76
77 if not svn_txn_id:
78 log.warning('Cannot store txn_id because it is empty')
79 return
80
81 redis_conn = get_redis_client()
82
83 store_key = get_txn_id_data_key(repo_path, svn_txn_id)
84 store_data = json.dumps(data_dict)
85 redis_conn.set(store_key, store_data)
86
87
88 def get_txn_id_from_store(repo_path, svn_txn_id, rm_on_read=False):
89 """
90 Reads txn_id from store and if present returns the data for callback manager
91 """
92 log.debug('svn-txn-id: %s, retrieving data', svn_txn_id)
93 redis_conn = get_redis_client()
94
95 store_key = get_txn_id_data_key(repo_path, svn_txn_id)
96 data = {}
97 redis_conn.get(store_key)
98 raw_data = 'not-set'
99 try:
100 raw_data = redis_conn.get(store_key)
101 if not raw_data:
102 raise ValueError(f'Failed to get txn_id metadata, from store: {store_key}')
103 data = json.loads(raw_data)
104 except Exception:
105 log.exception('Failed to get txn_id metadata: %s', raw_data)
106
107 if rm_on_read:
108 log.debug('Cleaning up txn_id at %s', store_key)
109 redis_conn.delete(store_key)
110
111 return data
@@ -1,5 +1,5 b''
1 [bumpversion]
1 [bumpversion]
2 current_version = 5.0.3
2 current_version = 5.1.0
3 message = release: Bump version {current_version} to {new_version}
3 message = release: Bump version {current_version} to {new_version}
4
4
5 [bumpversion:file:vcsserver/VERSION]
5 [bumpversion:file:vcsserver/VERSION]
@@ -1,139 +1,144 b''
1 # required for pushd to work..
1 # required for pushd to work..
2 SHELL = /bin/bash
2 SHELL = /bin/bash
3
3
4
4
5 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
5 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
6 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
6 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
7
7
8 .PHONY: clean
8 .PHONY: clean
9 ## Cleanup compiled and cache py files
9 ## Cleanup compiled and cache py files
10 clean:
10 clean:
11 make test-clean
11 make test-clean
12 find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
12 find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
13 find . -type d -name "build" -prune -exec rm -rf '{}' ';'
13 find . -type d -name "build" -prune -exec rm -rf '{}' ';'
14
14
15
15
16 .PHONY: test
16 .PHONY: test
17 ## run test-clean and tests
17 ## run test-clean and tests
18 test:
18 test:
19 make test-clean
19 make test-clean
20 make test-only
20 make test-only
21
21
22
22
23 .PHONY: test-clean
23 .PHONY: test-clean
24 ## run test-clean and tests
24 ## run test-clean and tests
25 test-clean:
25 test-clean:
26 rm -rf coverage.xml htmlcov junit.xml pylint.log result
26 rm -rf coverage.xml htmlcov junit.xml pylint.log result
27 find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
27 find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
28 find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
28 find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
29
29
30
30
31 .PHONY: test-only
31 .PHONY: test-only
32 ## Run tests only without cleanup
32 ## Run tests only without cleanup
33 test-only:
33 test-only:
34 PYTHONHASHSEED=random \
34 PYTHONHASHSEED=random \
35 py.test -x -vv -r xw -p no:sugar \
35 py.test -x -vv -r xw -p no:sugar \
36 --cov-report=term-missing --cov-report=html \
36 --cov-report=term-missing --cov-report=html \
37 --cov=vcsserver vcsserver
37 --cov=vcsserver vcsserver
38
38
39
39
40 .PHONY: ruff-check
40 .PHONY: ruff-check
41 ## run a ruff analysis
41 ## run a ruff analysis
42 ruff-check:
42 ruff-check:
43 ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev .
43 ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev .
44
44
45
46 .PHONY: pip-packages
45 .PHONY: pip-packages
47 ## Show outdated packages
46 ## Show outdated packages
48 pip-packages:
47 pip-packages:
49 python ${OUTDATED_PACKAGES}
48 python ${OUTDATED_PACKAGES}
50
49
51
50
52 .PHONY: build
51 .PHONY: build
53 ## Build sdist/egg
52 ## Build sdist/egg
54 build:
53 build:
55 python -m build
54 python -m build
56
55
57
56
58 .PHONY: dev-sh
57 .PHONY: dev-sh
59 ## make dev-sh
58 ## make dev-sh
60 dev-sh:
59 dev-sh:
61 sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
60 sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
62 sudo apt-get update
61 sudo apt-get update
63 sudo apt-get install -y zsh carapace-bin
62 sudo apt-get install -y zsh carapace-bin
64 rm -rf /home/rhodecode/.oh-my-zsh
63 rm -rf /home/rhodecode/.oh-my-zsh
65 curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
64 curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
66 echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
65 @echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
67 PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
66 @echo "${RC_DEV_CMD_HELP}"
67 @PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
68
69
70 .PHONY: dev-cleanup
71 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
72 dev-cleanup:
73 pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
74 rm -rf /tmp/*
68
75
69
76
70 .PHONY: dev-env
77 .PHONY: dev-env
71 ## make dev-env based on the requirements files and install develop of packages
78 ## make dev-env based on the requirements files and install develop of packages
72 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
79 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
73 dev-env:
80 dev-env:
81 sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
74 pip install build virtualenv
82 pip install build virtualenv
75 pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
83 pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
76 pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
84 pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
77 pip install -e .
85 pip install -e .
78
86
79
87
80 .PHONY: sh
88 .PHONY: sh
81 ## shortcut for make dev-sh dev-env
89 ## shortcut for make dev-sh dev-env
82 sh:
90 sh:
83 make dev-env
91 make dev-env
84 make dev-sh
92 make dev-sh
85
93
86
94
87 .PHONY: dev-srv
95 ## Allows changes of workers e.g make dev-srv-g workers=2
88 ## run develop server instance, docker exec -it $(docker ps -q --filter 'name=dev-enterprise-ce') /bin/bash
96 workers?=1
89 dev-srv:
90 pserve --reload .dev/dev.ini
91
97
92
98 .PHONY: dev-srv
93 .PHONY: dev-srv-g
99 ## run gunicorn web server with reloader, use workers=N to set multiworker mode
94 ## run gunicorn multi process workers
100 dev-srv:
95 dev-srv-g:
101 gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10010 --config=.dev/gunicorn_config.py --reload --workers=$(workers)
96 gunicorn --workers=4 --paste .dev/dev.ini --bind=0.0.0.0:10010 --config=.dev/gunicorn_config.py
97
102
98
103
99 # Default command on calling make
104 # Default command on calling make
100 .DEFAULT_GOAL := show-help
105 .DEFAULT_GOAL := show-help
101
106
102 .PHONY: show-help
107 .PHONY: show-help
103 show-help:
108 show-help:
104 @echo "$$(tput bold)Available rules:$$(tput sgr0)"
109 @echo "$$(tput bold)Available rules:$$(tput sgr0)"
105 @echo
110 @echo
106 @sed -n -e "/^## / { \
111 @sed -n -e "/^## / { \
107 h; \
112 h; \
108 s/.*//; \
113 s/.*//; \
109 :doc" \
114 :doc" \
110 -e "H; \
115 -e "H; \
111 n; \
116 n; \
112 s/^## //; \
117 s/^## //; \
113 t doc" \
118 t doc" \
114 -e "s/:.*//; \
119 -e "s/:.*//; \
115 G; \
120 G; \
116 s/\\n## /---/; \
121 s/\\n## /---/; \
117 s/\\n/ /g; \
122 s/\\n/ /g; \
118 p; \
123 p; \
119 }" ${MAKEFILE_LIST} \
124 }" ${MAKEFILE_LIST} \
120 | LC_ALL='C' sort --ignore-case \
125 | LC_ALL='C' sort --ignore-case \
121 | awk -F '---' \
126 | awk -F '---' \
122 -v ncol=$$(tput cols) \
127 -v ncol=$$(tput cols) \
123 -v indent=19 \
128 -v indent=19 \
124 -v col_on="$$(tput setaf 6)" \
129 -v col_on="$$(tput setaf 6)" \
125 -v col_off="$$(tput sgr0)" \
130 -v col_off="$$(tput sgr0)" \
126 '{ \
131 '{ \
127 printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
132 printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
128 n = split($$2, words, " "); \
133 n = split($$2, words, " "); \
129 line_length = ncol - indent; \
134 line_length = ncol - indent; \
130 for (i = 1; i <= n; i++) { \
135 for (i = 1; i <= n; i++) { \
131 line_length -= length(words[i]) + 1; \
136 line_length -= length(words[i]) + 1; \
132 if (line_length <= 0) { \
137 if (line_length <= 0) { \
133 line_length = ncol - indent - length(words[i]) - 1; \
138 line_length = ncol - indent - length(words[i]) - 1; \
134 printf "\n%*s ", -indent, " "; \
139 printf "\n%*s ", -indent, " "; \
135 } \
140 } \
136 printf "%s ", words[i]; \
141 printf "%s ", words[i]; \
137 } \
142 } \
138 printf "\n"; \
143 printf "\n"; \
139 }'
144 }'
@@ -1,204 +1,191 b''
1 #
2
1
3 ; #################################
2 ; #################################
4 ; RHODECODE VCSSERVER CONFIGURATION
3 ; RHODECODE VCSSERVER CONFIGURATION
5 ; #################################
4 ; #################################
6
5
7 [server:main]
6 [server:main]
8 ; COMMON HOST/IP CONFIG
7 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
8 ; Host port for gunicorn are controlled by gunicorn_conf.py
9 host = 0.0.0.0
9 host = 0.0.0.0
10 port = 10010
10 port = 10010
11
11
12 ; ##################################################
13 ; WAITRESS WSGI SERVER - Recommended for Development
14 ; ##################################################
15
16 ; use server type
17 use = egg:waitress#main
18
19 ; number of worker threads
20 threads = 5
21
22 ; MAX BODY SIZE 100GB
23 max_request_body_size = 107374182400
24
25 ; Use poll instead of select, fixes file descriptors limits problems.
26 ; May not work on old windows systems.
27 asyncore_use_poll = true
28
29
12
30 ; ###########################
13 ; ###########################
31 ; GUNICORN APPLICATION SERVER
14 ; GUNICORN APPLICATION SERVER
32 ; ###########################
15 ; ###########################
33
16
34 ; run with gunicorn --paste rhodecode.ini
17 ; run with gunicorn --config gunicorn_conf.py --paste vcsserver.ini
35
18
36 ; Module to use, this setting shouldn't be changed
19 ; Module to use, this setting shouldn't be changed
37 #use = egg:gunicorn#main
20 use = egg:gunicorn#main
38
21
39 [app:main]
22 [app:main]
40 ; The %(here)s variable will be replaced with the absolute path of parent directory
23 ; The %(here)s variable will be replaced with the absolute path of parent directory
41 ; of this file
24 ; of this file
42 ; Each option in the app:main can be override by an environmental variable
25 ; Each option in the app:main can be override by an environmental variable
43 ;
26 ;
44 ;To override an option:
27 ;To override an option:
45 ;
28 ;
46 ;RC_<KeyName>
29 ;RC_<KeyName>
47 ;Everything should be uppercase, . and - should be replaced by _.
30 ;Everything should be uppercase, . and - should be replaced by _.
48 ;For example, if you have these configuration settings:
31 ;For example, if you have these configuration settings:
49 ;rc_cache.repo_object.backend = foo
32 ;rc_cache.repo_object.backend = foo
50 ;can be overridden by
33 ;can be overridden by
51 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
34 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
52
35
53 use = egg:rhodecode-vcsserver
36 use = egg:rhodecode-vcsserver
54
37
55
38
56 ; #############
39 ; #############
57 ; DEBUG OPTIONS
40 ; DEBUG OPTIONS
58 ; #############
41 ; #############
59
42
60 # During development the we want to have the debug toolbar enabled
43 # During development the we want to have the debug toolbar enabled
61 pyramid.includes =
44 pyramid.includes =
62 pyramid_debugtoolbar
45 pyramid_debugtoolbar
63
46
64 debugtoolbar.hosts = 0.0.0.0/0
47 debugtoolbar.hosts = 0.0.0.0/0
65 debugtoolbar.exclude_prefixes =
48 debugtoolbar.exclude_prefixes =
66 /css
49 /css
67 /fonts
50 /fonts
68 /images
51 /images
69 /js
52 /js
70
53
71 ; #################
54 ; #################
72 ; END DEBUG OPTIONS
55 ; END DEBUG OPTIONS
73 ; #################
56 ; #################
74
57
75 ; Pyramid default locales, we need this to be set
58 ; Pyramid default locales, we need this to be set
76 #pyramid.default_locale_name = en
59 #pyramid.default_locale_name = en
77
60
78 ; default locale used by VCS systems
61 ; default locale used by VCS systems
79 #locale = en_US.UTF-8
62 #locale = en_US.UTF-8
80
63
81 ; path to binaries for vcsserver, it should be set by the installer
64 ; path to binaries (hg,git,svn) for vcsserver, it should be set by the installer
82 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
65 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
83 ; it can also be a path to nix-build output in case of development
66 ; or /usr/local/bin/rhodecode_bin/vcs_bin
84 core.binary_dir = ""
67 core.binary_dir =
68
69 ; Redis connection settings for svn integrations logic
70 ; This connection string needs to be the same on ce and vcsserver
71 vcs.svn.redis_conn = redis://redis:6379/0
85
72
86 ; Custom exception store path, defaults to TMPDIR
73 ; Custom exception store path, defaults to TMPDIR
87 ; This is used to store exception from RhodeCode in shared directory
74 ; This is used to store exception from RhodeCode in shared directory
88 #exception_tracker.store_path =
75 #exception_tracker.store_path =
89
76
90 ; #############
77 ; #############
91 ; DOGPILE CACHE
78 ; DOGPILE CACHE
92 ; #############
79 ; #############
93
80
94 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
81 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
95 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
82 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
96 #cache_dir = %(here)s/data
83 #cache_dir = %(here)s/data
97
84
98 ; ***************************************
85 ; ***************************************
99 ; `repo_object` cache, default file based
86 ; `repo_object` cache, default file based
100 ; ***************************************
87 ; ***************************************
101
88
102 ; `repo_object` cache settings for vcs methods for repositories
89 ; `repo_object` cache settings for vcs methods for repositories
103 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
90 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
104
91
105 ; cache auto-expires after N seconds
92 ; cache auto-expires after N seconds
106 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
93 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
107 #rc_cache.repo_object.expiration_time = 2592000
94 #rc_cache.repo_object.expiration_time = 2592000
108
95
109 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
96 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
110 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
97 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
111
98
112 ; ***********************************************************
99 ; ***********************************************************
113 ; `repo_object` cache with redis backend
100 ; `repo_object` cache with redis backend
114 ; recommended for larger instance, and for better performance
101 ; recommended for larger instance, and for better performance
115 ; ***********************************************************
102 ; ***********************************************************
116
103
117 ; `repo_object` cache settings for vcs methods for repositories
104 ; `repo_object` cache settings for vcs methods for repositories
118 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
105 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
119
106
120 ; cache auto-expires after N seconds
107 ; cache auto-expires after N seconds
121 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
108 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
122 #rc_cache.repo_object.expiration_time = 2592000
109 #rc_cache.repo_object.expiration_time = 2592000
123
110
124 ; redis_expiration_time needs to be greater then expiration_time
111 ; redis_expiration_time needs to be greater then expiration_time
125 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
112 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
126
113
127 #rc_cache.repo_object.arguments.host = localhost
114 #rc_cache.repo_object.arguments.host = localhost
128 #rc_cache.repo_object.arguments.port = 6379
115 #rc_cache.repo_object.arguments.port = 6379
129 #rc_cache.repo_object.arguments.db = 5
116 #rc_cache.repo_object.arguments.db = 5
130 #rc_cache.repo_object.arguments.socket_timeout = 30
117 #rc_cache.repo_object.arguments.socket_timeout = 30
131 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
118 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
132 #rc_cache.repo_object.arguments.distributed_lock = true
119 #rc_cache.repo_object.arguments.distributed_lock = true
133
120
134 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
121 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
135 #rc_cache.repo_object.arguments.lock_auto_renewal = true
122 #rc_cache.repo_object.arguments.lock_auto_renewal = true
136
123
137 ; Statsd client config, this is used to send metrics to statsd
124 ; Statsd client config, this is used to send metrics to statsd
138 ; We recommend setting statsd_exported and scrape them using Promethues
125 ; We recommend setting statsd_exported and scrape them using Promethues
139 #statsd.enabled = false
126 #statsd.enabled = false
140 #statsd.statsd_host = 0.0.0.0
127 #statsd.statsd_host = 0.0.0.0
141 #statsd.statsd_port = 8125
128 #statsd.statsd_port = 8125
142 #statsd.statsd_prefix =
129 #statsd.statsd_prefix =
143 #statsd.statsd_ipv6 = false
130 #statsd.statsd_ipv6 = false
144
131
145 ; configure logging automatically at server startup set to false
132 ; configure logging automatically at server startup set to false
146 ; to use the below custom logging config.
133 ; to use the below custom logging config.
147 ; RC_LOGGING_FORMATTER
134 ; RC_LOGGING_FORMATTER
148 ; RC_LOGGING_LEVEL
135 ; RC_LOGGING_LEVEL
149 ; env variables can control the settings for logging in case of autoconfigure
136 ; env variables can control the settings for logging in case of autoconfigure
150
137
151 #logging.autoconfigure = true
138 #logging.autoconfigure = true
152
139
153 ; specify your own custom logging config file to configure logging
140 ; specify your own custom logging config file to configure logging
154 #logging.logging_conf_file = /path/to/custom_logging.ini
141 #logging.logging_conf_file = /path/to/custom_logging.ini
155
142
156 ; #####################
143 ; #####################
157 ; LOGGING CONFIGURATION
144 ; LOGGING CONFIGURATION
158 ; #####################
145 ; #####################
159
146
160 [loggers]
147 [loggers]
161 keys = root, vcsserver
148 keys = root, vcsserver
162
149
163 [handlers]
150 [handlers]
164 keys = console
151 keys = console
165
152
166 [formatters]
153 [formatters]
167 keys = generic, json
154 keys = generic, json
168
155
169 ; #######
156 ; #######
170 ; LOGGERS
157 ; LOGGERS
171 ; #######
158 ; #######
172 [logger_root]
159 [logger_root]
173 level = NOTSET
160 level = NOTSET
174 handlers = console
161 handlers = console
175
162
176 [logger_vcsserver]
163 [logger_vcsserver]
177 level = DEBUG
164 level = DEBUG
178 handlers =
165 handlers =
179 qualname = vcsserver
166 qualname = vcsserver
180 propagate = 1
167 propagate = 1
181
168
182 ; ########
169 ; ########
183 ; HANDLERS
170 ; HANDLERS
184 ; ########
171 ; ########
185
172
186 [handler_console]
173 [handler_console]
187 class = StreamHandler
174 class = StreamHandler
188 args = (sys.stderr, )
175 args = (sys.stderr, )
189 level = DEBUG
176 level = DEBUG
190 ; To enable JSON formatted logs replace 'generic' with 'json'
177 ; To enable JSON formatted logs replace 'generic' with 'json'
191 ; This allows sending properly formatted logs to grafana loki or elasticsearch
178 ; This allows sending properly formatted logs to grafana loki or elasticsearch
192 formatter = generic
179 formatter = generic
193
180
194 ; ##########
181 ; ##########
195 ; FORMATTERS
182 ; FORMATTERS
196 ; ##########
183 ; ##########
197
184
198 [formatter_generic]
185 [formatter_generic]
199 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
186 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
200 datefmt = %Y-%m-%d %H:%M:%S
187 datefmt = %Y-%m-%d %H:%M:%S
201
188
202 [formatter_json]
189 [formatter_json]
203 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
190 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
204 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
191 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -1,167 +1,171 b''
1 #
2
1
3 ; #################################
2 ; #################################
4 ; RHODECODE VCSSERVER CONFIGURATION
3 ; RHODECODE VCSSERVER CONFIGURATION
5 ; #################################
4 ; #################################
6
5
7 [server:main]
6 [server:main]
8 ; COMMON HOST/IP CONFIG
7 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
9 host = 127.0.0.1
8 ; Host port for gunicorn are controlled by gunicorn_conf.py
9 host = 0.0.0.0
10 port = 10010
10 port = 10010
11
11
12
12
13 ; ###########################
13 ; ###########################
14 ; GUNICORN APPLICATION SERVER
14 ; GUNICORN APPLICATION SERVER
15 ; ###########################
15 ; ###########################
16
16
17 ; run with gunicorn --paste rhodecode.ini
17 ; run with gunicorn --config gunicorn_conf.py --paste vcsserver.ini
18
18
19 ; Module to use, this setting shouldn't be changed
19 ; Module to use, this setting shouldn't be changed
20 use = egg:gunicorn#main
20 use = egg:gunicorn#main
21
21
22 [app:main]
22 [app:main]
23 ; The %(here)s variable will be replaced with the absolute path of parent directory
23 ; The %(here)s variable will be replaced with the absolute path of parent directory
24 ; of this file
24 ; of this file
25 ; Each option in the app:main can be override by an environmental variable
25 ; Each option in the app:main can be override by an environmental variable
26 ;
26 ;
27 ;To override an option:
27 ;To override an option:
28 ;
28 ;
29 ;RC_<KeyName>
29 ;RC_<KeyName>
30 ;Everything should be uppercase, . and - should be replaced by _.
30 ;Everything should be uppercase, . and - should be replaced by _.
31 ;For example, if you have these configuration settings:
31 ;For example, if you have these configuration settings:
32 ;rc_cache.repo_object.backend = foo
32 ;rc_cache.repo_object.backend = foo
33 ;can be overridden by
33 ;can be overridden by
34 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
34 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
35
35
36 use = egg:rhodecode-vcsserver
36 use = egg:rhodecode-vcsserver
37
37
38 ; Pyramid default locales, we need this to be set
38 ; Pyramid default locales, we need this to be set
39 #pyramid.default_locale_name = en
39 #pyramid.default_locale_name = en
40
40
41 ; default locale used by VCS systems
41 ; default locale used by VCS systems
42 #locale = en_US.UTF-8
42 #locale = en_US.UTF-8
43
43
44 ; path to binaries for vcsserver, it should be set by the installer
44 ; path to binaries (hg,git,svn) for vcsserver, it should be set by the installer
45 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
45 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
46 ; it can also be a path to nix-build output in case of development
46 ; or /usr/local/bin/rhodecode_bin/vcs_bin
47 core.binary_dir = ""
47 core.binary_dir =
48
49 ; Redis connection settings for svn integrations logic
50 ; This connection string needs to be the same on ce and vcsserver
51 vcs.svn.redis_conn = redis://redis:6379/0
48
52
49 ; Custom exception store path, defaults to TMPDIR
53 ; Custom exception store path, defaults to TMPDIR
50 ; This is used to store exception from RhodeCode in shared directory
54 ; This is used to store exception from RhodeCode in shared directory
51 #exception_tracker.store_path =
55 #exception_tracker.store_path =
52
56
53 ; #############
57 ; #############
54 ; DOGPILE CACHE
58 ; DOGPILE CACHE
55 ; #############
59 ; #############
56
60
57 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
61 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
58 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
62 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
59 #cache_dir = %(here)s/data
63 #cache_dir = %(here)s/data
60
64
61 ; ***************************************
65 ; ***************************************
62 ; `repo_object` cache, default file based
66 ; `repo_object` cache, default file based
63 ; ***************************************
67 ; ***************************************
64
68
65 ; `repo_object` cache settings for vcs methods for repositories
69 ; `repo_object` cache settings for vcs methods for repositories
66 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
70 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
67
71
68 ; cache auto-expires after N seconds
72 ; cache auto-expires after N seconds
69 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
73 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
70 #rc_cache.repo_object.expiration_time = 2592000
74 #rc_cache.repo_object.expiration_time = 2592000
71
75
72 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
76 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
73 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
77 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
74
78
75 ; ***********************************************************
79 ; ***********************************************************
76 ; `repo_object` cache with redis backend
80 ; `repo_object` cache with redis backend
77 ; recommended for larger instance, and for better performance
81 ; recommended for larger instance, and for better performance
78 ; ***********************************************************
82 ; ***********************************************************
79
83
80 ; `repo_object` cache settings for vcs methods for repositories
84 ; `repo_object` cache settings for vcs methods for repositories
81 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
85 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
82
86
83 ; cache auto-expires after N seconds
87 ; cache auto-expires after N seconds
84 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
88 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
85 #rc_cache.repo_object.expiration_time = 2592000
89 #rc_cache.repo_object.expiration_time = 2592000
86
90
87 ; redis_expiration_time needs to be greater then expiration_time
91 ; redis_expiration_time needs to be greater then expiration_time
88 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
92 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
89
93
90 #rc_cache.repo_object.arguments.host = localhost
94 #rc_cache.repo_object.arguments.host = localhost
91 #rc_cache.repo_object.arguments.port = 6379
95 #rc_cache.repo_object.arguments.port = 6379
92 #rc_cache.repo_object.arguments.db = 5
96 #rc_cache.repo_object.arguments.db = 5
93 #rc_cache.repo_object.arguments.socket_timeout = 30
97 #rc_cache.repo_object.arguments.socket_timeout = 30
94 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
98 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
95 #rc_cache.repo_object.arguments.distributed_lock = true
99 #rc_cache.repo_object.arguments.distributed_lock = true
96
100
97 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
101 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
98 #rc_cache.repo_object.arguments.lock_auto_renewal = true
102 #rc_cache.repo_object.arguments.lock_auto_renewal = true
99
103
100 ; Statsd client config, this is used to send metrics to statsd
104 ; Statsd client config, this is used to send metrics to statsd
101 ; We recommend setting statsd_exported and scrape them using Promethues
105 ; We recommend setting statsd_exported and scrape them using Promethues
102 #statsd.enabled = false
106 #statsd.enabled = false
103 #statsd.statsd_host = 0.0.0.0
107 #statsd.statsd_host = 0.0.0.0
104 #statsd.statsd_port = 8125
108 #statsd.statsd_port = 8125
105 #statsd.statsd_prefix =
109 #statsd.statsd_prefix =
106 #statsd.statsd_ipv6 = false
110 #statsd.statsd_ipv6 = false
107
111
108 ; configure logging automatically at server startup set to false
112 ; configure logging automatically at server startup set to false
109 ; to use the below custom logging config.
113 ; to use the below custom logging config.
110 ; RC_LOGGING_FORMATTER
114 ; RC_LOGGING_FORMATTER
111 ; RC_LOGGING_LEVEL
115 ; RC_LOGGING_LEVEL
112 ; env variables can control the settings for logging in case of autoconfigure
116 ; env variables can control the settings for logging in case of autoconfigure
113
117
114 #logging.autoconfigure = true
118 #logging.autoconfigure = true
115
119
116 ; specify your own custom logging config file to configure logging
120 ; specify your own custom logging config file to configure logging
117 #logging.logging_conf_file = /path/to/custom_logging.ini
121 #logging.logging_conf_file = /path/to/custom_logging.ini
118
122
119 ; #####################
123 ; #####################
120 ; LOGGING CONFIGURATION
124 ; LOGGING CONFIGURATION
121 ; #####################
125 ; #####################
122
126
123 [loggers]
127 [loggers]
124 keys = root, vcsserver
128 keys = root, vcsserver
125
129
126 [handlers]
130 [handlers]
127 keys = console
131 keys = console
128
132
129 [formatters]
133 [formatters]
130 keys = generic, json
134 keys = generic, json
131
135
132 ; #######
136 ; #######
133 ; LOGGERS
137 ; LOGGERS
134 ; #######
138 ; #######
135 [logger_root]
139 [logger_root]
136 level = NOTSET
140 level = NOTSET
137 handlers = console
141 handlers = console
138
142
139 [logger_vcsserver]
143 [logger_vcsserver]
140 level = INFO
144 level = INFO
141 handlers =
145 handlers =
142 qualname = vcsserver
146 qualname = vcsserver
143 propagate = 1
147 propagate = 1
144
148
145 ; ########
149 ; ########
146 ; HANDLERS
150 ; HANDLERS
147 ; ########
151 ; ########
148
152
149 [handler_console]
153 [handler_console]
150 class = StreamHandler
154 class = StreamHandler
151 args = (sys.stderr, )
155 args = (sys.stderr, )
152 level = INFO
156 level = INFO
153 ; To enable JSON formatted logs replace 'generic' with 'json'
157 ; To enable JSON formatted logs replace 'generic' with 'json'
154 ; This allows sending properly formatted logs to grafana loki or elasticsearch
158 ; This allows sending properly formatted logs to grafana loki or elasticsearch
155 formatter = generic
159 formatter = generic
156
160
157 ; ##########
161 ; ##########
158 ; FORMATTERS
162 ; FORMATTERS
159 ; ##########
163 ; ##########
160
164
161 [formatter_generic]
165 [formatter_generic]
162 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
166 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
163 datefmt = %Y-%m-%d %H:%M:%S
167 datefmt = %Y-%m-%d %H:%M:%S
164
168
165 [formatter_json]
169 [formatter_json]
166 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
170 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
167 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
171 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -1,77 +1,102 b''
1 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
1 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
2
2
3 async-timeout==4.0.3
3 async-timeout==4.0.3
4 atomicwrites==1.4.1
4 atomicwrites==1.4.1
5 celery==5.3.6
5 celery==5.3.6
6 billiard==4.2.0
6 billiard==4.2.0
7 click==8.1.3
7 click==8.1.3
8 click-didyoumean==0.3.0
8 click-didyoumean==0.3.0
9 click==8.1.3
9 click==8.1.3
10 click-plugins==1.1.1
10 click-plugins==1.1.1
11 click==8.1.3
11 click==8.1.3
12 click-repl==0.2.0
12 click-repl==0.2.0
13 click==8.1.3
13 click==8.1.3
14 prompt-toolkit==3.0.38
14 prompt-toolkit==3.0.38
15 wcwidth==0.2.6
15 wcwidth==0.2.6
16 six==1.16.0
16 six==1.16.0
17 kombu==5.3.5
17 kombu==5.3.5
18 amqp==5.2.0
18 amqp==5.2.0
19 vine==5.1.0
19 vine==5.1.0
20 vine==5.1.0
20 vine==5.1.0
21 python-dateutil==2.8.2
21 python-dateutil==2.8.2
22 six==1.16.0
22 six==1.16.0
23 tzdata==2023.4
23 tzdata==2024.1
24 vine==5.1.0
24 vine==5.1.0
25 contextlib2==21.6.0
25 contextlib2==21.6.0
26 cov-core==1.15.0
26 dogpile.cache==1.3.3
27 coverage==7.2.3
28 diskcache==5.6.3
29 dogpile.cache==1.3.0
30 decorator==5.1.1
27 decorator==5.1.1
31 stevedore==5.1.0
28 stevedore==5.1.0
32 pbr==5.11.1
29 pbr==5.11.1
33 dulwich==0.21.6
30 dulwich==0.21.6
34 urllib3==1.26.14
31 urllib3==1.26.14
32 fsspec==2024.6.0
35 gunicorn==21.2.0
33 gunicorn==21.2.0
36 packaging==23.1
34 packaging==24.0
37 hg-evolve==11.0.2
35 hg-evolve==11.1.3
38 importlib-metadata==6.0.0
36 importlib-metadata==6.0.0
39 zipp==3.15.0
37 zipp==3.15.0
40 mercurial==6.3.3
38 mercurial==6.7.4
41 mock==5.0.2
42 more-itertools==9.1.0
39 more-itertools==9.1.0
43 msgpack==1.0.7
40 msgpack==1.0.8
44 orjson==3.9.13
41 orjson==3.10.3
45 psutil==5.9.8
42 psutil==5.9.8
46 py==1.11.0
43 py==1.11.0
47 pygit2==1.13.3
44 pygit2==1.13.3
48 cffi==1.16.0
45 cffi==1.16.0
49 pycparser==2.21
46 pycparser==2.21
50 pygments==2.15.1
47 pygments==2.15.1
51 pyparsing==3.1.1
48 pyparsing==3.1.1
52 pyramid==2.0.2
49 pyramid==2.0.2
53 hupper==1.12
50 hupper==1.12
54 plaster==1.1.2
51 plaster==1.1.2
55 plaster-pastedeploy==1.0.1
52 plaster-pastedeploy==1.0.1
56 pastedeploy==3.1.0
53 pastedeploy==3.1.0
57 plaster==1.1.2
54 plaster==1.1.2
58 translationstring==1.4
55 translationstring==1.4
59 venusian==3.0.0
56 venusian==3.0.0
60 webob==1.8.7
57 webob==1.8.7
61 zope.deprecation==5.0.0
58 zope.deprecation==5.0.0
62 zope.interface==6.1.0
59 zope.interface==6.3.0
63 redis==5.0.1
60 redis==5.0.4
64 async-timeout==4.0.3
61 async-timeout==4.0.3
65 repoze.lru==0.7
62 repoze.lru==0.7
63 s3fs==2024.6.0
64 aiobotocore==2.13.0
65 aiohttp==3.9.5
66 aiosignal==1.3.1
67 frozenlist==1.4.1
68 attrs==22.2.0
69 frozenlist==1.4.1
70 multidict==6.0.5
71 yarl==1.9.4
72 idna==3.4
73 multidict==6.0.5
74 aioitertools==0.11.0
75 botocore==1.34.106
76 jmespath==1.0.1
77 python-dateutil==2.8.2
78 six==1.16.0
79 urllib3==1.26.14
80 wrapt==1.16.0
81 aiohttp==3.9.5
82 aiosignal==1.3.1
83 frozenlist==1.4.1
84 attrs==22.2.0
85 frozenlist==1.4.1
86 multidict==6.0.5
87 yarl==1.9.4
88 idna==3.4
89 multidict==6.0.5
90 fsspec==2024.6.0
66 scandir==1.10.0
91 scandir==1.10.0
67 setproctitle==1.3.3
92 setproctitle==1.3.3
68 subvertpy==0.11.0
93 subvertpy==0.11.0
69 waitress==3.0.0
94 waitress==3.0.0
70 wcwidth==0.2.6
95 wcwidth==0.2.6
71
96
72
97
73 ## test related requirements
98 ## test related requirements
74 #-r requirements_test.txt
99 #-r requirements_test.txt
75
100
76 ## uncomment to add the debug libraries
101 ## uncomment to add the debug libraries
77 #-r requirements_debug.txt
102 #-r requirements_debug.txt
@@ -1,45 +1,48 b''
1 # test related requirements
1 # test related requirements
2
2 mock==5.1.0
3 cov-core==1.15.0
3 pytest-cov==4.1.0
4 coverage==7.2.3
4 coverage==7.4.3
5 mock==5.0.2
5 pytest==8.1.1
6 py==1.11.0
7 pytest-cov==4.0.0
8 coverage==7.2.3
9 pytest==7.3.1
10 attrs==22.2.0
11 iniconfig==2.0.0
6 iniconfig==2.0.0
12 packaging==23.1
7 packaging==24.0
13 pluggy==1.0.0
8 pluggy==1.4.0
9 pytest-env==1.1.3
10 pytest==8.1.1
11 iniconfig==2.0.0
12 packaging==24.0
13 pluggy==1.4.0
14 pytest-profiling==1.7.0
14 pytest-profiling==1.7.0
15 gprof2dot==2022.7.29
15 gprof2dot==2022.7.29
16 pytest==7.3.1
16 pytest==8.1.1
17 attrs==22.2.0
18 iniconfig==2.0.0
17 iniconfig==2.0.0
19 packaging==23.1
18 packaging==24.0
20 pluggy==1.0.0
19 pluggy==1.4.0
21 six==1.16.0
20 six==1.16.0
22 pytest-runner==6.0.0
21 pytest-rerunfailures==13.0
23 pytest-sugar==0.9.7
22 packaging==24.0
24 packaging==23.1
23 pytest==8.1.1
25 pytest==7.3.1
26 attrs==22.2.0
27 iniconfig==2.0.0
24 iniconfig==2.0.0
28 packaging==23.1
25 packaging==24.0
29 pluggy==1.0.0
26 pluggy==1.4.0
30 termcolor==2.3.0
27 pytest-runner==6.0.1
31 pytest-timeout==2.1.0
28 pytest-sugar==1.0.0
32 pytest==7.3.1
29 packaging==24.0
33 attrs==22.2.0
30 pytest==8.1.1
34 iniconfig==2.0.0
31 iniconfig==2.0.0
35 packaging==23.1
32 packaging==24.0
36 pluggy==1.0.0
33 pluggy==1.4.0
34 termcolor==2.4.0
35 pytest-timeout==2.3.1
36 pytest==8.1.1
37 iniconfig==2.0.0
38 packaging==24.0
39 pluggy==1.4.0
37 webtest==3.0.0
40 webtest==3.0.0
38 beautifulsoup4==4.11.2
41 beautifulsoup4==4.12.3
39 soupsieve==2.4
42 soupsieve==2.5
40 waitress==3.0.0
43 waitress==3.0.0
41 webob==1.8.7
44 webob==1.8.7
42
45
43 # RhodeCode test-data
46 # RhodeCode test-data
44 rc_testdata @ https://code.rhodecode.com/upstream/rc-testdata-dist/raw/77378e9097f700b4c1b9391b56199fe63566b5c9/rc_testdata-0.11.0.tar.gz#egg=rc_testdata
47 rc_testdata @ https://code.rhodecode.com/upstream/rc-testdata-dist/raw/77378e9097f700b4c1b9391b56199fe63566b5c9/rc_testdata-0.11.0.tar.gz#egg=rc_testdata
45 rc_testdata==0.11.0
48 rc_testdata==0.11.0
@@ -1,1 +1,1 b''
1 5.0.3 No newline at end of file
1 5.1.0 No newline at end of file
@@ -1,193 +1,187 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import os
17 import os
18 import sys
18 import sys
19 import tempfile
19 import tempfile
20 import logging
20 import logging
21 import urllib.parse
21 import urllib.parse
22
22
23 from vcsserver.lib.rc_cache.archive_cache import get_archival_cache_store
23 from vcsserver.lib.archive_cache import get_archival_cache_store
24
24
25 from vcsserver import exceptions
25 from vcsserver import exceptions
26 from vcsserver.exceptions import NoContentException
26 from vcsserver.exceptions import NoContentException
27 from vcsserver.hgcompat import archival
27 from vcsserver.hgcompat import archival
28 from vcsserver.str_utils import safe_bytes
28 from vcsserver.lib.str_utils import safe_bytes
29 from vcsserver.lib.exc_tracking import format_exc
29 from vcsserver.lib.exc_tracking import format_exc
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32
32
33 class RepoFactory:
33 class RepoFactory:
34 """
34 """
35 Utility to create instances of repository
35 Utility to create instances of repository
36
36
37 It provides internal caching of the `repo` object based on
37 It provides internal caching of the `repo` object based on
38 the :term:`call context`.
38 the :term:`call context`.
39 """
39 """
40 repo_type = None
40 repo_type = None
41
41
42 def __init__(self):
42 def __init__(self):
43 pass
43 pass
44
44
45 def _create_config(self, path, config):
45 def _create_config(self, path, config):
46 config = {}
46 config = {}
47 return config
47 return config
48
48
49 def _create_repo(self, wire, create):
49 def _create_repo(self, wire, create):
50 raise NotImplementedError()
50 raise NotImplementedError()
51
51
52 def repo(self, wire, create=False):
52 def repo(self, wire, create=False):
53 raise NotImplementedError()
53 raise NotImplementedError()
54
54
55
55
56 def obfuscate_qs(query_string):
56 def obfuscate_qs(query_string):
57 if query_string is None:
57 if query_string is None:
58 return None
58 return None
59
59
60 parsed = []
60 parsed = []
61 for k, v in urllib.parse.parse_qsl(query_string, keep_blank_values=True):
61 for k, v in urllib.parse.parse_qsl(query_string, keep_blank_values=True):
62 if k in ['auth_token', 'api_key']:
62 if k in ['auth_token', 'api_key']:
63 v = "*****"
63 v = "*****"
64 parsed.append((k, v))
64 parsed.append((k, v))
65
65
66 return '&'.join('{}{}'.format(
66 return '&'.join('{}{}'.format(
67 k, f'={v}' if v else '') for k, v in parsed)
67 k, f'={v}' if v else '') for k, v in parsed)
68
68
69
69
70 def raise_from_original(new_type, org_exc: Exception):
70 def raise_from_original(new_type, org_exc: Exception):
71 """
71 """
72 Raise a new exception type with original args and traceback.
72 Raise a new exception type with original args and traceback.
73 """
73 """
74 exc_info = sys.exc_info()
74 exc_info = sys.exc_info()
75 exc_type, exc_value, exc_traceback = exc_info
75 exc_type, exc_value, exc_traceback = exc_info
76 new_exc = new_type(*exc_value.args)
76 new_exc = new_type(*exc_value.args)
77
77
78 # store the original traceback into the new exc
78 # store the original traceback into the new exc
79 new_exc._org_exc_tb = format_exc(exc_info)
79 new_exc._org_exc_tb = format_exc(exc_info)
80
80
81 try:
81 try:
82 raise new_exc.with_traceback(exc_traceback)
82 raise new_exc.with_traceback(exc_traceback)
83 finally:
83 finally:
84 del exc_traceback
84 del exc_traceback
85
85
86
86
87 class ArchiveNode:
87 class ArchiveNode:
88 def __init__(self, path, mode, is_link, raw_bytes):
88 def __init__(self, path, mode, is_link, raw_bytes):
89 self.path = path
89 self.path = path
90 self.mode = mode
90 self.mode = mode
91 self.is_link = is_link
91 self.is_link = is_link
92 self.raw_bytes = raw_bytes
92 self.raw_bytes = raw_bytes
93
93
94
94
95 def store_archive_in_cache(node_walker, archive_key, kind, mtime, archive_at_path, archive_dir_name,
95 def store_archive_in_cache(node_walker, archive_key, kind, mtime, archive_at_path, archive_dir_name,
96 commit_id, write_metadata=True, extra_metadata=None, cache_config=None):
96 commit_id, write_metadata=True, extra_metadata=None, cache_config=None):
97 """
97 """
98 Function that would store generate archive and send it to a dedicated backend store
98 Function that would store generate archive and send it to a dedicated backend store
99 In here we use diskcache
99 In here we use diskcache
100
100
101 :param node_walker: a generator returning nodes to add to archive
101 :param node_walker: a generator returning nodes to add to archive
102 :param archive_key: key used to store the path
102 :param archive_key: key used to store the path
103 :param kind: archive kind
103 :param kind: archive kind
104 :param mtime: time of creation
104 :param mtime: time of creation
105 :param archive_at_path: default '/' the path at archive was started.
105 :param archive_at_path: default '/' the path at archive was started.
106 If this is not '/' it means it's a partial archive
106 If this is not '/' it means it's a partial archive
107 :param archive_dir_name: inside dir name when creating an archive
107 :param archive_dir_name: inside dir name when creating an archive
108 :param commit_id: commit sha of revision archive was created at
108 :param commit_id: commit sha of revision archive was created at
109 :param write_metadata:
109 :param write_metadata:
110 :param extra_metadata:
110 :param extra_metadata:
111 :param cache_config:
111 :param cache_config:
112
112
113 walker should be a file walker, for example,
113 walker should be a file walker, for example,
114 def node_walker():
114 def node_walker():
115 for file_info in files:
115 for file_info in files:
116 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
116 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
117 """
117 """
118 extra_metadata = extra_metadata or {}
118 extra_metadata = extra_metadata or {}
119
119
120 d_cache = get_archival_cache_store(config=cache_config)
120 d_cache = get_archival_cache_store(config=cache_config)
121
121
122 if archive_key in d_cache:
122 if archive_key in d_cache:
123 with d_cache as d_cache_reader:
123 reader, metadata = d_cache.fetch(archive_key)
124 reader, tag = d_cache_reader.get(archive_key, read=True, tag=True, retry=True)
124 return reader.name
125 return reader.name
126
125
127 archive_tmp_path = safe_bytes(tempfile.mkstemp()[1])
126 archive_tmp_path = safe_bytes(tempfile.mkstemp()[1])
128 log.debug('Creating new temp archive in %s', archive_tmp_path)
127 log.debug('Creating new temp archive in %s', archive_tmp_path)
129
128
130 if kind == "tgz":
129 if kind == "tgz":
131 archiver = archival.tarit(archive_tmp_path, mtime, b"gz")
130 archiver = archival.tarit(archive_tmp_path, mtime, b"gz")
132 elif kind == "tbz2":
131 elif kind == "tbz2":
133 archiver = archival.tarit(archive_tmp_path, mtime, b"bz2")
132 archiver = archival.tarit(archive_tmp_path, mtime, b"bz2")
134 elif kind == 'zip':
133 elif kind == 'zip':
135 archiver = archival.zipit(archive_tmp_path, mtime)
134 archiver = archival.zipit(archive_tmp_path, mtime)
136 else:
135 else:
137 raise exceptions.ArchiveException()(
136 raise exceptions.ArchiveException()(
138 f'Remote does not support: "{kind}" archive type.')
137 f'Remote does not support: "{kind}" archive type.')
139
138
140 for f in node_walker(commit_id, archive_at_path):
139 for f in node_walker(commit_id, archive_at_path):
141 f_path = os.path.join(safe_bytes(archive_dir_name), safe_bytes(f.path).lstrip(b'/'))
140 f_path = os.path.join(safe_bytes(archive_dir_name), safe_bytes(f.path).lstrip(b'/'))
141
142 try:
142 try:
143 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
143 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
144 except NoContentException:
144 except NoContentException:
145 # NOTE(marcink): this is a special case for SVN so we can create "empty"
145 # NOTE(marcink): this is a special case for SVN so we can create "empty"
146 # directories which are not supported by archiver
146 # directories which are not supported by archiver
147 archiver.addfile(os.path.join(f_path, b'.dir'), f.mode, f.is_link, b'')
147 archiver.addfile(os.path.join(f_path, b'.dir'), f.mode, f.is_link, b'')
148
148
149 metadata = dict([
150 ('commit_id', commit_id),
151 ('mtime', mtime),
152 ])
153 metadata.update(extra_metadata)
149 if write_metadata:
154 if write_metadata:
150 metadata = dict([
151 ('commit_id', commit_id),
152 ('mtime', mtime),
153 ])
154 metadata.update(extra_metadata)
155
156 meta = [safe_bytes(f"{f_name}:{value}") for f_name, value in metadata.items()]
155 meta = [safe_bytes(f"{f_name}:{value}") for f_name, value in metadata.items()]
157 f_path = os.path.join(safe_bytes(archive_dir_name), b'.archival.txt')
156 f_path = os.path.join(safe_bytes(archive_dir_name), b'.archival.txt')
158 archiver.addfile(f_path, 0o644, False, b'\n'.join(meta))
157 archiver.addfile(f_path, 0o644, False, b'\n'.join(meta))
159
158
160 archiver.done()
159 archiver.done()
161
160
162 # ensure set & get are atomic
161 with open(archive_tmp_path, 'rb') as archive_file:
163 with d_cache.transact():
162 add_result = d_cache.store(archive_key, archive_file, metadata=metadata)
164
163 if not add_result:
165 with open(archive_tmp_path, 'rb') as archive_file:
164 log.error('Failed to store cache for key=%s', archive_key)
166 add_result = d_cache.set(archive_key, archive_file, read=True, tag='db-name', retry=True)
167 if not add_result:
168 log.error('Failed to store cache for key=%s', archive_key)
169
165
170 os.remove(archive_tmp_path)
166 os.remove(archive_tmp_path)
171
167
172 reader, tag = d_cache.get(archive_key, read=True, tag=True, retry=True)
168 reader, metadata = d_cache.fetch(archive_key)
173 if not reader:
174 raise AssertionError(f'empty reader on key={archive_key} added={add_result}')
175
169
176 return reader.name
170 return reader.name
177
171
178
172
179 class BinaryEnvelope:
173 class BinaryEnvelope:
180 def __init__(self, val):
174 def __init__(self, val):
181 self.val = val
175 self.val = val
182
176
183
177
184 class BytesEnvelope(bytes):
178 class BytesEnvelope(bytes):
185 def __new__(cls, content):
179 def __new__(cls, content):
186 if isinstance(content, bytes):
180 if isinstance(content, bytes):
187 return super().__new__(cls, content)
181 return super().__new__(cls, content)
188 else:
182 else:
189 raise TypeError('BytesEnvelope content= param must be bytes. Use BinaryEnvelope to wrap other types')
183 raise TypeError('BytesEnvelope content= param must be bytes. Use BinaryEnvelope to wrap other types')
190
184
191
185
192 class BinaryBytesEnvelope(BytesEnvelope):
186 class BinaryBytesEnvelope(BytesEnvelope):
193 pass
187 pass
@@ -1,185 +1,185 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
2 #
3 # This program is free software: you can redistribute it and/or modify
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
5 # (only), as published by the Free Software Foundation.
6 #
6 #
7 # This program is distributed in the hope that it will be useful,
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
10 # GNU General Public License for more details.
11 #
11 #
12 # You should have received a copy of the GNU Affero General Public License
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
14 #
15 # This program is dual-licensed. If you wish to learn more about the
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
18
19 import os
19 import os
20 import textwrap
20 import textwrap
21 import string
21 import string
22 import functools
22 import functools
23 import logging
23 import logging
24 import tempfile
24 import tempfile
25 import logging.config
25 import logging.config
26
26
27 from vcsserver.type_utils import str2bool, aslist
27 from vcsserver.lib.type_utils import str2bool, aslist
28
28
29 log = logging.getLogger(__name__)
29 log = logging.getLogger(__name__)
30
30
31
31
32 # skip keys, that are set here, so we don't double process those
32 # skip keys, that are set here, so we don't double process those
33 set_keys = {
33 set_keys = {
34 '__file__': ''
34 '__file__': ''
35 }
35 }
36
36
37
37
38 class SettingsMaker:
38 class SettingsMaker:
39
39
40 def __init__(self, app_settings):
40 def __init__(self, app_settings):
41 self.settings = app_settings
41 self.settings = app_settings
42
42
43 @classmethod
43 @classmethod
44 def _bool_func(cls, input_val):
44 def _bool_func(cls, input_val):
45 if isinstance(input_val, bytes):
45 if isinstance(input_val, bytes):
46 # decode to str
46 # decode to str
47 input_val = input_val.decode('utf8')
47 input_val = input_val.decode('utf8')
48 return str2bool(input_val)
48 return str2bool(input_val)
49
49
50 @classmethod
50 @classmethod
51 def _int_func(cls, input_val):
51 def _int_func(cls, input_val):
52 return int(input_val)
52 return int(input_val)
53
53
54 @classmethod
54 @classmethod
55 def _float_func(cls, input_val):
55 def _float_func(cls, input_val):
56 return float(input_val)
56 return float(input_val)
57
57
58 @classmethod
58 @classmethod
59 def _list_func(cls, input_val, sep=','):
59 def _list_func(cls, input_val, sep=','):
60 return aslist(input_val, sep=sep)
60 return aslist(input_val, sep=sep)
61
61
62 @classmethod
62 @classmethod
63 def _string_func(cls, input_val, lower=True):
63 def _string_func(cls, input_val, lower=True):
64 if lower:
64 if lower:
65 input_val = input_val.lower()
65 input_val = input_val.lower()
66 return input_val
66 return input_val
67
67
68 @classmethod
68 @classmethod
69 def _string_no_quote_func(cls, input_val, lower=True):
69 def _string_no_quote_func(cls, input_val, lower=True):
70 """
70 """
71 Special case string function that detects if value is set to empty quote string
71 Special case string function that detects if value is set to empty quote string
72 e.g.
72 e.g.
73
73
74 core.binary_dir = ""
74 core.binary_dir = ""
75 """
75 """
76
76
77 input_val = cls._string_func(input_val, lower=lower)
77 input_val = cls._string_func(input_val, lower=lower)
78 if input_val in ['""', "''"]:
78 if input_val in ['""', "''"]:
79 return ''
79 return ''
80 return input_val
80 return input_val
81
81
82 @classmethod
82 @classmethod
83 def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
83 def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
84
84
85 # ensure we have our dir created
85 # ensure we have our dir created
86 if not os.path.isdir(input_val) and ensure_dir:
86 if not os.path.isdir(input_val) and ensure_dir:
87 os.makedirs(input_val, mode=mode, exist_ok=True)
87 os.makedirs(input_val, mode=mode, exist_ok=True)
88
88
89 if not os.path.isdir(input_val):
89 if not os.path.isdir(input_val):
90 raise Exception(f'Dir at {input_val} does not exist')
90 raise Exception(f'Dir at {input_val} does not exist')
91 return input_val
91 return input_val
92
92
93 @classmethod
93 @classmethod
94 def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
94 def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
95 dirname = os.path.dirname(input_val)
95 dirname = os.path.dirname(input_val)
96 cls._dir_func(dirname, ensure_dir=ensure_dir)
96 cls._dir_func(dirname, ensure_dir=ensure_dir)
97 return input_val
97 return input_val
98
98
99 @classmethod
99 @classmethod
100 def _key_transformator(cls, key):
100 def _key_transformator(cls, key):
101 return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
101 return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
102
102
103 def maybe_env_key(self, key):
103 def maybe_env_key(self, key):
104 # now maybe we have this KEY in env, search and use the value with higher priority.
104 # now maybe we have this KEY in env, search and use the value with higher priority.
105 transformed_key = self._key_transformator(key)
105 transformed_key = self._key_transformator(key)
106 envvar_value = os.environ.get(transformed_key)
106 envvar_value = os.environ.get(transformed_key)
107 if envvar_value:
107 if envvar_value:
108 log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
108 log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
109
109
110 return envvar_value
110 return envvar_value
111
111
112 def env_expand(self):
112 def env_expand(self):
113 replaced = {}
113 replaced = {}
114 for k, v in self.settings.items():
114 for k, v in self.settings.items():
115 if k not in set_keys:
115 if k not in set_keys:
116 envvar_value = self.maybe_env_key(k)
116 envvar_value = self.maybe_env_key(k)
117 if envvar_value:
117 if envvar_value:
118 replaced[k] = envvar_value
118 replaced[k] = envvar_value
119 set_keys[k] = envvar_value
119 set_keys[k] = envvar_value
120
120
121 # replace ALL keys updated
121 # replace ALL keys updated
122 self.settings.update(replaced)
122 self.settings.update(replaced)
123
123
124 def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
124 def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
125 """
125 """
126 Helper to enable debug on running instance
126 Helper to enable debug on running instance
127 :return:
127 :return:
128 """
128 """
129
129
130 if not str2bool(self.settings.get('logging.autoconfigure')):
130 if not str2bool(self.settings.get('logging.autoconfigure')):
131 log.info('logging configuration based on main .ini file')
131 log.info('logging configuration based on main .ini file')
132 return
132 return
133
133
134 if logging_conf is None:
134 if logging_conf is None:
135 logging_conf = self.settings.get('logging.logging_conf_file') or ''
135 logging_conf = self.settings.get('logging.logging_conf_file') or ''
136
136
137 if not os.path.isfile(logging_conf):
137 if not os.path.isfile(logging_conf):
138 log.error('Unable to setup logging based on %s, '
138 log.error('Unable to setup logging based on %s, '
139 'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
139 'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
140 return
140 return
141
141
142 with open(logging_conf, 'rt') as f:
142 with open(logging_conf, 'rt') as f:
143 ini_template = textwrap.dedent(f.read())
143 ini_template = textwrap.dedent(f.read())
144 ini_template = string.Template(ini_template).safe_substitute(
144 ini_template = string.Template(ini_template).safe_substitute(
145 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
145 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
146 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
146 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
147 )
147 )
148
148
149 with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
149 with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
150 log.info('Saved Temporary LOGGING config at %s', f.name)
150 log.info('Saved Temporary LOGGING config at %s', f.name)
151 f.write(ini_template)
151 f.write(ini_template)
152
152
153 logging.config.fileConfig(f.name)
153 logging.config.fileConfig(f.name)
154 os.remove(f.name)
154 os.remove(f.name)
155
155
156 def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
156 def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
157 input_val = self.settings.get(key, default)
157 input_val = self.settings.get(key, default)
158
158
159 if default_when_empty and not input_val:
159 if default_when_empty and not input_val:
160 # use default value when value is set in the config but it is empty
160 # use default value when value is set in the config but it is empty
161 input_val = default
161 input_val = default
162
162
163 parser_func = {
163 parser_func = {
164 'bool': self._bool_func,
164 'bool': self._bool_func,
165 'int': self._int_func,
165 'int': self._int_func,
166 'float': self._float_func,
166 'float': self._float_func,
167 'list': self._list_func,
167 'list': self._list_func,
168 'list:newline': functools.partial(self._list_func, sep='/n'),
168 'list:newline': functools.partial(self._list_func, sep='/n'),
169 'list:spacesep': functools.partial(self._list_func, sep=' '),
169 'list:spacesep': functools.partial(self._list_func, sep=' '),
170 'string': functools.partial(self._string_func, lower=lower),
170 'string': functools.partial(self._string_func, lower=lower),
171 'string:noquote': functools.partial(self._string_no_quote_func, lower=lower),
171 'string:noquote': functools.partial(self._string_no_quote_func, lower=lower),
172 'dir': self._dir_func,
172 'dir': self._dir_func,
173 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
173 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
174 'file': self._file_path_func,
174 'file': self._file_path_func,
175 'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
175 'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
176 None: lambda i: i
176 None: lambda i: i
177 }[parser]
177 }[parser]
178
178
179 envvar_value = self.maybe_env_key(key)
179 envvar_value = self.maybe_env_key(key)
180 if envvar_value:
180 if envvar_value:
181 input_val = envvar_value
181 input_val = envvar_value
182 set_keys[key] = input_val
182 set_keys[key] = input_val
183
183
184 self.settings[key] = parser_func(input_val)
184 self.settings[key] = parser_func(input_val)
185 return self.settings[key]
185 return self.settings[key]
@@ -1,296 +1,296 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import re
18 import re
19 import logging
19 import logging
20
20
21 from pyramid.config import Configurator
21 from pyramid.config import Configurator
22 from pyramid.response import Response, FileIter
22 from pyramid.response import Response, FileIter
23 from pyramid.httpexceptions import (
23 from pyramid.httpexceptions import (
24 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
24 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
25 HTTPUnprocessableEntity)
25 HTTPUnprocessableEntity)
26
26
27 from vcsserver.lib.rc_json import json
27 from vcsserver.lib.ext_json import json
28 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
28 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
29 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
29 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
30 from vcsserver.str_utils import safe_int
30 from vcsserver.lib.str_utils import safe_int
31
31
32 log = logging.getLogger(__name__)
32 log = logging.getLogger(__name__)
33
33
34
34
35 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' # +json ?
35 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' # +json ?
36 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
36 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
37
37
38
38
39 def write_response_error(http_exception, text=None):
39 def write_response_error(http_exception, text=None):
40 content_type = GIT_LFS_CONTENT_TYPE + '+json'
40 content_type = GIT_LFS_CONTENT_TYPE + '+json'
41 _exception = http_exception(content_type=content_type)
41 _exception = http_exception(content_type=content_type)
42 _exception.content_type = content_type
42 _exception.content_type = content_type
43 if text:
43 if text:
44 _exception.body = json.dumps({'message': text})
44 _exception.body = json.dumps({'message': text})
45 log.debug('LFS: writing response of type %s to client with text:%s',
45 log.debug('LFS: writing response of type %s to client with text:%s',
46 http_exception, text)
46 http_exception, text)
47 return _exception
47 return _exception
48
48
49
49
50 class AuthHeaderRequired:
50 class AuthHeaderRequired:
51 """
51 """
52 Decorator to check if request has proper auth-header
52 Decorator to check if request has proper auth-header
53 """
53 """
54
54
55 def __call__(self, func):
55 def __call__(self, func):
56 return get_cython_compat_decorator(self.__wrapper, func)
56 return get_cython_compat_decorator(self.__wrapper, func)
57
57
58 def __wrapper(self, func, *fargs, **fkwargs):
58 def __wrapper(self, func, *fargs, **fkwargs):
59 request = fargs[1]
59 request = fargs[1]
60 auth = request.authorization
60 auth = request.authorization
61 if not auth:
61 if not auth:
62 return write_response_error(HTTPForbidden)
62 return write_response_error(HTTPForbidden)
63 return func(*fargs[1:], **fkwargs)
63 return func(*fargs[1:], **fkwargs)
64
64
65
65
66 # views
66 # views
67
67
68 def lfs_objects(request):
68 def lfs_objects(request):
69 # indicate not supported, V1 API
69 # indicate not supported, V1 API
70 log.warning('LFS: v1 api not supported, reporting it back to client')
70 log.warning('LFS: v1 api not supported, reporting it back to client')
71 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
71 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
72
72
73
73
74 @AuthHeaderRequired()
74 @AuthHeaderRequired()
75 def lfs_objects_batch(request):
75 def lfs_objects_batch(request):
76 """
76 """
77 The client sends the following information to the Batch endpoint to transfer some objects:
77 The client sends the following information to the Batch endpoint to transfer some objects:
78
78
79 operation - Should be download or upload.
79 operation - Should be download or upload.
80 transfers - An optional Array of String identifiers for transfer
80 transfers - An optional Array of String identifiers for transfer
81 adapters that the client has configured. If omitted, the basic
81 adapters that the client has configured. If omitted, the basic
82 transfer adapter MUST be assumed by the server.
82 transfer adapter MUST be assumed by the server.
83 objects - An Array of objects to download.
83 objects - An Array of objects to download.
84 oid - String OID of the LFS object.
84 oid - String OID of the LFS object.
85 size - Integer byte size of the LFS object. Must be at least zero.
85 size - Integer byte size of the LFS object. Must be at least zero.
86 """
86 """
87 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
87 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
88 auth = request.authorization
88 auth = request.authorization
89 repo = request.matchdict.get('repo')
89 repo = request.matchdict.get('repo')
90 data = request.json
90 data = request.json
91 operation = data.get('operation')
91 operation = data.get('operation')
92 http_scheme = request.registry.git_lfs_http_scheme
92 http_scheme = request.registry.git_lfs_http_scheme
93
93
94 if operation not in ('download', 'upload'):
94 if operation not in ('download', 'upload'):
95 log.debug('LFS: unsupported operation:%s', operation)
95 log.debug('LFS: unsupported operation:%s', operation)
96 return write_response_error(
96 return write_response_error(
97 HTTPBadRequest, f'unsupported operation mode: `{operation}`')
97 HTTPBadRequest, f'unsupported operation mode: `{operation}`')
98
98
99 if 'objects' not in data:
99 if 'objects' not in data:
100 log.debug('LFS: missing objects data')
100 log.debug('LFS: missing objects data')
101 return write_response_error(
101 return write_response_error(
102 HTTPBadRequest, 'missing objects data')
102 HTTPBadRequest, 'missing objects data')
103
103
104 log.debug('LFS: handling operation of type: %s', operation)
104 log.debug('LFS: handling operation of type: %s', operation)
105
105
106 objects = []
106 objects = []
107 for o in data['objects']:
107 for o in data['objects']:
108 try:
108 try:
109 oid = o['oid']
109 oid = o['oid']
110 obj_size = o['size']
110 obj_size = o['size']
111 except KeyError:
111 except KeyError:
112 log.exception('LFS, failed to extract data')
112 log.exception('LFS, failed to extract data')
113 return write_response_error(
113 return write_response_error(
114 HTTPBadRequest, 'unsupported data in objects')
114 HTTPBadRequest, 'unsupported data in objects')
115
115
116 obj_data = {'oid': oid}
116 obj_data = {'oid': oid}
117 if http_scheme == 'http':
117 if http_scheme == 'http':
118 # Note(marcink): when using http, we might have a custom port
118 # Note(marcink): when using http, we might have a custom port
119 # so we skip setting it to http, url dispatch then wont generate a port in URL
119 # so we skip setting it to http, url dispatch then wont generate a port in URL
120 # for development we need this
120 # for development we need this
121 http_scheme = None
121 http_scheme = None
122
122
123 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
123 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
124 _scheme=http_scheme)
124 _scheme=http_scheme)
125 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
125 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
126 _scheme=http_scheme)
126 _scheme=http_scheme)
127 store = LFSOidStore(
127 store = LFSOidStore(
128 oid, repo, store_location=request.registry.git_lfs_store_path)
128 oid, repo, store_location=request.registry.git_lfs_store_path)
129 handler = OidHandler(
129 handler = OidHandler(
130 store, repo, auth, oid, obj_size, obj_data,
130 store, repo, auth, oid, obj_size, obj_data,
131 obj_href, obj_verify_href)
131 obj_href, obj_verify_href)
132
132
133 # this verifies also OIDs
133 # this verifies also OIDs
134 actions, errors = handler.exec_operation(operation)
134 actions, errors = handler.exec_operation(operation)
135 if errors:
135 if errors:
136 log.warning('LFS: got following errors: %s', errors)
136 log.warning('LFS: got following errors: %s', errors)
137 obj_data['errors'] = errors
137 obj_data['errors'] = errors
138
138
139 if actions:
139 if actions:
140 obj_data['actions'] = actions
140 obj_data['actions'] = actions
141
141
142 obj_data['size'] = obj_size
142 obj_data['size'] = obj_size
143 obj_data['authenticated'] = True
143 obj_data['authenticated'] = True
144 objects.append(obj_data)
144 objects.append(obj_data)
145
145
146 result = {'objects': objects, 'transfer': 'basic'}
146 result = {'objects': objects, 'transfer': 'basic'}
147 log.debug('LFS Response %s', safe_result(result))
147 log.debug('LFS Response %s', safe_result(result))
148
148
149 return result
149 return result
150
150
151
151
152 def lfs_objects_oid_upload(request):
152 def lfs_objects_oid_upload(request):
153 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
153 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
154 repo = request.matchdict.get('repo')
154 repo = request.matchdict.get('repo')
155 oid = request.matchdict.get('oid')
155 oid = request.matchdict.get('oid')
156 store = LFSOidStore(
156 store = LFSOidStore(
157 oid, repo, store_location=request.registry.git_lfs_store_path)
157 oid, repo, store_location=request.registry.git_lfs_store_path)
158 engine = store.get_engine(mode='wb')
158 engine = store.get_engine(mode='wb')
159 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
159 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
160
160
161 body = request.environ['wsgi.input']
161 body = request.environ['wsgi.input']
162
162
163 with engine as f:
163 with engine as f:
164 blksize = 64 * 1024 # 64kb
164 blksize = 64 * 1024 # 64kb
165 while True:
165 while True:
166 # read in chunks as stream comes in from Gunicorn
166 # read in chunks as stream comes in from Gunicorn
167 # this is a specific Gunicorn support function.
167 # this is a specific Gunicorn support function.
168 # might work differently on waitress
168 # might work differently on waitress
169 chunk = body.read(blksize)
169 chunk = body.read(blksize)
170 if not chunk:
170 if not chunk:
171 break
171 break
172 f.write(chunk)
172 f.write(chunk)
173
173
174 return {'upload': 'ok'}
174 return {'upload': 'ok'}
175
175
176
176
177 def lfs_objects_oid_download(request):
177 def lfs_objects_oid_download(request):
178 repo = request.matchdict.get('repo')
178 repo = request.matchdict.get('repo')
179 oid = request.matchdict.get('oid')
179 oid = request.matchdict.get('oid')
180
180
181 store = LFSOidStore(
181 store = LFSOidStore(
182 oid, repo, store_location=request.registry.git_lfs_store_path)
182 oid, repo, store_location=request.registry.git_lfs_store_path)
183 if not store.has_oid():
183 if not store.has_oid():
184 log.debug('LFS: oid %s does not exists in store', oid)
184 log.debug('LFS: oid %s does not exists in store', oid)
185 return write_response_error(
185 return write_response_error(
186 HTTPNotFound, f'requested file with oid `{oid}` not found in store')
186 HTTPNotFound, f'requested file with oid `{oid}` not found in store')
187
187
188 # TODO(marcink): support range header ?
188 # TODO(marcink): support range header ?
189 # Range: bytes=0-, `bytes=(\d+)\-.*`
189 # Range: bytes=0-, `bytes=(\d+)\-.*`
190
190
191 f = open(store.oid_path, 'rb')
191 f = open(store.oid_path, 'rb')
192 response = Response(
192 response = Response(
193 content_type='application/octet-stream', app_iter=FileIter(f))
193 content_type='application/octet-stream', app_iter=FileIter(f))
194 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
194 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
195 return response
195 return response
196
196
197
197
198 def lfs_objects_verify(request):
198 def lfs_objects_verify(request):
199 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
199 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
200 repo = request.matchdict.get('repo')
200 repo = request.matchdict.get('repo')
201
201
202 data = request.json
202 data = request.json
203 oid = data.get('oid')
203 oid = data.get('oid')
204 size = safe_int(data.get('size'))
204 size = safe_int(data.get('size'))
205
205
206 if not (oid and size):
206 if not (oid and size):
207 return write_response_error(
207 return write_response_error(
208 HTTPBadRequest, 'missing oid and size in request data')
208 HTTPBadRequest, 'missing oid and size in request data')
209
209
210 store = LFSOidStore(
210 store = LFSOidStore(
211 oid, repo, store_location=request.registry.git_lfs_store_path)
211 oid, repo, store_location=request.registry.git_lfs_store_path)
212 if not store.has_oid():
212 if not store.has_oid():
213 log.debug('LFS: oid %s does not exists in store', oid)
213 log.debug('LFS: oid %s does not exists in store', oid)
214 return write_response_error(
214 return write_response_error(
215 HTTPNotFound, f'oid `{oid}` does not exists in store')
215 HTTPNotFound, f'oid `{oid}` does not exists in store')
216
216
217 store_size = store.size_oid()
217 store_size = store.size_oid()
218 if store_size != size:
218 if store_size != size:
219 msg = 'requested file size mismatch store size:{} requested:{}'.format(
219 msg = 'requested file size mismatch store size:{} requested:{}'.format(
220 store_size, size)
220 store_size, size)
221 return write_response_error(
221 return write_response_error(
222 HTTPUnprocessableEntity, msg)
222 HTTPUnprocessableEntity, msg)
223
223
224 return {'message': {'size': 'ok', 'in_store': 'ok'}}
224 return {'message': {'size': 'ok', 'in_store': 'ok'}}
225
225
226
226
227 def lfs_objects_lock(request):
227 def lfs_objects_lock(request):
228 return write_response_error(
228 return write_response_error(
229 HTTPNotImplemented, 'GIT LFS locking api not supported')
229 HTTPNotImplemented, 'GIT LFS locking api not supported')
230
230
231
231
232 def not_found(request):
232 def not_found(request):
233 return write_response_error(
233 return write_response_error(
234 HTTPNotFound, 'request path not found')
234 HTTPNotFound, 'request path not found')
235
235
236
236
237 def lfs_disabled(request):
237 def lfs_disabled(request):
238 return write_response_error(
238 return write_response_error(
239 HTTPNotImplemented, 'GIT LFS disabled for this repo')
239 HTTPNotImplemented, 'GIT LFS disabled for this repo')
240
240
241
241
242 def git_lfs_app(config):
242 def git_lfs_app(config):
243
243
244 # v1 API deprecation endpoint
244 # v1 API deprecation endpoint
245 config.add_route('lfs_objects',
245 config.add_route('lfs_objects',
246 '/{repo:.*?[^/]}/info/lfs/objects')
246 '/{repo:.*?[^/]}/info/lfs/objects')
247 config.add_view(lfs_objects, route_name='lfs_objects',
247 config.add_view(lfs_objects, route_name='lfs_objects',
248 request_method='POST', renderer='json')
248 request_method='POST', renderer='json')
249
249
250 # locking API
250 # locking API
251 config.add_route('lfs_objects_lock',
251 config.add_route('lfs_objects_lock',
252 '/{repo:.*?[^/]}/info/lfs/locks')
252 '/{repo:.*?[^/]}/info/lfs/locks')
253 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
253 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
254 request_method=('POST', 'GET'), renderer='json')
254 request_method=('POST', 'GET'), renderer='json')
255
255
256 config.add_route('lfs_objects_lock_verify',
256 config.add_route('lfs_objects_lock_verify',
257 '/{repo:.*?[^/]}/info/lfs/locks/verify')
257 '/{repo:.*?[^/]}/info/lfs/locks/verify')
258 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
258 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
259 request_method=('POST', 'GET'), renderer='json')
259 request_method=('POST', 'GET'), renderer='json')
260
260
261 # batch API
261 # batch API
262 config.add_route('lfs_objects_batch',
262 config.add_route('lfs_objects_batch',
263 '/{repo:.*?[^/]}/info/lfs/objects/batch')
263 '/{repo:.*?[^/]}/info/lfs/objects/batch')
264 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
264 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
265 request_method='POST', renderer='json')
265 request_method='POST', renderer='json')
266
266
267 # oid upload/download API
267 # oid upload/download API
268 config.add_route('lfs_objects_oid',
268 config.add_route('lfs_objects_oid',
269 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
269 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
270 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
270 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
271 request_method='PUT', renderer='json')
271 request_method='PUT', renderer='json')
272 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
272 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
273 request_method='GET', renderer='json')
273 request_method='GET', renderer='json')
274
274
275 # verification API
275 # verification API
276 config.add_route('lfs_objects_verify',
276 config.add_route('lfs_objects_verify',
277 '/{repo:.*?[^/]}/info/lfs/verify')
277 '/{repo:.*?[^/]}/info/lfs/verify')
278 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
278 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
279 request_method='POST', renderer='json')
279 request_method='POST', renderer='json')
280
280
281 # not found handler for API
281 # not found handler for API
282 config.add_notfound_view(not_found, renderer='json')
282 config.add_notfound_view(not_found, renderer='json')
283
283
284
284
285 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
285 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
286 config = Configurator()
286 config = Configurator()
287 if git_lfs_enabled:
287 if git_lfs_enabled:
288 config.include(git_lfs_app)
288 config.include(git_lfs_app)
289 config.registry.git_lfs_store_path = git_lfs_store_path
289 config.registry.git_lfs_store_path = git_lfs_store_path
290 config.registry.git_lfs_http_scheme = git_lfs_http_scheme
290 config.registry.git_lfs_http_scheme = git_lfs_http_scheme
291 else:
291 else:
292 # not found handler for API, reporting disabled LFS support
292 # not found handler for API, reporting disabled LFS support
293 config.add_notfound_view(lfs_disabled, renderer='json')
293 config.add_notfound_view(lfs_disabled, renderer='json')
294
294
295 app = config.make_wsgi_app()
295 app = config.make_wsgi_app()
296 return app
296 return app
@@ -1,274 +1,274 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from webtest.app import TestApp as WebObTestApp
20 from webtest.app import TestApp as WebObTestApp
21
21
22 from vcsserver.lib.rc_json import json
22 from vcsserver.lib.ext_json import json
23 from vcsserver.str_utils import safe_bytes
23 from vcsserver.lib.str_utils import safe_bytes
24 from vcsserver.git_lfs.app import create_app
24 from vcsserver.git_lfs.app import create_app
25 from vcsserver.git_lfs.lib import LFSOidStore
25 from vcsserver.git_lfs.lib import LFSOidStore
26
26
27
27
28 @pytest.fixture(scope='function')
28 @pytest.fixture(scope='function')
29 def git_lfs_app(tmpdir):
29 def git_lfs_app(tmpdir):
30 custom_app = WebObTestApp(create_app(
30 custom_app = WebObTestApp(create_app(
31 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
31 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
32 git_lfs_http_scheme='http'))
32 git_lfs_http_scheme='http'))
33 custom_app._store = str(tmpdir)
33 custom_app._store = str(tmpdir)
34 return custom_app
34 return custom_app
35
35
36
36
37 @pytest.fixture(scope='function')
37 @pytest.fixture(scope='function')
38 def git_lfs_https_app(tmpdir):
38 def git_lfs_https_app(tmpdir):
39 custom_app = WebObTestApp(create_app(
39 custom_app = WebObTestApp(create_app(
40 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
40 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
41 git_lfs_http_scheme='https'))
41 git_lfs_http_scheme='https'))
42 custom_app._store = str(tmpdir)
42 custom_app._store = str(tmpdir)
43 return custom_app
43 return custom_app
44
44
45
45
46 @pytest.fixture()
46 @pytest.fixture()
47 def http_auth():
47 def http_auth():
48 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
48 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
49
49
50
50
51 class TestLFSApplication:
51 class TestLFSApplication:
52
52
53 def test_app_wrong_path(self, git_lfs_app):
53 def test_app_wrong_path(self, git_lfs_app):
54 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
54 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
55
55
56 def test_app_deprecated_endpoint(self, git_lfs_app):
56 def test_app_deprecated_endpoint(self, git_lfs_app):
57 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
57 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
58 assert response.status_code == 501
58 assert response.status_code == 501
59 assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
59 assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
60
60
61 def test_app_lock_verify_api_not_available(self, git_lfs_app):
61 def test_app_lock_verify_api_not_available(self, git_lfs_app):
62 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
62 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
63 assert response.status_code == 501
63 assert response.status_code == 501
64 assert json.loads(response.text) == {
64 assert json.loads(response.text) == {
65 'message': 'GIT LFS locking api not supported'}
65 'message': 'GIT LFS locking api not supported'}
66
66
67 def test_app_lock_api_not_available(self, git_lfs_app):
67 def test_app_lock_api_not_available(self, git_lfs_app):
68 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
68 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
69 assert response.status_code == 501
69 assert response.status_code == 501
70 assert json.loads(response.text) == {
70 assert json.loads(response.text) == {
71 'message': 'GIT LFS locking api not supported'}
71 'message': 'GIT LFS locking api not supported'}
72
72
73 def test_app_batch_api_missing_auth(self, git_lfs_app):
73 def test_app_batch_api_missing_auth(self, git_lfs_app):
74 git_lfs_app.post_json(
74 git_lfs_app.post_json(
75 '/repo/info/lfs/objects/batch', params={}, status=403)
75 '/repo/info/lfs/objects/batch', params={}, status=403)
76
76
77 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
77 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
78 response = git_lfs_app.post_json(
78 response = git_lfs_app.post_json(
79 '/repo/info/lfs/objects/batch', params={}, status=400,
79 '/repo/info/lfs/objects/batch', params={}, status=400,
80 extra_environ=http_auth)
80 extra_environ=http_auth)
81 assert json.loads(response.text) == {
81 assert json.loads(response.text) == {
82 'message': 'unsupported operation mode: `None`'}
82 'message': 'unsupported operation mode: `None`'}
83
83
84 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
84 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
85 response = git_lfs_app.post_json(
85 response = git_lfs_app.post_json(
86 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
86 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
87 status=400, extra_environ=http_auth)
87 status=400, extra_environ=http_auth)
88 assert json.loads(response.text) == {
88 assert json.loads(response.text) == {
89 'message': 'missing objects data'}
89 'message': 'missing objects data'}
90
90
91 def test_app_batch_api_unsupported_data_in_objects(
91 def test_app_batch_api_unsupported_data_in_objects(
92 self, git_lfs_app, http_auth):
92 self, git_lfs_app, http_auth):
93 params = {'operation': 'download',
93 params = {'operation': 'download',
94 'objects': [{}]}
94 'objects': [{}]}
95 response = git_lfs_app.post_json(
95 response = git_lfs_app.post_json(
96 '/repo/info/lfs/objects/batch', params=params, status=400,
96 '/repo/info/lfs/objects/batch', params=params, status=400,
97 extra_environ=http_auth)
97 extra_environ=http_auth)
98 assert json.loads(response.text) == {
98 assert json.loads(response.text) == {
99 'message': 'unsupported data in objects'}
99 'message': 'unsupported data in objects'}
100
100
101 def test_app_batch_api_download_missing_object(
101 def test_app_batch_api_download_missing_object(
102 self, git_lfs_app, http_auth):
102 self, git_lfs_app, http_auth):
103 params = {'operation': 'download',
103 params = {'operation': 'download',
104 'objects': [{'oid': '123', 'size': '1024'}]}
104 'objects': [{'oid': '123', 'size': '1024'}]}
105 response = git_lfs_app.post_json(
105 response = git_lfs_app.post_json(
106 '/repo/info/lfs/objects/batch', params=params,
106 '/repo/info/lfs/objects/batch', params=params,
107 extra_environ=http_auth)
107 extra_environ=http_auth)
108
108
109 expected_objects = [
109 expected_objects = [
110 {'authenticated': True,
110 {'authenticated': True,
111 'errors': {'error': {
111 'errors': {'error': {
112 'code': 404,
112 'code': 404,
113 'message': 'object: 123 does not exist in store'}},
113 'message': 'object: 123 does not exist in store'}},
114 'oid': '123',
114 'oid': '123',
115 'size': '1024'}
115 'size': '1024'}
116 ]
116 ]
117 assert json.loads(response.text) == {
117 assert json.loads(response.text) == {
118 'objects': expected_objects, 'transfer': 'basic'}
118 'objects': expected_objects, 'transfer': 'basic'}
119
119
120 def test_app_batch_api_download(self, git_lfs_app, http_auth):
120 def test_app_batch_api_download(self, git_lfs_app, http_auth):
121 oid = '456'
121 oid = '456'
122 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
122 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
123 if not os.path.isdir(os.path.dirname(oid_path)):
123 if not os.path.isdir(os.path.dirname(oid_path)):
124 os.makedirs(os.path.dirname(oid_path))
124 os.makedirs(os.path.dirname(oid_path))
125 with open(oid_path, 'wb') as f:
125 with open(oid_path, 'wb') as f:
126 f.write(safe_bytes('OID_CONTENT'))
126 f.write(safe_bytes('OID_CONTENT'))
127
127
128 params = {'operation': 'download',
128 params = {'operation': 'download',
129 'objects': [{'oid': oid, 'size': '1024'}]}
129 'objects': [{'oid': oid, 'size': '1024'}]}
130 response = git_lfs_app.post_json(
130 response = git_lfs_app.post_json(
131 '/repo/info/lfs/objects/batch', params=params,
131 '/repo/info/lfs/objects/batch', params=params,
132 extra_environ=http_auth)
132 extra_environ=http_auth)
133
133
134 expected_objects = [
134 expected_objects = [
135 {'authenticated': True,
135 {'authenticated': True,
136 'actions': {
136 'actions': {
137 'download': {
137 'download': {
138 'header': {'Authorization': 'Basic XXXXX'},
138 'header': {'Authorization': 'Basic XXXXX'},
139 'href': 'http://localhost/repo/info/lfs/objects/456'},
139 'href': 'http://localhost/repo/info/lfs/objects/456'},
140 },
140 },
141 'oid': '456',
141 'oid': '456',
142 'size': '1024'}
142 'size': '1024'}
143 ]
143 ]
144 assert json.loads(response.text) == {
144 assert json.loads(response.text) == {
145 'objects': expected_objects, 'transfer': 'basic'}
145 'objects': expected_objects, 'transfer': 'basic'}
146
146
147 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
147 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
148 params = {'operation': 'upload',
148 params = {'operation': 'upload',
149 'objects': [{'oid': '123', 'size': '1024'}]}
149 'objects': [{'oid': '123', 'size': '1024'}]}
150 response = git_lfs_app.post_json(
150 response = git_lfs_app.post_json(
151 '/repo/info/lfs/objects/batch', params=params,
151 '/repo/info/lfs/objects/batch', params=params,
152 extra_environ=http_auth)
152 extra_environ=http_auth)
153 expected_objects = [
153 expected_objects = [
154 {'authenticated': True,
154 {'authenticated': True,
155 'actions': {
155 'actions': {
156 'upload': {
156 'upload': {
157 'header': {'Authorization': 'Basic XXXXX',
157 'header': {'Authorization': 'Basic XXXXX',
158 'Transfer-Encoding': 'chunked'},
158 'Transfer-Encoding': 'chunked'},
159 'href': 'http://localhost/repo/info/lfs/objects/123'},
159 'href': 'http://localhost/repo/info/lfs/objects/123'},
160 'verify': {
160 'verify': {
161 'header': {'Authorization': 'Basic XXXXX'},
161 'header': {'Authorization': 'Basic XXXXX'},
162 'href': 'http://localhost/repo/info/lfs/verify'}
162 'href': 'http://localhost/repo/info/lfs/verify'}
163 },
163 },
164 'oid': '123',
164 'oid': '123',
165 'size': '1024'}
165 'size': '1024'}
166 ]
166 ]
167 assert json.loads(response.text) == {
167 assert json.loads(response.text) == {
168 'objects': expected_objects, 'transfer': 'basic'}
168 'objects': expected_objects, 'transfer': 'basic'}
169
169
170 def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
170 def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
171 params = {'operation': 'upload',
171 params = {'operation': 'upload',
172 'objects': [{'oid': '123', 'size': '1024'}]}
172 'objects': [{'oid': '123', 'size': '1024'}]}
173 response = git_lfs_https_app.post_json(
173 response = git_lfs_https_app.post_json(
174 '/repo/info/lfs/objects/batch', params=params,
174 '/repo/info/lfs/objects/batch', params=params,
175 extra_environ=http_auth)
175 extra_environ=http_auth)
176 expected_objects = [
176 expected_objects = [
177 {'authenticated': True,
177 {'authenticated': True,
178 'actions': {
178 'actions': {
179 'upload': {
179 'upload': {
180 'header': {'Authorization': 'Basic XXXXX',
180 'header': {'Authorization': 'Basic XXXXX',
181 'Transfer-Encoding': 'chunked'},
181 'Transfer-Encoding': 'chunked'},
182 'href': 'https://localhost/repo/info/lfs/objects/123'},
182 'href': 'https://localhost/repo/info/lfs/objects/123'},
183 'verify': {
183 'verify': {
184 'header': {'Authorization': 'Basic XXXXX'},
184 'header': {'Authorization': 'Basic XXXXX'},
185 'href': 'https://localhost/repo/info/lfs/verify'}
185 'href': 'https://localhost/repo/info/lfs/verify'}
186 },
186 },
187 'oid': '123',
187 'oid': '123',
188 'size': '1024'}
188 'size': '1024'}
189 ]
189 ]
190 assert json.loads(response.text) == {
190 assert json.loads(response.text) == {
191 'objects': expected_objects, 'transfer': 'basic'}
191 'objects': expected_objects, 'transfer': 'basic'}
192
192
193 def test_app_verify_api_missing_data(self, git_lfs_app):
193 def test_app_verify_api_missing_data(self, git_lfs_app):
194 params = {'oid': 'missing'}
194 params = {'oid': 'missing'}
195 response = git_lfs_app.post_json(
195 response = git_lfs_app.post_json(
196 '/repo/info/lfs/verify', params=params,
196 '/repo/info/lfs/verify', params=params,
197 status=400)
197 status=400)
198
198
199 assert json.loads(response.text) == {
199 assert json.loads(response.text) == {
200 'message': 'missing oid and size in request data'}
200 'message': 'missing oid and size in request data'}
201
201
202 def test_app_verify_api_missing_obj(self, git_lfs_app):
202 def test_app_verify_api_missing_obj(self, git_lfs_app):
203 params = {'oid': 'missing', 'size': '1024'}
203 params = {'oid': 'missing', 'size': '1024'}
204 response = git_lfs_app.post_json(
204 response = git_lfs_app.post_json(
205 '/repo/info/lfs/verify', params=params,
205 '/repo/info/lfs/verify', params=params,
206 status=404)
206 status=404)
207
207
208 assert json.loads(response.text) == {
208 assert json.loads(response.text) == {
209 'message': 'oid `missing` does not exists in store'}
209 'message': 'oid `missing` does not exists in store'}
210
210
211 def test_app_verify_api_size_mismatch(self, git_lfs_app):
211 def test_app_verify_api_size_mismatch(self, git_lfs_app):
212 oid = 'existing'
212 oid = 'existing'
213 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
213 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
214 if not os.path.isdir(os.path.dirname(oid_path)):
214 if not os.path.isdir(os.path.dirname(oid_path)):
215 os.makedirs(os.path.dirname(oid_path))
215 os.makedirs(os.path.dirname(oid_path))
216 with open(oid_path, 'wb') as f:
216 with open(oid_path, 'wb') as f:
217 f.write(safe_bytes('OID_CONTENT'))
217 f.write(safe_bytes('OID_CONTENT'))
218
218
219 params = {'oid': oid, 'size': '1024'}
219 params = {'oid': oid, 'size': '1024'}
220 response = git_lfs_app.post_json(
220 response = git_lfs_app.post_json(
221 '/repo/info/lfs/verify', params=params, status=422)
221 '/repo/info/lfs/verify', params=params, status=422)
222
222
223 assert json.loads(response.text) == {
223 assert json.loads(response.text) == {
224 'message': 'requested file size mismatch '
224 'message': 'requested file size mismatch '
225 'store size:11 requested:1024'}
225 'store size:11 requested:1024'}
226
226
227 def test_app_verify_api(self, git_lfs_app):
227 def test_app_verify_api(self, git_lfs_app):
228 oid = 'existing'
228 oid = 'existing'
229 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
229 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
230 if not os.path.isdir(os.path.dirname(oid_path)):
230 if not os.path.isdir(os.path.dirname(oid_path)):
231 os.makedirs(os.path.dirname(oid_path))
231 os.makedirs(os.path.dirname(oid_path))
232 with open(oid_path, 'wb') as f:
232 with open(oid_path, 'wb') as f:
233 f.write(safe_bytes('OID_CONTENT'))
233 f.write(safe_bytes('OID_CONTENT'))
234
234
235 params = {'oid': oid, 'size': 11}
235 params = {'oid': oid, 'size': 11}
236 response = git_lfs_app.post_json(
236 response = git_lfs_app.post_json(
237 '/repo/info/lfs/verify', params=params)
237 '/repo/info/lfs/verify', params=params)
238
238
239 assert json.loads(response.text) == {
239 assert json.loads(response.text) == {
240 'message': {'size': 'ok', 'in_store': 'ok'}}
240 'message': {'size': 'ok', 'in_store': 'ok'}}
241
241
242 def test_app_download_api_oid_not_existing(self, git_lfs_app):
242 def test_app_download_api_oid_not_existing(self, git_lfs_app):
243 oid = 'missing'
243 oid = 'missing'
244
244
245 response = git_lfs_app.get(
245 response = git_lfs_app.get(
246 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
246 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
247
247
248 assert json.loads(response.text) == {
248 assert json.loads(response.text) == {
249 'message': 'requested file with oid `missing` not found in store'}
249 'message': 'requested file with oid `missing` not found in store'}
250
250
251 def test_app_download_api(self, git_lfs_app):
251 def test_app_download_api(self, git_lfs_app):
252 oid = 'existing'
252 oid = 'existing'
253 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
253 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
254 if not os.path.isdir(os.path.dirname(oid_path)):
254 if not os.path.isdir(os.path.dirname(oid_path)):
255 os.makedirs(os.path.dirname(oid_path))
255 os.makedirs(os.path.dirname(oid_path))
256 with open(oid_path, 'wb') as f:
256 with open(oid_path, 'wb') as f:
257 f.write(safe_bytes('OID_CONTENT'))
257 f.write(safe_bytes('OID_CONTENT'))
258
258
259 response = git_lfs_app.get(
259 response = git_lfs_app.get(
260 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
260 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
261 assert response
261 assert response
262
262
263 def test_app_upload(self, git_lfs_app):
263 def test_app_upload(self, git_lfs_app):
264 oid = 'uploaded'
264 oid = 'uploaded'
265
265
266 response = git_lfs_app.put(
266 response = git_lfs_app.put(
267 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
267 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
268
268
269 assert json.loads(response.text) == {'upload': 'ok'}
269 assert json.loads(response.text) == {'upload': 'ok'}
270
270
271 # verify that we actually wrote that OID
271 # verify that we actually wrote that OID
272 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
272 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
273 assert os.path.isfile(oid_path)
273 assert os.path.isfile(oid_path)
274 assert 'CONTENT' == open(oid_path).read()
274 assert 'CONTENT' == open(oid_path).read()
@@ -1,142 +1,142 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from vcsserver.str_utils import safe_bytes
20 from vcsserver.lib.str_utils import safe_bytes
21 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
21 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
22
22
23
23
24 @pytest.fixture()
24 @pytest.fixture()
25 def lfs_store(tmpdir):
25 def lfs_store(tmpdir):
26 repo = 'test'
26 repo = 'test'
27 oid = '123456789'
27 oid = '123456789'
28 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
28 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
29 return store
29 return store
30
30
31
31
32 @pytest.fixture()
32 @pytest.fixture()
33 def oid_handler(lfs_store):
33 def oid_handler(lfs_store):
34 store = lfs_store
34 store = lfs_store
35 repo = store.repo
35 repo = store.repo
36 oid = store.oid
36 oid = store.oid
37
37
38 oid_handler = OidHandler(
38 oid_handler = OidHandler(
39 store=store, repo_name=repo, auth=('basic', 'xxxx'),
39 store=store, repo_name=repo, auth=('basic', 'xxxx'),
40 oid=oid,
40 oid=oid,
41 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
41 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
42 obj_verify_href='http://localhost/verify')
42 obj_verify_href='http://localhost/verify')
43 return oid_handler
43 return oid_handler
44
44
45
45
46 class TestOidHandler:
46 class TestOidHandler:
47
47
48 @pytest.mark.parametrize('exec_action', [
48 @pytest.mark.parametrize('exec_action', [
49 'download',
49 'download',
50 'upload',
50 'upload',
51 ])
51 ])
52 def test_exec_action(self, exec_action, oid_handler):
52 def test_exec_action(self, exec_action, oid_handler):
53 handler = oid_handler.exec_operation(exec_action)
53 handler = oid_handler.exec_operation(exec_action)
54 assert handler
54 assert handler
55
55
56 def test_exec_action_undefined(self, oid_handler):
56 def test_exec_action_undefined(self, oid_handler):
57 with pytest.raises(AttributeError):
57 with pytest.raises(AttributeError):
58 oid_handler.exec_operation('wrong')
58 oid_handler.exec_operation('wrong')
59
59
60 def test_download_oid_not_existing(self, oid_handler):
60 def test_download_oid_not_existing(self, oid_handler):
61 response, has_errors = oid_handler.exec_operation('download')
61 response, has_errors = oid_handler.exec_operation('download')
62
62
63 assert response is None
63 assert response is None
64 assert has_errors['error'] == {
64 assert has_errors['error'] == {
65 'code': 404,
65 'code': 404,
66 'message': 'object: 123456789 does not exist in store'}
66 'message': 'object: 123456789 does not exist in store'}
67
67
68 def test_download_oid(self, oid_handler):
68 def test_download_oid(self, oid_handler):
69 store = oid_handler.get_store()
69 store = oid_handler.get_store()
70 if not os.path.isdir(os.path.dirname(store.oid_path)):
70 if not os.path.isdir(os.path.dirname(store.oid_path)):
71 os.makedirs(os.path.dirname(store.oid_path))
71 os.makedirs(os.path.dirname(store.oid_path))
72
72
73 with open(store.oid_path, 'wb') as f:
73 with open(store.oid_path, 'wb') as f:
74 f.write(safe_bytes('CONTENT'))
74 f.write(safe_bytes('CONTENT'))
75
75
76 response, has_errors = oid_handler.exec_operation('download')
76 response, has_errors = oid_handler.exec_operation('download')
77
77
78 assert has_errors is None
78 assert has_errors is None
79 assert response['download'] == {
79 assert response['download'] == {
80 'header': {'Authorization': 'basic xxxx'},
80 'header': {'Authorization': 'basic xxxx'},
81 'href': 'http://localhost/handle_oid'
81 'href': 'http://localhost/handle_oid'
82 }
82 }
83
83
84 def test_upload_oid_that_exists(self, oid_handler):
84 def test_upload_oid_that_exists(self, oid_handler):
85 store = oid_handler.get_store()
85 store = oid_handler.get_store()
86 if not os.path.isdir(os.path.dirname(store.oid_path)):
86 if not os.path.isdir(os.path.dirname(store.oid_path)):
87 os.makedirs(os.path.dirname(store.oid_path))
87 os.makedirs(os.path.dirname(store.oid_path))
88
88
89 with open(store.oid_path, 'wb') as f:
89 with open(store.oid_path, 'wb') as f:
90 f.write(safe_bytes('CONTENT'))
90 f.write(safe_bytes('CONTENT'))
91 oid_handler.obj_size = 7
91 oid_handler.obj_size = 7
92 response, has_errors = oid_handler.exec_operation('upload')
92 response, has_errors = oid_handler.exec_operation('upload')
93 assert has_errors is None
93 assert has_errors is None
94 assert response is None
94 assert response is None
95
95
96 def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
96 def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
97 store = oid_handler.get_store()
97 store = oid_handler.get_store()
98 if not os.path.isdir(os.path.dirname(store.oid_path)):
98 if not os.path.isdir(os.path.dirname(store.oid_path)):
99 os.makedirs(os.path.dirname(store.oid_path))
99 os.makedirs(os.path.dirname(store.oid_path))
100
100
101 with open(store.oid_path, 'wb') as f:
101 with open(store.oid_path, 'wb') as f:
102 f.write(safe_bytes('CONTENT'))
102 f.write(safe_bytes('CONTENT'))
103
103
104 oid_handler.obj_size = 10240
104 oid_handler.obj_size = 10240
105 response, has_errors = oid_handler.exec_operation('upload')
105 response, has_errors = oid_handler.exec_operation('upload')
106 assert has_errors is None
106 assert has_errors is None
107 assert response['upload'] == {
107 assert response['upload'] == {
108 'header': {'Authorization': 'basic xxxx',
108 'header': {'Authorization': 'basic xxxx',
109 'Transfer-Encoding': 'chunked'},
109 'Transfer-Encoding': 'chunked'},
110 'href': 'http://localhost/handle_oid',
110 'href': 'http://localhost/handle_oid',
111 }
111 }
112
112
113 def test_upload_oid(self, oid_handler):
113 def test_upload_oid(self, oid_handler):
114 response, has_errors = oid_handler.exec_operation('upload')
114 response, has_errors = oid_handler.exec_operation('upload')
115 assert has_errors is None
115 assert has_errors is None
116 assert response['upload'] == {
116 assert response['upload'] == {
117 'header': {'Authorization': 'basic xxxx',
117 'header': {'Authorization': 'basic xxxx',
118 'Transfer-Encoding': 'chunked'},
118 'Transfer-Encoding': 'chunked'},
119 'href': 'http://localhost/handle_oid'
119 'href': 'http://localhost/handle_oid'
120 }
120 }
121
121
122
122
123 class TestLFSStore:
123 class TestLFSStore:
124 def test_write_oid(self, lfs_store):
124 def test_write_oid(self, lfs_store):
125 oid_location = lfs_store.oid_path
125 oid_location = lfs_store.oid_path
126
126
127 assert not os.path.isfile(oid_location)
127 assert not os.path.isfile(oid_location)
128
128
129 engine = lfs_store.get_engine(mode='wb')
129 engine = lfs_store.get_engine(mode='wb')
130 with engine as f:
130 with engine as f:
131 f.write(safe_bytes('CONTENT'))
131 f.write(safe_bytes('CONTENT'))
132
132
133 assert os.path.isfile(oid_location)
133 assert os.path.isfile(oid_location)
134
134
135 def test_detect_has_oid(self, lfs_store):
135 def test_detect_has_oid(self, lfs_store):
136
136
137 assert lfs_store.has_oid() is False
137 assert lfs_store.has_oid() is False
138 engine = lfs_store.get_engine(mode='wb')
138 engine = lfs_store.get_engine(mode='wb')
139 with engine as f:
139 with engine as f:
140 f.write(safe_bytes('CONTENT'))
140 f.write(safe_bytes('CONTENT'))
141
141
142 assert lfs_store.has_oid() is True
142 assert lfs_store.has_oid() is True
@@ -1,92 +1,92 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """
18 """
19 Mercurial libs compatibility
19 Mercurial libs compatibility
20 """
20 """
21
21
22 import mercurial
22 import mercurial
23 from mercurial import demandimport
23 from mercurial import demandimport
24
24
25 # patch demandimport, due to bug in mercurial when it always triggers
25 # patch demandimport, due to bug in mercurial when it always triggers
26 # demandimport.enable()
26 # demandimport.enable()
27 from vcsserver.str_utils import safe_bytes
27 from vcsserver.lib.str_utils import safe_bytes
28
28
29 demandimport.enable = lambda *args, **kwargs: 1
29 demandimport.enable = lambda *args, **kwargs: 1
30
30
31 from mercurial import ui
31 from mercurial import ui
32 from mercurial import patch
32 from mercurial import patch
33 from mercurial import config
33 from mercurial import config
34 from mercurial import extensions
34 from mercurial import extensions
35 from mercurial import scmutil
35 from mercurial import scmutil
36 from mercurial import archival
36 from mercurial import archival
37 from mercurial import discovery
37 from mercurial import discovery
38 from mercurial import unionrepo
38 from mercurial import unionrepo
39 from mercurial import localrepo
39 from mercurial import localrepo
40 from mercurial import merge as hg_merge
40 from mercurial import merge as hg_merge
41 from mercurial import subrepo
41 from mercurial import subrepo
42 from mercurial import subrepoutil
42 from mercurial import subrepoutil
43 from mercurial import tags as hg_tag
43 from mercurial import tags as hg_tag
44 from mercurial import util as hgutil
44 from mercurial import util as hgutil
45 from mercurial.commands import clone, pull
45 from mercurial.commands import clone, pull
46 from mercurial.node import nullid
46 from mercurial.node import nullid
47 from mercurial.context import memctx, memfilectx
47 from mercurial.context import memctx, memfilectx
48 from mercurial.error import (
48 from mercurial.error import (
49 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
49 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
50 RequirementError, ProgrammingError)
50 RequirementError, ProgrammingError)
51 from mercurial.hgweb import hgweb_mod
51 from mercurial.hgweb import hgweb_mod
52 from mercurial.localrepo import instance
52 from mercurial.localrepo import instance
53 from mercurial.match import match, alwaysmatcher, patternmatcher
53 from mercurial.match import match, alwaysmatcher, patternmatcher
54 from mercurial.mdiff import diffopts
54 from mercurial.mdiff import diffopts
55 from mercurial.node import bin, hex
55 from mercurial.node import bin, hex
56 from mercurial.encoding import tolocal
56 from mercurial.encoding import tolocal
57 from mercurial.discovery import findcommonoutgoing
57 from mercurial.discovery import findcommonoutgoing
58 from mercurial.hg import peer
58 from mercurial.hg import peer
59 from mercurial.httppeer import makepeer
59 from mercurial.httppeer import make_peer
60 from mercurial.utils.urlutil import url as hg_url
60 from mercurial.utils.urlutil import url as hg_url
61 from mercurial.scmutil import revrange, revsymbol
61 from mercurial.scmutil import revrange, revsymbol
62 from mercurial.node import nullrev
62 from mercurial.node import nullrev
63 from mercurial import exchange
63 from mercurial import exchange
64 from hgext import largefiles
64 from hgext import largefiles
65
65
66 # those authnadlers are patched for python 2.6.5 bug an
66 # those authnadlers are patched for python 2.6.5 bug an
67 # infinit looping when given invalid resources
67 # infinit looping when given invalid resources
68 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
68 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
69
69
70 # hg strip is in core now
70 # hg strip is in core now
71 from mercurial import strip as hgext_strip
71 from mercurial import strip as hgext_strip
72
72
73
73
74 def get_ctx(repo, ref):
74 def get_ctx(repo, ref):
75 if not isinstance(ref, int):
75 if not isinstance(ref, int):
76 ref = safe_bytes(ref)
76 ref = safe_bytes(ref)
77
77
78 try:
78 try:
79 ctx = repo[ref]
79 ctx = repo[ref]
80 return ctx
80 return ctx
81 except (ProgrammingError, TypeError):
81 except (ProgrammingError, TypeError):
82 # we're unable to find the rev using a regular lookup, we fallback
82 # we're unable to find the rev using a regular lookup, we fallback
83 # to slower, but backward compat revsymbol usage
83 # to slower, but backward compat revsymbol usage
84 pass
84 pass
85 except (LookupError, RepoLookupError):
85 except (LookupError, RepoLookupError):
86 # Similar case as above but only for refs that are not numeric
86 # Similar case as above but only for refs that are not numeric
87 if isinstance(ref, int):
87 if isinstance(ref, int):
88 raise
88 raise
89
89
90 ctx = revsymbol(repo, ref)
90 ctx = revsymbol(repo, ref)
91
91
92 return ctx
92 return ctx
@@ -1,230 +1,238 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18
19 import re
19 import re
20 import os
20 import os
21 import sys
21 import sys
22 import datetime
22 import datetime
23 import logging
23 import logging
24 import pkg_resources
24 import pkg_resources
25
25
26 import vcsserver
26 import vcsserver
27 import vcsserver.settings
27 import vcsserver.settings
28 from vcsserver.str_utils import safe_bytes
28 from vcsserver.lib.str_utils import safe_bytes
29
29
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32 HOOKS_DIR_MODE = 0o755
32 HOOKS_DIR_MODE = 0o755
33 HOOKS_FILE_MODE = 0o755
33 HOOKS_FILE_MODE = 0o755
34
34
35
35
36 def set_permissions_if_needed(path_to_check, perms: oct):
36 def set_permissions_if_needed(path_to_check, perms: oct):
37 # Get current permissions
37 # Get current permissions
38 current_permissions = os.stat(path_to_check).st_mode & 0o777 # Extract permission bits
38 current_permissions = os.stat(path_to_check).st_mode & 0o777 # Extract permission bits
39
39
40 # Check if current permissions are lower than required
40 # Check if current permissions are lower than required
41 if current_permissions < int(perms):
41 if current_permissions < int(perms):
42 # Change the permissions if they are lower than required
42 # Change the permissions if they are lower than required
43 os.chmod(path_to_check, perms)
43 os.chmod(path_to_check, perms)
44
44
45
45
46 def get_git_hooks_path(repo_path, bare):
46 def get_git_hooks_path(repo_path, bare):
47 hooks_path = os.path.join(repo_path, 'hooks')
47 hooks_path = os.path.join(repo_path, 'hooks')
48 if not bare:
48 if not bare:
49 hooks_path = os.path.join(repo_path, '.git', 'hooks')
49 hooks_path = os.path.join(repo_path, '.git', 'hooks')
50
50
51 return hooks_path
51 return hooks_path
52
52
53
53
54 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
54 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
55 """
55 """
56 Creates a RhodeCode hook inside a git repository
56 Creates a RhodeCode hook inside a git repository
57
57
58 :param repo_path: path to repository
58 :param repo_path: path to repository
59 :param bare: defines if repository is considered a bare git repo
59 :param bare: defines if repository is considered a bare git repo
60 :param executable: binary executable to put in the hooks
60 :param executable: binary executable to put in the hooks
61 :param force_create: Creates even if the same name hook exists
61 :param force_create: Creates even if the same name hook exists
62 """
62 """
63 executable = executable or sys.executable
63 executable = executable or sys.executable
64 hooks_path = get_git_hooks_path(repo_path, bare)
64 hooks_path = get_git_hooks_path(repo_path, bare)
65
65
66 # we always call it to ensure dir exists and it has a proper mode
66 # we always call it to ensure dir exists and it has a proper mode
67 if not os.path.exists(hooks_path):
67 if not os.path.exists(hooks_path):
68 # If it doesn't exist, create a new directory with the specified mode
68 # If it doesn't exist, create a new directory with the specified mode
69 os.makedirs(hooks_path, mode=HOOKS_DIR_MODE, exist_ok=True)
69 os.makedirs(hooks_path, mode=HOOKS_DIR_MODE, exist_ok=True)
70 # If it exists, change the directory's mode to the specified mode
70 # If it exists, change the directory's mode to the specified mode
71 set_permissions_if_needed(hooks_path, perms=HOOKS_DIR_MODE)
71 set_permissions_if_needed(hooks_path, perms=HOOKS_DIR_MODE)
72
72
73 tmpl_post = pkg_resources.resource_string(
73 tmpl_post = pkg_resources.resource_string(
74 'vcsserver', '/'.join(
74 'vcsserver', '/'.join(
75 ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
75 ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
76 tmpl_pre = pkg_resources.resource_string(
76 tmpl_pre = pkg_resources.resource_string(
77 'vcsserver', '/'.join(
77 'vcsserver', '/'.join(
78 ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
78 ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
79
79
80 path = '' # not used for now
80 path = '' # not used for now
81 timestamp = datetime.datetime.utcnow().isoformat()
81 timestamp = datetime.datetime.utcnow().isoformat()
82
82
83 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
83 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
84 log.debug('Installing git hook in repo %s', repo_path)
84 log.debug('Installing git hook in repo %s', repo_path)
85 _hook_file = os.path.join(hooks_path, f'{h_type}-receive')
85 _hook_file = os.path.join(hooks_path, f'{h_type}-receive')
86 _rhodecode_hook = check_rhodecode_hook(_hook_file)
86 _rhodecode_hook = check_rhodecode_hook(_hook_file)
87
87
88 if _rhodecode_hook or force_create:
88 if _rhodecode_hook or force_create:
89 log.debug('writing git %s hook file at %s !', h_type, _hook_file)
89 log.debug('writing git %s hook file at %s !', h_type, _hook_file)
90 env_expand = str([
91 ('RC_INI_FILE', vcsserver.CONFIG['__file__']),
92 ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR),
93 ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()),
94 ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()),
95 ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()),
96 ])
90 try:
97 try:
91 with open(_hook_file, 'wb') as f:
98 with open(_hook_file, 'wb') as f:
99 template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
92 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
100 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
93 template = template.replace(b'_DATE_', safe_bytes(timestamp))
101 template = template.replace(b'_DATE_', safe_bytes(timestamp))
94 template = template.replace(b'_ENV_', safe_bytes(executable))
102 template = template.replace(b'_ENV_', safe_bytes(executable))
95 template = template.replace(b'_PATH_', safe_bytes(path))
103 template = template.replace(b'_PATH_', safe_bytes(path))
96 f.write(template)
104 f.write(template)
97 set_permissions_if_needed(_hook_file, perms=HOOKS_FILE_MODE)
105 set_permissions_if_needed(_hook_file, perms=HOOKS_FILE_MODE)
98 except OSError:
106 except OSError:
99 log.exception('error writing hook file %s', _hook_file)
107 log.exception('error writing hook file %s', _hook_file)
100 else:
108 else:
101 log.debug('skipping writing hook file')
109 log.debug('skipping writing hook file')
102
110
103 return True
111 return True
104
112
105
113
106 def get_svn_hooks_path(repo_path):
114 def get_svn_hooks_path(repo_path):
107 hooks_path = os.path.join(repo_path, 'hooks')
115 hooks_path = os.path.join(repo_path, 'hooks')
108
116
109 return hooks_path
117 return hooks_path
110
118
111
119
112 def install_svn_hooks(repo_path, executable=None, force_create=False):
120 def install_svn_hooks(repo_path, executable=None, force_create=False):
113 """
121 """
114 Creates RhodeCode hooks inside a svn repository
122 Creates RhodeCode hooks inside a svn repository
115
123
116 :param repo_path: path to repository
124 :param repo_path: path to repository
117 :param executable: binary executable to put in the hooks
125 :param executable: binary executable to put in the hooks
118 :param force_create: Create even if same name hook exists
126 :param force_create: Create even if same name hook exists
119 """
127 """
120 executable = executable or sys.executable
128 executable = executable or sys.executable
121 hooks_path = get_svn_hooks_path(repo_path)
129 hooks_path = get_svn_hooks_path(repo_path)
122 if not os.path.isdir(hooks_path):
130 if not os.path.isdir(hooks_path):
123 os.makedirs(hooks_path, mode=0o777, exist_ok=True)
131 os.makedirs(hooks_path, mode=0o777, exist_ok=True)
124
132
125 tmpl_post = pkg_resources.resource_string(
133 tmpl_post = pkg_resources.resource_string(
126 'vcsserver', '/'.join(
134 'vcsserver', '/'.join(
127 ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
135 ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
128 tmpl_pre = pkg_resources.resource_string(
136 tmpl_pre = pkg_resources.resource_string(
129 'vcsserver', '/'.join(
137 'vcsserver', '/'.join(
130 ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
138 ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
131
139
132 path = '' # not used for now
140 path = '' # not used for now
133 timestamp = datetime.datetime.utcnow().isoformat()
141 timestamp = datetime.datetime.utcnow().isoformat()
134
142
135 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
143 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
136 log.debug('Installing svn hook in repo %s', repo_path)
144 log.debug('Installing svn hook in repo %s', repo_path)
137 _hook_file = os.path.join(hooks_path, f'{h_type}-commit')
145 _hook_file = os.path.join(hooks_path, f'{h_type}-commit')
138 _rhodecode_hook = check_rhodecode_hook(_hook_file)
146 _rhodecode_hook = check_rhodecode_hook(_hook_file)
139
147
140 if _rhodecode_hook or force_create:
148 if _rhodecode_hook or force_create:
141 log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
149 log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
142
150
143 env_expand = str([
151 env_expand = str([
152 ('RC_INI_FILE', vcsserver.CONFIG['__file__']),
144 ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR),
153 ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR),
145 ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()),
154 ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()),
146 ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()),
155 ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()),
147 ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()),
156 ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()),
148
149 ])
157 ])
150 try:
158 try:
151 with open(_hook_file, 'wb') as f:
159 with open(_hook_file, 'wb') as f:
160 template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
152 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
161 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
153 template = template.replace(b'_DATE_', safe_bytes(timestamp))
162 template = template.replace(b'_DATE_', safe_bytes(timestamp))
154 template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
155 template = template.replace(b'_ENV_', safe_bytes(executable))
163 template = template.replace(b'_ENV_', safe_bytes(executable))
156 template = template.replace(b'_PATH_', safe_bytes(path))
164 template = template.replace(b'_PATH_', safe_bytes(path))
157
165
158 f.write(template)
166 f.write(template)
159 os.chmod(_hook_file, 0o755)
167 os.chmod(_hook_file, 0o755)
160 except OSError:
168 except OSError:
161 log.exception('error writing hook file %s', _hook_file)
169 log.exception('error writing hook file %s', _hook_file)
162 else:
170 else:
163 log.debug('skipping writing hook file')
171 log.debug('skipping writing hook file')
164
172
165 return True
173 return True
166
174
167
175
168 def get_version_from_hook(hook_path):
176 def get_version_from_hook(hook_path):
169 version = b''
177 version = b''
170 hook_content = read_hook_content(hook_path)
178 hook_content = read_hook_content(hook_path)
171 matches = re.search(rb'RC_HOOK_VER\s*=\s*(.*)', hook_content)
179 matches = re.search(rb'RC_HOOK_VER\s*=\s*(.*)', hook_content)
172 if matches:
180 if matches:
173 try:
181 try:
174 version = matches.groups()[0]
182 version = matches.groups()[0]
175 log.debug('got version %s from hooks.', version)
183 log.debug('got version %s from hooks.', version)
176 except Exception:
184 except Exception:
177 log.exception("Exception while reading the hook version.")
185 log.exception("Exception while reading the hook version.")
178 return version.replace(b"'", b"")
186 return version.replace(b"'", b"")
179
187
180
188
181 def check_rhodecode_hook(hook_path):
189 def check_rhodecode_hook(hook_path):
182 """
190 """
183 Check if the hook was created by RhodeCode
191 Check if the hook was created by RhodeCode
184 """
192 """
185 if not os.path.exists(hook_path):
193 if not os.path.exists(hook_path):
186 return True
194 return True
187
195
188 log.debug('hook exists, checking if it is from RhodeCode')
196 log.debug('hook exists, checking if it is from RhodeCode')
189
197
190 version = get_version_from_hook(hook_path)
198 version = get_version_from_hook(hook_path)
191 if version:
199 if version:
192 return True
200 return True
193
201
194 return False
202 return False
195
203
196
204
197 def read_hook_content(hook_path) -> bytes:
205 def read_hook_content(hook_path) -> bytes:
198 content = b''
206 content = b''
199 if os.path.isfile(hook_path):
207 if os.path.isfile(hook_path):
200 with open(hook_path, 'rb') as f:
208 with open(hook_path, 'rb') as f:
201 content = f.read()
209 content = f.read()
202 return content
210 return content
203
211
204
212
205 def get_git_pre_hook_version(repo_path, bare):
213 def get_git_pre_hook_version(repo_path, bare):
206 hooks_path = get_git_hooks_path(repo_path, bare)
214 hooks_path = get_git_hooks_path(repo_path, bare)
207 _hook_file = os.path.join(hooks_path, 'pre-receive')
215 _hook_file = os.path.join(hooks_path, 'pre-receive')
208 version = get_version_from_hook(_hook_file)
216 version = get_version_from_hook(_hook_file)
209 return version
217 return version
210
218
211
219
212 def get_git_post_hook_version(repo_path, bare):
220 def get_git_post_hook_version(repo_path, bare):
213 hooks_path = get_git_hooks_path(repo_path, bare)
221 hooks_path = get_git_hooks_path(repo_path, bare)
214 _hook_file = os.path.join(hooks_path, 'post-receive')
222 _hook_file = os.path.join(hooks_path, 'post-receive')
215 version = get_version_from_hook(_hook_file)
223 version = get_version_from_hook(_hook_file)
216 return version
224 return version
217
225
218
226
219 def get_svn_pre_hook_version(repo_path):
227 def get_svn_pre_hook_version(repo_path):
220 hooks_path = get_svn_hooks_path(repo_path)
228 hooks_path = get_svn_hooks_path(repo_path)
221 _hook_file = os.path.join(hooks_path, 'pre-commit')
229 _hook_file = os.path.join(hooks_path, 'pre-commit')
222 version = get_version_from_hook(_hook_file)
230 version = get_version_from_hook(_hook_file)
223 return version
231 return version
224
232
225
233
226 def get_svn_post_hook_version(repo_path):
234 def get_svn_post_hook_version(repo_path):
227 hooks_path = get_svn_hooks_path(repo_path)
235 hooks_path = get_svn_hooks_path(repo_path)
228 _hook_file = os.path.join(hooks_path, 'post-commit')
236 _hook_file = os.path.join(hooks_path, 'post-commit')
229 version = get_version_from_hook(_hook_file)
237 version = get_version_from_hook(_hook_file)
230 return version
238 return version
@@ -1,51 +1,59 b''
1 #!_ENV_
1 #!_ENV_
2
2 import os
3 import os
3 import sys
4 import sys
4 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
5
6
6 if path_adjust:
7 if path_adjust:
7 sys.path = path_adjust
8 sys.path = path_adjust
8
9
10 # special trick to pass in some information from rc to hooks
11 # mod_dav strips ALL env vars and we can't even access things like PATH
12 for env_k, env_v in _OS_EXPAND_:
13 os.environ[env_k] = env_v
14
9 try:
15 try:
10 from vcsserver import hooks
16 from vcsserver import hooks
11 except ImportError:
17 except ImportError:
12 if os.environ.get('RC_DEBUG_GIT_HOOK'):
18 if os.environ.get('RC_DEBUG_GIT_HOOK'):
13 import traceback
19 import traceback
14 print(traceback.format_exc())
20 print(traceback.format_exc())
15 hooks = None
21 hooks = None
16
22
17
23
18 # TIMESTAMP: _DATE_
24 # TIMESTAMP: _DATE_
19 RC_HOOK_VER = '_TMPL_'
25 RC_HOOK_VER = '_TMPL_'
20
26
21
27
22 def main():
28 def main():
23 if hooks is None:
29 if hooks is None:
24 # exit with success if we cannot import vcsserver.hooks !!
30 # exit with success if we cannot import vcsserver.hooks !!
25 # this allows simply push to this repo even without rhodecode
31 # this allows simply push to this repo even without rhodecode
26 sys.exit(0)
32 sys.exit(0)
27
33
28 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
34 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
29 sys.exit(0)
35 sys.exit(0)
30
36
31 repo_path = os.getcwd()
37 repo_path = os.getcwd()
32 push_data = sys.stdin.readlines()
38 push_data = sys.stdin.readlines()
33 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
39
34 # os.environ is modified here by a subprocess call that
40 # os.environ is modified here by a subprocess call that
35 # runs git and later git executes this hook.
41 # runs git and later git executes this hook.
36 # Environ gets some additional info from rhodecode system
42 # Environ gets some additional info from rhodecode system
37 # like IP or username from basic-auth
43 # like IP or username from basic-auth
44
45 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
38 try:
46 try:
39 result = hooks.git_post_receive(repo_path, push_data, os.environ)
47 result = hooks.git_post_receive(repo_path, push_data, os.environ)
40 sys.exit(result)
48 sys.exit(result)
41 except Exception as error:
49 except Exception as error:
42 # TODO: johbo: Improve handling of this special case
50 # TODO: johbo: Improve handling of this special case
43 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
51 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
44 raise
52 raise
45 print(f'ERROR: {error}')
53 print(f'ERROR: {error}')
46 sys.exit(1)
54 sys.exit(1)
47 sys.exit(0)
55 sys.exit(0)
48
56
49
57
50 if __name__ == '__main__':
58 if __name__ == '__main__':
51 main()
59 main()
@@ -1,51 +1,59 b''
1 #!_ENV_
1 #!_ENV_
2
2 import os
3 import os
3 import sys
4 import sys
4 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
5
6
6 if path_adjust:
7 if path_adjust:
7 sys.path = path_adjust
8 sys.path = path_adjust
8
9
10 # special trick to pass in some information from rc to hooks
11 # mod_dav strips ALL env vars and we can't even access things like PATH
12 for env_k, env_v in _OS_EXPAND_:
13 os.environ[env_k] = env_v
14
9 try:
15 try:
10 from vcsserver import hooks
16 from vcsserver import hooks
11 except ImportError:
17 except ImportError:
12 if os.environ.get('RC_DEBUG_GIT_HOOK'):
18 if os.environ.get('RC_DEBUG_GIT_HOOK'):
13 import traceback
19 import traceback
14 print(traceback.format_exc())
20 print(traceback.format_exc())
15 hooks = None
21 hooks = None
16
22
17
23
18 # TIMESTAMP: _DATE_
24 # TIMESTAMP: _DATE_
19 RC_HOOK_VER = '_TMPL_'
25 RC_HOOK_VER = '_TMPL_'
20
26
21
27
22 def main():
28 def main():
23 if hooks is None:
29 if hooks is None:
24 # exit with success if we cannot import vcsserver.hooks !!
30 # exit with success if we cannot import vcsserver.hooks !!
25 # this allows simply push to this repo even without rhodecode
31 # this allows simply push to this repo even without rhodecode
26 sys.exit(0)
32 sys.exit(0)
27
33
28 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
34 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
29 sys.exit(0)
35 sys.exit(0)
30
36
31 repo_path = os.getcwd()
37 repo_path = os.getcwd()
32 push_data = sys.stdin.readlines()
38 push_data = sys.stdin.readlines()
33 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
39
34 # os.environ is modified here by a subprocess call that
40 # os.environ is modified here by a subprocess call that
35 # runs git and later git executes this hook.
41 # runs git and later git executes this hook.
36 # Environ gets some additional info from rhodecode system
42 # Environ gets some additional info from rhodecode system
37 # like IP or username from basic-auth
43 # like IP or username from basic-auth
44
45 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
38 try:
46 try:
39 result = hooks.git_pre_receive(repo_path, push_data, os.environ)
47 result = hooks.git_pre_receive(repo_path, push_data, os.environ)
40 sys.exit(result)
48 sys.exit(result)
41 except Exception as error:
49 except Exception as error:
42 # TODO: johbo: Improve handling of this special case
50 # TODO: johbo: Improve handling of this special case
43 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
51 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
44 raise
52 raise
45 print(f'ERROR: {error}')
53 print(f'ERROR: {error}')
46 sys.exit(1)
54 sys.exit(1)
47 sys.exit(0)
55 sys.exit(0)
48
56
49
57
50 if __name__ == '__main__':
58 if __name__ == '__main__':
51 main()
59 main()
@@ -1,54 +1,59 b''
1 #!_ENV_
1 #!_ENV_
2
2
3 import os
3 import os
4 import sys
4 import sys
5 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
6
6
7 if path_adjust:
7 if path_adjust:
8 sys.path = path_adjust
8 sys.path = path_adjust
9
9
10 # special trick to pass in some information from rc to hooks
11 # mod_dav strips ALL env vars and we can't even access things like PATH
12 for env_k, env_v in _OS_EXPAND_:
13 os.environ[env_k] = env_v
14
10 try:
15 try:
11 from vcsserver import hooks
16 from vcsserver import hooks
12 except ImportError:
17 except ImportError:
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
18 if os.environ.get('RC_DEBUG_SVN_HOOK'):
14 import traceback
19 import traceback
15 print(traceback.format_exc())
20 print(traceback.format_exc())
16 hooks = None
21 hooks = None
17
22
18
23
19 # TIMESTAMP: _DATE_
24 # TIMESTAMP: _DATE_
20 RC_HOOK_VER = '_TMPL_'
25 RC_HOOK_VER = '_TMPL_'
21
26
22
27
23 # special trick to pass in some information from rc to hooks
28 # special trick to pass in some information from rc to hooks
24 # mod_dav strips ALL env vars and we can't even access things like PATH
29 # mod_dav strips ALL env vars and we can't even access things like PATH
25 for env_k, env_v in _OS_EXPAND_:
30 for env_k, env_v in _OS_EXPAND_:
26 os.environ[env_k] = env_v
31 os.environ[env_k] = env_v
27
32
28 def main():
33 def main():
29 if hooks is None:
34 if hooks is None:
30 # exit with success if we cannot import vcsserver.hooks !!
35 # exit with success if we cannot import vcsserver.hooks !!
31 # this allows simply push to this repo even without rhodecode
36 # this allows simply push to this repo even without rhodecode
32 sys.exit(0)
37 sys.exit(0)
33
38
34 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
39 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
35 sys.exit(0)
40 sys.exit(0)
36 repo_path = os.getcwd()
41 cwd_repo_path = os.getcwd()
37 push_data = sys.argv[1:]
42 push_data = sys.argv[1:]
38
43
39 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
44 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
40
45
41 try:
46 try:
42 result = hooks.svn_post_commit(repo_path, push_data, os.environ)
47 result = hooks.svn_post_commit(cwd_repo_path, push_data, os.environ)
43 sys.exit(result)
48 sys.exit(result)
44 except Exception as error:
49 except Exception as error:
45 # TODO: johbo: Improve handling of this special case
50 # TODO: johbo: Improve handling of this special case
46 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
51 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
47 raise
52 raise
48 print(f'ERROR: {error}')
53 print(f'ERROR: {error}')
49 sys.exit(1)
54 sys.exit(1)
50 sys.exit(0)
55 sys.exit(0)
51
56
52
57
53 if __name__ == '__main__':
58 if __name__ == '__main__':
54 main()
59 main()
@@ -1,58 +1,62 b''
1 #!_ENV_
1 #!_ENV_
2
2
3 import os
3 import os
4 import sys
4 import sys
5 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
6
6
7 if path_adjust:
7 if path_adjust:
8 sys.path = path_adjust
8 sys.path = path_adjust
9
9
10 # special trick to pass in some information from rc to hooks
11 # mod_dav strips ALL env vars and we can't even access things like PATH
12 for env_k, env_v in _OS_EXPAND_:
13 os.environ[env_k] = env_v
14
10 try:
15 try:
11 from vcsserver import hooks
16 from vcsserver import hooks
12 except ImportError:
17 except ImportError:
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
18 if os.environ.get('RC_DEBUG_SVN_HOOK'):
14 import traceback
19 import traceback
15 print(traceback.format_exc())
20 print(traceback.format_exc())
16 hooks = None
21 hooks = None
17
22
18
23
19 # TIMESTAMP: _DATE_
24 # TIMESTAMP: _DATE_
20 RC_HOOK_VER = '_TMPL_'
25 RC_HOOK_VER = '_TMPL_'
21
26
22
27
23 # special trick to pass in some information from rc to hooks
28 # special trick to pass in some information from rc to hooks
24 # mod_dav strips ALL env vars and we can't even access things like PATH
29 # mod_dav strips ALL env vars and we can't even access things like PATH
25 for env_k, env_v in _OS_EXPAND_:
30 for env_k, env_v in _OS_EXPAND_:
26 os.environ[env_k] = env_v
31 os.environ[env_k] = env_v
27
32
28 def main():
33 def main():
29 if os.environ.get('SSH_READ_ONLY') == '1':
34 if os.environ.get('SSH_READ_ONLY') == '1':
30 sys.stderr.write('Only read-only access is allowed')
35 sys.stderr.write('Only read-only access is allowed')
31 sys.exit(1)
36 sys.exit(1)
32
37
33 if hooks is None:
38 if hooks is None:
34 # exit with success if we cannot import vcsserver.hooks !!
39 # exit with success if we cannot import vcsserver.hooks !!
35 # this allows simply push to this repo even without rhodecode
40 # this allows simply push to this repo even without rhodecode
36 sys.exit(0)
41 sys.exit(0)
37
42
38 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
43 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
39 sys.exit(0)
44 sys.exit(0)
40 repo_path = os.getcwd()
45 cwd_repo_path = os.getcwd()
41 push_data = sys.argv[1:]
46 push_data = sys.argv[1:]
42
47
43 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
48 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
44
45 try:
49 try:
46 result = hooks.svn_pre_commit(repo_path, push_data, os.environ)
50 result = hooks.svn_pre_commit(cwd_repo_path, push_data, os.environ)
47 sys.exit(result)
51 sys.exit(result)
48 except Exception as error:
52 except Exception as error:
49 # TODO: johbo: Improve handling of this special case
53 # TODO: johbo: Improve handling of this special case
50 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
54 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
51 raise
55 raise
52 print(f'ERROR: {error}')
56 print(f'ERROR: {error}')
53 sys.exit(1)
57 sys.exit(1)
54 sys.exit(0)
58 sys.exit(0)
55
59
56
60
57 if __name__ == '__main__':
61 if __name__ == '__main__':
58 main()
62 main()
@@ -1,826 +1,822 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import sys
20 import sys
21 import logging
21 import logging
22 import collections
22 import collections
23 import base64
23 import base64
24 import msgpack
24 import msgpack
25 import dataclasses
25 import dataclasses
26 import pygit2
26 import pygit2
27
27
28 import http.client
28 import http.client
29 from celery import Celery
29 from celery import Celery
30
30
31 import mercurial.scmutil
31 import mercurial.scmutil
32 import mercurial.node
32 import mercurial.node
33
33
34 from vcsserver.lib.rc_json import json
35 from vcsserver import exceptions, subprocessio, settings
34 from vcsserver import exceptions, subprocessio, settings
36 from vcsserver.str_utils import ascii_str, safe_str
35 from vcsserver.lib.ext_json import json
36 from vcsserver.lib.str_utils import ascii_str, safe_str
37 from vcsserver.lib.svn_txn_utils import get_txn_id_from_store
37 from vcsserver.remote.git_remote import Repository
38 from vcsserver.remote.git_remote import Repository
38
39
39 celery_app = Celery('__vcsserver__')
40 celery_app = Celery('__vcsserver__')
40 log = logging.getLogger(__name__)
41 log = logging.getLogger(__name__)
41
42
42
43
43 class HooksHttpClient:
44 class HooksHttpClient:
44 proto = 'msgpack.v1'
45 proto = 'msgpack.v1'
45 connection = None
46 connection = None
46
47
47 def __init__(self, hooks_uri):
48 def __init__(self, hooks_uri):
48 self.hooks_uri = hooks_uri
49 self.hooks_uri = hooks_uri
49
50
50 def __repr__(self):
51 def __repr__(self):
51 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
52 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
52
53
53 def __call__(self, method, extras):
54 def __call__(self, method, extras):
54 connection = http.client.HTTPConnection(self.hooks_uri)
55 connection = http.client.HTTPConnection(self.hooks_uri)
55 # binary msgpack body
56 # binary msgpack body
56 headers, body = self._serialize(method, extras)
57 headers, body = self._serialize(method, extras)
57 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
58 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
58
59
59 try:
60 try:
60 try:
61 try:
61 connection.request('POST', '/', body, headers)
62 connection.request('POST', '/', body, headers)
62 except Exception as error:
63 except Exception as error:
63 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
64 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
64 raise
65 raise
65
66
66 response = connection.getresponse()
67 response = connection.getresponse()
67 try:
68 try:
68 return msgpack.load(response)
69 return msgpack.load(response)
69 except Exception:
70 except Exception:
70 response_data = response.read()
71 response_data = response.read()
71 log.exception('Failed to decode hook response json data. '
72 log.exception('Failed to decode hook response json data. '
72 'response_code:%s, raw_data:%s',
73 'response_code:%s, raw_data:%s',
73 response.status, response_data)
74 response.status, response_data)
74 raise
75 raise
75 finally:
76 finally:
76 connection.close()
77 connection.close()
77
78
78 @classmethod
79 @classmethod
79 def _serialize(cls, hook_name, extras):
80 def _serialize(cls, hook_name, extras):
80 data = {
81 data = {
81 'method': hook_name,
82 'method': hook_name,
82 'extras': extras
83 'extras': extras
83 }
84 }
84 headers = {
85 headers = {
85 "rc-hooks-protocol": cls.proto,
86 "rc-hooks-protocol": cls.proto,
86 "Connection": "keep-alive"
87 "Connection": "keep-alive"
87 }
88 }
88 return headers, msgpack.packb(data)
89 return headers, msgpack.packb(data)
89
90
90
91
91 class HooksCeleryClient:
92 class HooksCeleryClient:
92 TASK_TIMEOUT = 60 # time in seconds
93 TASK_TIMEOUT = 60 # time in seconds
93
94
94 def __init__(self, queue, backend):
95 def __init__(self, queue, backend):
95 celery_app.config_from_object({
96 celery_app.config_from_object({
96 'broker_url': queue, 'result_backend': backend,
97 'broker_url': queue, 'result_backend': backend,
97 'broker_connection_retry_on_startup': True,
98 'broker_connection_retry_on_startup': True,
98 'task_serializer': 'msgpack',
99 'task_serializer': 'json',
99 'accept_content': ['json', 'msgpack'],
100 'accept_content': ['json', 'msgpack'],
100 'result_serializer': 'msgpack',
101 'result_serializer': 'json',
101 'result_accept_content': ['json', 'msgpack']
102 'result_accept_content': ['json', 'msgpack']
102 })
103 })
103 self.celery_app = celery_app
104 self.celery_app = celery_app
104
105
105 def __call__(self, method, extras):
106 def __call__(self, method, extras):
106 inquired_task = self.celery_app.signature(
107 inquired_task = self.celery_app.signature(
107 f'rhodecode.lib.celerylib.tasks.{method}'
108 f'rhodecode.lib.celerylib.tasks.{method}'
108 )
109 )
109 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
110 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
110
111
111
112
112 class HooksShadowRepoClient:
113 class HooksShadowRepoClient:
113
114
114 def __call__(self, hook_name, extras):
115 def __call__(self, hook_name, extras):
115 return {'output': '', 'status': 0}
116 return {'output': '', 'status': 0}
116
117
117
118
118 class RemoteMessageWriter:
119 class RemoteMessageWriter:
119 """Writer base class."""
120 """Writer base class."""
120 def write(self, message):
121 def write(self, message):
121 raise NotImplementedError()
122 raise NotImplementedError()
122
123
123
124
124 class HgMessageWriter(RemoteMessageWriter):
125 class HgMessageWriter(RemoteMessageWriter):
125 """Writer that knows how to send messages to mercurial clients."""
126 """Writer that knows how to send messages to mercurial clients."""
126
127
127 def __init__(self, ui):
128 def __init__(self, ui):
128 self.ui = ui
129 self.ui = ui
129
130
130 def write(self, message: str):
131 def write(self, message: str):
131 # TODO: Check why the quiet flag is set by default.
132 # TODO: Check why the quiet flag is set by default.
132 old = self.ui.quiet
133 old = self.ui.quiet
133 self.ui.quiet = False
134 self.ui.quiet = False
134 self.ui.status(message.encode('utf-8'))
135 self.ui.status(message.encode('utf-8'))
135 self.ui.quiet = old
136 self.ui.quiet = old
136
137
137
138
138 class GitMessageWriter(RemoteMessageWriter):
139 class GitMessageWriter(RemoteMessageWriter):
139 """Writer that knows how to send messages to git clients."""
140 """Writer that knows how to send messages to git clients."""
140
141
141 def __init__(self, stdout=None):
142 def __init__(self, stdout=None):
142 self.stdout = stdout or sys.stdout
143 self.stdout = stdout or sys.stdout
143
144
144 def write(self, message: str):
145 def write(self, message: str):
145 self.stdout.write(message)
146 self.stdout.write(message)
146
147
147
148
148 class SvnMessageWriter(RemoteMessageWriter):
149 class SvnMessageWriter(RemoteMessageWriter):
149 """Writer that knows how to send messages to svn clients."""
150 """Writer that knows how to send messages to svn clients."""
150
151
151 def __init__(self, stderr=None):
152 def __init__(self, stderr=None):
152 # SVN needs data sent to stderr for back-to-client messaging
153 # SVN needs data sent to stderr for back-to-client messaging
153 self.stderr = stderr or sys.stderr
154 self.stderr = stderr or sys.stderr
154
155
155 def write(self, message):
156 def write(self, message):
156 self.stderr.write(message)
157 self.stderr.write(message)
157
158
158
159
159 def _handle_exception(result):
160 def _handle_exception(result):
160 exception_class = result.get('exception')
161 exception_class = result.get('exception')
161 exception_traceback = result.get('exception_traceback')
162 exception_traceback = result.get('exception_traceback')
162 log.debug('Handling hook-call exception: %s', exception_class)
163 log.debug('Handling hook-call exception: %s', exception_class)
163
164
164 if exception_traceback:
165 if exception_traceback:
165 log.error('Got traceback from remote call:%s', exception_traceback)
166 log.error('Got traceback from remote call:%s', exception_traceback)
166
167
167 if exception_class == 'HTTPLockedRC':
168 if exception_class == 'HTTPLockedRC':
168 raise exceptions.RepositoryLockedException()(*result['exception_args'])
169 raise exceptions.RepositoryLockedException()(*result['exception_args'])
169 elif exception_class == 'HTTPBranchProtected':
170 elif exception_class == 'HTTPBranchProtected':
170 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
171 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
171 elif exception_class == 'RepositoryError':
172 elif exception_class == 'RepositoryError':
172 raise exceptions.VcsException()(*result['exception_args'])
173 raise exceptions.VcsException()(*result['exception_args'])
173 elif exception_class:
174 elif exception_class:
174 raise Exception(
175 raise Exception(
175 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
176 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
176 )
177 )
177
178
178
179
179 def _get_hooks_client(extras):
180 def _get_hooks_client(extras):
180 hooks_uri = extras.get('hooks_uri')
181 hooks_uri = extras.get('hooks_uri')
181 task_queue = extras.get('task_queue')
182 task_queue = extras.get('task_queue')
182 task_backend = extras.get('task_backend')
183 task_backend = extras.get('task_backend')
183 is_shadow_repo = extras.get('is_shadow_repo')
184 is_shadow_repo = extras.get('is_shadow_repo')
184
185
185 if hooks_uri:
186 if hooks_uri:
186 return HooksHttpClient(hooks_uri)
187 return HooksHttpClient(hooks_uri)
187 elif task_queue and task_backend:
188 elif task_queue and task_backend:
188 return HooksCeleryClient(task_queue, task_backend)
189 return HooksCeleryClient(task_queue, task_backend)
189 elif is_shadow_repo:
190 elif is_shadow_repo:
190 return HooksShadowRepoClient()
191 return HooksShadowRepoClient()
191 else:
192 else:
192 raise Exception("Hooks client not found!")
193 raise Exception("Hooks client not found!")
193
194
194
195
195 def _call_hook(hook_name, extras, writer):
196 def _call_hook(hook_name, extras, writer):
196 hooks_client = _get_hooks_client(extras)
197 hooks_client = _get_hooks_client(extras)
197 log.debug('Hooks, using client:%s', hooks_client)
198 log.debug('Hooks, using client:%s', hooks_client)
198 result = hooks_client(hook_name, extras)
199 result = hooks_client(hook_name, extras)
199 log.debug('Hooks got result: %s', result)
200 log.debug('Hooks got result: %s', result)
200 _handle_exception(result)
201 _handle_exception(result)
201 writer.write(result['output'])
202 writer.write(result['output'])
202
203
203 return result['status']
204 return result['status']
204
205
205
206
206 def _extras_from_ui(ui):
207 def _extras_from_ui(ui):
207 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
208 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
208 if not hook_data:
209 if not hook_data:
209 # maybe it's inside environ ?
210 # maybe it's inside environ ?
210 env_hook_data = os.environ.get('RC_SCM_DATA')
211 env_hook_data = os.environ.get('RC_SCM_DATA')
211 if env_hook_data:
212 if env_hook_data:
212 hook_data = env_hook_data
213 hook_data = env_hook_data
213
214
214 extras = {}
215 extras = {}
215 if hook_data:
216 if hook_data:
216 extras = json.loads(hook_data)
217 extras = json.loads(hook_data)
217 return extras
218 return extras
218
219
219
220
220 def _rev_range_hash(repo, node, check_heads=False):
221 def _rev_range_hash(repo, node, check_heads=False):
221 from vcsserver.hgcompat import get_ctx
222 from vcsserver.hgcompat import get_ctx
222
223
223 commits = []
224 commits = []
224 revs = []
225 revs = []
225 start = get_ctx(repo, node).rev()
226 start = get_ctx(repo, node).rev()
226 end = len(repo)
227 end = len(repo)
227 for rev in range(start, end):
228 for rev in range(start, end):
228 revs.append(rev)
229 revs.append(rev)
229 ctx = get_ctx(repo, rev)
230 ctx = get_ctx(repo, rev)
230 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
231 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
231 branch = safe_str(ctx.branch())
232 branch = safe_str(ctx.branch())
232 commits.append((commit_id, branch))
233 commits.append((commit_id, branch))
233
234
234 parent_heads = []
235 parent_heads = []
235 if check_heads:
236 if check_heads:
236 parent_heads = _check_heads(repo, start, end, revs)
237 parent_heads = _check_heads(repo, start, end, revs)
237 return commits, parent_heads
238 return commits, parent_heads
238
239
239
240
240 def _check_heads(repo, start, end, commits):
241 def _check_heads(repo, start, end, commits):
241 from vcsserver.hgcompat import get_ctx
242 from vcsserver.hgcompat import get_ctx
242 changelog = repo.changelog
243 changelog = repo.changelog
243 parents = set()
244 parents = set()
244
245
245 for new_rev in commits:
246 for new_rev in commits:
246 for p in changelog.parentrevs(new_rev):
247 for p in changelog.parentrevs(new_rev):
247 if p == mercurial.node.nullrev:
248 if p == mercurial.node.nullrev:
248 continue
249 continue
249 if p < start:
250 if p < start:
250 parents.add(p)
251 parents.add(p)
251
252
252 for p in parents:
253 for p in parents:
253 branch = get_ctx(repo, p).branch()
254 branch = get_ctx(repo, p).branch()
254 # The heads descending from that parent, on the same branch
255 # The heads descending from that parent, on the same branch
255 parent_heads = {p}
256 parent_heads = {p}
256 reachable = {p}
257 reachable = {p}
257 for x in range(p + 1, end):
258 for x in range(p + 1, end):
258 if get_ctx(repo, x).branch() != branch:
259 if get_ctx(repo, x).branch() != branch:
259 continue
260 continue
260 for pp in changelog.parentrevs(x):
261 for pp in changelog.parentrevs(x):
261 if pp in reachable:
262 if pp in reachable:
262 reachable.add(x)
263 reachable.add(x)
263 parent_heads.discard(pp)
264 parent_heads.discard(pp)
264 parent_heads.add(x)
265 parent_heads.add(x)
265 # More than one head? Suggest merging
266 # More than one head? Suggest merging
266 if len(parent_heads) > 1:
267 if len(parent_heads) > 1:
267 return list(parent_heads)
268 return list(parent_heads)
268
269
269 return []
270 return []
270
271
271
272
272 def _get_git_env():
273 def _get_git_env():
273 env = {}
274 env = {}
274 for k, v in os.environ.items():
275 for k, v in os.environ.items():
275 if k.startswith('GIT'):
276 if k.startswith('GIT'):
276 env[k] = v
277 env[k] = v
277
278
278 # serialized version
279 # serialized version
279 return [(k, v) for k, v in env.items()]
280 return [(k, v) for k, v in env.items()]
280
281
281
282
282 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
283 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
283 env = {}
284 env = {}
284 for k, v in os.environ.items():
285 for k, v in os.environ.items():
285 if k.startswith('HG'):
286 if k.startswith('HG'):
286 env[k] = v
287 env[k] = v
287
288
288 env['HG_NODE'] = old_rev
289 env['HG_NODE'] = old_rev
289 env['HG_NODE_LAST'] = new_rev
290 env['HG_NODE_LAST'] = new_rev
290 env['HG_TXNID'] = txnid
291 env['HG_TXNID'] = txnid
291 env['HG_PENDING'] = repo_path
292 env['HG_PENDING'] = repo_path
292
293
293 return [(k, v) for k, v in env.items()]
294 return [(k, v) for k, v in env.items()]
294
295
295
296
297 def _get_ini_settings(ini_file):
298 from vcsserver.http_main import sanitize_settings_and_apply_defaults
299 from vcsserver.lib.config_utils import get_app_config_lightweight, configure_and_store_settings
300
301 global_config = {'__file__': ini_file}
302 ini_settings = get_app_config_lightweight(ini_file)
303 sanitize_settings_and_apply_defaults(global_config, ini_settings)
304 configure_and_store_settings(global_config, ini_settings)
305
306 return ini_settings
307
308
296 def _fix_hooks_executables(ini_path=''):
309 def _fix_hooks_executables(ini_path=''):
297 """
310 """
298 This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns
311 This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns
299 especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail
312 especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail
300 because svn is not on PATH
313 because svn is not on PATH
301 """
314 """
302 from vcsserver.http_main import sanitize_settings_and_apply_defaults
315 # set defaults, in case we can't read from ini_file
303 from vcsserver.lib.config_utils import get_app_config_lightweight
304
305 core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin'
316 core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin'
306 if ini_path:
317 if ini_path:
307
318 ini_settings = _get_ini_settings(ini_path)
308 ini_settings = get_app_config_lightweight(ini_path)
309 ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings)
310 core_binary_dir = ini_settings['core.binary_dir']
319 core_binary_dir = ini_settings['core.binary_dir']
311
320
312 settings.BINARY_DIR = core_binary_dir
321 settings.BINARY_DIR = core_binary_dir
313
322
314
323
315 def repo_size(ui, repo, **kwargs):
324 def repo_size(ui, repo, **kwargs):
316 extras = _extras_from_ui(ui)
325 extras = _extras_from_ui(ui)
317 return _call_hook('repo_size', extras, HgMessageWriter(ui))
326 return _call_hook('repo_size', extras, HgMessageWriter(ui))
318
327
319
328
320 def pre_pull(ui, repo, **kwargs):
329 def pre_pull(ui, repo, **kwargs):
321 extras = _extras_from_ui(ui)
330 extras = _extras_from_ui(ui)
322 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
331 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
323
332
324
333
325 def pre_pull_ssh(ui, repo, **kwargs):
334 def pre_pull_ssh(ui, repo, **kwargs):
326 extras = _extras_from_ui(ui)
335 extras = _extras_from_ui(ui)
327 if extras and extras.get('SSH'):
336 if extras and extras.get('SSH'):
328 return pre_pull(ui, repo, **kwargs)
337 return pre_pull(ui, repo, **kwargs)
329 return 0
338 return 0
330
339
331
340
332 def post_pull(ui, repo, **kwargs):
341 def post_pull(ui, repo, **kwargs):
333 extras = _extras_from_ui(ui)
342 extras = _extras_from_ui(ui)
334 return _call_hook('post_pull', extras, HgMessageWriter(ui))
343 return _call_hook('post_pull', extras, HgMessageWriter(ui))
335
344
336
345
337 def post_pull_ssh(ui, repo, **kwargs):
346 def post_pull_ssh(ui, repo, **kwargs):
338 extras = _extras_from_ui(ui)
347 extras = _extras_from_ui(ui)
339 if extras and extras.get('SSH'):
348 if extras and extras.get('SSH'):
340 return post_pull(ui, repo, **kwargs)
349 return post_pull(ui, repo, **kwargs)
341 return 0
350 return 0
342
351
343
352
344 def pre_push(ui, repo, node=None, **kwargs):
353 def pre_push(ui, repo, node=None, **kwargs):
345 """
354 """
346 Mercurial pre_push hook
355 Mercurial pre_push hook
347 """
356 """
348 extras = _extras_from_ui(ui)
357 extras = _extras_from_ui(ui)
349 detect_force_push = extras.get('detect_force_push')
358 detect_force_push = extras.get('detect_force_push')
350
359
351 rev_data = []
360 rev_data = []
352 hook_type: str = safe_str(kwargs.get('hooktype'))
361 hook_type: str = safe_str(kwargs.get('hooktype'))
353
362
354 if node and hook_type == 'pretxnchangegroup':
363 if node and hook_type == 'pretxnchangegroup':
355 branches = collections.defaultdict(list)
364 branches = collections.defaultdict(list)
356 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
365 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
357 for commit_id, branch in commits:
366 for commit_id, branch in commits:
358 branches[branch].append(commit_id)
367 branches[branch].append(commit_id)
359
368
360 for branch, commits in branches.items():
369 for branch, commits in branches.items():
361 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
370 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
362 rev_data.append({
371 rev_data.append({
363 'total_commits': len(commits),
372 'total_commits': len(commits),
364 'old_rev': old_rev,
373 'old_rev': old_rev,
365 'new_rev': commits[-1],
374 'new_rev': commits[-1],
366 'ref': '',
375 'ref': '',
367 'type': 'branch',
376 'type': 'branch',
368 'name': branch,
377 'name': branch,
369 })
378 })
370
379
371 for push_ref in rev_data:
380 for push_ref in rev_data:
372 push_ref['multiple_heads'] = _heads
381 push_ref['multiple_heads'] = _heads
373
382
374 repo_path = os.path.join(
383 repo_path = os.path.join(
375 extras.get('repo_store', ''), extras.get('repository', ''))
384 extras.get('repo_store', ''), extras.get('repository', ''))
376 push_ref['hg_env'] = _get_hg_env(
385 push_ref['hg_env'] = _get_hg_env(
377 old_rev=push_ref['old_rev'],
386 old_rev=push_ref['old_rev'],
378 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
387 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
379 repo_path=repo_path)
388 repo_path=repo_path)
380
389
381 extras['hook_type'] = hook_type or 'pre_push'
390 extras['hook_type'] = hook_type or 'pre_push'
382 extras['commit_ids'] = rev_data
391 extras['commit_ids'] = rev_data
383
392
384 return _call_hook('pre_push', extras, HgMessageWriter(ui))
393 return _call_hook('pre_push', extras, HgMessageWriter(ui))
385
394
386
395
387 def pre_push_ssh(ui, repo, node=None, **kwargs):
396 def pre_push_ssh(ui, repo, node=None, **kwargs):
388 extras = _extras_from_ui(ui)
397 extras = _extras_from_ui(ui)
389 if extras.get('SSH'):
398 if extras.get('SSH'):
390 return pre_push(ui, repo, node, **kwargs)
399 return pre_push(ui, repo, node, **kwargs)
391
400
392 return 0
401 return 0
393
402
394
403
395 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
404 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
396 """
405 """
397 Mercurial pre_push hook for SSH
406 Mercurial pre_push hook for SSH
398 """
407 """
399 extras = _extras_from_ui(ui)
408 extras = _extras_from_ui(ui)
400 if extras.get('SSH'):
409 if extras.get('SSH'):
401 permission = extras['SSH_PERMISSIONS']
410 permission = extras['SSH_PERMISSIONS']
402
411
403 if 'repository.write' == permission or 'repository.admin' == permission:
412 if 'repository.write' == permission or 'repository.admin' == permission:
404 return 0
413 return 0
405
414
406 # non-zero ret code
415 # non-zero ret code
407 return 1
416 return 1
408
417
409 return 0
418 return 0
410
419
411
420
412 def post_push(ui, repo, node, **kwargs):
421 def post_push(ui, repo, node, **kwargs):
413 """
422 """
414 Mercurial post_push hook
423 Mercurial post_push hook
415 """
424 """
416 extras = _extras_from_ui(ui)
425 extras = _extras_from_ui(ui)
417
426
418 commit_ids = []
427 commit_ids = []
419 branches = []
428 branches = []
420 bookmarks = []
429 bookmarks = []
421 tags = []
430 tags = []
422 hook_type: str = safe_str(kwargs.get('hooktype'))
431 hook_type: str = safe_str(kwargs.get('hooktype'))
423
432
424 commits, _heads = _rev_range_hash(repo, node)
433 commits, _heads = _rev_range_hash(repo, node)
425 for commit_id, branch in commits:
434 for commit_id, branch in commits:
426 commit_ids.append(commit_id)
435 commit_ids.append(commit_id)
427 if branch not in branches:
436 if branch not in branches:
428 branches.append(branch)
437 branches.append(branch)
429
438
430 if hasattr(ui, '_rc_pushkey_bookmarks'):
439 if hasattr(ui, '_rc_pushkey_bookmarks'):
431 bookmarks = ui._rc_pushkey_bookmarks
440 bookmarks = ui._rc_pushkey_bookmarks
432
441
433 extras['hook_type'] = hook_type or 'post_push'
442 extras['hook_type'] = hook_type or 'post_push'
434 extras['commit_ids'] = commit_ids
443 extras['commit_ids'] = commit_ids
435
444
436 extras['new_refs'] = {
445 extras['new_refs'] = {
437 'branches': branches,
446 'branches': branches,
438 'bookmarks': bookmarks,
447 'bookmarks': bookmarks,
439 'tags': tags
448 'tags': tags
440 }
449 }
441
450
442 return _call_hook('post_push', extras, HgMessageWriter(ui))
451 return _call_hook('post_push', extras, HgMessageWriter(ui))
443
452
444
453
445 def post_push_ssh(ui, repo, node, **kwargs):
454 def post_push_ssh(ui, repo, node, **kwargs):
446 """
455 """
447 Mercurial post_push hook for SSH
456 Mercurial post_push hook for SSH
448 """
457 """
449 if _extras_from_ui(ui).get('SSH'):
458 if _extras_from_ui(ui).get('SSH'):
450 return post_push(ui, repo, node, **kwargs)
459 return post_push(ui, repo, node, **kwargs)
451 return 0
460 return 0
452
461
453
462
454 def key_push(ui, repo, **kwargs):
463 def key_push(ui, repo, **kwargs):
455 from vcsserver.hgcompat import get_ctx
464 from vcsserver.hgcompat import get_ctx
456
465
457 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
466 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
458 # store new bookmarks in our UI object propagated later to post_push
467 # store new bookmarks in our UI object propagated later to post_push
459 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
468 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
460 return
469 return
461
470
462
471
463 # backward compat
472 # backward compat
464 log_pull_action = post_pull
473 log_pull_action = post_pull
465
474
466 # backward compat
475 # backward compat
467 log_push_action = post_push
476 log_push_action = post_push
468
477
469
478
470 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
479 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
471 """
480 """
472 Old hook name: keep here for backward compatibility.
481 Old hook name: keep here for backward compatibility.
473
482
474 This is only required when the installed git hooks are not upgraded.
483 This is only required when the installed git hooks are not upgraded.
475 """
484 """
476 pass
485 pass
477
486
478
487
479 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
488 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
480 """
489 """
481 Old hook name: keep here for backward compatibility.
490 Old hook name: keep here for backward compatibility.
482
491
483 This is only required when the installed git hooks are not upgraded.
492 This is only required when the installed git hooks are not upgraded.
484 """
493 """
485 pass
494 pass
486
495
487
496
488 @dataclasses.dataclass
497 @dataclasses.dataclass
489 class HookResponse:
498 class HookResponse:
490 status: int
499 status: int
491 output: str
500 output: str
492
501
493
502
494 def git_pre_pull(extras) -> HookResponse:
503 def git_pre_pull(extras) -> HookResponse:
495 """
504 """
496 Pre pull hook.
505 Pre pull hook.
497
506
498 :param extras: dictionary containing the keys defined in simplevcs
507 :param extras: dictionary containing the keys defined in simplevcs
499 :type extras: dict
508 :type extras: dict
500
509
501 :return: status code of the hook. 0 for success.
510 :return: status code of the hook. 0 for success.
502 :rtype: int
511 :rtype: int
503 """
512 """
504
513
505 if 'pull' not in extras['hooks']:
514 if 'pull' not in extras['hooks']:
506 return HookResponse(0, '')
515 return HookResponse(0, '')
507
516
508 stdout = io.StringIO()
517 stdout = io.StringIO()
509 try:
518 try:
510 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
519 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
511
520
512 except Exception as error:
521 except Exception as error:
513 log.exception('Failed to call pre_pull hook')
522 log.exception('Failed to call pre_pull hook')
514 status_code = 128
523 status_code = 128
515 stdout.write(f'ERROR: {error}\n')
524 stdout.write(f'ERROR: {error}\n')
516
525
517 return HookResponse(status_code, stdout.getvalue())
526 return HookResponse(status_code, stdout.getvalue())
518
527
519
528
520 def git_post_pull(extras) -> HookResponse:
529 def git_post_pull(extras) -> HookResponse:
521 """
530 """
522 Post pull hook.
531 Post pull hook.
523
532
524 :param extras: dictionary containing the keys defined in simplevcs
533 :param extras: dictionary containing the keys defined in simplevcs
525 :type extras: dict
534 :type extras: dict
526
535
527 :return: status code of the hook. 0 for success.
536 :return: status code of the hook. 0 for success.
528 :rtype: int
537 :rtype: int
529 """
538 """
530 if 'pull' not in extras['hooks']:
539 if 'pull' not in extras['hooks']:
531 return HookResponse(0, '')
540 return HookResponse(0, '')
532
541
533 stdout = io.StringIO()
542 stdout = io.StringIO()
534 try:
543 try:
535 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
544 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
536 except Exception as error:
545 except Exception as error:
537 status = 128
546 status = 128
538 stdout.write(f'ERROR: {error}\n')
547 stdout.write(f'ERROR: {error}\n')
539
548
540 return HookResponse(status, stdout.getvalue())
549 return HookResponse(status, stdout.getvalue())
541
550
542
551
543 def _parse_git_ref_lines(revision_lines):
552 def _parse_git_ref_lines(revision_lines):
544 rev_data = []
553 rev_data = []
545 for revision_line in revision_lines or []:
554 for revision_line in revision_lines or []:
546 old_rev, new_rev, ref = revision_line.strip().split(' ')
555 old_rev, new_rev, ref = revision_line.strip().split(' ')
547 ref_data = ref.split('/', 2)
556 ref_data = ref.split('/', 2)
548 if ref_data[1] in ('tags', 'heads'):
557 if ref_data[1] in ('tags', 'heads'):
549 rev_data.append({
558 rev_data.append({
550 # NOTE(marcink):
559 # NOTE(marcink):
551 # we're unable to tell total_commits for git at this point
560 # we're unable to tell total_commits for git at this point
552 # but we set the variable for consistency with GIT
561 # but we set the variable for consistency with GIT
553 'total_commits': -1,
562 'total_commits': -1,
554 'old_rev': old_rev,
563 'old_rev': old_rev,
555 'new_rev': new_rev,
564 'new_rev': new_rev,
556 'ref': ref,
565 'ref': ref,
557 'type': ref_data[1],
566 'type': ref_data[1],
558 'name': ref_data[2],
567 'name': ref_data[2],
559 })
568 })
560 return rev_data
569 return rev_data
561
570
562
571
563 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
572 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
564 """
573 """
565 Pre push hook.
574 Pre push hook.
566
575
567 :return: status code of the hook. 0 for success.
576 :return: status code of the hook. 0 for success.
568 """
577 """
569 extras = json.loads(env['RC_SCM_DATA'])
578 extras = json.loads(env['RC_SCM_DATA'])
570 rev_data = _parse_git_ref_lines(revision_lines)
579 rev_data = _parse_git_ref_lines(revision_lines)
571 if 'push' not in extras['hooks']:
580 if 'push' not in extras['hooks']:
572 return 0
581 return 0
573 _fix_hooks_executables()
582 _fix_hooks_executables(env.get('RC_INI_FILE'))
574
583
575 empty_commit_id = '0' * 40
584 empty_commit_id = '0' * 40
576
585
577 detect_force_push = extras.get('detect_force_push')
586 detect_force_push = extras.get('detect_force_push')
578
587
579 for push_ref in rev_data:
588 for push_ref in rev_data:
580 # store our git-env which holds the temp store
589 # store our git-env which holds the temp store
581 push_ref['git_env'] = _get_git_env()
590 push_ref['git_env'] = _get_git_env()
582 push_ref['pruned_sha'] = ''
591 push_ref['pruned_sha'] = ''
583 if not detect_force_push:
592 if not detect_force_push:
584 # don't check for forced-push when we don't need to
593 # don't check for forced-push when we don't need to
585 continue
594 continue
586
595
587 type_ = push_ref['type']
596 type_ = push_ref['type']
588 new_branch = push_ref['old_rev'] == empty_commit_id
597 new_branch = push_ref['old_rev'] == empty_commit_id
589 delete_branch = push_ref['new_rev'] == empty_commit_id
598 delete_branch = push_ref['new_rev'] == empty_commit_id
590 if type_ == 'heads' and not (new_branch or delete_branch):
599 if type_ == 'heads' and not (new_branch or delete_branch):
591 old_rev = push_ref['old_rev']
600 old_rev = push_ref['old_rev']
592 new_rev = push_ref['new_rev']
601 new_rev = push_ref['new_rev']
593 cmd = [settings.GIT_EXECUTABLE(), 'rev-list', old_rev, f'^{new_rev}']
602 cmd = [settings.GIT_EXECUTABLE(), 'rev-list', old_rev, f'^{new_rev}']
594 stdout, stderr = subprocessio.run_command(
603 stdout, stderr = subprocessio.run_command(
595 cmd, env=os.environ.copy())
604 cmd, env=os.environ.copy())
596 # means we're having some non-reachable objects, this forced push was used
605 # means we're having some non-reachable objects, this forced push was used
597 if stdout:
606 if stdout:
598 push_ref['pruned_sha'] = stdout.splitlines()
607 push_ref['pruned_sha'] = stdout.splitlines()
599
608
600 extras['hook_type'] = 'pre_receive'
609 extras['hook_type'] = 'pre_receive'
601 extras['commit_ids'] = rev_data
610 extras['commit_ids'] = rev_data
602
611
603 stdout = sys.stdout
612 stdout = sys.stdout
604 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
613 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
605
614
606 return status_code
615 return status_code
607
616
608
617
609 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
618 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
610 """
619 """
611 Post push hook.
620 Post push hook.
612
621
613 :return: status code of the hook. 0 for success.
622 :return: status code of the hook. 0 for success.
614 """
623 """
615 extras = json.loads(env['RC_SCM_DATA'])
624 extras = json.loads(env['RC_SCM_DATA'])
616 if 'push' not in extras['hooks']:
625 if 'push' not in extras['hooks']:
617 return 0
626 return 0
618
627
619 _fix_hooks_executables()
628 _fix_hooks_executables(env.get('RC_INI_FILE'))
620
629
621 rev_data = _parse_git_ref_lines(revision_lines)
630 rev_data = _parse_git_ref_lines(revision_lines)
622
631
623 git_revs = []
632 git_revs = []
624
633
625 # N.B.(skreft): it is ok to just call git, as git before calling a
634 # N.B.(skreft): it is ok to just call git, as git before calling a
626 # subcommand sets the PATH environment variable so that it point to the
635 # subcommand sets the PATH environment variable so that it point to the
627 # correct version of the git executable.
636 # correct version of the git executable.
628 empty_commit_id = '0' * 40
637 empty_commit_id = '0' * 40
629 branches = []
638 branches = []
630 tags = []
639 tags = []
631 for push_ref in rev_data:
640 for push_ref in rev_data:
632 type_ = push_ref['type']
641 type_ = push_ref['type']
633
642
634 if type_ == 'heads':
643 if type_ == 'heads':
635 # starting new branch case
644 # starting new branch case
636 if push_ref['old_rev'] == empty_commit_id:
645 if push_ref['old_rev'] == empty_commit_id:
637 push_ref_name = push_ref['name']
646 push_ref_name = push_ref['name']
638
647
639 if push_ref_name not in branches:
648 if push_ref_name not in branches:
640 branches.append(push_ref_name)
649 branches.append(push_ref_name)
641
650
642 need_head_set = ''
651 need_head_set = ''
643 with Repository(os.getcwd()) as repo:
652 with Repository(os.getcwd()) as repo:
644 try:
653 try:
645 repo.head
654 repo.head
646 except pygit2.GitError:
655 except pygit2.GitError:
647 need_head_set = f'refs/heads/{push_ref_name}'
656 need_head_set = f'refs/heads/{push_ref_name}'
648
657
649 if need_head_set:
658 if need_head_set:
650 repo.set_head(need_head_set)
659 repo.set_head(need_head_set)
651 print(f"Setting default branch to {push_ref_name}")
660 print(f"Setting default branch to {push_ref_name}")
652
661
653 cmd = [settings.GIT_EXECUTABLE(), 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
662 cmd = [settings.GIT_EXECUTABLE(), 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
654 stdout, stderr = subprocessio.run_command(
663 stdout, stderr = subprocessio.run_command(
655 cmd, env=os.environ.copy())
664 cmd, env=os.environ.copy())
656 heads = safe_str(stdout)
665 heads = safe_str(stdout)
657 heads = heads.replace(push_ref['ref'], '')
666 heads = heads.replace(push_ref['ref'], '')
658 heads = ' '.join(head for head
667 heads = ' '.join(head for head
659 in heads.splitlines() if head) or '.'
668 in heads.splitlines() if head) or '.'
660 cmd = [settings.GIT_EXECUTABLE(), 'log', '--reverse',
669 cmd = [settings.GIT_EXECUTABLE(), 'log', '--reverse',
661 '--pretty=format:%H', '--', push_ref['new_rev'],
670 '--pretty=format:%H', '--', push_ref['new_rev'],
662 '--not', heads]
671 '--not', heads]
663 stdout, stderr = subprocessio.run_command(
672 stdout, stderr = subprocessio.run_command(
664 cmd, env=os.environ.copy())
673 cmd, env=os.environ.copy())
665 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
674 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
666
675
667 # delete branch case
676 # delete branch case
668 elif push_ref['new_rev'] == empty_commit_id:
677 elif push_ref['new_rev'] == empty_commit_id:
669 git_revs.append(f'delete_branch=>{push_ref["name"]}')
678 git_revs.append(f'delete_branch=>{push_ref["name"]}')
670 else:
679 else:
671 if push_ref['name'] not in branches:
680 if push_ref['name'] not in branches:
672 branches.append(push_ref['name'])
681 branches.append(push_ref['name'])
673
682
674 cmd = [settings.GIT_EXECUTABLE(), 'log',
683 cmd = [settings.GIT_EXECUTABLE(), 'log',
675 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
684 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
676 '--reverse', '--pretty=format:%H']
685 '--reverse', '--pretty=format:%H']
677 stdout, stderr = subprocessio.run_command(
686 stdout, stderr = subprocessio.run_command(
678 cmd, env=os.environ.copy())
687 cmd, env=os.environ.copy())
679 # we get bytes from stdout, we need str to be consistent
688 # we get bytes from stdout, we need str to be consistent
680 log_revs = list(map(ascii_str, stdout.splitlines()))
689 log_revs = list(map(ascii_str, stdout.splitlines()))
681 git_revs.extend(log_revs)
690 git_revs.extend(log_revs)
682
691
683 # Pure pygit2 impl. but still 2-3x slower :/
692 # Pure pygit2 impl. but still 2-3x slower :/
684 # results = []
693 # results = []
685 #
694 #
686 # with Repository(os.getcwd()) as repo:
695 # with Repository(os.getcwd()) as repo:
687 # repo_new_rev = repo[push_ref['new_rev']]
696 # repo_new_rev = repo[push_ref['new_rev']]
688 # repo_old_rev = repo[push_ref['old_rev']]
697 # repo_old_rev = repo[push_ref['old_rev']]
689 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
698 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
690 #
699 #
691 # for commit in walker:
700 # for commit in walker:
692 # if commit.id == repo_old_rev.id:
701 # if commit.id == repo_old_rev.id:
693 # break
702 # break
694 # results.append(commit.id.hex)
703 # results.append(commit.id.hex)
695 # # reverse the order, can't use GIT_SORT_REVERSE
704 # # reverse the order, can't use GIT_SORT_REVERSE
696 # log_revs = results[::-1]
705 # log_revs = results[::-1]
697
706
698 elif type_ == 'tags':
707 elif type_ == 'tags':
699 if push_ref['name'] not in tags:
708 if push_ref['name'] not in tags:
700 tags.append(push_ref['name'])
709 tags.append(push_ref['name'])
701 git_revs.append(f'tag=>{push_ref["name"]}')
710 git_revs.append(f'tag=>{push_ref["name"]}')
702
711
703 extras['hook_type'] = 'post_receive'
712 extras['hook_type'] = 'post_receive'
704 extras['commit_ids'] = git_revs
713 extras['commit_ids'] = git_revs
705 extras['new_refs'] = {
714 extras['new_refs'] = {
706 'branches': branches,
715 'branches': branches,
707 'bookmarks': [],
716 'bookmarks': [],
708 'tags': tags,
717 'tags': tags,
709 }
718 }
710
719
711 stdout = sys.stdout
720 stdout = sys.stdout
712
721
713 if 'repo_size' in extras['hooks']:
722 if 'repo_size' in extras['hooks']:
714 try:
723 try:
715 _call_hook('repo_size', extras, GitMessageWriter(stdout))
724 _call_hook('repo_size', extras, GitMessageWriter(stdout))
716 except Exception:
725 except Exception:
717 pass
726 pass
718
727
719 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
728 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
720 return status_code
729 return status_code
721
730
722
731
723 def _get_extras_from_txn_id(path, txn_id):
732 def get_extras_from_txn_id(repo_path, txn_id):
724 _fix_hooks_executables()
733 extras = get_txn_id_from_store(repo_path, txn_id)
725
726 extras = {}
727 try:
728 cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
729 '-t', txn_id,
730 '--revprop', path, 'rc-scm-extras']
731 stdout, stderr = subprocessio.run_command(
732 cmd, env=os.environ.copy())
733 extras = json.loads(base64.urlsafe_b64decode(stdout))
734 except Exception:
735 log.exception('Failed to extract extras info from txn_id')
736
737 return extras
738
739
740 def _get_extras_from_commit_id(commit_id, path):
741 _fix_hooks_executables()
742
743 extras = {}
744 try:
745 cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
746 '-r', commit_id,
747 '--revprop', path, 'rc-scm-extras']
748 stdout, stderr = subprocessio.run_command(
749 cmd, env=os.environ.copy())
750 extras = json.loads(base64.urlsafe_b64decode(stdout))
751 except Exception:
752 log.exception('Failed to extract extras info from commit_id')
753
754 return extras
734 return extras
755
735
756
736
757 def svn_pre_commit(repo_path, commit_data, env):
737 def svn_pre_commit(repo_path, commit_data, env):
758
738
759 path, txn_id = commit_data
739 path, txn_id = commit_data
760 branches = []
740 branches = []
761 tags = []
741 tags = []
762
742
763 if env.get('RC_SCM_DATA'):
743 if env.get('RC_SCM_DATA'):
764 extras = json.loads(env['RC_SCM_DATA'])
744 extras = json.loads(env['RC_SCM_DATA'])
765 else:
745 else:
746 ini_path = env.get('RC_INI_FILE')
747 if ini_path:
748 _get_ini_settings(ini_path)
766 # fallback method to read from TXN-ID stored data
749 # fallback method to read from TXN-ID stored data
767 extras = _get_extras_from_txn_id(path, txn_id)
750 extras = get_extras_from_txn_id(path, txn_id)
768 if not extras:
751
769 return 0
752 if not extras:
753 raise ValueError('SVN-PRE-COMMIT: Failed to extract context data in called extras for hook execution')
754
755 if extras.get('rc_internal_commit'):
756 # special marker for internal commit, we don't call hooks client
757 return 0
770
758
771 extras['hook_type'] = 'pre_commit'
759 extras['hook_type'] = 'pre_commit'
772 extras['commit_ids'] = [txn_id]
760 extras['commit_ids'] = [txn_id]
773 extras['txn_id'] = txn_id
761 extras['txn_id'] = txn_id
774 extras['new_refs'] = {
762 extras['new_refs'] = {
775 'total_commits': 1,
763 'total_commits': 1,
776 'branches': branches,
764 'branches': branches,
777 'bookmarks': [],
765 'bookmarks': [],
778 'tags': tags,
766 'tags': tags,
779 }
767 }
780
768
781 return _call_hook('pre_push', extras, SvnMessageWriter())
769 return _call_hook('pre_push', extras, SvnMessageWriter())
782
770
783
771
784 def svn_post_commit(repo_path, commit_data, env):
772 def svn_post_commit(repo_path, commit_data, env):
785 """
773 """
786 commit_data is path, rev, txn_id
774 commit_data is path, rev, txn_id
787 """
775 """
788
776
789 if len(commit_data) == 3:
777 if len(commit_data) == 3:
790 path, commit_id, txn_id = commit_data
778 path, commit_id, txn_id = commit_data
791 elif len(commit_data) == 2:
779 elif len(commit_data) == 2:
792 log.error('Failed to extract txn_id from commit_data using legacy method. '
780 log.error('Failed to extract txn_id from commit_data using legacy method. '
793 'Some functionality might be limited')
781 'Some functionality might be limited')
794 path, commit_id = commit_data
782 path, commit_id = commit_data
795 txn_id = None
783 txn_id = None
796 else:
784 else:
797 return 0
785 return 0
798
786
799 branches = []
787 branches = []
800 tags = []
788 tags = []
801
789
802 if env.get('RC_SCM_DATA'):
790 if env.get('RC_SCM_DATA'):
803 extras = json.loads(env['RC_SCM_DATA'])
791 extras = json.loads(env['RC_SCM_DATA'])
804 else:
792 else:
793 ini_path = env.get('RC_INI_FILE')
794 if ini_path:
795 _get_ini_settings(ini_path)
805 # fallback method to read from TXN-ID stored data
796 # fallback method to read from TXN-ID stored data
806 extras = _get_extras_from_commit_id(commit_id, path)
797 extras = get_extras_from_txn_id(path, txn_id)
807 if not extras:
798
808 return 0
799 if not extras and txn_id:
800 raise ValueError('SVN-POST-COMMIT: Failed to extract context data in called extras for hook execution')
801
802 if extras.get('rc_internal_commit'):
803 # special marker for internal commit, we don't call hooks client
804 return 0
809
805
810 extras['hook_type'] = 'post_commit'
806 extras['hook_type'] = 'post_commit'
811 extras['commit_ids'] = [commit_id]
807 extras['commit_ids'] = [commit_id]
812 extras['txn_id'] = txn_id
808 extras['txn_id'] = txn_id
813 extras['new_refs'] = {
809 extras['new_refs'] = {
814 'branches': branches,
810 'branches': branches,
815 'bookmarks': [],
811 'bookmarks': [],
816 'tags': tags,
812 'tags': tags,
817 'total_commits': 1,
813 'total_commits': 1,
818 }
814 }
819
815
820 if 'repo_size' in extras['hooks']:
816 if 'repo_size' in extras['hooks']:
821 try:
817 try:
822 _call_hook('repo_size', extras, SvnMessageWriter())
818 _call_hook('repo_size', extras, SvnMessageWriter())
823 except Exception:
819 except Exception:
824 pass
820 pass
825
821
826 return _call_hook('post_push', extras, SvnMessageWriter())
822 return _call_hook('post_push', extras, SvnMessageWriter())
@@ -1,774 +1,763 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import platform
20 import platform
21 import sys
21 import sys
22 import locale
22 import locale
23 import logging
23 import logging
24 import uuid
24 import uuid
25 import time
25 import time
26 import wsgiref.util
26 import wsgiref.util
27 import tempfile
27 import tempfile
28 import psutil
28 import psutil
29
29
30 from itertools import chain
30 from itertools import chain
31
31
32 import msgpack
32 import msgpack
33 import configparser
33 import configparser
34
34
35 from pyramid.config import Configurator
35 from pyramid.config import Configurator
36 from pyramid.wsgi import wsgiapp
36 from pyramid.wsgi import wsgiapp
37 from pyramid.response import Response
37 from pyramid.response import Response
38
38
39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
40 from vcsserver.lib.rc_json import json
40
41 from vcsserver.config.settings_maker import SettingsMaker
41 from vcsserver.config.settings_maker import SettingsMaker
42 from vcsserver.str_utils import safe_int
42
43 from vcsserver.lib.statsd_client import StatsdClient
44 from vcsserver.tweens.request_wrapper import get_headers_call_context
43 from vcsserver.tweens.request_wrapper import get_headers_call_context
45
44
46 import vcsserver
45 from vcsserver import remote_wsgi, scm_app, hgpatches
47 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
46 from vcsserver.server import VcsServer
48 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
47 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
49 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
48 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
50 from vcsserver.echo_stub.echo_app import EchoApp
49 from vcsserver.echo_stub.echo_app import EchoApp
51 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
50 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
52 from vcsserver.lib.exc_tracking import store_exception, format_exc
51 from vcsserver.lib.exc_tracking import store_exception, format_exc
53 from vcsserver.server import VcsServer
52 from vcsserver.lib.str_utils import safe_int
53 from vcsserver.lib.statsd_client import StatsdClient
54 from vcsserver.lib.ext_json import json
55 from vcsserver.lib.config_utils import configure_and_store_settings
56
54
57
55 strict_vcs = True
58 strict_vcs = True
56
59
57 git_import_err = None
60 git_import_err = None
58 try:
61 try:
59 from vcsserver.remote.git_remote import GitFactory, GitRemote
62 from vcsserver.remote.git_remote import GitFactory, GitRemote
60 except ImportError as e:
63 except ImportError as e:
61 GitFactory = None
64 GitFactory = None
62 GitRemote = None
65 GitRemote = None
63 git_import_err = e
66 git_import_err = e
64 if strict_vcs:
67 if strict_vcs:
65 raise
68 raise
66
69
67
70
68 hg_import_err = None
71 hg_import_err = None
69 try:
72 try:
70 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
73 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
71 except ImportError as e:
74 except ImportError as e:
72 MercurialFactory = None
75 MercurialFactory = None
73 HgRemote = None
76 HgRemote = None
74 hg_import_err = e
77 hg_import_err = e
75 if strict_vcs:
78 if strict_vcs:
76 raise
79 raise
77
80
78
81
79 svn_import_err = None
82 svn_import_err = None
80 try:
83 try:
81 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
84 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
82 except ImportError as e:
85 except ImportError as e:
83 SubversionFactory = None
86 SubversionFactory = None
84 SvnRemote = None
87 SvnRemote = None
85 svn_import_err = e
88 svn_import_err = e
86 if strict_vcs:
89 if strict_vcs:
87 raise
90 raise
88
91
89 log = logging.getLogger(__name__)
92 log = logging.getLogger(__name__)
90
93
91 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
94 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
92 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
95 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
93
96
94 try:
97 try:
95 locale.setlocale(locale.LC_ALL, '')
98 locale.setlocale(locale.LC_ALL, '')
96 except locale.Error as e:
99 except locale.Error as e:
97 log.error(
100 log.error('LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
98 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
99 os.environ['LC_ALL'] = 'C'
101 os.environ['LC_ALL'] = 'C'
100
102
101
103
102 def _is_request_chunked(environ):
104 def _is_request_chunked(environ):
103 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
105 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
104 return stream
106 return stream
105
107
106
108
107 def log_max_fd():
109 def log_max_fd():
108 try:
110 try:
109 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
111 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
110 log.info('Max file descriptors value: %s', maxfd)
112 log.info('Max file descriptors value: %s', maxfd)
111 except Exception:
113 except Exception:
112 pass
114 pass
113
115
114
116
115 class VCS:
117 class VCS:
116 def __init__(self, locale_conf=None, cache_config=None):
118 def __init__(self, locale_conf=None, cache_config=None):
117 self.locale = locale_conf
119 self.locale = locale_conf
118 self.cache_config = cache_config
120 self.cache_config = cache_config
119 self._configure_locale()
121 self._configure_locale()
120
122
121 log_max_fd()
123 log_max_fd()
122
124
123 if GitFactory and GitRemote:
125 if GitFactory and GitRemote:
124 git_factory = GitFactory()
126 git_factory = GitFactory()
125 self._git_remote = GitRemote(git_factory)
127 self._git_remote = GitRemote(git_factory)
126 else:
128 else:
127 log.error("Git client import failed: %s", git_import_err)
129 log.error("Git client import failed: %s", git_import_err)
128
130
129 if MercurialFactory and HgRemote:
131 if MercurialFactory and HgRemote:
130 hg_factory = MercurialFactory()
132 hg_factory = MercurialFactory()
131 self._hg_remote = HgRemote(hg_factory)
133 self._hg_remote = HgRemote(hg_factory)
132 else:
134 else:
133 log.error("Mercurial client import failed: %s", hg_import_err)
135 log.error("Mercurial client import failed: %s", hg_import_err)
134
136
135 if SubversionFactory and SvnRemote:
137 if SubversionFactory and SvnRemote:
136 svn_factory = SubversionFactory()
138 svn_factory = SubversionFactory()
137
139
138 # hg factory is used for svn url validation
140 # hg factory is used for svn url validation
139 hg_factory = MercurialFactory()
141 hg_factory = MercurialFactory()
140 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
142 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
141 else:
143 else:
142 log.error("Subversion client import failed: %s", svn_import_err)
144 log.error("Subversion client import failed: %s", svn_import_err)
143
145
144 self._vcsserver = VcsServer()
146 self._vcsserver = VcsServer()
145
147
146 def _configure_locale(self):
148 def _configure_locale(self):
147 if self.locale:
149 if self.locale:
148 log.info('Settings locale: `LC_ALL` to %s', self.locale)
150 log.info('Settings locale: `LC_ALL` to %s', self.locale)
149 else:
151 else:
150 log.info('Configuring locale subsystem based on environment variables')
152 log.info('Configuring locale subsystem based on environment variables')
151 try:
153 try:
152 # If self.locale is the empty string, then the locale
154 # If self.locale is the empty string, then the locale
153 # module will use the environment variables. See the
155 # module will use the environment variables. See the
154 # documentation of the package `locale`.
156 # documentation of the package `locale`.
155 locale.setlocale(locale.LC_ALL, self.locale)
157 locale.setlocale(locale.LC_ALL, self.locale)
156
158
157 language_code, encoding = locale.getlocale()
159 language_code, encoding = locale.getlocale()
158 log.info(
160 log.info(
159 'Locale set to language code "%s" with encoding "%s".',
161 'Locale set to language code "%s" with encoding "%s".',
160 language_code, encoding)
162 language_code, encoding)
161 except locale.Error:
163 except locale.Error:
162 log.exception('Cannot set locale, not configuring the locale system')
164 log.exception('Cannot set locale, not configuring the locale system')
163
165
164
166
165 class WsgiProxy:
167 class WsgiProxy:
166 def __init__(self, wsgi):
168 def __init__(self, wsgi):
167 self.wsgi = wsgi
169 self.wsgi = wsgi
168
170
169 def __call__(self, environ, start_response):
171 def __call__(self, environ, start_response):
170 input_data = environ['wsgi.input'].read()
172 input_data = environ['wsgi.input'].read()
171 input_data = msgpack.unpackb(input_data)
173 input_data = msgpack.unpackb(input_data)
172
174
173 error = None
175 error = None
174 try:
176 try:
175 data, status, headers = self.wsgi.handle(
177 data, status, headers = self.wsgi.handle(
176 input_data['environment'], input_data['input_data'],
178 input_data['environment'], input_data['input_data'],
177 *input_data['args'], **input_data['kwargs'])
179 *input_data['args'], **input_data['kwargs'])
178 except Exception as e:
180 except Exception as e:
179 data, status, headers = [], None, None
181 data, status, headers = [], None, None
180 error = {
182 error = {
181 'message': str(e),
183 'message': str(e),
182 '_vcs_kind': getattr(e, '_vcs_kind', None)
184 '_vcs_kind': getattr(e, '_vcs_kind', None)
183 }
185 }
184
186
185 start_response(200, {})
187 start_response(200, {})
186 return self._iterator(error, status, headers, data)
188 return self._iterator(error, status, headers, data)
187
189
188 def _iterator(self, error, status, headers, data):
190 def _iterator(self, error, status, headers, data):
189 initial_data = [
191 initial_data = [
190 error,
192 error,
191 status,
193 status,
192 headers,
194 headers,
193 ]
195 ]
194
196
195 for d in chain(initial_data, data):
197 for d in chain(initial_data, data):
196 yield msgpack.packb(d)
198 yield msgpack.packb(d)
197
199
198
200
199 def not_found(request):
201 def not_found(request):
200 return {'status': '404 NOT FOUND'}
202 return {'status': '404 NOT FOUND'}
201
203
202
204
203 class VCSViewPredicate:
205 class VCSViewPredicate:
204 def __init__(self, val, config):
206 def __init__(self, val, config):
205 self.remotes = val
207 self.remotes = val
206
208
207 def text(self):
209 def text(self):
208 return f'vcs view method = {list(self.remotes.keys())}'
210 return f'vcs view method = {list(self.remotes.keys())}'
209
211
210 phash = text
212 phash = text
211
213
212 def __call__(self, context, request):
214 def __call__(self, context, request):
213 """
215 """
214 View predicate that returns true if given backend is supported by
216 View predicate that returns true if given backend is supported by
215 defined remotes.
217 defined remotes.
216 """
218 """
217 backend = request.matchdict.get('backend')
219 backend = request.matchdict.get('backend')
218 return backend in self.remotes
220 return backend in self.remotes
219
221
220
222
221 class HTTPApplication:
223 class HTTPApplication:
222 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
224 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
223
225
224 remote_wsgi = remote_wsgi
226 remote_wsgi = remote_wsgi
225 _use_echo_app = False
227 _use_echo_app = False
226
228
227 def __init__(self, settings=None, global_config=None):
229 def __init__(self, settings=None, global_config=None):
228
230
229 self.config = Configurator(settings=settings)
231 self.config = Configurator(settings=settings)
230 # Init our statsd at very start
232 # Init our statsd at very start
231 self.config.registry.statsd = StatsdClient.statsd
233 self.config.registry.statsd = StatsdClient.statsd
232 self.config.registry.vcs_call_context = {}
234 self.config.registry.vcs_call_context = {}
233
235
234 self.global_config = global_config
236 self.global_config = global_config
235 self.config.include('vcsserver.lib.rc_cache')
237 self.config.include('vcsserver.lib.rc_cache')
236 self.config.include('vcsserver.lib.rc_cache.archive_cache')
238 self.config.include('vcsserver.lib.archive_cache')
237
239
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
240 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
241 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
240 self._remotes = {
242 self._remotes = {
241 'hg': vcs._hg_remote,
243 'hg': vcs._hg_remote,
242 'git': vcs._git_remote,
244 'git': vcs._git_remote,
243 'svn': vcs._svn_remote,
245 'svn': vcs._svn_remote,
244 'server': vcs._vcsserver,
246 'server': vcs._vcsserver,
245 }
247 }
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
248 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
247 self._use_echo_app = True
249 self._use_echo_app = True
248 log.warning("Using EchoApp for VCS operations.")
250 log.warning("Using EchoApp for VCS operations.")
249 self.remote_wsgi = remote_wsgi_stub
251 self.remote_wsgi = remote_wsgi_stub
250
252
251 self._configure_settings(global_config, settings)
253 configure_and_store_settings(global_config, settings)
252
254
253 self._configure()
255 self._configure()
254
256
255 def _configure_settings(self, global_config, app_settings):
256 """
257 Configure the settings module.
258 """
259 settings_merged = global_config.copy()
260 settings_merged.update(app_settings)
261
262 binary_dir = app_settings['core.binary_dir']
263
264 settings.BINARY_DIR = binary_dir
265
266 # Store the settings to make them available to other modules.
267 vcsserver.PYRAMID_SETTINGS = settings_merged
268 vcsserver.CONFIG = settings_merged
269
270 def _configure(self):
257 def _configure(self):
271 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
258 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
272
259
273 self.config.add_route('service', '/_service')
260 self.config.add_route('service', '/_service')
274 self.config.add_route('status', '/status')
261 self.config.add_route('status', '/status')
275 self.config.add_route('hg_proxy', '/proxy/hg')
262 self.config.add_route('hg_proxy', '/proxy/hg')
276 self.config.add_route('git_proxy', '/proxy/git')
263 self.config.add_route('git_proxy', '/proxy/git')
277
264
278 # rpc methods
265 # rpc methods
279 self.config.add_route('vcs', '/{backend}')
266 self.config.add_route('vcs', '/{backend}')
280
267
281 # streaming rpc remote methods
268 # streaming rpc remote methods
282 self.config.add_route('vcs_stream', '/{backend}/stream')
269 self.config.add_route('vcs_stream', '/{backend}/stream')
283
270
284 # vcs operations clone/push as streaming
271 # vcs operations clone/push as streaming
285 self.config.add_route('stream_git', '/stream/git/*repo_name')
272 self.config.add_route('stream_git', '/stream/git/*repo_name')
286 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
273 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
287
274
288 self.config.add_view(self.status_view, route_name='status', renderer='json')
275 self.config.add_view(self.status_view, route_name='status', renderer='json')
289 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
276 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
290
277
291 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
278 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
292 self.config.add_view(self.git_proxy(), route_name='git_proxy')
279 self.config.add_view(self.git_proxy(), route_name='git_proxy')
293 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
280 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
294 vcs_view=self._remotes)
281 vcs_view=self._remotes)
295 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
282 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
296 vcs_view=self._remotes)
283 vcs_view=self._remotes)
297
284
298 self.config.add_view(self.hg_stream(), route_name='stream_hg')
285 self.config.add_view(self.hg_stream(), route_name='stream_hg')
299 self.config.add_view(self.git_stream(), route_name='stream_git')
286 self.config.add_view(self.git_stream(), route_name='stream_git')
300
287
301 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
288 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
302
289
303 self.config.add_notfound_view(not_found, renderer='json')
290 self.config.add_notfound_view(not_found, renderer='json')
304
291
305 self.config.add_view(self.handle_vcs_exception, context=Exception)
292 self.config.add_view(self.handle_vcs_exception, context=Exception)
306
293
307 self.config.add_tween(
294 self.config.add_tween(
308 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
295 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
309 )
296 )
310 self.config.add_request_method(
297 self.config.add_request_method(
311 'vcsserver.lib.request_counter.get_request_counter',
298 'vcsserver.lib.request_counter.get_request_counter',
312 'request_count')
299 'request_count')
313
300
314 def wsgi_app(self):
301 def wsgi_app(self):
315 return self.config.make_wsgi_app()
302 return self.config.make_wsgi_app()
316
303
317 def _vcs_view_params(self, request):
304 def _vcs_view_params(self, request):
318 remote = self._remotes[request.matchdict['backend']]
305 remote = self._remotes[request.matchdict['backend']]
319 payload = msgpack.unpackb(request.body, use_list=True)
306 payload = msgpack.unpackb(request.body, use_list=True)
320
307
321 method = payload.get('method')
308 method = payload.get('method')
322 params = payload['params']
309 params = payload['params']
323 wire = params.get('wire')
310 wire = params.get('wire')
324 args = params.get('args')
311 args = params.get('args')
325 kwargs = params.get('kwargs')
312 kwargs = params.get('kwargs')
326 context_uid = None
313 context_uid = None
327
314
328 request.registry.vcs_call_context = {
315 request.registry.vcs_call_context = {
329 'method': method,
316 'method': method,
330 'repo_name': payload.get('_repo_name'),
317 'repo_name': payload.get('_repo_name'),
331 }
318 }
332
319
333 if wire:
320 if wire:
334 try:
321 try:
335 wire['context'] = context_uid = uuid.UUID(wire['context'])
322 wire['context'] = context_uid = uuid.UUID(wire['context'])
336 except KeyError:
323 except KeyError:
337 pass
324 pass
338 args.insert(0, wire)
325 args.insert(0, wire)
339 repo_state_uid = wire.get('repo_state_uid') if wire else None
326 repo_state_uid = wire.get('repo_state_uid') if wire else None
340
327
341 # NOTE(marcink): trading complexity for slight performance
328 # NOTE(marcink): trading complexity for slight performance
342 if log.isEnabledFor(logging.DEBUG):
329 if log.isEnabledFor(logging.DEBUG):
343 # also we SKIP printing out any of those methods args since they maybe excessive
330 # also we SKIP printing out any of those methods args since they maybe excessive
344 just_args_methods = {
331 just_args_methods = {
345 'commitctx': ('content', 'removed', 'updated'),
332 'commitctx': ('content', 'removed', 'updated'),
346 'commit': ('content', 'removed', 'updated')
333 'commit': ('content', 'removed', 'updated')
347 }
334 }
348 if method in just_args_methods:
335 if method in just_args_methods:
349 skip_args = just_args_methods[method]
336 skip_args = just_args_methods[method]
350 call_args = ''
337 call_args = ''
351 call_kwargs = {}
338 call_kwargs = {}
352 for k in kwargs:
339 for k in kwargs:
353 if k in skip_args:
340 if k in skip_args:
354 # replace our skip key with dummy
341 # replace our skip key with dummy
355 call_kwargs[k] = f'RemovedParam({k})'
342 call_kwargs[k] = f'RemovedParam({k})'
356 else:
343 else:
357 call_kwargs[k] = kwargs[k]
344 call_kwargs[k] = kwargs[k]
358 else:
345 else:
359 call_args = args[1:]
346 call_args = args[1:]
360 call_kwargs = kwargs
347 call_kwargs = kwargs
361
348
362 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
349 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
363 method, call_args, call_kwargs, context_uid, repo_state_uid)
350 method, call_args, call_kwargs, context_uid, repo_state_uid)
364
351
365 statsd = request.registry.statsd
352 statsd = request.registry.statsd
366 if statsd:
353 if statsd:
367 statsd.incr(
354 statsd.incr(
368 'vcsserver_method_total', tags=[
355 'vcsserver_method_total', tags=[
369 f"method:{method}",
356 f"method:{method}",
370 ])
357 ])
371 return payload, remote, method, args, kwargs
358 return payload, remote, method, args, kwargs
372
359
373 def vcs_view(self, request):
360 def vcs_view(self, request):
374
361
375 payload, remote, method, args, kwargs = self._vcs_view_params(request)
362 payload, remote, method, args, kwargs = self._vcs_view_params(request)
376 payload_id = payload.get('id')
363 payload_id = payload.get('id')
377
364
378 try:
365 try:
379 resp = getattr(remote, method)(*args, **kwargs)
366 resp = getattr(remote, method)(*args, **kwargs)
380 except Exception as e:
367 except Exception as e:
381 exc_info = list(sys.exc_info())
368 exc_info = list(sys.exc_info())
382 exc_type, exc_value, exc_traceback = exc_info
369 exc_type, exc_value, exc_traceback = exc_info
383
370
384 org_exc = getattr(e, '_org_exc', None)
371 org_exc = getattr(e, '_org_exc', None)
385 org_exc_name = None
372 org_exc_name = None
386 org_exc_tb = ''
373 org_exc_tb = ''
387 if org_exc:
374 if org_exc:
388 org_exc_name = org_exc.__class__.__name__
375 org_exc_name = org_exc.__class__.__name__
389 org_exc_tb = getattr(e, '_org_exc_tb', '')
376 org_exc_tb = getattr(e, '_org_exc_tb', '')
390 # replace our "faked" exception with our org
377 # replace our "faked" exception with our org
391 exc_info[0] = org_exc.__class__
378 exc_info[0] = org_exc.__class__
392 exc_info[1] = org_exc
379 exc_info[1] = org_exc
393
380
394 should_store_exc = True
381 should_store_exc = True
395 if org_exc:
382 if org_exc:
396 def get_exc_fqn(_exc_obj):
383 def get_exc_fqn(_exc_obj):
397 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
384 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
398 return module_name + '.' + org_exc_name
385 return module_name + '.' + org_exc_name
399
386
400 exc_fqn = get_exc_fqn(org_exc)
387 exc_fqn = get_exc_fqn(org_exc)
401
388
402 if exc_fqn in ['mercurial.error.RepoLookupError',
389 if exc_fqn in ['mercurial.error.RepoLookupError',
403 'vcsserver.exceptions.RefNotFoundException']:
390 'vcsserver.exceptions.RefNotFoundException']:
404 should_store_exc = False
391 should_store_exc = False
405
392
406 if should_store_exc:
393 if should_store_exc:
407 store_exception(id(exc_info), exc_info, request_path=request.path)
394 store_exception(id(exc_info), exc_info, request_path=request.path)
408
395
409 tb_info = format_exc(exc_info)
396 tb_info = format_exc(exc_info)
410
397
411 type_ = e.__class__.__name__
398 type_ = e.__class__.__name__
412 if type_ not in self.ALLOWED_EXCEPTIONS:
399 if type_ not in self.ALLOWED_EXCEPTIONS:
413 type_ = None
400 type_ = None
414
401
415 resp = {
402 resp = {
416 'id': payload_id,
403 'id': payload_id,
417 'error': {
404 'error': {
418 'message': str(e),
405 'message': str(e),
419 'traceback': tb_info,
406 'traceback': tb_info,
420 'org_exc': org_exc_name,
407 'org_exc': org_exc_name,
421 'org_exc_tb': org_exc_tb,
408 'org_exc_tb': org_exc_tb,
422 'type': type_
409 'type': type_
423 }
410 }
424 }
411 }
425
412
426 try:
413 try:
427 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
414 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
428 except AttributeError:
415 except AttributeError:
429 pass
416 pass
430 else:
417 else:
431 resp = {
418 resp = {
432 'id': payload_id,
419 'id': payload_id,
433 'result': resp
420 'result': resp
434 }
421 }
435 log.debug('Serving data for method %s', method)
422 log.debug('Serving data for method %s', method)
436 return resp
423 return resp
437
424
438 def vcs_stream_view(self, request):
425 def vcs_stream_view(self, request):
439 payload, remote, method, args, kwargs = self._vcs_view_params(request)
426 payload, remote, method, args, kwargs = self._vcs_view_params(request)
440 # this method has a stream: marker we remove it here
427 # this method has a stream: marker we remove it here
441 method = method.split('stream:')[-1]
428 method = method.split('stream:')[-1]
442 chunk_size = safe_int(payload.get('chunk_size')) or 4096
429 chunk_size = safe_int(payload.get('chunk_size')) or 4096
443
430
444 resp = getattr(remote, method)(*args, **kwargs)
431 resp = getattr(remote, method)(*args, **kwargs)
445
432
446 def get_chunked_data(method_resp):
433 def get_chunked_data(method_resp):
447 stream = io.BytesIO(method_resp)
434 stream = io.BytesIO(method_resp)
448 while 1:
435 while 1:
449 chunk = stream.read(chunk_size)
436 chunk = stream.read(chunk_size)
450 if not chunk:
437 if not chunk:
451 break
438 break
452 yield chunk
439 yield chunk
453
440
454 response = Response(app_iter=get_chunked_data(resp))
441 response = Response(app_iter=get_chunked_data(resp))
455 response.content_type = 'application/octet-stream'
442 response.content_type = 'application/octet-stream'
456
443
457 return response
444 return response
458
445
459 def status_view(self, request):
446 def status_view(self, request):
460 import vcsserver
447 import vcsserver
461 _platform_id = platform.uname()[1] or 'instance'
448 _platform_id = platform.uname()[1] or 'instance'
462
449
463 return {
450 return {
464 "status": "OK",
451 "status": "OK",
465 "vcsserver_version": vcsserver.get_version(),
452 "vcsserver_version": vcsserver.get_version(),
466 "platform": _platform_id,
453 "platform": _platform_id,
467 "pid": os.getpid(),
454 "pid": os.getpid(),
468 }
455 }
469
456
470 def service_view(self, request):
457 def service_view(self, request):
471 import vcsserver
458 import vcsserver
472
459
473 payload = msgpack.unpackb(request.body, use_list=True)
460 payload = msgpack.unpackb(request.body, use_list=True)
474 server_config, app_config = {}, {}
461 server_config, app_config = {}, {}
475
462
476 try:
463 try:
477 path = self.global_config['__file__']
464 path = self.global_config['__file__']
478 config = configparser.RawConfigParser()
465 config = configparser.RawConfigParser()
479
466
480 config.read(path)
467 config.read(path)
481
468
482 if config.has_section('server:main'):
469 if config.has_section('server:main'):
483 server_config = dict(config.items('server:main'))
470 server_config = dict(config.items('server:main'))
484 if config.has_section('app:main'):
471 if config.has_section('app:main'):
485 app_config = dict(config.items('app:main'))
472 app_config = dict(config.items('app:main'))
486
473
487 except Exception:
474 except Exception:
488 log.exception('Failed to read .ini file for display')
475 log.exception('Failed to read .ini file for display')
489
476
490 environ = list(os.environ.items())
477 environ = list(os.environ.items())
491
478
492 resp = {
479 resp = {
493 'id': payload.get('id'),
480 'id': payload.get('id'),
494 'result': dict(
481 'result': dict(
495 version=vcsserver.get_version(),
482 version=vcsserver.get_version(),
496 config=server_config,
483 config=server_config,
497 app_config=app_config,
484 app_config=app_config,
498 environ=environ,
485 environ=environ,
499 payload=payload,
486 payload=payload,
500 )
487 )
501 }
488 }
502 return resp
489 return resp
503
490
504 def _msgpack_renderer_factory(self, info):
491 def _msgpack_renderer_factory(self, info):
505
492
506 def _render(value, system):
493 def _render(value, system):
507 bin_type = False
494 bin_type = False
508 res = value.get('result')
495 res = value.get('result')
509 if isinstance(res, BytesEnvelope):
496 if isinstance(res, BytesEnvelope):
510 log.debug('Result is wrapped in BytesEnvelope type')
497 log.debug('Result is wrapped in BytesEnvelope type')
511 bin_type = True
498 bin_type = True
512 elif isinstance(res, BinaryEnvelope):
499 elif isinstance(res, BinaryEnvelope):
513 log.debug('Result is wrapped in BinaryEnvelope type')
500 log.debug('Result is wrapped in BinaryEnvelope type')
514 value['result'] = res.val
501 value['result'] = res.val
515 bin_type = True
502 bin_type = True
516
503
517 request = system.get('request')
504 request = system.get('request')
518 if request is not None:
505 if request is not None:
519 response = request.response
506 response = request.response
520 ct = response.content_type
507 ct = response.content_type
521 if ct == response.default_content_type:
508 if ct == response.default_content_type:
522 response.content_type = 'application/x-msgpack'
509 response.content_type = 'application/x-msgpack'
523 if bin_type:
510 if bin_type:
524 response.content_type = 'application/x-msgpack-bin'
511 response.content_type = 'application/x-msgpack-bin'
525
512
526 return msgpack.packb(value, use_bin_type=bin_type)
513 return msgpack.packb(value, use_bin_type=bin_type)
527 return _render
514 return _render
528
515
529 def set_env_from_config(self, environ, config):
516 def set_env_from_config(self, environ, config):
530 dict_conf = {}
517 dict_conf = {}
531 try:
518 try:
532 for elem in config:
519 for elem in config:
533 if elem[0] == 'rhodecode':
520 if elem[0] == 'rhodecode':
534 dict_conf = json.loads(elem[2])
521 dict_conf = json.loads(elem[2])
535 break
522 break
536 except Exception:
523 except Exception:
537 log.exception('Failed to fetch SCM CONFIG')
524 log.exception('Failed to fetch SCM CONFIG')
538 return
525 return
539
526
540 username = dict_conf.get('username')
527 username = dict_conf.get('username')
541 if username:
528 if username:
542 environ['REMOTE_USER'] = username
529 environ['REMOTE_USER'] = username
543 # mercurial specific, some extension api rely on this
530 # mercurial specific, some extension api rely on this
544 environ['HGUSER'] = username
531 environ['HGUSER'] = username
545
532
546 ip = dict_conf.get('ip')
533 ip = dict_conf.get('ip')
547 if ip:
534 if ip:
548 environ['REMOTE_HOST'] = ip
535 environ['REMOTE_HOST'] = ip
549
536
550 if _is_request_chunked(environ):
537 if _is_request_chunked(environ):
551 # set the compatibility flag for webob
538 # set the compatibility flag for webob
552 environ['wsgi.input_terminated'] = True
539 environ['wsgi.input_terminated'] = True
553
540
554 def hg_proxy(self):
541 def hg_proxy(self):
555 @wsgiapp
542 @wsgiapp
556 def _hg_proxy(environ, start_response):
543 def _hg_proxy(environ, start_response):
557 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
544 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
558 return app(environ, start_response)
545 return app(environ, start_response)
559 return _hg_proxy
546 return _hg_proxy
560
547
561 def git_proxy(self):
548 def git_proxy(self):
562 @wsgiapp
549 @wsgiapp
563 def _git_proxy(environ, start_response):
550 def _git_proxy(environ, start_response):
564 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
551 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
565 return app(environ, start_response)
552 return app(environ, start_response)
566 return _git_proxy
553 return _git_proxy
567
554
568 def hg_stream(self):
555 def hg_stream(self):
569 if self._use_echo_app:
556 if self._use_echo_app:
570 @wsgiapp
557 @wsgiapp
571 def _hg_stream(environ, start_response):
558 def _hg_stream(environ, start_response):
572 app = EchoApp('fake_path', 'fake_name', None)
559 app = EchoApp('fake_path', 'fake_name', None)
573 return app(environ, start_response)
560 return app(environ, start_response)
574 return _hg_stream
561 return _hg_stream
575 else:
562 else:
576 @wsgiapp
563 @wsgiapp
577 def _hg_stream(environ, start_response):
564 def _hg_stream(environ, start_response):
578 log.debug('http-app: handling hg stream')
565 log.debug('http-app: handling hg stream')
579 call_context = get_headers_call_context(environ)
566 call_context = get_headers_call_context(environ)
580
567
581 repo_path = call_context['repo_path']
568 repo_path = call_context['repo_path']
582 repo_name = call_context['repo_name']
569 repo_name = call_context['repo_name']
583 config = call_context['repo_config']
570 config = call_context['repo_config']
584
571
585 app = scm_app.create_hg_wsgi_app(
572 app = scm_app.create_hg_wsgi_app(
586 repo_path, repo_name, config)
573 repo_path, repo_name, config)
587
574
588 # Consistent path information for hgweb
575 # Consistent path information for hgweb
589 environ['PATH_INFO'] = call_context['path_info']
576 environ['PATH_INFO'] = call_context['path_info']
590 environ['REPO_NAME'] = repo_name
577 environ['REPO_NAME'] = repo_name
591 self.set_env_from_config(environ, config)
578 self.set_env_from_config(environ, config)
592
579
593 log.debug('http-app: starting app handler '
580 log.debug('http-app: starting app handler '
594 'with %s and process request', app)
581 'with %s and process request', app)
595 return app(environ, ResponseFilter(start_response))
582 return app(environ, ResponseFilter(start_response))
596 return _hg_stream
583 return _hg_stream
597
584
598 def git_stream(self):
585 def git_stream(self):
599 if self._use_echo_app:
586 if self._use_echo_app:
600 @wsgiapp
587 @wsgiapp
601 def _git_stream(environ, start_response):
588 def _git_stream(environ, start_response):
602 app = EchoApp('fake_path', 'fake_name', None)
589 app = EchoApp('fake_path', 'fake_name', None)
603 return app(environ, start_response)
590 return app(environ, start_response)
604 return _git_stream
591 return _git_stream
605 else:
592 else:
606 @wsgiapp
593 @wsgiapp
607 def _git_stream(environ, start_response):
594 def _git_stream(environ, start_response):
608 log.debug('http-app: handling git stream')
595 log.debug('http-app: handling git stream')
609
596
610 call_context = get_headers_call_context(environ)
597 call_context = get_headers_call_context(environ)
611
598
612 repo_path = call_context['repo_path']
599 repo_path = call_context['repo_path']
613 repo_name = call_context['repo_name']
600 repo_name = call_context['repo_name']
614 config = call_context['repo_config']
601 config = call_context['repo_config']
615
602
616 environ['PATH_INFO'] = call_context['path_info']
603 environ['PATH_INFO'] = call_context['path_info']
617 self.set_env_from_config(environ, config)
604 self.set_env_from_config(environ, config)
618
605
619 content_type = environ.get('CONTENT_TYPE', '')
606 content_type = environ.get('CONTENT_TYPE', '')
620
607
621 path = environ['PATH_INFO']
608 path = environ['PATH_INFO']
622 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
609 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
623 log.debug(
610 log.debug(
624 'LFS: Detecting if request `%s` is LFS server path based '
611 'LFS: Detecting if request `%s` is LFS server path based '
625 'on content type:`%s`, is_lfs:%s',
612 'on content type:`%s`, is_lfs:%s',
626 path, content_type, is_lfs_request)
613 path, content_type, is_lfs_request)
627
614
628 if not is_lfs_request:
615 if not is_lfs_request:
629 # fallback detection by path
616 # fallback detection by path
630 if GIT_LFS_PROTO_PAT.match(path):
617 if GIT_LFS_PROTO_PAT.match(path):
631 is_lfs_request = True
618 is_lfs_request = True
632 log.debug(
619 log.debug(
633 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
620 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
634 path, is_lfs_request)
621 path, is_lfs_request)
635
622
636 if is_lfs_request:
623 if is_lfs_request:
637 app = scm_app.create_git_lfs_wsgi_app(
624 app = scm_app.create_git_lfs_wsgi_app(
638 repo_path, repo_name, config)
625 repo_path, repo_name, config)
639 else:
626 else:
640 app = scm_app.create_git_wsgi_app(
627 app = scm_app.create_git_wsgi_app(
641 repo_path, repo_name, config)
628 repo_path, repo_name, config)
642
629
643 log.debug('http-app: starting app handler '
630 log.debug('http-app: starting app handler '
644 'with %s and process request', app)
631 'with %s and process request', app)
645
632
646 return app(environ, start_response)
633 return app(environ, start_response)
647
634
648 return _git_stream
635 return _git_stream
649
636
650 def handle_vcs_exception(self, exception, request):
637 def handle_vcs_exception(self, exception, request):
651 _vcs_kind = getattr(exception, '_vcs_kind', '')
638 _vcs_kind = getattr(exception, '_vcs_kind', '')
652
639
653 if _vcs_kind == 'repo_locked':
640 if _vcs_kind == 'repo_locked':
654 headers_call_context = get_headers_call_context(request.environ)
641 headers_call_context = get_headers_call_context(request.environ)
655 status_code = safe_int(headers_call_context['locked_status_code'])
642 status_code = safe_int(headers_call_context['locked_status_code'])
656
643
657 return HTTPRepoLocked(
644 return HTTPRepoLocked(
658 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
645 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
659
646
660 elif _vcs_kind == 'repo_branch_protected':
647 elif _vcs_kind == 'repo_branch_protected':
661 # Get custom repo-branch-protected status code if present.
648 # Get custom repo-branch-protected status code if present.
662 return HTTPRepoBranchProtected(
649 return HTTPRepoBranchProtected(
663 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
650 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
664
651
665 exc_info = request.exc_info
652 exc_info = request.exc_info
666 store_exception(id(exc_info), exc_info)
653 store_exception(id(exc_info), exc_info)
667
654
668 traceback_info = 'unavailable'
655 traceback_info = 'unavailable'
669 if request.exc_info:
656 if request.exc_info:
670 traceback_info = format_exc(request.exc_info)
657 traceback_info = format_exc(request.exc_info)
671
658
672 log.error(
659 log.error(
673 'error occurred handling this request for path: %s, \n%s',
660 'error occurred handling this request for path: %s, \n%s',
674 request.path, traceback_info)
661 request.path, traceback_info)
675
662
676 statsd = request.registry.statsd
663 statsd = request.registry.statsd
677 if statsd:
664 if statsd:
678 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
665 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
679 statsd.incr('vcsserver_exception_total',
666 statsd.incr('vcsserver_exception_total',
680 tags=[f"type:{exc_type}"])
667 tags=[f"type:{exc_type}"])
681 raise exception
668 raise exception
682
669
683
670
684 class ResponseFilter:
671 class ResponseFilter:
685
672
686 def __init__(self, start_response):
673 def __init__(self, start_response):
687 self._start_response = start_response
674 self._start_response = start_response
688
675
689 def __call__(self, status, response_headers, exc_info=None):
676 def __call__(self, status, response_headers, exc_info=None):
690 headers = tuple(
677 headers = tuple(
691 (h, v) for h, v in response_headers
678 (h, v) for h, v in response_headers
692 if not wsgiref.util.is_hop_by_hop(h))
679 if not wsgiref.util.is_hop_by_hop(h))
693 return self._start_response(status, headers, exc_info)
680 return self._start_response(status, headers, exc_info)
694
681
695
682
696 def sanitize_settings_and_apply_defaults(global_config, settings):
683 def sanitize_settings_and_apply_defaults(global_config, settings):
697 _global_settings_maker = SettingsMaker(global_config)
684 _global_settings_maker = SettingsMaker(global_config)
698 settings_maker = SettingsMaker(settings)
685 settings_maker = SettingsMaker(settings)
699
686
700 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
687 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
701
688
702 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
689 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
703 settings_maker.enable_logging(logging_conf)
690 settings_maker.enable_logging(logging_conf)
704
691
705 # Default includes, possible to change as a user
692 # Default includes, possible to change as a user
706 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
693 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
707 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
694 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
708
695
709 settings_maker.make_setting('__file__', global_config.get('__file__'))
696 settings_maker.make_setting('__file__', global_config.get('__file__'))
710
697
711 settings_maker.make_setting('pyramid.default_locale_name', 'en')
698 settings_maker.make_setting('pyramid.default_locale_name', 'en')
712 settings_maker.make_setting('locale', 'en_US.UTF-8')
699 settings_maker.make_setting('locale', 'en_US.UTF-8')
713
700
714 settings_maker.make_setting(
701 settings_maker.make_setting(
715 'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin',
702 'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin',
716 default_when_empty=True, parser='string:noquote')
703 default_when_empty=True, parser='string:noquote')
717
704
705 settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
706
718 temp_store = tempfile.gettempdir()
707 temp_store = tempfile.gettempdir()
719 default_cache_dir = os.path.join(temp_store, 'rc_cache')
708 default_cache_dir = os.path.join(temp_store, 'rc_cache')
720 # save default, cache dir, and use it for all backends later.
709 # save default, cache dir, and use it for all backends later.
721 default_cache_dir = settings_maker.make_setting(
710 default_cache_dir = settings_maker.make_setting(
722 'cache_dir',
711 'cache_dir',
723 default=default_cache_dir, default_when_empty=True,
712 default=default_cache_dir, default_when_empty=True,
724 parser='dir:ensured')
713 parser='dir:ensured')
725
714
726 # exception store cache
715 # exception store cache
727 settings_maker.make_setting(
716 settings_maker.make_setting(
728 'exception_tracker.store_path',
717 'exception_tracker.store_path',
729 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
718 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
730 parser='dir:ensured'
719 parser='dir:ensured'
731 )
720 )
732
721
733 # repo_object cache defaults
722 # repo_object cache defaults
734 settings_maker.make_setting(
723 settings_maker.make_setting(
735 'rc_cache.repo_object.backend',
724 'rc_cache.repo_object.backend',
736 default='dogpile.cache.rc.file_namespace',
725 default='dogpile.cache.rc.file_namespace',
737 parser='string')
726 parser='string')
738 settings_maker.make_setting(
727 settings_maker.make_setting(
739 'rc_cache.repo_object.expiration_time',
728 'rc_cache.repo_object.expiration_time',
740 default=30 * 24 * 60 * 60, # 30days
729 default=30 * 24 * 60 * 60, # 30days
741 parser='int')
730 parser='int')
742 settings_maker.make_setting(
731 settings_maker.make_setting(
743 'rc_cache.repo_object.arguments.filename',
732 'rc_cache.repo_object.arguments.filename',
744 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
733 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
745 parser='string')
734 parser='string')
746
735
747 # statsd
736 # statsd
748 settings_maker.make_setting('statsd.enabled', False, parser='bool')
737 settings_maker.make_setting('statsd.enabled', False, parser='bool')
749 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
738 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
750 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
739 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
751 settings_maker.make_setting('statsd.statsd_prefix', '')
740 settings_maker.make_setting('statsd.statsd_prefix', '')
752 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
741 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
753
742
754 settings_maker.env_expand()
743 settings_maker.env_expand()
755
744
756
745
757 def main(global_config, **settings):
746 def main(global_config, **settings):
758 start_time = time.time()
747 start_time = time.time()
759 log.info('Pyramid app config starting')
748 log.info('Pyramid app config starting')
760
749
761 if MercurialFactory:
750 if MercurialFactory:
762 hgpatches.patch_largefiles_capabilities()
751 hgpatches.patch_largefiles_capabilities()
763 hgpatches.patch_subrepo_type_mapping()
752 hgpatches.patch_subrepo_type_mapping()
764
753
765 # Fill in and sanitize the defaults & do ENV expansion
754 # Fill in and sanitize the defaults & do ENV expansion
766 sanitize_settings_and_apply_defaults(global_config, settings)
755 sanitize_settings_and_apply_defaults(global_config, settings)
767
756
768 # init and bootstrap StatsdClient
757 # init and bootstrap StatsdClient
769 StatsdClient.setup(settings)
758 StatsdClient.setup(settings)
770
759
771 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
760 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
772 total_time = time.time() - start_time
761 total_time = time.time() - start_time
773 log.info('Pyramid app created and configured in %.2fs', total_time)
762 log.info('Pyramid app created and configured in %.2fs', total_time)
774 return pyramid_app
763 return pyramid_app
@@ -1,243 +1,237 b''
1 '''
1 '''
2 This library is provided to allow standard python logging
2 This library is provided to allow standard python logging
3 to output log data as JSON formatted strings
3 to output log data as JSON formatted strings
4 '''
4 '''
5 import logging
5 import logging
6 import json
7 import re
6 import re
8 from datetime import date, datetime, time, tzinfo, timedelta
7 from datetime import date, datetime, time, tzinfo, timedelta
9 import traceback
8 import traceback
10 import importlib
9 import importlib
11
10
12 from inspect import istraceback
11 from inspect import istraceback
13
12
14 from collections import OrderedDict
13 from collections import OrderedDict
15
14
16
15 from ...logging_formatter import _inject_req_id, ExceptionAwareFormatter
17 def _inject_req_id(record, *args, **kwargs):
16 from ...ext_json import sjson as json
18 return record
19
20
21 ExceptionAwareFormatter = logging.Formatter
22
23
17
24 ZERO = timedelta(0)
18 ZERO = timedelta(0)
25 HOUR = timedelta(hours=1)
19 HOUR = timedelta(hours=1)
26
20
27
21
28 class UTC(tzinfo):
22 class UTC(tzinfo):
29 """UTC"""
23 """UTC"""
30
24
31 def utcoffset(self, dt):
25 def utcoffset(self, dt):
32 return ZERO
26 return ZERO
33
27
34 def tzname(self, dt):
28 def tzname(self, dt):
35 return "UTC"
29 return "UTC"
36
30
37 def dst(self, dt):
31 def dst(self, dt):
38 return ZERO
32 return ZERO
39
33
40 utc = UTC()
34 utc = UTC()
41
35
42
36
43 # skip natural LogRecord attributes
37 # skip natural LogRecord attributes
44 # http://docs.python.org/library/logging.html#logrecord-attributes
38 # http://docs.python.org/library/logging.html#logrecord-attributes
45 RESERVED_ATTRS = (
39 RESERVED_ATTRS = (
46 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
40 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
47 'funcName', 'levelname', 'levelno', 'lineno', 'module',
41 'funcName', 'levelname', 'levelno', 'lineno', 'module',
48 'msecs', 'message', 'msg', 'name', 'pathname', 'process',
42 'msecs', 'message', 'msg', 'name', 'pathname', 'process',
49 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName')
43 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName')
50
44
51
45
52 def merge_record_extra(record, target, reserved):
46 def merge_record_extra(record, target, reserved):
53 """
47 """
54 Merges extra attributes from LogRecord object into target dictionary
48 Merges extra attributes from LogRecord object into target dictionary
55
49
56 :param record: logging.LogRecord
50 :param record: logging.LogRecord
57 :param target: dict to update
51 :param target: dict to update
58 :param reserved: dict or list with reserved keys to skip
52 :param reserved: dict or list with reserved keys to skip
59 """
53 """
60 for key, value in record.__dict__.items():
54 for key, value in record.__dict__.items():
61 # this allows to have numeric keys
55 # this allows to have numeric keys
62 if (key not in reserved
56 if (key not in reserved
63 and not (hasattr(key, "startswith")
57 and not (hasattr(key, "startswith")
64 and key.startswith('_'))):
58 and key.startswith('_'))):
65 target[key] = value
59 target[key] = value
66 return target
60 return target
67
61
68
62
69 class JsonEncoder(json.JSONEncoder):
63 class JsonEncoder(json.JSONEncoder):
70 """
64 """
71 A custom encoder extending the default JSONEncoder
65 A custom encoder extending the default JSONEncoder
72 """
66 """
73
67
74 def default(self, obj):
68 def default(self, obj):
75 if isinstance(obj, (date, datetime, time)):
69 if isinstance(obj, (date, datetime, time)):
76 return self.format_datetime_obj(obj)
70 return self.format_datetime_obj(obj)
77
71
78 elif istraceback(obj):
72 elif istraceback(obj):
79 return ''.join(traceback.format_tb(obj)).strip()
73 return ''.join(traceback.format_tb(obj)).strip()
80
74
81 elif type(obj) == Exception \
75 elif type(obj) == Exception \
82 or isinstance(obj, Exception) \
76 or isinstance(obj, Exception) \
83 or type(obj) == type:
77 or type(obj) == type:
84 return str(obj)
78 return str(obj)
85
79
86 try:
80 try:
87 return super().default(obj)
81 return super().default(obj)
88
82
89 except TypeError:
83 except TypeError:
90 try:
84 try:
91 return str(obj)
85 return str(obj)
92
86
93 except Exception:
87 except Exception:
94 return None
88 return None
95
89
96 def format_datetime_obj(self, obj):
90 def format_datetime_obj(self, obj):
97 return obj.isoformat()
91 return obj.isoformat()
98
92
99
93
100 class JsonFormatter(ExceptionAwareFormatter):
94 class JsonFormatter(ExceptionAwareFormatter):
101 """
95 """
102 A custom formatter to format logging records as json strings.
96 A custom formatter to format logging records as json strings.
103 Extra values will be formatted as str() if not supported by
97 Extra values will be formatted as str() if not supported by
104 json default encoder
98 json default encoder
105 """
99 """
106
100
107 def __init__(self, *args, **kwargs):
101 def __init__(self, *args, **kwargs):
108 """
102 """
109 :param json_default: a function for encoding non-standard objects
103 :param json_default: a function for encoding non-standard objects
110 as outlined in http://docs.python.org/2/library/json.html
104 as outlined in http://docs.python.org/2/library/json.html
111 :param json_encoder: optional custom encoder
105 :param json_encoder: optional custom encoder
112 :param json_serializer: a :meth:`json.dumps`-compatible callable
106 :param json_serializer: a :meth:`json.dumps`-compatible callable
113 that will be used to serialize the log record.
107 that will be used to serialize the log record.
114 :param json_indent: an optional :meth:`json.dumps`-compatible numeric value
108 :param json_indent: an optional :meth:`json.dumps`-compatible numeric value
115 that will be used to customize the indent of the output json.
109 that will be used to customize the indent of the output json.
116 :param prefix: an optional string prefix added at the beginning of
110 :param prefix: an optional string prefix added at the beginning of
117 the formatted string
111 the formatted string
118 :param json_indent: indent parameter for json.dumps
112 :param json_indent: indent parameter for json.dumps
119 :param json_ensure_ascii: ensure_ascii parameter for json.dumps
113 :param json_ensure_ascii: ensure_ascii parameter for json.dumps
120 :param reserved_attrs: an optional list of fields that will be skipped when
114 :param reserved_attrs: an optional list of fields that will be skipped when
121 outputting json log record. Defaults to all log record attributes:
115 outputting json log record. Defaults to all log record attributes:
122 http://docs.python.org/library/logging.html#logrecord-attributes
116 http://docs.python.org/library/logging.html#logrecord-attributes
123 :param timestamp: an optional string/boolean field to add a timestamp when
117 :param timestamp: an optional string/boolean field to add a timestamp when
124 outputting the json log record. If string is passed, timestamp will be added
118 outputting the json log record. If string is passed, timestamp will be added
125 to log record using string as key. If True boolean is passed, timestamp key
119 to log record using string as key. If True boolean is passed, timestamp key
126 will be "timestamp". Defaults to False/off.
120 will be "timestamp". Defaults to False/off.
127 """
121 """
128 self.json_default = self._str_to_fn(kwargs.pop("json_default", None))
122 self.json_default = self._str_to_fn(kwargs.pop("json_default", None))
129 self.json_encoder = self._str_to_fn(kwargs.pop("json_encoder", None))
123 self.json_encoder = self._str_to_fn(kwargs.pop("json_encoder", None))
130 self.json_serializer = self._str_to_fn(kwargs.pop("json_serializer", json.dumps))
124 self.json_serializer = self._str_to_fn(kwargs.pop("json_serializer", json.dumps))
131 self.json_indent = kwargs.pop("json_indent", None)
125 self.json_indent = kwargs.pop("json_indent", None)
132 self.json_ensure_ascii = kwargs.pop("json_ensure_ascii", True)
126 self.json_ensure_ascii = kwargs.pop("json_ensure_ascii", True)
133 self.prefix = kwargs.pop("prefix", "")
127 self.prefix = kwargs.pop("prefix", "")
134 reserved_attrs = kwargs.pop("reserved_attrs", RESERVED_ATTRS)
128 reserved_attrs = kwargs.pop("reserved_attrs", RESERVED_ATTRS)
135 self.reserved_attrs = dict(list(zip(reserved_attrs, reserved_attrs)))
129 self.reserved_attrs = dict(list(zip(reserved_attrs, reserved_attrs)))
136 self.timestamp = kwargs.pop("timestamp", True)
130 self.timestamp = kwargs.pop("timestamp", True)
137
131
138 # super(JsonFormatter, self).__init__(*args, **kwargs)
132 # super(JsonFormatter, self).__init__(*args, **kwargs)
139 logging.Formatter.__init__(self, *args, **kwargs)
133 logging.Formatter.__init__(self, *args, **kwargs)
140 if not self.json_encoder and not self.json_default:
134 if not self.json_encoder and not self.json_default:
141 self.json_encoder = JsonEncoder
135 self.json_encoder = JsonEncoder
142
136
143 self._required_fields = self.parse()
137 self._required_fields = self.parse()
144 self._skip_fields = dict(list(zip(self._required_fields,
138 self._skip_fields = dict(list(zip(self._required_fields,
145 self._required_fields)))
139 self._required_fields)))
146 self._skip_fields.update(self.reserved_attrs)
140 self._skip_fields.update(self.reserved_attrs)
147
141
148 def _str_to_fn(self, fn_as_str):
142 def _str_to_fn(self, fn_as_str):
149 """
143 """
150 If the argument is not a string, return whatever was passed in.
144 If the argument is not a string, return whatever was passed in.
151 Parses a string such as package.module.function, imports the module
145 Parses a string such as package.module.function, imports the module
152 and returns the function.
146 and returns the function.
153
147
154 :param fn_as_str: The string to parse. If not a string, return it.
148 :param fn_as_str: The string to parse. If not a string, return it.
155 """
149 """
156 if not isinstance(fn_as_str, str):
150 if not isinstance(fn_as_str, str):
157 return fn_as_str
151 return fn_as_str
158
152
159 path, _, function = fn_as_str.rpartition('.')
153 path, _, function = fn_as_str.rpartition('.')
160 module = importlib.import_module(path)
154 module = importlib.import_module(path)
161 return getattr(module, function)
155 return getattr(module, function)
162
156
163 def parse(self):
157 def parse(self):
164 """
158 """
165 Parses format string looking for substitutions
159 Parses format string looking for substitutions
166
160
167 This method is responsible for returning a list of fields (as strings)
161 This method is responsible for returning a list of fields (as strings)
168 to include in all log messages.
162 to include in all log messages.
169 """
163 """
170 standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE)
164 standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE)
171 return standard_formatters.findall(self._fmt)
165 return standard_formatters.findall(self._fmt)
172
166
173 def add_fields(self, log_record, record, message_dict):
167 def add_fields(self, log_record, record, message_dict):
174 """
168 """
175 Override this method to implement custom logic for adding fields.
169 Override this method to implement custom logic for adding fields.
176 """
170 """
177 for field in self._required_fields:
171 for field in self._required_fields:
178 log_record[field] = record.__dict__.get(field)
172 log_record[field] = record.__dict__.get(field)
179 log_record.update(message_dict)
173 log_record.update(message_dict)
180 merge_record_extra(record, log_record, reserved=self._skip_fields)
174 merge_record_extra(record, log_record, reserved=self._skip_fields)
181
175
182 if self.timestamp:
176 if self.timestamp:
183 key = self.timestamp if type(self.timestamp) == str else 'timestamp'
177 key = self.timestamp if type(self.timestamp) == str else 'timestamp'
184 log_record[key] = datetime.fromtimestamp(record.created, tz=utc)
178 log_record[key] = datetime.fromtimestamp(record.created, tz=utc)
185
179
186 def process_log_record(self, log_record):
180 def process_log_record(self, log_record):
187 """
181 """
188 Override this method to implement custom logic
182 Override this method to implement custom logic
189 on the possibly ordered dictionary.
183 on the possibly ordered dictionary.
190 """
184 """
191 return log_record
185 return log_record
192
186
193 def jsonify_log_record(self, log_record):
187 def jsonify_log_record(self, log_record):
194 """Returns a json string of the log record."""
188 """Returns a json string of the log record."""
195 return self.json_serializer(log_record,
189 return self.json_serializer(log_record,
196 default=self.json_default,
190 default=self.json_default,
197 cls=self.json_encoder,
191 cls=self.json_encoder,
198 indent=self.json_indent,
192 indent=self.json_indent,
199 ensure_ascii=self.json_ensure_ascii)
193 ensure_ascii=self.json_ensure_ascii)
200
194
201 def serialize_log_record(self, log_record):
195 def serialize_log_record(self, log_record):
202 """Returns the final representation of the log record."""
196 """Returns the final representation of the log record."""
203 return "{}{}".format(self.prefix, self.jsonify_log_record(log_record))
197 return "{}{}".format(self.prefix, self.jsonify_log_record(log_record))
204
198
205 def format(self, record):
199 def format(self, record):
206 """Formats a log record and serializes to json"""
200 """Formats a log record and serializes to json"""
207 message_dict = {}
201 message_dict = {}
208 # FIXME: logging.LogRecord.msg and logging.LogRecord.message in typeshed
202 # FIXME: logging.LogRecord.msg and logging.LogRecord.message in typeshed
209 # are always type of str. We shouldn't need to override that.
203 # are always type of str. We shouldn't need to override that.
210 if isinstance(record.msg, dict):
204 if isinstance(record.msg, dict):
211 message_dict = record.msg
205 message_dict = record.msg
212 record.message = None
206 record.message = None
213 else:
207 else:
214 record.message = record.getMessage()
208 record.message = record.getMessage()
215 # only format time if needed
209 # only format time if needed
216 if "asctime" in self._required_fields:
210 if "asctime" in self._required_fields:
217 record.asctime = self.formatTime(record, self.datefmt)
211 record.asctime = self.formatTime(record, self.datefmt)
218
212
219 # Display formatted exception, but allow overriding it in the
213 # Display formatted exception, but allow overriding it in the
220 # user-supplied dict.
214 # user-supplied dict.
221 if record.exc_info and not message_dict.get('exc_info'):
215 if record.exc_info and not message_dict.get('exc_info'):
222 message_dict['exc_info'] = self.formatException(record.exc_info)
216 message_dict['exc_info'] = self.formatException(record.exc_info)
223 if not message_dict.get('exc_info') and record.exc_text:
217 if not message_dict.get('exc_info') and record.exc_text:
224 message_dict['exc_info'] = record.exc_text
218 message_dict['exc_info'] = record.exc_text
225 # Display formatted record of stack frames
219 # Display formatted record of stack frames
226 # default format is a string returned from :func:`traceback.print_stack`
220 # default format is a string returned from :func:`traceback.print_stack`
227 try:
221 try:
228 if record.stack_info and not message_dict.get('stack_info'):
222 if record.stack_info and not message_dict.get('stack_info'):
229 message_dict['stack_info'] = self.formatStack(record.stack_info)
223 message_dict['stack_info'] = self.formatStack(record.stack_info)
230 except AttributeError:
224 except AttributeError:
231 # Python2.7 doesn't have stack_info.
225 # Python2.7 doesn't have stack_info.
232 pass
226 pass
233
227
234 try:
228 try:
235 log_record = OrderedDict()
229 log_record = OrderedDict()
236 except NameError:
230 except NameError:
237 log_record = {}
231 log_record = {}
238
232
239 _inject_req_id(record, with_prefix=False)
233 _inject_req_id(record, with_prefix=False)
240 self.add_fields(log_record, record, message_dict)
234 self.add_fields(log_record, record, message_dict)
241 log_record = self.process_log_record(log_record)
235 log_record = self.process_log_record(log_record)
242
236
243 return self.serialize_log_record(log_record)
237 return self.serialize_log_record(log_record)
@@ -1,394 +1,402 b''
1
1
2 import threading
2 import threading
3 import weakref
3 import weakref
4 from base64 import b64encode
4 from base64 import b64encode
5 from logging import getLogger
5 from logging import getLogger
6 from os import urandom
6 from os import urandom
7 from typing import Union
7 from typing import Union
8
8
9 from redis import StrictRedis
9 from redis import StrictRedis
10
10
11 __version__ = '4.0.0'
11 __version__ = '4.0.0'
12
12
13 loggers = {
13 loggers = {
14 k: getLogger("vcsserver." + ".".join((__name__, k)))
14 k: getLogger("vcsserver." + ".".join((__name__, k)))
15 for k in [
15 for k in [
16 "acquire",
16 "acquire",
17 "refresh.thread.start",
17 "refresh.thread.start",
18 "refresh.thread.stop",
18 "refresh.thread.stop",
19 "refresh.thread.exit",
19 "refresh.thread.exit",
20 "refresh.start",
20 "refresh.start",
21 "refresh.shutdown",
21 "refresh.shutdown",
22 "refresh.exit",
22 "refresh.exit",
23 "release",
23 "release",
24 ]
24 ]
25 }
25 }
26
26
27 text_type = str
27 text_type = str
28 binary_type = bytes
28 binary_type = bytes
29
29
30
30
31 # Check if the id match. If not, return an error code.
31 # Check if the id match. If not, return an error code.
32 UNLOCK_SCRIPT = b"""
32 UNLOCK_SCRIPT = b"""
33 if redis.call("get", KEYS[1]) ~= ARGV[1] then
33 if redis.call("get", KEYS[1]) ~= ARGV[1] then
34 return 1
34 return 1
35 else
35 else
36 redis.call("del", KEYS[2])
36 redis.call("del", KEYS[2])
37 redis.call("lpush", KEYS[2], 1)
37 redis.call("lpush", KEYS[2], 1)
38 redis.call("pexpire", KEYS[2], ARGV[2])
38 redis.call("pexpire", KEYS[2], ARGV[2])
39 redis.call("del", KEYS[1])
39 redis.call("del", KEYS[1])
40 return 0
40 return 0
41 end
41 end
42 """
42 """
43
43
44 # Covers both cases when key doesn't exist and doesn't equal to lock's id
44 # Covers both cases when key doesn't exist and doesn't equal to lock's id
45 EXTEND_SCRIPT = b"""
45 EXTEND_SCRIPT = b"""
46 if redis.call("get", KEYS[1]) ~= ARGV[1] then
46 if redis.call("get", KEYS[1]) ~= ARGV[1] then
47 return 1
47 return 1
48 elseif redis.call("ttl", KEYS[1]) < 0 then
48 elseif redis.call("ttl", KEYS[1]) < 0 then
49 return 2
49 return 2
50 else
50 else
51 redis.call("expire", KEYS[1], ARGV[2])
51 redis.call("expire", KEYS[1], ARGV[2])
52 return 0
52 return 0
53 end
53 end
54 """
54 """
55
55
56 RESET_SCRIPT = b"""
56 RESET_SCRIPT = b"""
57 redis.call('del', KEYS[2])
57 redis.call('del', KEYS[2])
58 redis.call('lpush', KEYS[2], 1)
58 redis.call('lpush', KEYS[2], 1)
59 redis.call('pexpire', KEYS[2], ARGV[2])
59 redis.call('pexpire', KEYS[2], ARGV[2])
60 return redis.call('del', KEYS[1])
60 return redis.call('del', KEYS[1])
61 """
61 """
62
62
63 RESET_ALL_SCRIPT = b"""
63 RESET_ALL_SCRIPT = b"""
64 local locks = redis.call('keys', 'lock:*')
64 local locks = redis.call('keys', 'lock:*')
65 local signal
65 local signal
66 for _, lock in pairs(locks) do
66 for _, lock in pairs(locks) do
67 signal = 'lock-signal:' .. string.sub(lock, 6)
67 signal = 'lock-signal:' .. string.sub(lock, 6)
68 redis.call('del', signal)
68 redis.call('del', signal)
69 redis.call('lpush', signal, 1)
69 redis.call('lpush', signal, 1)
70 redis.call('expire', signal, 1)
70 redis.call('expire', signal, 1)
71 redis.call('del', lock)
71 redis.call('del', lock)
72 end
72 end
73 return #locks
73 return #locks
74 """
74 """
75
75
76
76
77 class AlreadyAcquired(RuntimeError):
77 class AlreadyAcquired(RuntimeError):
78 pass
78 pass
79
79
80
80
81 class NotAcquired(RuntimeError):
81 class NotAcquired(RuntimeError):
82 pass
82 pass
83
83
84
84
85 class AlreadyStarted(RuntimeError):
85 class AlreadyStarted(RuntimeError):
86 pass
86 pass
87
87
88
88
89 class TimeoutNotUsable(RuntimeError):
89 class TimeoutNotUsable(RuntimeError):
90 pass
90 pass
91
91
92
92
93 class InvalidTimeout(RuntimeError):
93 class InvalidTimeout(RuntimeError):
94 pass
94 pass
95
95
96
96
97 class TimeoutTooLarge(RuntimeError):
97 class TimeoutTooLarge(RuntimeError):
98 pass
98 pass
99
99
100
100
101 class NotExpirable(RuntimeError):
101 class NotExpirable(RuntimeError):
102 pass
102 pass
103
103
104
104
105 class Lock:
105 class Lock:
106 """
106 """
107 A Lock context manager implemented via redis SETNX/BLPOP.
107 A Lock context manager implemented via redis SETNX/BLPOP.
108 """
108 """
109
109
110 unlock_script = None
110 unlock_script = None
111 extend_script = None
111 extend_script = None
112 reset_script = None
112 reset_script = None
113 reset_all_script = None
113 reset_all_script = None
114 blocking = None
114
115
115 _lock_renewal_interval: float
116 _lock_renewal_interval: float
116 _lock_renewal_thread: Union[threading.Thread, None]
117 _lock_renewal_thread: Union[threading.Thread, None]
117
118
118 def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
119 def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000, blocking=True):
119 """
120 """
120 :param redis_client:
121 :param redis_client:
121 An instance of :class:`~StrictRedis`.
122 An instance of :class:`~StrictRedis`.
122 :param name:
123 :param name:
123 The name (redis key) the lock should have.
124 The name (redis key) the lock should have.
124 :param expire:
125 :param expire:
125 The lock expiry time in seconds. If left at the default (None)
126 The lock expiry time in seconds. If left at the default (None)
126 the lock will not expire.
127 the lock will not expire.
127 :param id:
128 :param id:
128 The ID (redis value) the lock should have. A random value is
129 The ID (redis value) the lock should have. A random value is
129 generated when left at the default.
130 generated when left at the default.
130
131
131 Note that if you specify this then the lock is marked as "held". Acquires
132 Note that if you specify this then the lock is marked as "held". Acquires
132 won't be possible.
133 won't be possible.
133 :param auto_renewal:
134 :param auto_renewal:
134 If set to ``True``, Lock will automatically renew the lock so that it
135 If set to ``True``, Lock will automatically renew the lock so that it
135 doesn't expire for as long as the lock is held (acquire() called
136 doesn't expire for as long as the lock is held (acquire() called
136 or running in a context manager).
137 or running in a context manager).
137
138
138 Implementation note: Renewal will happen using a daemon thread with
139 Implementation note: Renewal will happen using a daemon thread with
139 an interval of ``expire*2/3``. If wishing to use a different renewal
140 an interval of ``expire*2/3``. If wishing to use a different renewal
140 time, subclass Lock, call ``super().__init__()`` then set
141 time, subclass Lock, call ``super().__init__()`` then set
141 ``self._lock_renewal_interval`` to your desired interval.
142 ``self._lock_renewal_interval`` to your desired interval.
142 :param strict:
143 :param strict:
143 If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
144 If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
144 :param signal_expire:
145 :param signal_expire:
145 Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
146 Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
147 :param blocking:
148 Boolean value specifying whether lock should be blocking or not.
149 Used in `__enter__` method.
146 """
150 """
147 if strict and not isinstance(redis_client, StrictRedis):
151 if strict and not isinstance(redis_client, StrictRedis):
148 raise ValueError("redis_client must be instance of StrictRedis. "
152 raise ValueError("redis_client must be instance of StrictRedis. "
149 "Use strict=False if you know what you're doing.")
153 "Use strict=False if you know what you're doing.")
150 if auto_renewal and expire is None:
154 if auto_renewal and expire is None:
151 raise ValueError("Expire may not be None when auto_renewal is set")
155 raise ValueError("Expire may not be None when auto_renewal is set")
152
156
153 self._client = redis_client
157 self._client = redis_client
154
158
155 if expire:
159 if expire:
156 expire = int(expire)
160 expire = int(expire)
157 if expire < 0:
161 if expire < 0:
158 raise ValueError("A negative expire is not acceptable.")
162 raise ValueError("A negative expire is not acceptable.")
159 else:
163 else:
160 expire = None
164 expire = None
161 self._expire = expire
165 self._expire = expire
162
166
163 self._signal_expire = signal_expire
167 self._signal_expire = signal_expire
164 if id is None:
168 if id is None:
165 self._id = b64encode(urandom(18)).decode('ascii')
169 self._id = b64encode(urandom(18)).decode('ascii')
166 elif isinstance(id, binary_type):
170 elif isinstance(id, binary_type):
167 try:
171 try:
168 self._id = id.decode('ascii')
172 self._id = id.decode('ascii')
169 except UnicodeDecodeError:
173 except UnicodeDecodeError:
170 self._id = b64encode(id).decode('ascii')
174 self._id = b64encode(id).decode('ascii')
171 elif isinstance(id, text_type):
175 elif isinstance(id, text_type):
172 self._id = id
176 self._id = id
173 else:
177 else:
174 raise TypeError(f"Incorrect type for `id`. Must be bytes/str not {type(id)}.")
178 raise TypeError(f"Incorrect type for `id`. Must be bytes/str not {type(id)}.")
175 self._name = 'lock:' + name
179 self._name = 'lock:' + name
176 self._signal = 'lock-signal:' + name
180 self._signal = 'lock-signal:' + name
177 self._lock_renewal_interval = (float(expire) * 2 / 3
181 self._lock_renewal_interval = (float(expire) * 2 / 3
178 if auto_renewal
182 if auto_renewal
179 else None)
183 else None)
180 self._lock_renewal_thread = None
184 self._lock_renewal_thread = None
181
185
186 self.blocking = blocking
187
182 self.register_scripts(redis_client)
188 self.register_scripts(redis_client)
183
189
184 @classmethod
190 @classmethod
185 def register_scripts(cls, redis_client):
191 def register_scripts(cls, redis_client):
186 global reset_all_script
192 global reset_all_script
187 if reset_all_script is None:
193 if reset_all_script is None:
188 cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
194 cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
189 cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
195 cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
190 cls.reset_script = redis_client.register_script(RESET_SCRIPT)
196 cls.reset_script = redis_client.register_script(RESET_SCRIPT)
191 cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
197 cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
192 reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
198 reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
193
199
194 @property
200 @property
195 def _held(self):
201 def _held(self):
196 return self.id == self.get_owner_id()
202 return self.id == self.get_owner_id()
197
203
198 def reset(self):
204 def reset(self):
199 """
205 """
200 Forcibly deletes the lock. Use this with care.
206 Forcibly deletes the lock. Use this with care.
201 """
207 """
202 self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
208 self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
203
209
204 @property
210 @property
205 def id(self):
211 def id(self):
206 return self._id
212 return self._id
207
213
208 def get_owner_id(self):
214 def get_owner_id(self):
209 owner_id = self._client.get(self._name)
215 owner_id = self._client.get(self._name)
210 if isinstance(owner_id, binary_type):
216 if isinstance(owner_id, binary_type):
211 owner_id = owner_id.decode('ascii', 'replace')
217 owner_id = owner_id.decode('ascii', 'replace')
212 return owner_id
218 return owner_id
213
219
214 def acquire(self, blocking=True, timeout=None):
220 def acquire(self, blocking=True, timeout=None):
215 """
221 """
216 :param blocking:
222 :param blocking:
217 Boolean value specifying whether lock should be blocking or not.
223 Boolean value specifying whether lock should be blocking or not.
218 :param timeout:
224 :param timeout:
219 An integer value specifying the maximum number of seconds to block.
225 An integer value specifying the maximum number of seconds to block.
220 """
226 """
221 logger = loggers["acquire"]
227 logger = loggers["acquire"]
222
228
223 logger.debug("Getting blocking: %s acquire on %r ...", blocking, self._name)
229 logger.debug("Getting blocking: %s acquire on %r ...", blocking, self._name)
224
230
225 if self._held:
231 if self._held:
226 owner_id = self.get_owner_id()
232 owner_id = self.get_owner_id()
227 raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
233 raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
228
234
229 if not blocking and timeout is not None:
235 if not blocking and timeout is not None:
230 raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
236 raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
231
237
232 if timeout:
238 if timeout:
233 timeout = int(timeout)
239 timeout = int(timeout)
234 if timeout < 0:
240 if timeout < 0:
235 raise InvalidTimeout(f"Timeout ({timeout}) cannot be less than or equal to 0")
241 raise InvalidTimeout(f"Timeout ({timeout}) cannot be less than or equal to 0")
236
242
237 if self._expire and not self._lock_renewal_interval and timeout > self._expire:
243 if self._expire and not self._lock_renewal_interval and timeout > self._expire:
238 raise TimeoutTooLarge(f"Timeout ({timeout}) cannot be greater than expire ({self._expire})")
244 raise TimeoutTooLarge(f"Timeout ({timeout}) cannot be greater than expire ({self._expire})")
239
245
240 busy = True
246 busy = True
241 blpop_timeout = timeout or self._expire or 0
247 blpop_timeout = timeout or self._expire or 0
242 timed_out = False
248 timed_out = False
243 while busy:
249 while busy:
244 busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
250 busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
245 if busy:
251 if busy:
246 if timed_out:
252 if timed_out:
247 return False
253 return False
248 elif blocking:
254 elif blocking:
249 timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
255 timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
250 else:
256 else:
251 logger.warning("Failed to acquire Lock(%r).", self._name)
257 logger.warning("Failed to acquire Lock(%r).", self._name)
252 return False
258 return False
253
259
254 logger.debug("Acquired Lock(%r).", self._name)
260 logger.debug("Acquired Lock(%r).", self._name)
255 if self._lock_renewal_interval is not None:
261 if self._lock_renewal_interval is not None:
256 self._start_lock_renewer()
262 self._start_lock_renewer()
257 return True
263 return True
258
264
259 def extend(self, expire=None):
265 def extend(self, expire=None):
260 """
266 """
261 Extends expiration time of the lock.
267 Extends expiration time of the lock.
262
268
263 :param expire:
269 :param expire:
264 New expiration time. If ``None`` - `expire` provided during
270 New expiration time. If ``None`` - `expire` provided during
265 lock initialization will be taken.
271 lock initialization will be taken.
266 """
272 """
267 if expire:
273 if expire:
268 expire = int(expire)
274 expire = int(expire)
269 if expire < 0:
275 if expire < 0:
270 raise ValueError("A negative expire is not acceptable.")
276 raise ValueError("A negative expire is not acceptable.")
271 elif self._expire is not None:
277 elif self._expire is not None:
272 expire = self._expire
278 expire = self._expire
273 else:
279 else:
274 raise TypeError(
280 raise TypeError(
275 "To extend a lock 'expire' must be provided as an "
281 "To extend a lock 'expire' must be provided as an "
276 "argument to extend() method or at initialization time."
282 "argument to extend() method or at initialization time."
277 )
283 )
278
284
279 error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
285 error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
280 if error == 1:
286 if error == 1:
281 raise NotAcquired(f"Lock {self._name} is not acquired or it already expired.")
287 raise NotAcquired(f"Lock {self._name} is not acquired or it already expired.")
282 elif error == 2:
288 elif error == 2:
283 raise NotExpirable(f"Lock {self._name} has no assigned expiration time")
289 raise NotExpirable(f"Lock {self._name} has no assigned expiration time")
284 elif error:
290 elif error:
285 raise RuntimeError(f"Unsupported error code {error} from EXTEND script")
291 raise RuntimeError(f"Unsupported error code {error} from EXTEND script")
286
292
287 @staticmethod
293 @staticmethod
288 def _lock_renewer(name, lockref, interval, stop):
294 def _lock_renewer(name, lockref, interval, stop):
289 """
295 """
290 Renew the lock key in redis every `interval` seconds for as long
296 Renew the lock key in redis every `interval` seconds for as long
291 as `self._lock_renewal_thread.should_exit` is False.
297 as `self._lock_renewal_thread.should_exit` is False.
292 """
298 """
293 while not stop.wait(timeout=interval):
299 while not stop.wait(timeout=interval):
294 loggers["refresh.thread.start"].debug("Refreshing Lock(%r).", name)
300 loggers["refresh.thread.start"].debug("Refreshing Lock(%r).", name)
295 lock: "Lock" = lockref()
301 lock: "Lock" = lockref()
296 if lock is None:
302 if lock is None:
297 loggers["refresh.thread.stop"].debug(
303 loggers["refresh.thread.stop"].debug(
298 "Stopping loop because Lock(%r) was garbage collected.", name
304 "Stopping loop because Lock(%r) was garbage collected.", name
299 )
305 )
300 break
306 break
301 lock.extend(expire=lock._expire)
307 lock.extend(expire=lock._expire)
302 del lock
308 del lock
303 loggers["refresh.thread.exit"].debug("Exiting renewal thread for Lock(%r).", name)
309 loggers["refresh.thread.exit"].debug("Exiting renewal thread for Lock(%r).", name)
304
310
305 def _start_lock_renewer(self):
311 def _start_lock_renewer(self):
306 """
312 """
307 Starts the lock refresher thread.
313 Starts the lock refresher thread.
308 """
314 """
309 if self._lock_renewal_thread is not None:
315 if self._lock_renewal_thread is not None:
310 raise AlreadyStarted("Lock refresh thread already started")
316 raise AlreadyStarted("Lock refresh thread already started")
311
317
312 loggers["refresh.start"].debug(
318 loggers["refresh.start"].debug(
313 "Starting renewal thread for Lock(%r). Refresh interval: %s seconds.",
319 "Starting renewal thread for Lock(%r). Refresh interval: %s seconds.",
314 self._name, self._lock_renewal_interval
320 self._name, self._lock_renewal_interval
315 )
321 )
316 self._lock_renewal_stop = threading.Event()
322 self._lock_renewal_stop = threading.Event()
317 self._lock_renewal_thread = threading.Thread(
323 self._lock_renewal_thread = threading.Thread(
318 group=None,
324 group=None,
319 target=self._lock_renewer,
325 target=self._lock_renewer,
320 kwargs={
326 kwargs={
321 'name': self._name,
327 'name': self._name,
322 'lockref': weakref.ref(self),
328 'lockref': weakref.ref(self),
323 'interval': self._lock_renewal_interval,
329 'interval': self._lock_renewal_interval,
324 'stop': self._lock_renewal_stop,
330 'stop': self._lock_renewal_stop,
325 },
331 },
326 )
332 )
327 self._lock_renewal_thread.daemon = True
333 self._lock_renewal_thread.daemon = True
328 self._lock_renewal_thread.start()
334 self._lock_renewal_thread.start()
329
335
330 def _stop_lock_renewer(self):
336 def _stop_lock_renewer(self):
331 """
337 """
332 Stop the lock renewer.
338 Stop the lock renewer.
333
339
334 This signals the renewal thread and waits for its exit.
340 This signals the renewal thread and waits for its exit.
335 """
341 """
336 if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
342 if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
337 return
343 return
338 loggers["refresh.shutdown"].debug("Signaling renewal thread for Lock(%r) to exit.", self._name)
344 loggers["refresh.shutdown"].debug("Signaling renewal thread for Lock(%r) to exit.", self._name)
339 self._lock_renewal_stop.set()
345 self._lock_renewal_stop.set()
340 self._lock_renewal_thread.join()
346 self._lock_renewal_thread.join()
341 self._lock_renewal_thread = None
347 self._lock_renewal_thread = None
342 loggers["refresh.exit"].debug("Renewal thread for Lock(%r) exited.", self._name)
348 loggers["refresh.exit"].debug("Renewal thread for Lock(%r) exited.", self._name)
343
349
344 def __enter__(self):
350 def __enter__(self):
345 acquired = self.acquire(blocking=True)
351 acquired = self.acquire(blocking=self.blocking)
346 if not acquired:
352 if not acquired:
347 raise AssertionError(f"Lock({self._name}) wasn't acquired, but blocking=True was used!")
353 if self.blocking:
354 raise AssertionError(f"Lock({self._name}) wasn't acquired, but blocking=True was used!")
355 raise NotAcquired(f"Lock({self._name}) is not acquired or it already expired.")
348 return self
356 return self
349
357
350 def __exit__(self, exc_type=None, exc_value=None, traceback=None):
358 def __exit__(self, exc_type=None, exc_value=None, traceback=None):
351 self.release()
359 self.release()
352
360
353 def release(self):
361 def release(self):
354 """Releases the lock, that was acquired with the same object.
362 """Releases the lock, that was acquired with the same object.
355
363
356 .. note::
364 .. note::
357
365
358 If you want to release a lock that you acquired in a different place you have two choices:
366 If you want to release a lock that you acquired in a different place you have two choices:
359
367
360 * Use ``Lock("name", id=id_from_other_place).release()``
368 * Use ``Lock("name", id=id_from_other_place).release()``
361 * Use ``Lock("name").reset()``
369 * Use ``Lock("name").reset()``
362 """
370 """
363 if self._lock_renewal_thread is not None:
371 if self._lock_renewal_thread is not None:
364 self._stop_lock_renewer()
372 self._stop_lock_renewer()
365 loggers["release"].debug("Releasing Lock(%r).", self._name)
373 loggers["release"].debug("Releasing Lock(%r).", self._name)
366 error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
374 error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
367 if error == 1:
375 if error == 1:
368 raise NotAcquired(f"Lock({self._name}) is not acquired or it already expired.")
376 raise NotAcquired(f"Lock({self._name}) is not acquired or it already expired.")
369 elif error:
377 elif error:
370 raise RuntimeError(f"Unsupported error code {error} from EXTEND script.")
378 raise RuntimeError(f"Unsupported error code {error} from EXTEND script.")
371
379
372 def locked(self):
380 def locked(self):
373 """
381 """
374 Return true if the lock is acquired.
382 Return true if the lock is acquired.
375
383
376 Checks that lock with same name already exists. This method returns true, even if
384 Checks that lock with same name already exists. This method returns true, even if
377 lock have another id.
385 lock have another id.
378 """
386 """
379 return self._client.exists(self._name) == 1
387 return self._client.exists(self._name) == 1
380
388
381
389
382 reset_all_script = None
390 reset_all_script = None
383
391
384
392
385 def reset_all(redis_client):
393 def reset_all(redis_client):
386 """
394 """
387 Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
395 Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
388
396
389 :param redis_client:
397 :param redis_client:
390 An instance of :class:`~StrictRedis`.
398 An instance of :class:`~StrictRedis`.
391 """
399 """
392 Lock.register_scripts(redis_client)
400 Lock.register_scripts(redis_client)
393
401
394 reset_all_script(client=redis_client) # noqa
402 reset_all_script(client=redis_client) # noqa
@@ -1,50 +1,51 b''
1 import logging
1 import logging
2
2
3 from .stream import TCPStatsClient, UnixSocketStatsClient # noqa
3 from .stream import TCPStatsClient, UnixSocketStatsClient # noqa
4 from .udp import StatsClient # noqa
4 from .udp import StatsClient # noqa
5
5
6 HOST = 'localhost'
6 HOST = 'localhost'
7 PORT = 8125
7 PORT = 8125
8 IPV6 = False
8 IPV6 = False
9 PREFIX = None
9 PREFIX = None
10 MAXUDPSIZE = 512
10 MAXUDPSIZE = 512
11
11
12 log = logging.getLogger('rhodecode.statsd')
12 log = logging.getLogger('rhodecode.statsd')
13
13
14
14
15 def statsd_config(config, prefix='statsd.'):
15 def statsd_config(config, prefix='statsd.'):
16 _config = {}
16 _config = {}
17 for key in config.keys():
17 for key in list(config.keys()):
18 if key.startswith(prefix):
18 if key.startswith(prefix):
19 _config[key[len(prefix):]] = config[key]
19 _config[key[len(prefix):]] = config[key]
20 return _config
20 return _config
21
21
22
22
23 def client_from_config(configuration, prefix='statsd.', **kwargs):
23 def client_from_config(configuration, prefix='statsd.', **kwargs):
24 from pyramid.settings import asbool
24 from pyramid.settings import asbool
25
25
26 _config = statsd_config(configuration, prefix)
26 _config = statsd_config(configuration, prefix)
27 statsd_flag = _config.get('enabled')
27 statsd_enabled = asbool(_config.pop('enabled', False))
28 statsd_enabled = asbool(_config.pop('enabled', False))
28 if not statsd_enabled:
29 if not statsd_enabled:
29 log.debug('statsd client not enabled by statsd.enabled = flag, skipping...')
30 log.debug('statsd client not enabled by statsd.enabled = %s flag, skipping...', statsd_flag)
30 return
31 return
31
32
32 host = _config.pop('statsd_host', HOST)
33 host = _config.pop('statsd_host', HOST)
33 port = _config.pop('statsd_port', PORT)
34 port = _config.pop('statsd_port', PORT)
34 prefix = _config.pop('statsd_prefix', PREFIX)
35 prefix = _config.pop('statsd_prefix', PREFIX)
35 maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
36 maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
36 ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
37 ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
37 log.debug('configured statsd client %s:%s', host, port)
38 log.debug('configured statsd client %s:%s', host, port)
38
39
39 try:
40 try:
40 client = StatsClient(
41 client = StatsClient(
41 host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
42 host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
42 except Exception:
43 except Exception:
43 log.exception('StatsD is enabled, but failed to connect to statsd server, fallback: disable statsd')
44 log.exception('StatsD is enabled, but failed to connect to statsd server, fallback: disable statsd')
44 client = None
45 client = None
45
46
46 return client
47 return client
47
48
48
49
49 def get_statsd_client(request):
50 def get_statsd_client(request):
50 return client_from_config(request.registry.settings)
51 return client_from_config(request.registry.settings)
@@ -1,40 +1,58 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
2 #
3 # This program is free software: you can redistribute it and/or modify
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
5 # (only), as published by the Free Software Foundation.
6 #
6 #
7 # This program is distributed in the hope that it will be useful,
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
10 # GNU General Public License for more details.
11 #
11 #
12 # You should have received a copy of the GNU Affero General Public License
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
14 #
15 # This program is dual-licensed. If you wish to learn more about the
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 import os
18 import os
19 import vcsserver
20 import vcsserver.settings
19
21
20
22
21 def get_config(ini_path, **kwargs):
23 def get_config(ini_path, **kwargs):
22 import configparser
24 import configparser
23 parser = configparser.ConfigParser(**kwargs)
25 parser = configparser.ConfigParser(**kwargs)
24 parser.read(ini_path)
26 parser.read(ini_path)
25 return parser
27 return parser
26
28
27
29
28 def get_app_config_lightweight(ini_path):
30 def get_app_config_lightweight(ini_path):
29 parser = get_config(ini_path)
31 parser = get_config(ini_path)
30 parser.set('app:main', 'here', os.getcwd())
32 parser.set('app:main', 'here', os.getcwd())
31 parser.set('app:main', '__file__', ini_path)
33 parser.set('app:main', '__file__', ini_path)
32 return dict(parser.items('app:main'))
34 return dict(parser.items('app:main'))
33
35
34
36
35 def get_app_config(ini_path):
37 def get_app_config(ini_path):
36 """
38 """
37 This loads the app context and provides a heavy type iniliaziation of config
39 This loads the app context and provides a heavy type iniliaziation of config
38 """
40 """
39 from paste.deploy.loadwsgi import appconfig
41 from paste.deploy.loadwsgi import appconfig
40 return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
42 return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
43
44
45 def configure_and_store_settings(global_config, app_settings):
46 """
47 Configure the settings module.
48 """
49 settings_merged = global_config.copy()
50 settings_merged.update(app_settings)
51
52 binary_dir = app_settings['core.binary_dir']
53
54 vcsserver.settings.BINARY_DIR = binary_dir
55
56 # Store the settings to make them available to other modules.
57 vcsserver.PYRAMID_SETTINGS = settings_merged
58 vcsserver.CONFIG = settings_merged
@@ -1,2 +1,14 b''
1 # use orjson by default
1 import json as stdlib_json
2 import orjson as json
2
3 try:
4 # we keep simplejson for having dump functionality still
5 # orjson doesn't support it
6 import simplejson as sjson
7 except ImportError:
8 sjson = stdlib_json
9
10 try:
11 import orjson
12 import orjson as json
13 except ImportError:
14 json = stdlib_json
@@ -1,53 +1,63 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
2 #
3 # This program is free software: you can redistribute it and/or modify
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
5 # (only), as published by the Free Software Foundation.
6 #
6 #
7 # This program is distributed in the hope that it will be useful,
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
10 # GNU General Public License for more details.
11 #
11 #
12 # You should have received a copy of the GNU Affero General Public License
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
14 #
15 # This program is dual-licensed. If you wish to learn more about the
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
18
19 import sys
19 import sys
20 import logging
20 import logging
21
21
22
22
23 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
23 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
24
24
25 # Sequences
25 # Sequences
26 RESET_SEQ = "\033[0m"
26 RESET_SEQ = "\033[0m"
27 COLOR_SEQ = "\033[0;%dm"
27 COLOR_SEQ = "\033[0;%dm"
28 BOLD_SEQ = "\033[1m"
28 BOLD_SEQ = "\033[1m"
29
29
30 COLORS = {
30 COLORS = {
31 'CRITICAL': MAGENTA,
31 'CRITICAL': MAGENTA,
32 'ERROR': RED,
32 'ERROR': RED,
33 'WARNING': CYAN,
33 'WARNING': CYAN,
34 'INFO': GREEN,
34 'INFO': GREEN,
35 'DEBUG': BLUE,
35 'DEBUG': BLUE,
36 'SQL': YELLOW
36 'SQL': YELLOW
37 }
37 }
38
38
39
39
40 def _inject_req_id(record, *args, **kwargs):
41 return record
42
43
44 class ExceptionAwareFormatter(logging.Formatter):
45 pass
46
47
40 class ColorFormatter(logging.Formatter):
48 class ColorFormatter(logging.Formatter):
41
49
42 def format(self, record):
50 def format(self, record):
43 """
51 """
44 Change record's levelname to use with COLORS enum
52 Changes record's levelname to use with COLORS enum
45 """
53 """
46 def_record = super().format(record)
54 def_record = super().format(record)
47
55
48 levelname = record.levelname
56 levelname = record.levelname
49 start = COLOR_SEQ % (COLORS[levelname])
57 start = COLOR_SEQ % (COLORS[levelname])
50 end = RESET_SEQ
58 end = RESET_SEQ
51
59
52 colored_record = ''.join([start, def_record, end])
60 colored_record = ''.join([start, def_record, end])
53 return colored_record
61 return colored_record
62
63
@@ -1,63 +1,63 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18
19 import logging
19 import logging
20
20
21 from repoze.lru import LRUCache
21 from repoze.lru import LRUCache
22
22
23 from vcsserver.str_utils import safe_str
23 from vcsserver.lib.str_utils import safe_str
24
24
25 log = logging.getLogger(__name__)
25 log = logging.getLogger(__name__)
26
26
27
27
28 class LRUDict(LRUCache):
28 class LRUDict(LRUCache):
29 """
29 """
30 Wrapper to provide partial dict access
30 Wrapper to provide partial dict access
31 """
31 """
32
32
33 def __setitem__(self, key, value):
33 def __setitem__(self, key, value):
34 return self.put(key, value)
34 return self.put(key, value)
35
35
36 def __getitem__(self, key):
36 def __getitem__(self, key):
37 return self.get(key)
37 return self.get(key)
38
38
39 def __contains__(self, key):
39 def __contains__(self, key):
40 return bool(self.get(key))
40 return bool(self.get(key))
41
41
42 def __delitem__(self, key):
42 def __delitem__(self, key):
43 del self.data[key]
43 del self.data[key]
44
44
45 def keys(self):
45 def keys(self):
46 return list(self.data.keys())
46 return list(self.data.keys())
47
47
48
48
49 class LRUDictDebug(LRUDict):
49 class LRUDictDebug(LRUDict):
50 """
50 """
51 Wrapper to provide some debug options
51 Wrapper to provide some debug options
52 """
52 """
53 def _report_keys(self):
53 def _report_keys(self):
54 elems_cnt = f'{len(list(self.keys()))}/{self.size}'
54 elems_cnt = f'{len(list(self.keys()))}/{self.size}'
55 # trick for pformat print it more nicely
55 # trick for pformat print it more nicely
56 fmt = '\n'
56 fmt = '\n'
57 for cnt, elem in enumerate(self.keys()):
57 for cnt, elem in enumerate(self.keys()):
58 fmt += f'{cnt+1} - {safe_str(elem)}\n'
58 fmt += f'{cnt+1} - {safe_str(elem)}\n'
59 log.debug('current LRU keys (%s):%s', elems_cnt, fmt)
59 log.debug('current LRU keys (%s):%s', elems_cnt, fmt)
60
60
61 def __getitem__(self, key):
61 def __getitem__(self, key):
62 self._report_keys()
62 self._report_keys()
63 return self.get(key)
63 return self.get(key)
@@ -1,303 +1,303 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 #import errno
18 #import errno
19 import fcntl
19 import fcntl
20 import functools
20 import functools
21 import logging
21 import logging
22 import os
22 import os
23 import pickle
23 import pickle
24 #import time
24 #import time
25
25
26 #import gevent
26 #import gevent
27 import msgpack
27 import msgpack
28 import redis
28 import redis
29
29
30 flock_org = fcntl.flock
30 flock_org = fcntl.flock
31 from typing import Union
31 from typing import Union
32
32
33 from dogpile.cache.api import Deserializer, Serializer
33 from dogpile.cache.api import Deserializer, Serializer
34 from dogpile.cache.backends import file as file_backend
34 from dogpile.cache.backends import file as file_backend
35 from dogpile.cache.backends import memory as memory_backend
35 from dogpile.cache.backends import memory as memory_backend
36 from dogpile.cache.backends import redis as redis_backend
36 from dogpile.cache.backends import redis as redis_backend
37 from dogpile.cache.backends.file import FileLock
37 from dogpile.cache.backends.file import FileLock
38 from dogpile.cache.util import memoized_property
38 from dogpile.cache.util import memoized_property
39
39
40 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
40 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
41 from vcsserver.str_utils import safe_bytes, safe_str
41 from vcsserver.lib.str_utils import safe_bytes, safe_str
42 from vcsserver.type_utils import str2bool
42 from vcsserver.lib.type_utils import str2bool
43
43
44 _default_max_size = 1024
44 _default_max_size = 1024
45
45
46 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
47
47
48
48
49 class LRUMemoryBackend(memory_backend.MemoryBackend):
49 class LRUMemoryBackend(memory_backend.MemoryBackend):
50 key_prefix = 'lru_mem_backend'
50 key_prefix = 'lru_mem_backend'
51 pickle_values = False
51 pickle_values = False
52
52
53 def __init__(self, arguments):
53 def __init__(self, arguments):
54 self.max_size = arguments.pop('max_size', _default_max_size)
54 self.max_size = arguments.pop('max_size', _default_max_size)
55
55
56 LRUDictClass = LRUDict
56 LRUDictClass = LRUDict
57 if arguments.pop('log_key_count', None):
57 if arguments.pop('log_key_count', None):
58 LRUDictClass = LRUDictDebug
58 LRUDictClass = LRUDictDebug
59
59
60 arguments['cache_dict'] = LRUDictClass(self.max_size)
60 arguments['cache_dict'] = LRUDictClass(self.max_size)
61 super().__init__(arguments)
61 super().__init__(arguments)
62
62
63 def __repr__(self):
63 def __repr__(self):
64 return f'{self.__class__}(maxsize=`{self.max_size}`)'
64 return f'{self.__class__}(maxsize=`{self.max_size}`)'
65
65
66 def __str__(self):
66 def __str__(self):
67 return self.__repr__()
67 return self.__repr__()
68
68
69 def delete(self, key):
69 def delete(self, key):
70 try:
70 try:
71 del self._cache[key]
71 del self._cache[key]
72 except KeyError:
72 except KeyError:
73 # we don't care if key isn't there at deletion
73 # we don't care if key isn't there at deletion
74 pass
74 pass
75
75
76 def list_keys(self, prefix):
76 def list_keys(self, prefix):
77 return list(self._cache.keys())
77 return list(self._cache.keys())
78
78
79 def delete_multi(self, keys):
79 def delete_multi(self, keys):
80 for key in keys:
80 for key in keys:
81 self.delete(key)
81 self.delete(key)
82
82
83 def delete_multi_by_prefix(self, prefix):
83 def delete_multi_by_prefix(self, prefix):
84 cache_keys = self.list_keys(prefix=prefix)
84 cache_keys = self.list_keys(prefix=prefix)
85 num_affected_keys = len(cache_keys)
85 num_affected_keys = len(cache_keys)
86 if num_affected_keys:
86 if num_affected_keys:
87 self.delete_multi(cache_keys)
87 self.delete_multi(cache_keys)
88 return num_affected_keys
88 return num_affected_keys
89
89
90
90
91 class PickleSerializer:
91 class PickleSerializer:
92 serializer: None | Serializer = staticmethod( # type: ignore
92 serializer: None | Serializer = staticmethod( # type: ignore
93 functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL)
93 functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL)
94 )
94 )
95 deserializer: None | Deserializer = staticmethod( # type: ignore
95 deserializer: None | Deserializer = staticmethod( # type: ignore
96 functools.partial(pickle.loads)
96 functools.partial(pickle.loads)
97 )
97 )
98
98
99
99
100 class MsgPackSerializer:
100 class MsgPackSerializer:
101 serializer: None | Serializer = staticmethod( # type: ignore
101 serializer: None | Serializer = staticmethod( # type: ignore
102 msgpack.packb
102 msgpack.packb
103 )
103 )
104 deserializer: None | Deserializer = staticmethod( # type: ignore
104 deserializer: None | Deserializer = staticmethod( # type: ignore
105 functools.partial(msgpack.unpackb, use_list=False)
105 functools.partial(msgpack.unpackb, use_list=False)
106 )
106 )
107
107
108
108
109 class CustomLockFactory(FileLock):
109 class CustomLockFactory(FileLock):
110
110
111 pass
111 pass
112
112
113
113
114 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
114 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
115 key_prefix = 'file_backend'
115 key_prefix = 'file_backend'
116
116
117 def __init__(self, arguments):
117 def __init__(self, arguments):
118 arguments['lock_factory'] = CustomLockFactory
118 arguments['lock_factory'] = CustomLockFactory
119 db_file = arguments.get('filename')
119 db_file = arguments.get('filename')
120
120
121 log.debug('initialing cache-backend=%s db in %s', self.__class__.__name__, db_file)
121 log.debug('initialing cache-backend=%s db in %s', self.__class__.__name__, db_file)
122 db_file_dir = os.path.dirname(db_file)
122 db_file_dir = os.path.dirname(db_file)
123 if not os.path.isdir(db_file_dir):
123 if not os.path.isdir(db_file_dir):
124 os.makedirs(db_file_dir)
124 os.makedirs(db_file_dir)
125
125
126 try:
126 try:
127 super().__init__(arguments)
127 super().__init__(arguments)
128 except Exception:
128 except Exception:
129 log.exception('Failed to initialize db at: %s', db_file)
129 log.exception('Failed to initialize db at: %s', db_file)
130 raise
130 raise
131
131
132 def __repr__(self):
132 def __repr__(self):
133 return f'{self.__class__}(file=`{self.filename}`)'
133 return f'{self.__class__}(file=`{self.filename}`)'
134
134
135 def __str__(self):
135 def __str__(self):
136 return self.__repr__()
136 return self.__repr__()
137
137
138 def _get_keys_pattern(self, prefix: bytes = b''):
138 def _get_keys_pattern(self, prefix: bytes = b''):
139 return b'%b:%b' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
139 return b'%b:%b' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
140
140
141 def list_keys(self, prefix: bytes = b''):
141 def list_keys(self, prefix: bytes = b''):
142 prefix = self._get_keys_pattern(prefix)
142 prefix = self._get_keys_pattern(prefix)
143
143
144 def cond(dbm_key: bytes):
144 def cond(dbm_key: bytes):
145 if not prefix:
145 if not prefix:
146 return True
146 return True
147
147
148 if dbm_key.startswith(prefix):
148 if dbm_key.startswith(prefix):
149 return True
149 return True
150 return False
150 return False
151
151
152 with self._dbm_file(True) as dbm:
152 with self._dbm_file(True) as dbm:
153 try:
153 try:
154 return list(filter(cond, dbm.keys()))
154 return list(filter(cond, dbm.keys()))
155 except Exception:
155 except Exception:
156 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
156 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
157 raise
157 raise
158
158
159 def delete_multi_by_prefix(self, prefix):
159 def delete_multi_by_prefix(self, prefix):
160 cache_keys = self.list_keys(prefix=prefix)
160 cache_keys = self.list_keys(prefix=prefix)
161 num_affected_keys = len(cache_keys)
161 num_affected_keys = len(cache_keys)
162 if num_affected_keys:
162 if num_affected_keys:
163 self.delete_multi(cache_keys)
163 self.delete_multi(cache_keys)
164 return num_affected_keys
164 return num_affected_keys
165
165
166 def get_store(self):
166 def get_store(self):
167 return self.filename
167 return self.filename
168
168
169
169
170 class BaseRedisBackend(redis_backend.RedisBackend):
170 class BaseRedisBackend(redis_backend.RedisBackend):
171 key_prefix = ''
171 key_prefix = ''
172
172
173 def __init__(self, arguments):
173 def __init__(self, arguments):
174 self.db_conn = arguments.get('host', '') or arguments.get('url', '') or 'redis-host'
174 self.db_conn = arguments.get('host', '') or arguments.get('url', '') or 'redis-host'
175 super().__init__(arguments)
175 super().__init__(arguments)
176
176
177 self._lock_timeout = self.lock_timeout
177 self._lock_timeout = self.lock_timeout
178 self._lock_auto_renewal = str2bool(arguments.pop("lock_auto_renewal", True))
178 self._lock_auto_renewal = str2bool(arguments.pop("lock_auto_renewal", True))
179
179
180 if self._lock_auto_renewal and not self._lock_timeout:
180 if self._lock_auto_renewal and not self._lock_timeout:
181 # set default timeout for auto_renewal
181 # set default timeout for auto_renewal
182 self._lock_timeout = 30
182 self._lock_timeout = 30
183
183
184 def __repr__(self):
184 def __repr__(self):
185 return f'{self.__class__}(conn=`{self.db_conn}`)'
185 return f'{self.__class__}(conn=`{self.db_conn}`)'
186
186
187 def __str__(self):
187 def __str__(self):
188 return self.__repr__()
188 return self.__repr__()
189
189
190 def _create_client(self):
190 def _create_client(self):
191 args = {}
191 args = {}
192
192
193 if self.url is not None:
193 if self.url is not None:
194 args.update(url=self.url)
194 args.update(url=self.url)
195
195
196 else:
196 else:
197 args.update(
197 args.update(
198 host=self.host, password=self.password,
198 host=self.host, password=self.password,
199 port=self.port, db=self.db
199 port=self.port, db=self.db
200 )
200 )
201
201
202 connection_pool = redis.ConnectionPool(**args)
202 connection_pool = redis.ConnectionPool(**args)
203 self.writer_client = redis.StrictRedis(
203 self.writer_client = redis.StrictRedis(
204 connection_pool=connection_pool
204 connection_pool=connection_pool
205 )
205 )
206 self.reader_client = self.writer_client
206 self.reader_client = self.writer_client
207
207
208 def _get_keys_pattern(self, prefix: bytes = b''):
208 def _get_keys_pattern(self, prefix: bytes = b''):
209 return b'%b:%b*' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
209 return b'%b:%b*' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
210
210
211 def list_keys(self, prefix: bytes = b''):
211 def list_keys(self, prefix: bytes = b''):
212 prefix = self._get_keys_pattern(prefix)
212 prefix = self._get_keys_pattern(prefix)
213 return self.reader_client.keys(prefix)
213 return self.reader_client.keys(prefix)
214
214
215 def delete_multi_by_prefix(self, prefix, use_lua=False):
215 def delete_multi_by_prefix(self, prefix, use_lua=False):
216 if use_lua:
216 if use_lua:
217 # high efficient LUA script to delete ALL keys by prefix...
217 # high efficient LUA script to delete ALL keys by prefix...
218 lua = """local keys = redis.call('keys', ARGV[1])
218 lua = """local keys = redis.call('keys', ARGV[1])
219 for i=1,#keys,5000 do
219 for i=1,#keys,5000 do
220 redis.call('del', unpack(keys, i, math.min(i+(5000-1), #keys)))
220 redis.call('del', unpack(keys, i, math.min(i+(5000-1), #keys)))
221 end
221 end
222 return #keys"""
222 return #keys"""
223 num_affected_keys = self.writer_client.eval(
223 num_affected_keys = self.writer_client.eval(
224 lua,
224 lua,
225 0,
225 0,
226 f"{prefix}*")
226 f"{prefix}*")
227 else:
227 else:
228 cache_keys = self.list_keys(prefix=prefix)
228 cache_keys = self.list_keys(prefix=prefix)
229 num_affected_keys = len(cache_keys)
229 num_affected_keys = len(cache_keys)
230 if num_affected_keys:
230 if num_affected_keys:
231 self.delete_multi(cache_keys)
231 self.delete_multi(cache_keys)
232 return num_affected_keys
232 return num_affected_keys
233
233
234 def get_store(self):
234 def get_store(self):
235 return self.reader_client.connection_pool
235 return self.reader_client.connection_pool
236
236
237 def get_mutex(self, key):
237 def get_mutex(self, key):
238 if self.distributed_lock:
238 if self.distributed_lock:
239 lock_key = f'_lock_{safe_str(key)}'
239 lock_key = f'_lock_{safe_str(key)}'
240 return get_mutex_lock(
240 return get_mutex_lock(
241 self.writer_client, lock_key,
241 self.writer_client, lock_key,
242 self._lock_timeout,
242 self._lock_timeout,
243 auto_renewal=self._lock_auto_renewal
243 auto_renewal=self._lock_auto_renewal
244 )
244 )
245 else:
245 else:
246 return None
246 return None
247
247
248
248
249 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
249 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
250 key_prefix = 'redis_pickle_backend'
250 key_prefix = 'redis_pickle_backend'
251 pass
251 pass
252
252
253
253
254 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
254 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
255 key_prefix = 'redis_msgpack_backend'
255 key_prefix = 'redis_msgpack_backend'
256 pass
256 pass
257
257
258
258
259 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
259 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
260 from vcsserver.lib._vendor import redis_lock
260 from vcsserver.lib._vendor import redis_lock
261
261
262 class _RedisLockWrapper:
262 class _RedisLockWrapper:
263 """LockWrapper for redis_lock"""
263 """LockWrapper for redis_lock"""
264
264
265 @classmethod
265 @classmethod
266 def get_lock(cls):
266 def get_lock(cls):
267 return redis_lock.Lock(
267 return redis_lock.Lock(
268 redis_client=client,
268 redis_client=client,
269 name=lock_key,
269 name=lock_key,
270 expire=lock_timeout,
270 expire=lock_timeout,
271 auto_renewal=auto_renewal,
271 auto_renewal=auto_renewal,
272 strict=True,
272 strict=True,
273 )
273 )
274
274
275 def __repr__(self):
275 def __repr__(self):
276 return f"{self.__class__.__name__}:{lock_key}"
276 return f"{self.__class__.__name__}:{lock_key}"
277
277
278 def __str__(self):
278 def __str__(self):
279 return f"{self.__class__.__name__}:{lock_key}"
279 return f"{self.__class__.__name__}:{lock_key}"
280
280
281 def __init__(self):
281 def __init__(self):
282 self.lock = self.get_lock()
282 self.lock = self.get_lock()
283 self.lock_key = lock_key
283 self.lock_key = lock_key
284
284
285 def acquire(self, wait=True):
285 def acquire(self, wait=True):
286 log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
286 log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
287 try:
287 try:
288 acquired = self.lock.acquire(wait)
288 acquired = self.lock.acquire(wait)
289 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
289 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
290 return acquired
290 return acquired
291 except redis_lock.AlreadyAcquired:
291 except redis_lock.AlreadyAcquired:
292 return False
292 return False
293 except redis_lock.AlreadyStarted:
293 except redis_lock.AlreadyStarted:
294 # refresh thread exists, but it also means we acquired the lock
294 # refresh thread exists, but it also means we acquired the lock
295 return True
295 return True
296
296
297 def release(self):
297 def release(self):
298 try:
298 try:
299 self.lock.release()
299 self.lock.release()
300 except redis_lock.NotAcquired:
300 except redis_lock.NotAcquired:
301 pass
301 pass
302
302
303 return _RedisLockWrapper()
303 return _RedisLockWrapper()
@@ -1,245 +1,245 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import functools
18 import functools
19 import logging
19 import logging
20 import os
20 import os
21 import threading
21 import threading
22 import time
22 import time
23
23
24 import decorator
24 import decorator
25 from dogpile.cache import CacheRegion
25 from dogpile.cache import CacheRegion
26
26
27
27
28 from vcsserver.utils import sha1
28 from vcsserver.utils import sha1
29 from vcsserver.str_utils import safe_bytes
29 from vcsserver.lib.str_utils import safe_bytes
30 from vcsserver.type_utils import str2bool # noqa :required by imports from .utils
30 from vcsserver.lib.type_utils import str2bool # noqa :required by imports from .utils
31
31
32 from . import region_meta
32 from . import region_meta
33
33
34 log = logging.getLogger(__name__)
34 log = logging.getLogger(__name__)
35
35
36
36
37 class RhodeCodeCacheRegion(CacheRegion):
37 class RhodeCodeCacheRegion(CacheRegion):
38
38
39 def __repr__(self):
39 def __repr__(self):
40 return f'`{self.__class__.__name__}(name={self.name}, backend={self.backend.__class__})`'
40 return f'`{self.__class__.__name__}(name={self.name}, backend={self.backend.__class__})`'
41
41
42 def conditional_cache_on_arguments(
42 def conditional_cache_on_arguments(
43 self, namespace=None,
43 self, namespace=None,
44 expiration_time=None,
44 expiration_time=None,
45 should_cache_fn=None,
45 should_cache_fn=None,
46 to_str=str,
46 to_str=str,
47 function_key_generator=None,
47 function_key_generator=None,
48 condition=True):
48 condition=True):
49 """
49 """
50 Custom conditional decorator, that will not touch any dogpile internals if
50 Custom conditional decorator, that will not touch any dogpile internals if
51 condition isn't meet. This works a bit different from should_cache_fn
51 condition isn't meet. This works a bit different from should_cache_fn
52 And it's faster in cases we don't ever want to compute cached values
52 And it's faster in cases we don't ever want to compute cached values
53 """
53 """
54 expiration_time_is_callable = callable(expiration_time)
54 expiration_time_is_callable = callable(expiration_time)
55 if not namespace:
55 if not namespace:
56 namespace = getattr(self, '_default_namespace', None)
56 namespace = getattr(self, '_default_namespace', None)
57
57
58 if function_key_generator is None:
58 if function_key_generator is None:
59 function_key_generator = self.function_key_generator
59 function_key_generator = self.function_key_generator
60
60
61 def get_or_create_for_user_func(func_key_generator, user_func, *arg, **kw):
61 def get_or_create_for_user_func(func_key_generator, user_func, *arg, **kw):
62
62
63 if not condition:
63 if not condition:
64 log.debug('Calling un-cached method:%s', user_func.__name__)
64 log.debug('Calling un-cached method:%s', user_func.__name__)
65 start = time.time()
65 start = time.time()
66 result = user_func(*arg, **kw)
66 result = user_func(*arg, **kw)
67 total = time.time() - start
67 total = time.time() - start
68 log.debug('un-cached method:%s took %.4fs', user_func.__name__, total)
68 log.debug('un-cached method:%s took %.4fs', user_func.__name__, total)
69 return result
69 return result
70
70
71 key = func_key_generator(*arg, **kw)
71 key = func_key_generator(*arg, **kw)
72
72
73 timeout = expiration_time() if expiration_time_is_callable \
73 timeout = expiration_time() if expiration_time_is_callable \
74 else expiration_time
74 else expiration_time
75
75
76 log.debug('Calling cached method:`%s`', user_func.__name__)
76 log.debug('Calling cached method:`%s`', user_func.__name__)
77 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
77 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
78
78
79 def cache_decorator(user_func):
79 def cache_decorator(user_func):
80 if to_str is str:
80 if to_str is str:
81 # backwards compatible
81 # backwards compatible
82 key_generator = function_key_generator(namespace, user_func)
82 key_generator = function_key_generator(namespace, user_func)
83 else:
83 else:
84 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
84 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
85
85
86 def refresh(*arg, **kw):
86 def refresh(*arg, **kw):
87 """
87 """
88 Like invalidate, but regenerates the value instead
88 Like invalidate, but regenerates the value instead
89 """
89 """
90 key = key_generator(*arg, **kw)
90 key = key_generator(*arg, **kw)
91 value = user_func(*arg, **kw)
91 value = user_func(*arg, **kw)
92 self.set(key, value)
92 self.set(key, value)
93 return value
93 return value
94
94
95 def invalidate(*arg, **kw):
95 def invalidate(*arg, **kw):
96 key = key_generator(*arg, **kw)
96 key = key_generator(*arg, **kw)
97 self.delete(key)
97 self.delete(key)
98
98
99 def set_(value, *arg, **kw):
99 def set_(value, *arg, **kw):
100 key = key_generator(*arg, **kw)
100 key = key_generator(*arg, **kw)
101 self.set(key, value)
101 self.set(key, value)
102
102
103 def get(*arg, **kw):
103 def get(*arg, **kw):
104 key = key_generator(*arg, **kw)
104 key = key_generator(*arg, **kw)
105 return self.get(key)
105 return self.get(key)
106
106
107 user_func.set = set_
107 user_func.set = set_
108 user_func.invalidate = invalidate
108 user_func.invalidate = invalidate
109 user_func.get = get
109 user_func.get = get
110 user_func.refresh = refresh
110 user_func.refresh = refresh
111 user_func.key_generator = key_generator
111 user_func.key_generator = key_generator
112 user_func.original = user_func
112 user_func.original = user_func
113
113
114 # Use `decorate` to preserve the signature of :param:`user_func`.
114 # Use `decorate` to preserve the signature of :param:`user_func`.
115 return decorator.decorate(user_func, functools.partial(
115 return decorator.decorate(user_func, functools.partial(
116 get_or_create_for_user_func, key_generator))
116 get_or_create_for_user_func, key_generator))
117
117
118 return cache_decorator
118 return cache_decorator
119
119
120
120
121 def make_region(*arg, **kw):
121 def make_region(*arg, **kw):
122 return RhodeCodeCacheRegion(*arg, **kw)
122 return RhodeCodeCacheRegion(*arg, **kw)
123
123
124
124
125 def get_default_cache_settings(settings, prefixes=None):
125 def get_default_cache_settings(settings, prefixes=None):
126 prefixes = prefixes or []
126 prefixes = prefixes or []
127 cache_settings = {}
127 cache_settings = {}
128 for key in settings.keys():
128 for key in settings.keys():
129 for prefix in prefixes:
129 for prefix in prefixes:
130 if key.startswith(prefix):
130 if key.startswith(prefix):
131 name = key.split(prefix)[1].strip()
131 name = key.split(prefix)[1].strip()
132 val = settings[key]
132 val = settings[key]
133 if isinstance(val, str):
133 if isinstance(val, str):
134 val = val.strip()
134 val = val.strip()
135 cache_settings[name] = val
135 cache_settings[name] = val
136 return cache_settings
136 return cache_settings
137
137
138
138
139 def compute_key_from_params(*args):
139 def compute_key_from_params(*args):
140 """
140 """
141 Helper to compute key from given params to be used in cache manager
141 Helper to compute key from given params to be used in cache manager
142 """
142 """
143 return sha1(safe_bytes("_".join(map(str, args))))
143 return sha1(safe_bytes("_".join(map(str, args))))
144
144
145
145
146 def custom_key_generator(backend, namespace, fn):
146 def custom_key_generator(backend, namespace, fn):
147 func_name = fn.__name__
147 func_name = fn.__name__
148
148
149 def generate_key(*args):
149 def generate_key(*args):
150 backend_pref = getattr(backend, 'key_prefix', None) or 'backend_prefix'
150 backend_pref = getattr(backend, 'key_prefix', None) or 'backend_prefix'
151 namespace_pref = namespace or 'default_namespace'
151 namespace_pref = namespace or 'default_namespace'
152 arg_key = compute_key_from_params(*args)
152 arg_key = compute_key_from_params(*args)
153 final_key = f"{backend_pref}:{namespace_pref}:{func_name}_{arg_key}"
153 final_key = f"{backend_pref}:{namespace_pref}:{func_name}_{arg_key}"
154
154
155 return final_key
155 return final_key
156
156
157 return generate_key
157 return generate_key
158
158
159
159
160 def backend_key_generator(backend):
160 def backend_key_generator(backend):
161 """
161 """
162 Special wrapper that also sends over the backend to the key generator
162 Special wrapper that also sends over the backend to the key generator
163 """
163 """
164 def wrapper(namespace, fn):
164 def wrapper(namespace, fn):
165 return custom_key_generator(backend, namespace, fn)
165 return custom_key_generator(backend, namespace, fn)
166 return wrapper
166 return wrapper
167
167
168
168
169 def get_or_create_region(region_name, region_namespace: str = None, use_async_runner=False):
169 def get_or_create_region(region_name, region_namespace: str = None, use_async_runner=False):
170 from .backends import FileNamespaceBackend
170 from .backends import FileNamespaceBackend
171 from . import async_creation_runner
171 from . import async_creation_runner
172
172
173 region_obj = region_meta.dogpile_cache_regions.get(region_name)
173 region_obj = region_meta.dogpile_cache_regions.get(region_name)
174 if not region_obj:
174 if not region_obj:
175 reg_keys = list(region_meta.dogpile_cache_regions.keys())
175 reg_keys = list(region_meta.dogpile_cache_regions.keys())
176 raise OSError(f'Region `{region_name}` not in configured: {reg_keys}.')
176 raise OSError(f'Region `{region_name}` not in configured: {reg_keys}.')
177
177
178 region_uid_name = f'{region_name}:{region_namespace}'
178 region_uid_name = f'{region_name}:{region_namespace}'
179
179
180 # Special case for ONLY the FileNamespaceBackend backend. We register one-file-per-region
180 # Special case for ONLY the FileNamespaceBackend backend. We register one-file-per-region
181 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
181 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
182 if not region_namespace:
182 if not region_namespace:
183 raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
183 raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
184
184
185 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
185 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
186 if region_exist:
186 if region_exist:
187 log.debug('Using already configured region: %s', region_namespace)
187 log.debug('Using already configured region: %s', region_namespace)
188 return region_exist
188 return region_exist
189
189
190 expiration_time = region_obj.expiration_time
190 expiration_time = region_obj.expiration_time
191
191
192 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
192 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
193 namespace_cache_dir = cache_dir
193 namespace_cache_dir = cache_dir
194
194
195 # we default the namespace_cache_dir to our default cache dir.
195 # we default the namespace_cache_dir to our default cache dir.
196 # however, if this backend is configured with filename= param, we prioritize that
196 # however, if this backend is configured with filename= param, we prioritize that
197 # so all caches within that particular region, even those namespaced end up in the same path
197 # so all caches within that particular region, even those namespaced end up in the same path
198 if region_obj.actual_backend.filename:
198 if region_obj.actual_backend.filename:
199 namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
199 namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
200
200
201 if not os.path.isdir(namespace_cache_dir):
201 if not os.path.isdir(namespace_cache_dir):
202 os.makedirs(namespace_cache_dir)
202 os.makedirs(namespace_cache_dir)
203 new_region = make_region(
203 new_region = make_region(
204 name=region_uid_name,
204 name=region_uid_name,
205 function_key_generator=backend_key_generator(region_obj.actual_backend)
205 function_key_generator=backend_key_generator(region_obj.actual_backend)
206 )
206 )
207
207
208 namespace_filename = os.path.join(
208 namespace_filename = os.path.join(
209 namespace_cache_dir, f"{region_name}_{region_namespace}.cache_db")
209 namespace_cache_dir, f"{region_name}_{region_namespace}.cache_db")
210 # special type that allows 1db per namespace
210 # special type that allows 1db per namespace
211 new_region.configure(
211 new_region.configure(
212 backend='dogpile.cache.rc.file_namespace',
212 backend='dogpile.cache.rc.file_namespace',
213 expiration_time=expiration_time,
213 expiration_time=expiration_time,
214 arguments={"filename": namespace_filename}
214 arguments={"filename": namespace_filename}
215 )
215 )
216
216
217 # create and save in region caches
217 # create and save in region caches
218 log.debug('configuring new region: %s', region_uid_name)
218 log.debug('configuring new region: %s', region_uid_name)
219 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
219 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
220
220
221 region_obj._default_namespace = region_namespace
221 region_obj._default_namespace = region_namespace
222 if use_async_runner:
222 if use_async_runner:
223 region_obj.async_creation_runner = async_creation_runner
223 region_obj.async_creation_runner = async_creation_runner
224 return region_obj
224 return region_obj
225
225
226
226
227 def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str) -> int:
227 def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str) -> int:
228 from . import CLEAR_DELETE, CLEAR_INVALIDATE
228 from . import CLEAR_DELETE, CLEAR_INVALIDATE
229
229
230 if not isinstance(cache_region, RhodeCodeCacheRegion):
230 if not isinstance(cache_region, RhodeCodeCacheRegion):
231 cache_region = get_or_create_region(cache_region, cache_namespace_uid)
231 cache_region = get_or_create_region(cache_region, cache_namespace_uid)
232 log.debug('clearing cache region: %s [prefix:%s] with method=%s',
232 log.debug('clearing cache region: %s [prefix:%s] with method=%s',
233 cache_region, cache_namespace_uid, method)
233 cache_region, cache_namespace_uid, method)
234
234
235 num_affected_keys = 0
235 num_affected_keys = 0
236
236
237 if method == CLEAR_INVALIDATE:
237 if method == CLEAR_INVALIDATE:
238 # NOTE: The CacheRegion.invalidate() method’s default mode of
238 # NOTE: The CacheRegion.invalidate() method’s default mode of
239 # operation is to set a timestamp local to this CacheRegion in this Python process only.
239 # operation is to set a timestamp local to this CacheRegion in this Python process only.
240 # It does not impact other Python processes or regions as the timestamp is only stored locally in memory.
240 # It does not impact other Python processes or regions as the timestamp is only stored locally in memory.
241 cache_region.invalidate(hard=True)
241 cache_region.invalidate(hard=True)
242
242
243 if method == CLEAR_DELETE:
243 if method == CLEAR_DELETE:
244 num_affected_keys = cache_region.backend.delete_multi_by_prefix(prefix=cache_namespace_uid)
244 num_affected_keys = cache_region.backend.delete_multi_by_prefix(prefix=cache_namespace_uid)
245 return num_affected_keys
245 return num_affected_keys
@@ -1,158 +1,158 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import typing
18 import typing
19 import base64
19 import base64
20 import logging
20 import logging
21
21
22
22
23 log = logging.getLogger(__name__)
23 log = logging.getLogger(__name__)
24
24
25
25
26 def safe_int(val, default=None) -> int:
26 def safe_int(val, default=None) -> int:
27 """
27 """
28 Returns int() of val if val is not convertable to int use default
28 Returns int() of val if val is not convertable to int use default
29 instead
29 instead
30
30
31 :param val:
31 :param val:
32 :param default:
32 :param default:
33 """
33 """
34
34
35 try:
35 try:
36 val = int(val)
36 val = int(val)
37 except (ValueError, TypeError):
37 except (ValueError, TypeError):
38 val = default
38 val = default
39
39
40 return val
40 return val
41
41
42
42
43 def base64_to_str(text) -> str:
43 def base64_to_str(text) -> str:
44 return safe_str(base64.encodebytes(safe_bytes(text))).strip()
44 return safe_str(base64.encodebytes(safe_bytes(text))).strip()
45
45
46
46
47 def get_default_encodings() -> list[str]:
47 def get_default_encodings() -> list[str]:
48 return ['utf8']
48 return ['utf8']
49
49
50
50
51 def safe_str(str_, to_encoding=None) -> str:
51 def safe_str(str_, to_encoding=None) -> str:
52 """
52 """
53 safe str function. Does few trick to turn unicode_ into string
53 safe str function. Does few trick to turn unicode_ into string
54
54
55 :param str_: str to encode
55 :param str_: str to encode
56 :param to_encoding: encode to this type UTF8 default
56 :param to_encoding: encode to this type UTF8 default
57 """
57 """
58 if isinstance(str_, str):
58 if isinstance(str_, str):
59 return str_
59 return str_
60
60
61 # if it's bytes cast to str
61 # if it's bytes cast to str
62 if not isinstance(str_, bytes):
62 if not isinstance(str_, bytes):
63 return str(str_)
63 return str(str_)
64
64
65 to_encoding = to_encoding or get_default_encodings()
65 to_encoding = to_encoding or get_default_encodings()
66 if not isinstance(to_encoding, (list, tuple)):
66 if not isinstance(to_encoding, (list, tuple)):
67 to_encoding = [to_encoding]
67 to_encoding = [to_encoding]
68
68
69 for enc in to_encoding:
69 for enc in to_encoding:
70 try:
70 try:
71 return str(str_, enc)
71 return str(str_, enc)
72 except UnicodeDecodeError:
72 except UnicodeDecodeError:
73 pass
73 pass
74
74
75 return str(str_, to_encoding[0], 'replace')
75 return str(str_, to_encoding[0], 'replace')
76
76
77
77
78 def safe_bytes(str_, from_encoding=None) -> bytes:
78 def safe_bytes(str_, from_encoding=None) -> bytes:
79 """
79 """
80 safe bytes function. Does few trick to turn str_ into bytes string:
80 safe bytes function. Does few trick to turn str_ into bytes string:
81
81
82 :param str_: string to decode
82 :param str_: string to decode
83 :param from_encoding: encode from this type UTF8 default
83 :param from_encoding: encode from this type UTF8 default
84 """
84 """
85 if isinstance(str_, bytes):
85 if isinstance(str_, bytes):
86 return str_
86 return str_
87
87
88 if not isinstance(str_, str):
88 if not isinstance(str_, str):
89 raise ValueError(f'safe_bytes cannot convert other types than str: got: {type(str_)}')
89 raise ValueError(f'safe_bytes cannot convert other types than str: got: {type(str_)}')
90
90
91 from_encoding = from_encoding or get_default_encodings()
91 from_encoding = from_encoding or get_default_encodings()
92 if not isinstance(from_encoding, (list, tuple)):
92 if not isinstance(from_encoding, (list, tuple)):
93 from_encoding = [from_encoding]
93 from_encoding = [from_encoding]
94
94
95 for enc in from_encoding:
95 for enc in from_encoding:
96 try:
96 try:
97 return str_.encode(enc)
97 return str_.encode(enc)
98 except UnicodeDecodeError:
98 except UnicodeDecodeError:
99 pass
99 pass
100
100
101 return str_.encode(from_encoding[0], 'replace')
101 return str_.encode(from_encoding[0], 'replace')
102
102
103
103
104 def ascii_bytes(str_, allow_bytes=False) -> bytes:
104 def ascii_bytes(str_, allow_bytes=False) -> bytes:
105 """
105 """
106 Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
106 Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
107 Fails with UnicodeError on invalid input.
107 Fails with UnicodeError on invalid input.
108 This should be used where encoding and "safe" ambiguity should be avoided.
108 This should be used where encoding and "safe" ambiguity should be avoided.
109 Where strings already have been encoded in other ways but still are unicode
109 Where strings already have been encoded in other ways but still are unicode
110 string - for example to hex, base64, json, urlencoding, or are known to be
110 string - for example to hex, base64, json, urlencoding, or are known to be
111 identifiers.
111 identifiers.
112 """
112 """
113 if allow_bytes and isinstance(str_, bytes):
113 if allow_bytes and isinstance(str_, bytes):
114 return str_
114 return str_
115
115
116 if not isinstance(str_, str):
116 if not isinstance(str_, str):
117 raise ValueError(f'ascii_bytes cannot convert other types than str: got: {type(str_)}')
117 raise ValueError(f'ascii_bytes cannot convert other types than str: got: {type(str_)}')
118 return str_.encode('ascii')
118 return str_.encode('ascii')
119
119
120
120
121 def ascii_str(str_) -> str:
121 def ascii_str(str_) -> str:
122 """
122 """
123 Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
123 Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
124 Fails with UnicodeError on invalid input.
124 Fails with UnicodeError on invalid input.
125 This should be used where encoding and "safe" ambiguity should be avoided.
125 This should be used where encoding and "safe" ambiguity should be avoided.
126 Where strings are encoded but also in other ways are known to be ASCII, and
126 Where strings are encoded but also in other ways are known to be ASCII, and
127 where a unicode string is wanted without caring about encoding. For example
127 where a unicode string is wanted without caring about encoding. For example
128 to hex, base64, urlencoding, or are known to be identifiers.
128 to hex, base64, urlencoding, or are known to be identifiers.
129 """
129 """
130
130
131 if not isinstance(str_, bytes):
131 if not isinstance(str_, bytes):
132 raise ValueError(f'ascii_str cannot convert other types than bytes: got: {type(str_)}')
132 raise ValueError(f'ascii_str cannot convert other types than bytes: got: {type(str_)}')
133 return str_.decode('ascii')
133 return str_.decode('ascii')
134
134
135
135
136 def convert_to_str(data):
136 def convert_to_str(data):
137 if isinstance(data, bytes):
137 if isinstance(data, bytes):
138 return safe_str(data)
138 return safe_str(data)
139 elif isinstance(data, tuple):
139 elif isinstance(data, tuple):
140 return tuple(convert_to_str(item) for item in data)
140 return tuple(convert_to_str(item) for item in data)
141 elif isinstance(data, list):
141 elif isinstance(data, list):
142 return list(convert_to_str(item) for item in data)
142 return list(convert_to_str(item) for item in data)
143 else:
143 else:
144 return data
144 return data
145
145
146
146
147 def splitnewlines(text: bytes):
147 def splitnewlines(text: bytes):
148 """
148 """
149 like splitlines, but only split on newlines.
149 like splitlines, but only split on newlines.
150 """
150 """
151
151
152 lines = [_l + b'\n' for _l in text.split(b'\n')]
152 lines = [_l + b'\n' for _l in text.split(b'\n')]
153 if lines:
153 if lines:
154 if lines[-1] == b'\n':
154 if lines[-1] == b'\n':
155 lines.pop()
155 lines.pop()
156 else:
156 else:
157 lines[-1] = lines[-1][:-1]
157 lines[-1] = lines[-1][:-1]
158 return lines No newline at end of file
158 return lines
@@ -1,160 +1,160 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import tempfile
19 import tempfile
20
20
21 from svn import client
21 from svn import client
22 from svn import core
22 from svn import core
23 from svn import ra
23 from svn import ra
24
24
25 from mercurial import error
25 from mercurial import error
26
26
27 from vcsserver.str_utils import safe_bytes
27 from vcsserver.lib.str_utils import safe_bytes
28
28
29 core.svn_config_ensure(None)
29 core.svn_config_ensure(None)
30 svn_config = core.svn_config_get_config(None)
30 svn_config = core.svn_config_get_config(None)
31
31
32
32
33 class RaCallbacks(ra.Callbacks):
33 class RaCallbacks(ra.Callbacks):
34 @staticmethod
34 @staticmethod
35 def open_tmp_file(pool): # pragma: no cover
35 def open_tmp_file(pool): # pragma: no cover
36 (fd, fn) = tempfile.mkstemp()
36 (fd, fn) = tempfile.mkstemp()
37 os.close(fd)
37 os.close(fd)
38 return fn
38 return fn
39
39
40 @staticmethod
40 @staticmethod
41 def get_client_string(pool):
41 def get_client_string(pool):
42 return b'RhodeCode-subversion-url-checker'
42 return b'RhodeCode-subversion-url-checker'
43
43
44
44
45 class SubversionException(Exception):
45 class SubversionException(Exception):
46 pass
46 pass
47
47
48
48
49 class SubversionConnectionException(SubversionException):
49 class SubversionConnectionException(SubversionException):
50 """Exception raised when a generic error occurs when connecting to a repository."""
50 """Exception raised when a generic error occurs when connecting to a repository."""
51
51
52
52
53 def normalize_url(url):
53 def normalize_url(url):
54 if not url:
54 if not url:
55 return url
55 return url
56 if url.startswith(b'svn+http://') or url.startswith(b'svn+https://'):
56 if url.startswith(b'svn+http://') or url.startswith(b'svn+https://'):
57 url = url[4:]
57 url = url[4:]
58 url = url.rstrip(b'/')
58 url = url.rstrip(b'/')
59 return url
59 return url
60
60
61
61
62 def _create_auth_baton(pool):
62 def _create_auth_baton(pool):
63 """Create a Subversion authentication baton. """
63 """Create a Subversion authentication baton. """
64 # Give the client context baton a suite of authentication
64 # Give the client context baton a suite of authentication
65 # providers.h
65 # providers.h
66 platform_specific = [
66 platform_specific = [
67 'svn_auth_get_gnome_keyring_simple_provider',
67 'svn_auth_get_gnome_keyring_simple_provider',
68 'svn_auth_get_gnome_keyring_ssl_client_cert_pw_provider',
68 'svn_auth_get_gnome_keyring_ssl_client_cert_pw_provider',
69 'svn_auth_get_keychain_simple_provider',
69 'svn_auth_get_keychain_simple_provider',
70 'svn_auth_get_keychain_ssl_client_cert_pw_provider',
70 'svn_auth_get_keychain_ssl_client_cert_pw_provider',
71 'svn_auth_get_kwallet_simple_provider',
71 'svn_auth_get_kwallet_simple_provider',
72 'svn_auth_get_kwallet_ssl_client_cert_pw_provider',
72 'svn_auth_get_kwallet_ssl_client_cert_pw_provider',
73 'svn_auth_get_ssl_client_cert_file_provider',
73 'svn_auth_get_ssl_client_cert_file_provider',
74 'svn_auth_get_windows_simple_provider',
74 'svn_auth_get_windows_simple_provider',
75 'svn_auth_get_windows_ssl_server_trust_provider',
75 'svn_auth_get_windows_ssl_server_trust_provider',
76 ]
76 ]
77
77
78 providers = []
78 providers = []
79
79
80 for p in platform_specific:
80 for p in platform_specific:
81 if getattr(core, p, None) is not None:
81 if getattr(core, p, None) is not None:
82 try:
82 try:
83 providers.append(getattr(core, p)())
83 providers.append(getattr(core, p)())
84 except RuntimeError:
84 except RuntimeError:
85 pass
85 pass
86
86
87 providers += [
87 providers += [
88 client.get_simple_provider(),
88 client.get_simple_provider(),
89 client.get_username_provider(),
89 client.get_username_provider(),
90 client.get_ssl_client_cert_file_provider(),
90 client.get_ssl_client_cert_file_provider(),
91 client.get_ssl_client_cert_pw_file_provider(),
91 client.get_ssl_client_cert_pw_file_provider(),
92 client.get_ssl_server_trust_file_provider(),
92 client.get_ssl_server_trust_file_provider(),
93 ]
93 ]
94
94
95 return core.svn_auth_open(providers, pool)
95 return core.svn_auth_open(providers, pool)
96
96
97
97
98 class SubversionRepo:
98 class SubversionRepo:
99 """Wrapper for a Subversion repository.
99 """Wrapper for a Subversion repository.
100
100
101 It uses the SWIG Python bindings, see above for requirements.
101 It uses the SWIG Python bindings, see above for requirements.
102 """
102 """
103 def __init__(self, svn_url: bytes = b'', username: bytes = b'', password: bytes = b''):
103 def __init__(self, svn_url: bytes = b'', username: bytes = b'', password: bytes = b''):
104
104
105 self.username = username
105 self.username = username
106 self.password = password
106 self.password = password
107 self.svn_url = core.svn_path_canonicalize(svn_url)
107 self.svn_url = core.svn_path_canonicalize(svn_url)
108
108
109 self.auth_baton_pool = core.Pool()
109 self.auth_baton_pool = core.Pool()
110 self.auth_baton = _create_auth_baton(self.auth_baton_pool)
110 self.auth_baton = _create_auth_baton(self.auth_baton_pool)
111 # self.init_ra_and_client() assumes that a pool already exists
111 # self.init_ra_and_client() assumes that a pool already exists
112 self.pool = core.Pool()
112 self.pool = core.Pool()
113
113
114 self.ra = self.init_ra_and_client()
114 self.ra = self.init_ra_and_client()
115 self.uuid = ra.get_uuid(self.ra, self.pool)
115 self.uuid = ra.get_uuid(self.ra, self.pool)
116
116
117 def init_ra_and_client(self):
117 def init_ra_and_client(self):
118 """Initializes the RA and client layers, because sometimes getting
118 """Initializes the RA and client layers, because sometimes getting
119 unified diffs runs the remote server out of open files.
119 unified diffs runs the remote server out of open files.
120 """
120 """
121
121
122 if self.username:
122 if self.username:
123 core.svn_auth_set_parameter(self.auth_baton,
123 core.svn_auth_set_parameter(self.auth_baton,
124 core.SVN_AUTH_PARAM_DEFAULT_USERNAME,
124 core.SVN_AUTH_PARAM_DEFAULT_USERNAME,
125 self.username)
125 self.username)
126 if self.password:
126 if self.password:
127 core.svn_auth_set_parameter(self.auth_baton,
127 core.svn_auth_set_parameter(self.auth_baton,
128 core.SVN_AUTH_PARAM_DEFAULT_PASSWORD,
128 core.SVN_AUTH_PARAM_DEFAULT_PASSWORD,
129 self.password)
129 self.password)
130
130
131 callbacks = RaCallbacks()
131 callbacks = RaCallbacks()
132 callbacks.auth_baton = self.auth_baton
132 callbacks.auth_baton = self.auth_baton
133
133
134 try:
134 try:
135 return ra.open2(self.svn_url, callbacks, svn_config, self.pool)
135 return ra.open2(self.svn_url, callbacks, svn_config, self.pool)
136 except SubversionException as e:
136 except SubversionException as e:
137 # e.child contains a detailed error messages
137 # e.child contains a detailed error messages
138 msglist = []
138 msglist = []
139 svn_exc = e
139 svn_exc = e
140 while svn_exc:
140 while svn_exc:
141 if svn_exc.args[0]:
141 if svn_exc.args[0]:
142 msglist.append(svn_exc.args[0])
142 msglist.append(svn_exc.args[0])
143 svn_exc = svn_exc.child
143 svn_exc = svn_exc.child
144 msg = '\n'.join(msglist)
144 msg = '\n'.join(msglist)
145 raise SubversionConnectionException(msg)
145 raise SubversionConnectionException(msg)
146
146
147
147
148 class svnremoterepo:
148 class svnremoterepo:
149 """ the dumb wrapper for actual Subversion repositories """
149 """ the dumb wrapper for actual Subversion repositories """
150
150
151 def __init__(self, username: bytes = b'', password: bytes = b'', svn_url: bytes = b''):
151 def __init__(self, username: bytes = b'', password: bytes = b'', svn_url: bytes = b''):
152 self.username = username or b''
152 self.username = username or b''
153 self.password = password or b''
153 self.password = password or b''
154 self.path = normalize_url(svn_url)
154 self.path = normalize_url(svn_url)
155
155
156 def svn(self):
156 def svn(self):
157 try:
157 try:
158 return SubversionRepo(self.path, self.username, self.password)
158 return SubversionRepo(self.path, self.username, self.password)
159 except SubversionConnectionException as e:
159 except SubversionConnectionException as e:
160 raise error.Abort(safe_bytes(e))
160 raise error.Abort(safe_bytes(e))
1 NO CONTENT: file renamed from vcsserver/type_utils.py to vcsserver/lib/type_utils.py
NO CONTENT: file renamed from vcsserver/type_utils.py to vcsserver/lib/type_utils.py
@@ -1,417 +1,417 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """Handles the Git smart protocol."""
18 """Handles the Git smart protocol."""
19
19
20 import os
20 import os
21 import socket
21 import socket
22 import logging
22 import logging
23
23
24 import dulwich.protocol
24 import dulwich.protocol
25 from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
25 from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
26 from webob import Request, Response, exc
26 from webob import Request, Response, exc
27
27
28 from vcsserver.lib.rc_json import json
28 from vcsserver.lib.ext_json import json
29 from vcsserver import hooks, subprocessio
29 from vcsserver import hooks, subprocessio
30 from vcsserver.str_utils import ascii_bytes
30 from vcsserver.lib.str_utils import ascii_bytes
31
31
32
32
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 class FileWrapper:
36 class FileWrapper:
37 """File wrapper that ensures how much data is read from it."""
37 """File wrapper that ensures how much data is read from it."""
38
38
39 def __init__(self, fd, content_length):
39 def __init__(self, fd, content_length):
40 self.fd = fd
40 self.fd = fd
41 self.content_length = content_length
41 self.content_length = content_length
42 self.remain = content_length
42 self.remain = content_length
43
43
44 def read(self, size):
44 def read(self, size):
45 if size <= self.remain:
45 if size <= self.remain:
46 try:
46 try:
47 data = self.fd.read(size)
47 data = self.fd.read(size)
48 except socket.error:
48 except socket.error:
49 raise IOError(self)
49 raise IOError(self)
50 self.remain -= size
50 self.remain -= size
51 elif self.remain:
51 elif self.remain:
52 data = self.fd.read(self.remain)
52 data = self.fd.read(self.remain)
53 self.remain = 0
53 self.remain = 0
54 else:
54 else:
55 data = None
55 data = None
56 return data
56 return data
57
57
58 def __repr__(self):
58 def __repr__(self):
59 return '<FileWrapper {} len: {}, read: {}>'.format(
59 return '<FileWrapper {} len: {}, read: {}>'.format(
60 self.fd, self.content_length, self.content_length - self.remain
60 self.fd, self.content_length, self.content_length - self.remain
61 )
61 )
62
62
63
63
64 class GitRepository:
64 class GitRepository:
65 """WSGI app for handling Git smart protocol endpoints."""
65 """WSGI app for handling Git smart protocol endpoints."""
66
66
67 git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
67 git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
68 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
68 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
69 valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
69 valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
70
70
71 # The last bytes are the SHA1 of the first 12 bytes.
71 # The last bytes are the SHA1 of the first 12 bytes.
72 EMPTY_PACK = (
72 EMPTY_PACK = (
73 b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
73 b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
74 b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
74 b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
75 )
75 )
76 FLUSH_PACKET = b"0000"
76 FLUSH_PACKET = b"0000"
77
77
78 SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
78 SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
79
79
80 def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
80 def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
81 files = frozenset(f.lower() for f in os.listdir(content_path))
81 files = frozenset(f.lower() for f in os.listdir(content_path))
82 valid_dir_signature = self.git_folder_signature.issubset(files)
82 valid_dir_signature = self.git_folder_signature.issubset(files)
83
83
84 if not valid_dir_signature:
84 if not valid_dir_signature:
85 raise OSError(f'{content_path} missing git signature')
85 raise OSError(f'{content_path} missing git signature')
86
86
87 self.content_path = content_path
87 self.content_path = content_path
88 self.repo_name = repo_name
88 self.repo_name = repo_name
89 self.extras = extras
89 self.extras = extras
90 self.git_path = git_path
90 self.git_path = git_path
91 self.update_server_info = update_server_info
91 self.update_server_info = update_server_info
92
92
93 def _get_fixedpath(self, path):
93 def _get_fixedpath(self, path):
94 """
94 """
95 Small fix for repo_path
95 Small fix for repo_path
96
96
97 :param path:
97 :param path:
98 """
98 """
99 path = path.split(self.repo_name, 1)[-1]
99 path = path.split(self.repo_name, 1)[-1]
100 if path.startswith('.git'):
100 if path.startswith('.git'):
101 # for bare repos we still get the .git prefix inside, we skip it
101 # for bare repos we still get the .git prefix inside, we skip it
102 # here, and remove from the service command
102 # here, and remove from the service command
103 path = path[4:]
103 path = path[4:]
104
104
105 return path.strip('/')
105 return path.strip('/')
106
106
107 def inforefs(self, request, unused_environ):
107 def inforefs(self, request, unused_environ):
108 """
108 """
109 WSGI Response producer for HTTP GET Git Smart
109 WSGI Response producer for HTTP GET Git Smart
110 HTTP /info/refs request.
110 HTTP /info/refs request.
111 """
111 """
112
112
113 git_command = request.GET.get('service')
113 git_command = request.GET.get('service')
114 if git_command not in self.commands:
114 if git_command not in self.commands:
115 log.debug('command %s not allowed', git_command)
115 log.debug('command %s not allowed', git_command)
116 return exc.HTTPForbidden()
116 return exc.HTTPForbidden()
117
117
118 # please, resist the urge to add '\n' to git capture and increment
118 # please, resist the urge to add '\n' to git capture and increment
119 # line count by 1.
119 # line count by 1.
120 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
120 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
121 # a part of protocol.
121 # a part of protocol.
122 # The code in Git client not only does NOT need '\n', but actually
122 # The code in Git client not only does NOT need '\n', but actually
123 # blows up if you sprinkle "flush" (0000) as "0001\n".
123 # blows up if you sprinkle "flush" (0000) as "0001\n".
124 # It reads binary, per number of bytes specified.
124 # It reads binary, per number of bytes specified.
125 # if you do add '\n' as part of data, count it.
125 # if you do add '\n' as part of data, count it.
126 server_advert = f'# service={git_command}\n'
126 server_advert = f'# service={git_command}\n'
127 packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
127 packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
128 try:
128 try:
129 gitenv = dict(os.environ)
129 gitenv = dict(os.environ)
130 # forget all configs
130 # forget all configs
131 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
131 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
132 command = [self.git_path, git_command[4:], '--stateless-rpc',
132 command = [self.git_path, git_command[4:], '--stateless-rpc',
133 '--advertise-refs', self.content_path]
133 '--advertise-refs', self.content_path]
134 out = subprocessio.SubprocessIOChunker(
134 out = subprocessio.SubprocessIOChunker(
135 command,
135 command,
136 env=gitenv,
136 env=gitenv,
137 starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
137 starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
138 shell=False
138 shell=False
139 )
139 )
140 except OSError:
140 except OSError:
141 log.exception('Error processing command')
141 log.exception('Error processing command')
142 raise exc.HTTPExpectationFailed()
142 raise exc.HTTPExpectationFailed()
143
143
144 resp = Response()
144 resp = Response()
145 resp.content_type = f'application/x-{git_command}-advertisement'
145 resp.content_type = f'application/x-{git_command}-advertisement'
146 resp.charset = None
146 resp.charset = None
147 resp.app_iter = out
147 resp.app_iter = out
148
148
149 return resp
149 return resp
150
150
151 def _get_want_capabilities(self, request):
151 def _get_want_capabilities(self, request):
152 """Read the capabilities found in the first want line of the request."""
152 """Read the capabilities found in the first want line of the request."""
153 pos = request.body_file_seekable.tell()
153 pos = request.body_file_seekable.tell()
154 first_line = request.body_file_seekable.readline()
154 first_line = request.body_file_seekable.readline()
155 request.body_file_seekable.seek(pos)
155 request.body_file_seekable.seek(pos)
156
156
157 return frozenset(
157 return frozenset(
158 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
158 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
159
159
160 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
160 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
161 """
161 """
162 Construct a response with an empty PACK file.
162 Construct a response with an empty PACK file.
163
163
164 We use an empty PACK file, as that would trigger the failure of the pull
164 We use an empty PACK file, as that would trigger the failure of the pull
165 or clone command.
165 or clone command.
166
166
167 We also print in the error output a message explaining why the command
167 We also print in the error output a message explaining why the command
168 was aborted.
168 was aborted.
169
169
170 If additionally, the user is accepting messages we send them the output
170 If additionally, the user is accepting messages we send them the output
171 of the pre-pull hook.
171 of the pre-pull hook.
172
172
173 Note that for clients not supporting side-band we just send them the
173 Note that for clients not supporting side-band we just send them the
174 emtpy PACK file.
174 emtpy PACK file.
175 """
175 """
176
176
177 if self.SIDE_BAND_CAPS.intersection(capabilities):
177 if self.SIDE_BAND_CAPS.intersection(capabilities):
178 response = []
178 response = []
179 proto = dulwich.protocol.Protocol(None, response.append)
179 proto = dulwich.protocol.Protocol(None, response.append)
180 proto.write_pkt_line(dulwich.protocol.NAK_LINE)
180 proto.write_pkt_line(dulwich.protocol.NAK_LINE)
181
181
182 self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
182 self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
183 # N.B.(skreft): Do not change the sideband channel to 3, as that
183 # N.B.(skreft): Do not change the sideband channel to 3, as that
184 # produces a fatal error in the client:
184 # produces a fatal error in the client:
185 # fatal: error in sideband demultiplexer
185 # fatal: error in sideband demultiplexer
186 proto.write_sideband(
186 proto.write_sideband(
187 dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
187 dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
188 ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
188 ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
189 proto.write_sideband(
189 proto.write_sideband(
190 dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
190 dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
191 ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
191 ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
192
192
193 # writes b"0000" as default
193 # writes b"0000" as default
194 proto.write_pkt_line(None)
194 proto.write_pkt_line(None)
195
195
196 return response
196 return response
197 else:
197 else:
198 return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
198 return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
199
199
200 def _build_post_pull_response(self, response, capabilities, start_message, end_message):
200 def _build_post_pull_response(self, response, capabilities, start_message, end_message):
201 """
201 """
202 Given a list response we inject the post-pull messages.
202 Given a list response we inject the post-pull messages.
203
203
204 We only inject the messages if the client supports sideband, and the
204 We only inject the messages if the client supports sideband, and the
205 response has the format:
205 response has the format:
206 0008NAK\n...0000
206 0008NAK\n...0000
207
207
208 Note that we do not check the no-progress capability as by default, git
208 Note that we do not check the no-progress capability as by default, git
209 sends it, which effectively would block all messages.
209 sends it, which effectively would block all messages.
210 """
210 """
211
211
212 if not self.SIDE_BAND_CAPS.intersection(capabilities):
212 if not self.SIDE_BAND_CAPS.intersection(capabilities):
213 return response
213 return response
214
214
215 if not start_message and not end_message:
215 if not start_message and not end_message:
216 return response
216 return response
217
217
218 try:
218 try:
219 iter(response)
219 iter(response)
220 # iterator probably will work, we continue
220 # iterator probably will work, we continue
221 except TypeError:
221 except TypeError:
222 raise TypeError(f'response must be an iterator: got {type(response)}')
222 raise TypeError(f'response must be an iterator: got {type(response)}')
223 if isinstance(response, (list, tuple)):
223 if isinstance(response, (list, tuple)):
224 raise TypeError(f'response must be an iterator: got {type(response)}')
224 raise TypeError(f'response must be an iterator: got {type(response)}')
225
225
226 def injected_response():
226 def injected_response():
227
227
228 do_loop = 1
228 do_loop = 1
229 header_injected = 0
229 header_injected = 0
230 next_item = None
230 next_item = None
231 has_item = False
231 has_item = False
232 item = b''
232 item = b''
233
233
234 while do_loop:
234 while do_loop:
235
235
236 try:
236 try:
237 next_item = next(response)
237 next_item = next(response)
238 except StopIteration:
238 except StopIteration:
239 do_loop = 0
239 do_loop = 0
240
240
241 if has_item:
241 if has_item:
242 # last item ! alter it now
242 # last item ! alter it now
243 if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
243 if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
244 new_response = [item[:-4]]
244 new_response = [item[:-4]]
245 new_response.extend(self._get_messages(end_message, capabilities))
245 new_response.extend(self._get_messages(end_message, capabilities))
246 new_response.append(self.FLUSH_PACKET)
246 new_response.append(self.FLUSH_PACKET)
247 item = b''.join(new_response)
247 item = b''.join(new_response)
248
248
249 yield item
249 yield item
250
250
251 has_item = True
251 has_item = True
252 item = next_item
252 item = next_item
253
253
254 # alter item if it's the initial chunk
254 # alter item if it's the initial chunk
255 if not header_injected and item.startswith(b'0008NAK\n'):
255 if not header_injected and item.startswith(b'0008NAK\n'):
256 new_response = [b'0008NAK\n']
256 new_response = [b'0008NAK\n']
257 new_response.extend(self._get_messages(start_message, capabilities))
257 new_response.extend(self._get_messages(start_message, capabilities))
258 new_response.append(item[8:])
258 new_response.append(item[8:])
259 item = b''.join(new_response)
259 item = b''.join(new_response)
260 header_injected = 1
260 header_injected = 1
261
261
262 return injected_response()
262 return injected_response()
263
263
264 def _write_sideband_to_proto(self, proto, data, capabilities):
264 def _write_sideband_to_proto(self, proto, data, capabilities):
265 """
265 """
266 Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
266 Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
267
267
268 We do not use dulwich's write_sideband directly as it only supports
268 We do not use dulwich's write_sideband directly as it only supports
269 side-band-64k.
269 side-band-64k.
270 """
270 """
271 if not data:
271 if not data:
272 return
272 return
273
273
274 # N.B.(skreft): The values below are explained in the pack protocol
274 # N.B.(skreft): The values below are explained in the pack protocol
275 # documentation, section Packfile Data.
275 # documentation, section Packfile Data.
276 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
276 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
277 if CAPABILITY_SIDE_BAND_64K in capabilities:
277 if CAPABILITY_SIDE_BAND_64K in capabilities:
278 chunk_size = 65515
278 chunk_size = 65515
279 elif CAPABILITY_SIDE_BAND in capabilities:
279 elif CAPABILITY_SIDE_BAND in capabilities:
280 chunk_size = 995
280 chunk_size = 995
281 else:
281 else:
282 return
282 return
283
283
284 chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
284 chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
285
285
286 for chunk in chunker:
286 for chunk in chunker:
287 proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
287 proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
288
288
289 def _get_messages(self, data, capabilities):
289 def _get_messages(self, data, capabilities):
290 """Return a list with packets for sending data in sideband number 2."""
290 """Return a list with packets for sending data in sideband number 2."""
291 response = []
291 response = []
292 proto = dulwich.protocol.Protocol(None, response.append)
292 proto = dulwich.protocol.Protocol(None, response.append)
293
293
294 self._write_sideband_to_proto(proto, data, capabilities)
294 self._write_sideband_to_proto(proto, data, capabilities)
295
295
296 return response
296 return response
297
297
298 def backend(self, request, environ):
298 def backend(self, request, environ):
299 """
299 """
300 WSGI Response producer for HTTP POST Git Smart HTTP requests.
300 WSGI Response producer for HTTP POST Git Smart HTTP requests.
301 Reads commands and data from HTTP POST's body.
301 Reads commands and data from HTTP POST's body.
302 returns an iterator obj with contents of git command's
302 returns an iterator obj with contents of git command's
303 response to stdout
303 response to stdout
304 """
304 """
305 # TODO(skreft): think how we could detect an HTTPLockedException, as
305 # TODO(skreft): think how we could detect an HTTPLockedException, as
306 # we probably want to have the same mechanism used by mercurial and
306 # we probably want to have the same mechanism used by mercurial and
307 # simplevcs.
307 # simplevcs.
308 # For that we would need to parse the output of the command looking for
308 # For that we would need to parse the output of the command looking for
309 # some signs of the HTTPLockedError, parse the data and reraise it in
309 # some signs of the HTTPLockedError, parse the data and reraise it in
310 # pygrack. However, that would interfere with the streaming.
310 # pygrack. However, that would interfere with the streaming.
311 #
311 #
312 # Now the output of a blocked push is:
312 # Now the output of a blocked push is:
313 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
313 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
314 # POST git-receive-pack (1047 bytes)
314 # POST git-receive-pack (1047 bytes)
315 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
315 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
316 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
316 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
317 # ! [remote rejected] master -> master (pre-receive hook declined)
317 # ! [remote rejected] master -> master (pre-receive hook declined)
318 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
318 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
319
319
320 git_command = self._get_fixedpath(request.path_info)
320 git_command = self._get_fixedpath(request.path_info)
321 if git_command not in self.commands:
321 if git_command not in self.commands:
322 log.debug('command %s not allowed', git_command)
322 log.debug('command %s not allowed', git_command)
323 return exc.HTTPForbidden()
323 return exc.HTTPForbidden()
324
324
325 capabilities = None
325 capabilities = None
326 if git_command == 'git-upload-pack':
326 if git_command == 'git-upload-pack':
327 capabilities = self._get_want_capabilities(request)
327 capabilities = self._get_want_capabilities(request)
328
328
329 if 'CONTENT_LENGTH' in environ:
329 if 'CONTENT_LENGTH' in environ:
330 inputstream = FileWrapper(request.body_file_seekable,
330 inputstream = FileWrapper(request.body_file_seekable,
331 request.content_length)
331 request.content_length)
332 else:
332 else:
333 inputstream = request.body_file_seekable
333 inputstream = request.body_file_seekable
334
334
335 resp = Response()
335 resp = Response()
336 resp.content_type = f'application/x-{git_command}-result'
336 resp.content_type = f'application/x-{git_command}-result'
337 resp.charset = None
337 resp.charset = None
338
338
339 pre_pull_messages = ''
339 pre_pull_messages = ''
340 # Upload-pack == clone
340 # Upload-pack == clone
341 if git_command == 'git-upload-pack':
341 if git_command == 'git-upload-pack':
342 hook_response = hooks.git_pre_pull(self.extras)
342 hook_response = hooks.git_pre_pull(self.extras)
343 if hook_response.status != 0:
343 if hook_response.status != 0:
344 pre_pull_messages = hook_response.output
344 pre_pull_messages = hook_response.output
345 resp.app_iter = self._build_failed_pre_pull_response(
345 resp.app_iter = self._build_failed_pre_pull_response(
346 capabilities, pre_pull_messages)
346 capabilities, pre_pull_messages)
347 return resp
347 return resp
348
348
349 gitenv = dict(os.environ)
349 gitenv = dict(os.environ)
350 # forget all configs
350 # forget all configs
351 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
351 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
352 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
352 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
353 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
353 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
354 self.content_path]
354 self.content_path]
355 log.debug('handling cmd %s', cmd)
355 log.debug('handling cmd %s', cmd)
356
356
357 out = subprocessio.SubprocessIOChunker(
357 out = subprocessio.SubprocessIOChunker(
358 cmd,
358 cmd,
359 input_stream=inputstream,
359 input_stream=inputstream,
360 env=gitenv,
360 env=gitenv,
361 cwd=self.content_path,
361 cwd=self.content_path,
362 shell=False,
362 shell=False,
363 fail_on_stderr=False,
363 fail_on_stderr=False,
364 fail_on_return_code=False
364 fail_on_return_code=False
365 )
365 )
366
366
367 if self.update_server_info and git_command == 'git-receive-pack':
367 if self.update_server_info and git_command == 'git-receive-pack':
368 # We need to fully consume the iterator here, as the
368 # We need to fully consume the iterator here, as the
369 # update-server-info command needs to be run after the push.
369 # update-server-info command needs to be run after the push.
370 out = list(out)
370 out = list(out)
371
371
372 # Updating refs manually after each push.
372 # Updating refs manually after each push.
373 # This is required as some clients are exposing Git repos internally
373 # This is required as some clients are exposing Git repos internally
374 # with the dumb protocol.
374 # with the dumb protocol.
375 cmd = [self.git_path, 'update-server-info']
375 cmd = [self.git_path, 'update-server-info']
376 log.debug('handling cmd %s', cmd)
376 log.debug('handling cmd %s', cmd)
377 output = subprocessio.SubprocessIOChunker(
377 output = subprocessio.SubprocessIOChunker(
378 cmd,
378 cmd,
379 input_stream=inputstream,
379 input_stream=inputstream,
380 env=gitenv,
380 env=gitenv,
381 cwd=self.content_path,
381 cwd=self.content_path,
382 shell=False,
382 shell=False,
383 fail_on_stderr=False,
383 fail_on_stderr=False,
384 fail_on_return_code=False
384 fail_on_return_code=False
385 )
385 )
386 # Consume all the output so the subprocess finishes
386 # Consume all the output so the subprocess finishes
387 for _ in output:
387 for _ in output:
388 pass
388 pass
389
389
390 # Upload-pack == clone
390 # Upload-pack == clone
391 if git_command == 'git-upload-pack':
391 if git_command == 'git-upload-pack':
392 hook_response = hooks.git_post_pull(self.extras)
392 hook_response = hooks.git_post_pull(self.extras)
393 post_pull_messages = hook_response.output
393 post_pull_messages = hook_response.output
394 resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
394 resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
395 else:
395 else:
396 resp.app_iter = out
396 resp.app_iter = out
397
397
398 return resp
398 return resp
399
399
400 def __call__(self, environ, start_response):
400 def __call__(self, environ, start_response):
401 request = Request(environ)
401 request = Request(environ)
402 _path = self._get_fixedpath(request.path_info)
402 _path = self._get_fixedpath(request.path_info)
403 if _path.startswith('info/refs'):
403 if _path.startswith('info/refs'):
404 app = self.inforefs
404 app = self.inforefs
405 else:
405 else:
406 app = self.backend
406 app = self.backend
407
407
408 try:
408 try:
409 resp = app(request, environ)
409 resp = app(request, environ)
410 except exc.HTTPException as error:
410 except exc.HTTPException as error:
411 log.exception('HTTP Error')
411 log.exception('HTTP Error')
412 resp = error
412 resp = error
413 except Exception:
413 except Exception:
414 log.exception('Unknown error')
414 log.exception('Unknown error')
415 resp = exc.HTTPInternalServerError()
415 resp = exc.HTTPInternalServerError()
416
416
417 return resp(environ, start_response)
417 return resp(environ, start_response)
@@ -1,1519 +1,1526 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import collections
18 import collections
19 import logging
19 import logging
20 import os
20 import os
21 import re
21 import re
22 import stat
22 import stat
23 import traceback
23 import traceback
24 import urllib.request
24 import urllib.request
25 import urllib.parse
25 import urllib.parse
26 import urllib.error
26 import urllib.error
27 from functools import wraps
27 from functools import wraps
28
28
29 import more_itertools
29 import more_itertools
30 import pygit2
30 import pygit2
31 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
32 from pygit2 import index as LibGit2Index
32 from pygit2 import index as LibGit2Index
33 from dulwich import index, objects
33 from dulwich import index, objects
34 from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
34 from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
35 from dulwich.errors import (
35 from dulwich.errors import (
36 NotGitRepository, ChecksumMismatch, WrongObjectException,
36 NotGitRepository, ChecksumMismatch, WrongObjectException,
37 MissingCommitError, ObjectMissing, HangupException,
37 MissingCommitError, ObjectMissing, HangupException,
38 UnexpectedCommandError)
38 UnexpectedCommandError)
39 from dulwich.repo import Repo as DulwichRepo
39 from dulwich.repo import Repo as DulwichRepo
40
40
41 import rhodecode
41 import rhodecode
42 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver import exceptions, settings, subprocessio
43 from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes, convert_to_str, splitnewlines
43 from vcsserver.lib.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes, convert_to_str, splitnewlines
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
45 from vcsserver.hgcompat import (
45 from vcsserver.hgcompat import (
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
47 from vcsserver.git_lfs.lib import LFSOidStore
47 from vcsserver.git_lfs.lib import LFSOidStore
48 from vcsserver.vcs_base import RemoteBase
48 from vcsserver.vcs_base import RemoteBase
49
49
50 DIR_STAT = stat.S_IFDIR
50 DIR_STAT = stat.S_IFDIR
51 FILE_MODE = stat.S_IFMT
51 FILE_MODE = stat.S_IFMT
52 GIT_LINK = objects.S_IFGITLINK
52 GIT_LINK = objects.S_IFGITLINK
53 PEELED_REF_MARKER = b'^{}'
53 PEELED_REF_MARKER = b'^{}'
54 HEAD_MARKER = b'HEAD'
54 HEAD_MARKER = b'HEAD'
55
55
56 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
57
57
58
58
59 def reraise_safe_exceptions(func):
59 def reraise_safe_exceptions(func):
60 """Converts Dulwich exceptions to something neutral."""
60 """Converts Dulwich exceptions to something neutral."""
61
61
62 @wraps(func)
62 @wraps(func)
63 def wrapper(*args, **kwargs):
63 def wrapper(*args, **kwargs):
64 try:
64 try:
65 return func(*args, **kwargs)
65 return func(*args, **kwargs)
66 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
66 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
67 exc = exceptions.LookupException(org_exc=e)
67 exc = exceptions.LookupException(org_exc=e)
68 raise exc(safe_str(e))
68 raise exc(safe_str(e))
69 except (HangupException, UnexpectedCommandError) as e:
69 except (HangupException, UnexpectedCommandError) as e:
70 exc = exceptions.VcsException(org_exc=e)
70 exc = exceptions.VcsException(org_exc=e)
71 raise exc(safe_str(e))
71 raise exc(safe_str(e))
72 except Exception:
72 except Exception:
73 # NOTE(marcink): because of how dulwich handles some exceptions
73 # NOTE(marcink): because of how dulwich handles some exceptions
74 # (KeyError on empty repos), we cannot track this and catch all
74 # (KeyError on empty repos), we cannot track this and catch all
75 # exceptions, it's an exceptions from other handlers
75 # exceptions, it's an exceptions from other handlers
76 #if not hasattr(e, '_vcs_kind'):
76 #if not hasattr(e, '_vcs_kind'):
77 #log.exception("Unhandled exception in git remote call")
77 #log.exception("Unhandled exception in git remote call")
78 #raise_from_original(exceptions.UnhandledException)
78 #raise_from_original(exceptions.UnhandledException)
79 raise
79 raise
80 return wrapper
80 return wrapper
81
81
82
82
83 class Repo(DulwichRepo):
83 class Repo(DulwichRepo):
84 """
84 """
85 A wrapper for dulwich Repo class.
85 A wrapper for dulwich Repo class.
86
86
87 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
87 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
88 "Too many open files" error. We need to close all opened file descriptors
88 "Too many open files" error. We need to close all opened file descriptors
89 once the repo object is destroyed.
89 once the repo object is destroyed.
90 """
90 """
91 def __del__(self):
91 def __del__(self):
92 if hasattr(self, 'object_store'):
92 if hasattr(self, 'object_store'):
93 self.close()
93 self.close()
94
94
95
95
96 class Repository(LibGit2Repo):
96 class Repository(LibGit2Repo):
97
97
98 def __enter__(self):
98 def __enter__(self):
99 return self
99 return self
100
100
101 def __exit__(self, exc_type, exc_val, exc_tb):
101 def __exit__(self, exc_type, exc_val, exc_tb):
102 self.free()
102 self.free()
103
103
104
104
105 class GitFactory(RepoFactory):
105 class GitFactory(RepoFactory):
106 repo_type = 'git'
106 repo_type = 'git'
107
107
108 def _create_repo(self, wire, create, use_libgit2=False):
108 def _create_repo(self, wire, create, use_libgit2=False):
109 if use_libgit2:
109 if use_libgit2:
110 repo = Repository(safe_bytes(wire['path']))
110 repo = Repository(safe_bytes(wire['path']))
111 else:
111 else:
112 # dulwich mode
112 # dulwich mode
113 repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
113 repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
114 repo = Repo(repo_path)
114 repo = Repo(repo_path)
115
115
116 log.debug('repository created: got GIT object: %s', repo)
116 log.debug('repository created: got GIT object: %s', repo)
117 return repo
117 return repo
118
118
119 def repo(self, wire, create=False, use_libgit2=False):
119 def repo(self, wire, create=False, use_libgit2=False):
120 """
120 """
121 Get a repository instance for the given path.
121 Get a repository instance for the given path.
122 """
122 """
123 return self._create_repo(wire, create, use_libgit2)
123 return self._create_repo(wire, create, use_libgit2)
124
124
125 def repo_libgit2(self, wire):
125 def repo_libgit2(self, wire):
126 return self.repo(wire, use_libgit2=True)
126 return self.repo(wire, use_libgit2=True)
127
127
128
128
129 def create_signature_from_string(author_str, **kwargs):
129 def create_signature_from_string(author_str, **kwargs):
130 """
130 """
131 Creates a pygit2.Signature object from a string of the format 'Name <email>'.
131 Creates a pygit2.Signature object from a string of the format 'Name <email>'.
132
132
133 :param author_str: String of the format 'Name <email>'
133 :param author_str: String of the format 'Name <email>'
134 :return: pygit2.Signature object
134 :return: pygit2.Signature object
135 """
135 """
136 match = re.match(r'^(.+) <(.+)>$', author_str)
136 match = re.match(r'^(.+) <(.+)>$', author_str)
137 if match is None:
137 if match is None:
138 raise ValueError(f"Invalid format: {author_str}")
138 raise ValueError(f"Invalid format: {author_str}")
139
139
140 name, email = match.groups()
140 name, email = match.groups()
141 return pygit2.Signature(name, email, **kwargs)
141 return pygit2.Signature(name, email, **kwargs)
142
142
143
143
144 def get_obfuscated_url(url_obj):
144 def get_obfuscated_url(url_obj):
145 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
145 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
146 url_obj.query = obfuscate_qs(url_obj.query)
146 url_obj.query = obfuscate_qs(url_obj.query)
147 obfuscated_uri = str(url_obj)
147 obfuscated_uri = str(url_obj)
148 return obfuscated_uri
148 return obfuscated_uri
149
149
150
150
151 class GitRemote(RemoteBase):
151 class GitRemote(RemoteBase):
152
152
153 def __init__(self, factory):
153 def __init__(self, factory):
154 self._factory = factory
154 self._factory = factory
155 self._bulk_methods = {
155 self._bulk_methods = {
156 "date": self.date,
156 "date": self.date,
157 "author": self.author,
157 "author": self.author,
158 "branch": self.branch,
158 "branch": self.branch,
159 "message": self.message,
159 "message": self.message,
160 "parents": self.parents,
160 "parents": self.parents,
161 "_commit": self.revision,
161 "_commit": self.revision,
162 }
162 }
163 self._bulk_file_methods = {
163 self._bulk_file_methods = {
164 "size": self.get_node_size,
164 "size": self.get_node_size,
165 "data": self.get_node_data,
165 "data": self.get_node_data,
166 "flags": self.get_node_flags,
166 "flags": self.get_node_flags,
167 "is_binary": self.get_node_is_binary,
167 "is_binary": self.get_node_is_binary,
168 "md5": self.md5_hash
168 "md5": self.md5_hash
169 }
169 }
170
170
171 def _wire_to_config(self, wire):
171 def _wire_to_config(self, wire):
172 if 'config' in wire:
172 if 'config' in wire:
173 return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
173 return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
174 return {}
174 return {}
175
175
176 def _remote_conf(self, config):
176 def _remote_conf(self, config):
177 params = [
177 params = [
178 '-c', 'core.askpass=""',
178 '-c', 'core.askpass=""',
179 ]
179 ]
180 config_attrs = {
180 config_attrs = {
181 'vcs_ssl_dir': 'http.sslCAinfo={}',
181 'vcs_ssl_dir': 'http.sslCAinfo={}',
182 'vcs_git_lfs_store_location': 'lfs.storage={}'
182 'vcs_git_lfs_store_location': 'lfs.storage={}'
183 }
183 }
184 for key, param in config_attrs.items():
184 for key, param in config_attrs.items():
185 if value := config.get(key):
185 if value := config.get(key):
186 params.extend(['-c', param.format(value)])
186 params.extend(['-c', param.format(value)])
187 return params
187 return params
188
188
189 @reraise_safe_exceptions
189 @reraise_safe_exceptions
190 def discover_git_version(self):
190 def discover_git_version(self):
191 stdout, _ = self.run_git_command(
191 stdout, _ = self.run_git_command(
192 {}, ['--version'], _bare=True, _safe=True)
192 {}, ['--version'], _bare=True, _safe=True)
193 prefix = b'git version'
193 prefix = b'git version'
194 if stdout.startswith(prefix):
194 if stdout.startswith(prefix):
195 stdout = stdout[len(prefix):]
195 stdout = stdout[len(prefix):]
196 return safe_str(stdout.strip())
196 return safe_str(stdout.strip())
197
197
198 @reraise_safe_exceptions
198 @reraise_safe_exceptions
199 def is_empty(self, wire):
199 def is_empty(self, wire):
200 repo_init = self._factory.repo_libgit2(wire)
200 repo_init = self._factory.repo_libgit2(wire)
201 with repo_init as repo:
201 with repo_init as repo:
202 try:
202 try:
203 has_head = repo.head.name
203 has_head = repo.head.name
204 if has_head:
204 if has_head:
205 return False
205 return False
206
206
207 # NOTE(marcink): check again using more expensive method
207 # NOTE(marcink): check again using more expensive method
208 return repo.is_empty
208 return repo.is_empty
209 except Exception:
209 except Exception:
210 pass
210 pass
211
211
212 return True
212 return True
213
213
214 @reraise_safe_exceptions
214 @reraise_safe_exceptions
215 def assert_correct_path(self, wire):
215 def assert_correct_path(self, wire):
216 cache_on, context_uid, repo_id = self._cache_on(wire)
216 cache_on, context_uid, repo_id = self._cache_on(wire)
217 region = self._region(wire)
217 region = self._region(wire)
218
218
219 @region.conditional_cache_on_arguments(condition=cache_on)
219 @region.conditional_cache_on_arguments(condition=cache_on)
220 def _assert_correct_path(_context_uid, _repo_id, fast_check):
220 def _assert_correct_path(_context_uid, _repo_id, fast_check):
221 if fast_check:
221 if fast_check:
222 path = safe_str(wire['path'])
222 path = safe_str(wire['path'])
223 if pygit2.discover_repository(path):
223 if pygit2.discover_repository(path):
224 return True
224 return True
225 return False
225 return False
226 else:
226 else:
227 try:
227 try:
228 repo_init = self._factory.repo_libgit2(wire)
228 repo_init = self._factory.repo_libgit2(wire)
229 with repo_init:
229 with repo_init:
230 pass
230 pass
231 except pygit2.GitError:
231 except pygit2.GitError:
232 path = wire.get('path')
232 path = wire.get('path')
233 tb = traceback.format_exc()
233 tb = traceback.format_exc()
234 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
234 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
235 return False
235 return False
236 return True
236 return True
237
237
238 return _assert_correct_path(context_uid, repo_id, True)
238 return _assert_correct_path(context_uid, repo_id, True)
239
239
240 @reraise_safe_exceptions
240 @reraise_safe_exceptions
241 def bare(self, wire):
241 def bare(self, wire):
242 repo_init = self._factory.repo_libgit2(wire)
242 repo_init = self._factory.repo_libgit2(wire)
243 with repo_init as repo:
243 with repo_init as repo:
244 return repo.is_bare
244 return repo.is_bare
245
245
246 @reraise_safe_exceptions
246 @reraise_safe_exceptions
247 def get_node_data(self, wire, commit_id, path):
247 def get_node_data(self, wire, commit_id, path):
248 repo_init = self._factory.repo_libgit2(wire)
248 repo_init = self._factory.repo_libgit2(wire)
249 with repo_init as repo:
249 with repo_init as repo:
250 commit = repo[commit_id]
250 commit = repo[commit_id]
251 blob_obj = commit.tree[path]
251 blob_obj = commit.tree[path]
252
252
253 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
253 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
254 raise exceptions.LookupException()(
254 raise exceptions.LookupException()(
255 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
255 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
256
256
257 return BytesEnvelope(blob_obj.data)
257 return BytesEnvelope(blob_obj.data)
258
258
259 @reraise_safe_exceptions
259 @reraise_safe_exceptions
260 def get_node_size(self, wire, commit_id, path):
260 def get_node_size(self, wire, commit_id, path):
261 repo_init = self._factory.repo_libgit2(wire)
261 repo_init = self._factory.repo_libgit2(wire)
262 with repo_init as repo:
262 with repo_init as repo:
263 commit = repo[commit_id]
263 commit = repo[commit_id]
264 blob_obj = commit.tree[path]
264 blob_obj = commit.tree[path]
265
265
266 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
266 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
267 raise exceptions.LookupException()(
267 raise exceptions.LookupException()(
268 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
268 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
269
269
270 return blob_obj.size
270 return blob_obj.size
271
271
272 @reraise_safe_exceptions
272 @reraise_safe_exceptions
273 def get_node_flags(self, wire, commit_id, path):
273 def get_node_flags(self, wire, commit_id, path):
274 repo_init = self._factory.repo_libgit2(wire)
274 repo_init = self._factory.repo_libgit2(wire)
275 with repo_init as repo:
275 with repo_init as repo:
276 commit = repo[commit_id]
276 commit = repo[commit_id]
277 blob_obj = commit.tree[path]
277 blob_obj = commit.tree[path]
278
278
279 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
279 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
280 raise exceptions.LookupException()(
280 raise exceptions.LookupException()(
281 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
281 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
282
282
283 return blob_obj.filemode
283 return blob_obj.filemode
284
284
285 @reraise_safe_exceptions
285 @reraise_safe_exceptions
286 def get_node_is_binary(self, wire, commit_id, path):
286 def get_node_is_binary(self, wire, commit_id, path):
287 repo_init = self._factory.repo_libgit2(wire)
287 repo_init = self._factory.repo_libgit2(wire)
288 with repo_init as repo:
288 with repo_init as repo:
289 commit = repo[commit_id]
289 commit = repo[commit_id]
290 blob_obj = commit.tree[path]
290 blob_obj = commit.tree[path]
291
291
292 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
292 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
293 raise exceptions.LookupException()(
293 raise exceptions.LookupException()(
294 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
294 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
295
295
296 return blob_obj.is_binary
296 return blob_obj.is_binary
297
297
298 @reraise_safe_exceptions
298 @reraise_safe_exceptions
299 def blob_as_pretty_string(self, wire, sha):
299 def blob_as_pretty_string(self, wire, sha):
300 repo_init = self._factory.repo_libgit2(wire)
300 repo_init = self._factory.repo_libgit2(wire)
301 with repo_init as repo:
301 with repo_init as repo:
302 blob_obj = repo[sha]
302 blob_obj = repo[sha]
303 return BytesEnvelope(blob_obj.data)
303 return BytesEnvelope(blob_obj.data)
304
304
305 @reraise_safe_exceptions
305 @reraise_safe_exceptions
306 def blob_raw_length(self, wire, sha):
306 def blob_raw_length(self, wire, sha):
307 cache_on, context_uid, repo_id = self._cache_on(wire)
307 cache_on, context_uid, repo_id = self._cache_on(wire)
308 region = self._region(wire)
308 region = self._region(wire)
309
309
310 @region.conditional_cache_on_arguments(condition=cache_on)
310 @region.conditional_cache_on_arguments(condition=cache_on)
311 def _blob_raw_length(_repo_id, _sha):
311 def _blob_raw_length(_repo_id, _sha):
312
312
313 repo_init = self._factory.repo_libgit2(wire)
313 repo_init = self._factory.repo_libgit2(wire)
314 with repo_init as repo:
314 with repo_init as repo:
315 blob = repo[sha]
315 blob = repo[sha]
316 return blob.size
316 return blob.size
317
317
318 return _blob_raw_length(repo_id, sha)
318 return _blob_raw_length(repo_id, sha)
319
319
320 def _parse_lfs_pointer(self, raw_content):
320 def _parse_lfs_pointer(self, raw_content):
321 spec_string = b'version https://git-lfs.github.com/spec'
321 spec_string = b'version https://git-lfs.github.com/spec'
322 if raw_content and raw_content.startswith(spec_string):
322 if raw_content and raw_content.startswith(spec_string):
323
323
324 pattern = re.compile(rb"""
324 pattern = re.compile(rb"""
325 (?:\n)?
325 (?:\n)?
326 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
326 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
327 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
327 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
328 ^size[ ](?P<oid_size>[0-9]+)\n
328 ^size[ ](?P<oid_size>[0-9]+)\n
329 (?:\n)?
329 (?:\n)?
330 """, re.VERBOSE | re.MULTILINE)
330 """, re.VERBOSE | re.MULTILINE)
331 match = pattern.match(raw_content)
331 match = pattern.match(raw_content)
332 if match:
332 if match:
333 return match.groupdict()
333 return match.groupdict()
334
334
335 return {}
335 return {}
336
336
337 @reraise_safe_exceptions
337 @reraise_safe_exceptions
338 def is_large_file(self, wire, commit_id):
338 def is_large_file(self, wire, commit_id):
339 cache_on, context_uid, repo_id = self._cache_on(wire)
339 cache_on, context_uid, repo_id = self._cache_on(wire)
340 region = self._region(wire)
340 region = self._region(wire)
341
341
342 @region.conditional_cache_on_arguments(condition=cache_on)
342 @region.conditional_cache_on_arguments(condition=cache_on)
343 def _is_large_file(_repo_id, _sha):
343 def _is_large_file(_repo_id, _sha):
344 repo_init = self._factory.repo_libgit2(wire)
344 repo_init = self._factory.repo_libgit2(wire)
345 with repo_init as repo:
345 with repo_init as repo:
346 blob = repo[commit_id]
346 blob = repo[commit_id]
347 if blob.is_binary:
347 if blob.is_binary:
348 return {}
348 return {}
349
349
350 return self._parse_lfs_pointer(blob.data)
350 return self._parse_lfs_pointer(blob.data)
351
351
352 return _is_large_file(repo_id, commit_id)
352 return _is_large_file(repo_id, commit_id)
353
353
354 @reraise_safe_exceptions
354 @reraise_safe_exceptions
355 def is_binary(self, wire, tree_id):
355 def is_binary(self, wire, tree_id):
356 cache_on, context_uid, repo_id = self._cache_on(wire)
356 cache_on, context_uid, repo_id = self._cache_on(wire)
357 region = self._region(wire)
357 region = self._region(wire)
358
358
359 @region.conditional_cache_on_arguments(condition=cache_on)
359 @region.conditional_cache_on_arguments(condition=cache_on)
360 def _is_binary(_repo_id, _tree_id):
360 def _is_binary(_repo_id, _tree_id):
361 repo_init = self._factory.repo_libgit2(wire)
361 repo_init = self._factory.repo_libgit2(wire)
362 with repo_init as repo:
362 with repo_init as repo:
363 blob_obj = repo[tree_id]
363 blob_obj = repo[tree_id]
364 return blob_obj.is_binary
364 return blob_obj.is_binary
365
365
366 return _is_binary(repo_id, tree_id)
366 return _is_binary(repo_id, tree_id)
367
367
368 @reraise_safe_exceptions
368 @reraise_safe_exceptions
369 def md5_hash(self, wire, commit_id, path):
369 def md5_hash(self, wire, commit_id, path):
370 cache_on, context_uid, repo_id = self._cache_on(wire)
370 cache_on, context_uid, repo_id = self._cache_on(wire)
371 region = self._region(wire)
371 region = self._region(wire)
372
372
373 @region.conditional_cache_on_arguments(condition=cache_on)
373 @region.conditional_cache_on_arguments(condition=cache_on)
374 def _md5_hash(_repo_id, _commit_id, _path):
374 def _md5_hash(_repo_id, _commit_id, _path):
375 repo_init = self._factory.repo_libgit2(wire)
375 repo_init = self._factory.repo_libgit2(wire)
376 with repo_init as repo:
376 with repo_init as repo:
377 commit = repo[_commit_id]
377 commit = repo[_commit_id]
378 blob_obj = commit.tree[_path]
378 blob_obj = commit.tree[_path]
379
379
380 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
380 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
381 raise exceptions.LookupException()(
381 raise exceptions.LookupException()(
382 f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
382 f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
383
383
384 return ''
384 return ''
385
385
386 return _md5_hash(repo_id, commit_id, path)
386 return _md5_hash(repo_id, commit_id, path)
387
387
388 @reraise_safe_exceptions
388 @reraise_safe_exceptions
389 def in_largefiles_store(self, wire, oid):
389 def in_largefiles_store(self, wire, oid):
390 conf = self._wire_to_config(wire)
390 conf = self._wire_to_config(wire)
391 repo_init = self._factory.repo_libgit2(wire)
391 repo_init = self._factory.repo_libgit2(wire)
392 with repo_init as repo:
392 with repo_init as repo:
393 repo_name = repo.path
393 repo_name = repo.path
394
394
395 store_location = conf.get('vcs_git_lfs_store_location')
395 store_location = conf.get('vcs_git_lfs_store_location')
396 if store_location:
396 if store_location:
397
397
398 store = LFSOidStore(
398 store = LFSOidStore(
399 oid=oid, repo=repo_name, store_location=store_location)
399 oid=oid, repo=repo_name, store_location=store_location)
400 return store.has_oid()
400 return store.has_oid()
401
401
402 return False
402 return False
403
403
404 @reraise_safe_exceptions
404 @reraise_safe_exceptions
405 def store_path(self, wire, oid):
405 def store_path(self, wire, oid):
406 conf = self._wire_to_config(wire)
406 conf = self._wire_to_config(wire)
407 repo_init = self._factory.repo_libgit2(wire)
407 repo_init = self._factory.repo_libgit2(wire)
408 with repo_init as repo:
408 with repo_init as repo:
409 repo_name = repo.path
409 repo_name = repo.path
410
410
411 store_location = conf.get('vcs_git_lfs_store_location')
411 store_location = conf.get('vcs_git_lfs_store_location')
412 if store_location:
412 if store_location:
413 store = LFSOidStore(
413 store = LFSOidStore(
414 oid=oid, repo=repo_name, store_location=store_location)
414 oid=oid, repo=repo_name, store_location=store_location)
415 return store.oid_path
415 return store.oid_path
416 raise ValueError(f'Unable to fetch oid with path {oid}')
416 raise ValueError(f'Unable to fetch oid with path {oid}')
417
417
418 @reraise_safe_exceptions
418 @reraise_safe_exceptions
419 def bulk_request(self, wire, rev, pre_load):
419 def bulk_request(self, wire, rev, pre_load):
420 cache_on, context_uid, repo_id = self._cache_on(wire)
420 cache_on, context_uid, repo_id = self._cache_on(wire)
421 region = self._region(wire)
421 region = self._region(wire)
422
422
423 @region.conditional_cache_on_arguments(condition=cache_on)
423 @region.conditional_cache_on_arguments(condition=cache_on)
424 def _bulk_request(_repo_id, _rev, _pre_load):
424 def _bulk_request(_repo_id, _rev, _pre_load):
425 result = {}
425 result = {}
426 for attr in pre_load:
426 for attr in pre_load:
427 try:
427 try:
428 method = self._bulk_methods[attr]
428 method = self._bulk_methods[attr]
429 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
429 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
430 args = [wire, rev]
430 args = [wire, rev]
431 result[attr] = method(*args)
431 result[attr] = method(*args)
432 except KeyError as e:
432 except KeyError as e:
433 raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
433 raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
434 return result
434 return result
435
435
436 return _bulk_request(repo_id, rev, sorted(pre_load))
436 return _bulk_request(repo_id, rev, sorted(pre_load))
437
437
438 @reraise_safe_exceptions
438 @reraise_safe_exceptions
439 def bulk_file_request(self, wire, commit_id, path, pre_load):
439 def bulk_file_request(self, wire, commit_id, path, pre_load):
440 cache_on, context_uid, repo_id = self._cache_on(wire)
440 cache_on, context_uid, repo_id = self._cache_on(wire)
441 region = self._region(wire)
441 region = self._region(wire)
442
442
443 @region.conditional_cache_on_arguments(condition=cache_on)
443 @region.conditional_cache_on_arguments(condition=cache_on)
444 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
444 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
445 result = {}
445 result = {}
446 for attr in pre_load:
446 for attr in pre_load:
447 try:
447 try:
448 method = self._bulk_file_methods[attr]
448 method = self._bulk_file_methods[attr]
449 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
449 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
450 result[attr] = method(wire, _commit_id, _path)
450 result[attr] = method(wire, _commit_id, _path)
451 except KeyError as e:
451 except KeyError as e:
452 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
452 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
453 return result
453 return result
454
454
455 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
455 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
456
456
457 def _build_opener(self, url: str):
457 def _build_opener(self, url: str):
458 handlers = []
458 handlers = []
459 url_obj = url_parser(safe_bytes(url))
459 url_obj = url_parser(safe_bytes(url))
460 authinfo = url_obj.authinfo()[1]
460 authinfo = url_obj.authinfo()[1]
461
461
462 if authinfo:
462 if authinfo:
463 # create a password manager
463 # create a password manager
464 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
464 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
465 passmgr.add_password(*convert_to_str(authinfo))
465 passmgr.add_password(*convert_to_str(authinfo))
466
466
467 handlers.extend((httpbasicauthhandler(passmgr),
467 handlers.extend((httpbasicauthhandler(passmgr),
468 httpdigestauthhandler(passmgr)))
468 httpdigestauthhandler(passmgr)))
469
469
470 return urllib.request.build_opener(*handlers)
470 return urllib.request.build_opener(*handlers)
471
471
472 @reraise_safe_exceptions
472 @reraise_safe_exceptions
473 def check_url(self, url, config):
473 def check_url(self, url, config):
474 url_obj = url_parser(safe_bytes(url))
474 url_obj = url_parser(safe_bytes(url))
475
475
476 test_uri = safe_str(url_obj.authinfo()[0])
476 test_uri = safe_str(url_obj.authinfo()[0])
477 obfuscated_uri = get_obfuscated_url(url_obj)
477 obfuscated_uri = get_obfuscated_url(url_obj)
478
478
479 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
479 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
480
480
481 if not test_uri.endswith('info/refs'):
481 if not test_uri.endswith('info/refs'):
482 test_uri = test_uri.rstrip('/') + '/info/refs'
482 test_uri = test_uri.rstrip('/') + '/info/refs'
483
483
484 o = self._build_opener(url=url)
484 o = self._build_opener(url=url)
485 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
485 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
486
486
487 q = {"service": 'git-upload-pack'}
487 q = {"service": 'git-upload-pack'}
488 qs = f'?{urllib.parse.urlencode(q)}'
488 qs = f'?{urllib.parse.urlencode(q)}'
489 cu = f"{test_uri}{qs}"
489 cu = f"{test_uri}{qs}"
490
490
491 try:
491 try:
492 req = urllib.request.Request(cu, None, {})
492 req = urllib.request.Request(cu, None, {})
493 log.debug("Trying to open URL %s", obfuscated_uri)
493 log.debug("Trying to open URL %s", obfuscated_uri)
494 resp = o.open(req)
494 resp = o.open(req)
495 if resp.code != 200:
495 if resp.code != 200:
496 raise exceptions.URLError()('Return Code is not 200')
496 raise exceptions.URLError()('Return Code is not 200')
497 except Exception as e:
497 except Exception as e:
498 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
498 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
499 # means it cannot be cloned
499 # means it cannot be cloned
500 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
500 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
501
501
502 # now detect if it's proper git repo
502 # now detect if it's proper git repo
503 gitdata: bytes = resp.read()
503 gitdata: bytes = resp.read()
504
504
505 if b'service=git-upload-pack' in gitdata:
505 if b'service=git-upload-pack' in gitdata:
506 pass
506 pass
507 elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
507 elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
508 # old style git can return some other format!
508 # old style git can return some other format!
509 pass
509 pass
510 else:
510 else:
511 e = None
511 e = None
512 raise exceptions.URLError(e)(
512 raise exceptions.URLError(e)(
513 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
513 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
514
514
515 return True
515 return True
516
516
517 @reraise_safe_exceptions
517 @reraise_safe_exceptions
518 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
518 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
519 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
519 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
520 remote_refs = self.pull(wire, url, apply_refs=False)
520 remote_refs = self.pull(wire, url, apply_refs=False)
521 repo = self._factory.repo(wire)
521 repo = self._factory.repo(wire)
522 if isinstance(valid_refs, list):
522 if isinstance(valid_refs, list):
523 valid_refs = tuple(valid_refs)
523 valid_refs = tuple(valid_refs)
524
524
525 for k in remote_refs:
525 for k in remote_refs:
526 # only parse heads/tags and skip so called deferred tags
526 # only parse heads/tags and skip so called deferred tags
527 if k.startswith(valid_refs) and not k.endswith(deferred):
527 if k.startswith(valid_refs) and not k.endswith(deferred):
528 repo[k] = remote_refs[k]
528 repo[k] = remote_refs[k]
529
529
530 if update_after_clone:
530 if update_after_clone:
531 # we want to checkout HEAD
531 # we want to checkout HEAD
532 repo["HEAD"] = remote_refs["HEAD"]
532 repo["HEAD"] = remote_refs["HEAD"]
533 index.build_index_from_tree(repo.path, repo.index_path(),
533 index.build_index_from_tree(repo.path, repo.index_path(),
534 repo.object_store, repo["HEAD"].tree)
534 repo.object_store, repo["HEAD"].tree)
535
535
536 @reraise_safe_exceptions
536 @reraise_safe_exceptions
537 def branch(self, wire, commit_id):
537 def branch(self, wire, commit_id):
538 cache_on, context_uid, repo_id = self._cache_on(wire)
538 cache_on, context_uid, repo_id = self._cache_on(wire)
539 region = self._region(wire)
539 region = self._region(wire)
540
540
541 @region.conditional_cache_on_arguments(condition=cache_on)
541 @region.conditional_cache_on_arguments(condition=cache_on)
542 def _branch(_context_uid, _repo_id, _commit_id):
542 def _branch(_context_uid, _repo_id, _commit_id):
543 regex = re.compile('^refs/heads')
543 regex = re.compile('^refs/heads')
544
544
545 def filter_with(ref):
545 def filter_with(ref):
546 return regex.match(ref[0]) and ref[1] == _commit_id
546 return regex.match(ref[0]) and ref[1] == _commit_id
547
547
548 branches = list(filter(filter_with, list(self.get_refs(wire).items())))
548 branches = list(filter(filter_with, list(self.get_refs(wire).items())))
549 return [x[0].split('refs/heads/')[-1] for x in branches]
549 return [x[0].split('refs/heads/')[-1] for x in branches]
550
550
551 return _branch(context_uid, repo_id, commit_id)
551 return _branch(context_uid, repo_id, commit_id)
552
552
553 @reraise_safe_exceptions
553 @reraise_safe_exceptions
554 def delete_branch(self, wire, branch_name):
555 repo_init = self._factory.repo_libgit2(wire)
556 with repo_init as repo:
557 if branch := repo.lookup_branch(branch_name):
558 branch.delete()
559
560 @reraise_safe_exceptions
554 def commit_branches(self, wire, commit_id):
561 def commit_branches(self, wire, commit_id):
555 cache_on, context_uid, repo_id = self._cache_on(wire)
562 cache_on, context_uid, repo_id = self._cache_on(wire)
556 region = self._region(wire)
563 region = self._region(wire)
557
564
558 @region.conditional_cache_on_arguments(condition=cache_on)
565 @region.conditional_cache_on_arguments(condition=cache_on)
559 def _commit_branches(_context_uid, _repo_id, _commit_id):
566 def _commit_branches(_context_uid, _repo_id, _commit_id):
560 repo_init = self._factory.repo_libgit2(wire)
567 repo_init = self._factory.repo_libgit2(wire)
561 with repo_init as repo:
568 with repo_init as repo:
562 branches = [x for x in repo.branches.with_commit(_commit_id)]
569 branches = [x for x in repo.branches.with_commit(_commit_id)]
563 return branches
570 return branches
564
571
565 return _commit_branches(context_uid, repo_id, commit_id)
572 return _commit_branches(context_uid, repo_id, commit_id)
566
573
567 @reraise_safe_exceptions
574 @reraise_safe_exceptions
568 def add_object(self, wire, content):
575 def add_object(self, wire, content):
569 repo_init = self._factory.repo_libgit2(wire)
576 repo_init = self._factory.repo_libgit2(wire)
570 with repo_init as repo:
577 with repo_init as repo:
571 blob = objects.Blob()
578 blob = objects.Blob()
572 blob.set_raw_string(content)
579 blob.set_raw_string(content)
573 repo.object_store.add_object(blob)
580 repo.object_store.add_object(blob)
574 return blob.id
581 return blob.id
575
582
576 @reraise_safe_exceptions
583 @reraise_safe_exceptions
577 def create_commit(self, wire, author, committer, message, branch, new_tree_id,
584 def create_commit(self, wire, author, committer, message, branch, new_tree_id,
578 date_args: list[int, int] = None,
585 date_args: list[int, int] = None,
579 parents: list | None = None):
586 parents: list | None = None):
580
587
581 repo_init = self._factory.repo_libgit2(wire)
588 repo_init = self._factory.repo_libgit2(wire)
582 with repo_init as repo:
589 with repo_init as repo:
583
590
584 if date_args:
591 if date_args:
585 current_time, offset = date_args
592 current_time, offset = date_args
586
593
587 kw = {
594 kw = {
588 'time': current_time,
595 'time': current_time,
589 'offset': offset
596 'offset': offset
590 }
597 }
591 author = create_signature_from_string(author, **kw)
598 author = create_signature_from_string(author, **kw)
592 committer = create_signature_from_string(committer, **kw)
599 committer = create_signature_from_string(committer, **kw)
593
600
594 tree = new_tree_id
601 tree = new_tree_id
595 if isinstance(tree, (bytes, str)):
602 if isinstance(tree, (bytes, str)):
596 # validate this tree is in the repo...
603 # validate this tree is in the repo...
597 tree = repo[safe_str(tree)].id
604 tree = repo[safe_str(tree)].id
598
605
599 if parents:
606 if parents:
600 # run via sha's and validate them in repo
607 # run via sha's and validate them in repo
601 parents = [repo[c].id for c in parents]
608 parents = [repo[c].id for c in parents]
602 else:
609 else:
603 parents = []
610 parents = []
604 # ensure we COMMIT on top of given branch head
611 # ensure we COMMIT on top of given branch head
605 # check if this repo has ANY branches, otherwise it's a new branch case we need to make
612 # check if this repo has ANY branches, otherwise it's a new branch case we need to make
606 if branch in repo.branches.local:
613 if branch in repo.branches.local:
607 parents += [repo.branches[branch].target]
614 parents += [repo.branches[branch].target]
608 elif [x for x in repo.branches.local]:
615 elif [x for x in repo.branches.local]:
609 parents += [repo.head.target]
616 parents += [repo.head.target]
610 #else:
617 #else:
611 # in case we want to commit on new branch we create it on top of HEAD
618 # in case we want to commit on new branch we create it on top of HEAD
612 #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
619 #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
613
620
614 # # Create a new commit
621 # # Create a new commit
615 commit_oid = repo.create_commit(
622 commit_oid = repo.create_commit(
616 f'refs/heads/{branch}', # the name of the reference to update
623 f'refs/heads/{branch}', # the name of the reference to update
617 author, # the author of the commit
624 author, # the author of the commit
618 committer, # the committer of the commit
625 committer, # the committer of the commit
619 message, # the commit message
626 message, # the commit message
620 tree, # the tree produced by the index
627 tree, # the tree produced by the index
621 parents # list of parents for the new commit, usually just one,
628 parents # list of parents for the new commit, usually just one,
622 )
629 )
623
630
624 new_commit_id = safe_str(commit_oid)
631 new_commit_id = safe_str(commit_oid)
625
632
626 return new_commit_id
633 return new_commit_id
627
634
628 @reraise_safe_exceptions
635 @reraise_safe_exceptions
629 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
636 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
630
637
631 def mode2pygit(mode):
638 def mode2pygit(mode):
632 """
639 """
633 git only supports two filemode 644 and 755
640 git only supports two filemode 644 and 755
634
641
635 0o100755 -> 33261
642 0o100755 -> 33261
636 0o100644 -> 33188
643 0o100644 -> 33188
637 """
644 """
638 return {
645 return {
639 0o100644: pygit2.GIT_FILEMODE_BLOB,
646 0o100644: pygit2.GIT_FILEMODE_BLOB,
640 0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
647 0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
641 0o120000: pygit2.GIT_FILEMODE_LINK
648 0o120000: pygit2.GIT_FILEMODE_LINK
642 }.get(mode) or pygit2.GIT_FILEMODE_BLOB
649 }.get(mode) or pygit2.GIT_FILEMODE_BLOB
643
650
644 repo_init = self._factory.repo_libgit2(wire)
651 repo_init = self._factory.repo_libgit2(wire)
645 with repo_init as repo:
652 with repo_init as repo:
646 repo_index = repo.index
653 repo_index = repo.index
647
654
648 commit_parents = None
655 commit_parents = None
649 if commit_tree and commit_data['parents']:
656 if commit_tree and commit_data['parents']:
650 commit_parents = commit_data['parents']
657 commit_parents = commit_data['parents']
651 parent_commit = repo[commit_parents[0]]
658 parent_commit = repo[commit_parents[0]]
652 repo_index.read_tree(parent_commit.tree)
659 repo_index.read_tree(parent_commit.tree)
653
660
654 for pathspec in updated:
661 for pathspec in updated:
655 blob_id = repo.create_blob(pathspec['content'])
662 blob_id = repo.create_blob(pathspec['content'])
656 ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
663 ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
657 repo_index.add(ie)
664 repo_index.add(ie)
658
665
659 for pathspec in removed:
666 for pathspec in removed:
660 repo_index.remove(pathspec)
667 repo_index.remove(pathspec)
661
668
662 # Write changes to the index
669 # Write changes to the index
663 repo_index.write()
670 repo_index.write()
664
671
665 # Create a tree from the updated index
672 # Create a tree from the updated index
666 written_commit_tree = repo_index.write_tree()
673 written_commit_tree = repo_index.write_tree()
667
674
668 new_tree_id = written_commit_tree
675 new_tree_id = written_commit_tree
669
676
670 author = commit_data['author']
677 author = commit_data['author']
671 committer = commit_data['committer']
678 committer = commit_data['committer']
672 message = commit_data['message']
679 message = commit_data['message']
673
680
674 date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
681 date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
675
682
676 new_commit_id = self.create_commit(wire, author, committer, message, branch,
683 new_commit_id = self.create_commit(wire, author, committer, message, branch,
677 new_tree_id, date_args=date_args, parents=commit_parents)
684 new_tree_id, date_args=date_args, parents=commit_parents)
678
685
679 # libgit2, ensure the branch is there and exists
686 # libgit2, ensure the branch is there and exists
680 self.create_branch(wire, branch, new_commit_id)
687 self.create_branch(wire, branch, new_commit_id)
681
688
682 # libgit2, set new ref to this created commit
689 # libgit2, set new ref to this created commit
683 self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
690 self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
684
691
685 return new_commit_id
692 return new_commit_id
686
693
687 @reraise_safe_exceptions
694 @reraise_safe_exceptions
688 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
695 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
689 if url != 'default' and '://' not in url:
696 if url != 'default' and '://' not in url:
690 client = LocalGitClient(url)
697 client = LocalGitClient(url)
691 else:
698 else:
692 url_obj = url_parser(safe_bytes(url))
699 url_obj = url_parser(safe_bytes(url))
693 o = self._build_opener(url)
700 o = self._build_opener(url)
694 url = url_obj.authinfo()[0]
701 url = url_obj.authinfo()[0]
695 client = HttpGitClient(base_url=url, opener=o)
702 client = HttpGitClient(base_url=url, opener=o)
696 repo = self._factory.repo(wire)
703 repo = self._factory.repo(wire)
697
704
698 determine_wants = repo.object_store.determine_wants_all
705 determine_wants = repo.object_store.determine_wants_all
699
706
700 if refs:
707 if refs:
701 refs: list[bytes] = [ascii_bytes(x) for x in refs]
708 refs: list[bytes] = [ascii_bytes(x) for x in refs]
702
709
703 def determine_wants_requested(_remote_refs):
710 def determine_wants_requested(_remote_refs):
704 determined = []
711 determined = []
705 for ref_name, ref_hash in _remote_refs.items():
712 for ref_name, ref_hash in _remote_refs.items():
706 bytes_ref_name = safe_bytes(ref_name)
713 bytes_ref_name = safe_bytes(ref_name)
707
714
708 if bytes_ref_name in refs:
715 if bytes_ref_name in refs:
709 bytes_ref_hash = safe_bytes(ref_hash)
716 bytes_ref_hash = safe_bytes(ref_hash)
710 determined.append(bytes_ref_hash)
717 determined.append(bytes_ref_hash)
711 return determined
718 return determined
712
719
713 # swap with our custom requested wants
720 # swap with our custom requested wants
714 determine_wants = determine_wants_requested
721 determine_wants = determine_wants_requested
715
722
716 try:
723 try:
717 remote_refs = client.fetch(
724 remote_refs = client.fetch(
718 path=url, target=repo, determine_wants=determine_wants)
725 path=url, target=repo, determine_wants=determine_wants)
719
726
720 except NotGitRepository as e:
727 except NotGitRepository as e:
721 log.warning(
728 log.warning(
722 'Trying to fetch from "%s" failed, not a Git repository.', url)
729 'Trying to fetch from "%s" failed, not a Git repository.', url)
723 # Exception can contain unicode which we convert
730 # Exception can contain unicode which we convert
724 raise exceptions.AbortException(e)(repr(e))
731 raise exceptions.AbortException(e)(repr(e))
725
732
726 # mikhail: client.fetch() returns all the remote refs, but fetches only
733 # mikhail: client.fetch() returns all the remote refs, but fetches only
727 # refs filtered by `determine_wants` function. We need to filter result
734 # refs filtered by `determine_wants` function. We need to filter result
728 # as well
735 # as well
729 if refs:
736 if refs:
730 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
737 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
731
738
732 if apply_refs:
739 if apply_refs:
733 # TODO: johbo: Needs proper test coverage with a git repository
740 # TODO: johbo: Needs proper test coverage with a git repository
734 # that contains a tag object, so that we would end up with
741 # that contains a tag object, so that we would end up with
735 # a peeled ref at this point.
742 # a peeled ref at this point.
736 for k in remote_refs:
743 for k in remote_refs:
737 if k.endswith(PEELED_REF_MARKER):
744 if k.endswith(PEELED_REF_MARKER):
738 log.debug("Skipping peeled reference %s", k)
745 log.debug("Skipping peeled reference %s", k)
739 continue
746 continue
740 repo[k] = remote_refs[k]
747 repo[k] = remote_refs[k]
741
748
742 if refs and not update_after:
749 if refs and not update_after:
743 # update to ref
750 # update to ref
744 # mikhail: explicitly set the head to the last ref.
751 # mikhail: explicitly set the head to the last ref.
745 update_to_ref = refs[-1]
752 update_to_ref = refs[-1]
746 if isinstance(update_after, str):
753 if isinstance(update_after, str):
747 update_to_ref = update_after
754 update_to_ref = update_after
748
755
749 repo[HEAD_MARKER] = remote_refs[update_to_ref]
756 repo[HEAD_MARKER] = remote_refs[update_to_ref]
750
757
751 if update_after:
758 if update_after:
752 # we want to check out HEAD
759 # we want to check out HEAD
753 repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
760 repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
754 index.build_index_from_tree(repo.path, repo.index_path(),
761 index.build_index_from_tree(repo.path, repo.index_path(),
755 repo.object_store, repo[HEAD_MARKER].tree)
762 repo.object_store, repo[HEAD_MARKER].tree)
756
763
757 if isinstance(remote_refs, FetchPackResult):
764 if isinstance(remote_refs, FetchPackResult):
758 return remote_refs.refs
765 return remote_refs.refs
759 return remote_refs
766 return remote_refs
760
767
761 @reraise_safe_exceptions
768 @reraise_safe_exceptions
762 def sync_fetch(self, wire, url, refs=None, all_refs=False, **kwargs):
769 def sync_fetch(self, wire, url, refs=None, all_refs=False, **kwargs):
763 self._factory.repo(wire)
770 self._factory.repo(wire)
764 if refs and not isinstance(refs, (list, tuple)):
771 if refs and not isinstance(refs, (list, tuple)):
765 refs = [refs]
772 refs = [refs]
766
773
767 config = self._wire_to_config(wire)
774 config = self._wire_to_config(wire)
768 # get all remote refs we'll use to fetch later
775 # get all remote refs we'll use to fetch later
769 cmd = ['ls-remote']
776 cmd = ['ls-remote']
770 if not all_refs:
777 if not all_refs:
771 cmd += ['--heads', '--tags']
778 cmd += ['--heads', '--tags']
772 cmd += [url]
779 cmd += [url]
773 output, __ = self.run_git_command(
780 output, __ = self.run_git_command(
774 wire, cmd, fail_on_stderr=False,
781 wire, cmd, fail_on_stderr=False,
775 _copts=self._remote_conf(config),
782 _copts=self._remote_conf(config),
776 extra_env={'GIT_TERMINAL_PROMPT': '0'})
783 extra_env={'GIT_TERMINAL_PROMPT': '0'})
777
784
778 remote_refs = collections.OrderedDict()
785 remote_refs = collections.OrderedDict()
779 fetch_refs = []
786 fetch_refs = []
780
787
781 for ref_line in output.splitlines():
788 for ref_line in output.splitlines():
782 sha, ref = ref_line.split(b'\t')
789 sha, ref = ref_line.split(b'\t')
783 sha = sha.strip()
790 sha = sha.strip()
784 if ref in remote_refs:
791 if ref in remote_refs:
785 # duplicate, skip
792 # duplicate, skip
786 continue
793 continue
787 if ref.endswith(PEELED_REF_MARKER):
794 if ref.endswith(PEELED_REF_MARKER):
788 log.debug("Skipping peeled reference %s", ref)
795 log.debug("Skipping peeled reference %s", ref)
789 continue
796 continue
790 # don't sync HEAD
797 # don't sync HEAD
791 if ref in [HEAD_MARKER]:
798 if ref in [HEAD_MARKER]:
792 continue
799 continue
793
800
794 remote_refs[ref] = sha
801 remote_refs[ref] = sha
795
802
796 if refs and sha in refs:
803 if refs and sha in refs:
797 # we filter fetch using our specified refs
804 # we filter fetch using our specified refs
798 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
805 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
799 elif not refs:
806 elif not refs:
800 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
807 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
801 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
808 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
802
809
803 if fetch_refs:
810 if fetch_refs:
804 for chunk in more_itertools.chunked(fetch_refs, 128):
811 for chunk in more_itertools.chunked(fetch_refs, 128):
805 fetch_refs_chunks = list(chunk)
812 fetch_refs_chunks = list(chunk)
806 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
813 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
807 self.run_git_command(
814 self.run_git_command(
808 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
815 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
809 fail_on_stderr=False,
816 fail_on_stderr=False,
810 _copts=self._remote_conf(config),
817 _copts=self._remote_conf(config),
811 extra_env={'GIT_TERMINAL_PROMPT': '0'})
818 extra_env={'GIT_TERMINAL_PROMPT': '0'})
812 if kwargs.get('sync_large_objects'):
819 if kwargs.get('sync_large_objects'):
813 self.run_git_command(
820 self.run_git_command(
814 wire, ['lfs', 'fetch', url, '--all'],
821 wire, ['lfs', 'fetch', url, '--all'],
815 fail_on_stderr=False,
822 fail_on_stderr=False,
816 _copts=self._remote_conf(config),
823 _copts=self._remote_conf(config),
817 )
824 )
818
825
819 return remote_refs
826 return remote_refs
820
827
821 @reraise_safe_exceptions
828 @reraise_safe_exceptions
822 def sync_push(self, wire, url, refs=None, **kwargs):
829 def sync_push(self, wire, url, refs=None, **kwargs):
823 if not self.check_url(url, wire):
830 if not self.check_url(url, wire):
824 return
831 return
825 config = self._wire_to_config(wire)
832 config = self._wire_to_config(wire)
826 self._factory.repo(wire)
833 self._factory.repo(wire)
827 self.run_git_command(
834 self.run_git_command(
828 wire, ['push', url, '--mirror'], fail_on_stderr=False,
835 wire, ['push', url, '--mirror'], fail_on_stderr=False,
829 _copts=self._remote_conf(config),
836 _copts=self._remote_conf(config),
830 extra_env={'GIT_TERMINAL_PROMPT': '0'})
837 extra_env={'GIT_TERMINAL_PROMPT': '0'})
831 if kwargs.get('sync_large_objects'):
838 if kwargs.get('sync_large_objects'):
832 self.run_git_command(
839 self.run_git_command(
833 wire, ['lfs', 'push', url, '--all'],
840 wire, ['lfs', 'push', url, '--all'],
834 fail_on_stderr=False,
841 fail_on_stderr=False,
835 _copts=self._remote_conf(config),
842 _copts=self._remote_conf(config),
836 )
843 )
837
844
838 @reraise_safe_exceptions
845 @reraise_safe_exceptions
839 def get_remote_refs(self, wire, url):
846 def get_remote_refs(self, wire, url):
840 repo = Repo(url)
847 repo = Repo(url)
841 return repo.get_refs()
848 return repo.get_refs()
842
849
843 @reraise_safe_exceptions
850 @reraise_safe_exceptions
844 def get_description(self, wire):
851 def get_description(self, wire):
845 repo = self._factory.repo(wire)
852 repo = self._factory.repo(wire)
846 return repo.get_description()
853 return repo.get_description()
847
854
848 @reraise_safe_exceptions
855 @reraise_safe_exceptions
849 def get_missing_revs(self, wire, rev1, rev2, other_repo_path):
856 def get_missing_revs(self, wire, rev1, rev2, other_repo_path):
850 origin_repo_path = wire['path']
857 origin_repo_path = wire['path']
851 repo = self._factory.repo(wire)
858 repo = self._factory.repo(wire)
852 # fetch from other_repo_path to our origin repo
859 # fetch from other_repo_path to our origin repo
853 LocalGitClient(thin_packs=False).fetch(other_repo_path, repo)
860 LocalGitClient(thin_packs=False).fetch(other_repo_path, repo)
854
861
855 wire_remote = wire.copy()
862 wire_remote = wire.copy()
856 wire_remote['path'] = other_repo_path
863 wire_remote['path'] = other_repo_path
857 repo_remote = self._factory.repo(wire_remote)
864 repo_remote = self._factory.repo(wire_remote)
858
865
859 # fetch from origin_repo_path to our remote repo
866 # fetch from origin_repo_path to our remote repo
860 LocalGitClient(thin_packs=False).fetch(origin_repo_path, repo_remote)
867 LocalGitClient(thin_packs=False).fetch(origin_repo_path, repo_remote)
861
868
862 revs = [
869 revs = [
863 x.commit.id
870 x.commit.id
864 for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
871 for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
865 return revs
872 return revs
866
873
867 @reraise_safe_exceptions
874 @reraise_safe_exceptions
868 def get_object(self, wire, sha, maybe_unreachable=False):
875 def get_object(self, wire, sha, maybe_unreachable=False):
869 cache_on, context_uid, repo_id = self._cache_on(wire)
876 cache_on, context_uid, repo_id = self._cache_on(wire)
870 region = self._region(wire)
877 region = self._region(wire)
871
878
872 @region.conditional_cache_on_arguments(condition=cache_on)
879 @region.conditional_cache_on_arguments(condition=cache_on)
873 def _get_object(_context_uid, _repo_id, _sha):
880 def _get_object(_context_uid, _repo_id, _sha):
874 repo_init = self._factory.repo_libgit2(wire)
881 repo_init = self._factory.repo_libgit2(wire)
875 with repo_init as repo:
882 with repo_init as repo:
876
883
877 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
884 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
878 try:
885 try:
879 commit = repo.revparse_single(sha)
886 commit = repo.revparse_single(sha)
880 except KeyError:
887 except KeyError:
881 # NOTE(marcink): KeyError doesn't give us any meaningful information
888 # NOTE(marcink): KeyError doesn't give us any meaningful information
882 # here, we instead give something more explicit
889 # here, we instead give something more explicit
883 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
890 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
884 raise exceptions.LookupException(e)(missing_commit_err)
891 raise exceptions.LookupException(e)(missing_commit_err)
885 except ValueError as e:
892 except ValueError as e:
886 raise exceptions.LookupException(e)(missing_commit_err)
893 raise exceptions.LookupException(e)(missing_commit_err)
887
894
888 is_tag = False
895 is_tag = False
889 if isinstance(commit, pygit2.Tag):
896 if isinstance(commit, pygit2.Tag):
890 commit = repo.get(commit.target)
897 commit = repo.get(commit.target)
891 is_tag = True
898 is_tag = True
892
899
893 check_dangling = True
900 check_dangling = True
894 if is_tag:
901 if is_tag:
895 check_dangling = False
902 check_dangling = False
896
903
897 if check_dangling and maybe_unreachable:
904 if check_dangling and maybe_unreachable:
898 check_dangling = False
905 check_dangling = False
899
906
900 # we used a reference and it parsed means we're not having a dangling commit
907 # we used a reference and it parsed means we're not having a dangling commit
901 if sha != commit.hex:
908 if sha != commit.hex:
902 check_dangling = False
909 check_dangling = False
903
910
904 if check_dangling:
911 if check_dangling:
905 # check for dangling commit
912 # check for dangling commit
906 for branch in repo.branches.with_commit(commit.hex):
913 for branch in repo.branches.with_commit(commit.hex):
907 if branch:
914 if branch:
908 break
915 break
909 else:
916 else:
910 # NOTE(marcink): Empty error doesn't give us any meaningful information
917 # NOTE(marcink): Empty error doesn't give us any meaningful information
911 # here, we instead give something more explicit
918 # here, we instead give something more explicit
912 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
919 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
913 raise exceptions.LookupException(e)(missing_commit_err)
920 raise exceptions.LookupException(e)(missing_commit_err)
914
921
915 commit_id = commit.hex
922 commit_id = commit.hex
916 type_str = commit.type_str
923 type_str = commit.type_str
917
924
918 return {
925 return {
919 'id': commit_id,
926 'id': commit_id,
920 'type': type_str,
927 'type': type_str,
921 'commit_id': commit_id,
928 'commit_id': commit_id,
922 'idx': 0
929 'idx': 0
923 }
930 }
924
931
925 return _get_object(context_uid, repo_id, sha)
932 return _get_object(context_uid, repo_id, sha)
926
933
927 @reraise_safe_exceptions
934 @reraise_safe_exceptions
928 def get_refs(self, wire):
935 def get_refs(self, wire):
929 cache_on, context_uid, repo_id = self._cache_on(wire)
936 cache_on, context_uid, repo_id = self._cache_on(wire)
930 region = self._region(wire)
937 region = self._region(wire)
931
938
932 @region.conditional_cache_on_arguments(condition=cache_on)
939 @region.conditional_cache_on_arguments(condition=cache_on)
933 def _get_refs(_context_uid, _repo_id):
940 def _get_refs(_context_uid, _repo_id):
934
941
935 repo_init = self._factory.repo_libgit2(wire)
942 repo_init = self._factory.repo_libgit2(wire)
936 with repo_init as repo:
943 with repo_init as repo:
937 regex = re.compile('^refs/(heads|tags)/')
944 regex = re.compile('^refs/(heads|tags)/')
938 return {x.name: x.target.hex for x in
945 return {x.name: x.target.hex for x in
939 [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
946 [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
940
947
941 return _get_refs(context_uid, repo_id)
948 return _get_refs(context_uid, repo_id)
942
949
943 @reraise_safe_exceptions
950 @reraise_safe_exceptions
944 def get_branch_pointers(self, wire):
951 def get_branch_pointers(self, wire):
945 cache_on, context_uid, repo_id = self._cache_on(wire)
952 cache_on, context_uid, repo_id = self._cache_on(wire)
946 region = self._region(wire)
953 region = self._region(wire)
947
954
948 @region.conditional_cache_on_arguments(condition=cache_on)
955 @region.conditional_cache_on_arguments(condition=cache_on)
949 def _get_branch_pointers(_context_uid, _repo_id):
956 def _get_branch_pointers(_context_uid, _repo_id):
950
957
951 repo_init = self._factory.repo_libgit2(wire)
958 repo_init = self._factory.repo_libgit2(wire)
952 regex = re.compile('^refs/heads')
959 regex = re.compile('^refs/heads')
953 with repo_init as repo:
960 with repo_init as repo:
954 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
961 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
955 return {x.target.hex: x.shorthand for x in branches}
962 return {x.target.hex: x.shorthand for x in branches}
956
963
957 return _get_branch_pointers(context_uid, repo_id)
964 return _get_branch_pointers(context_uid, repo_id)
958
965
959 @reraise_safe_exceptions
966 @reraise_safe_exceptions
960 def head(self, wire, show_exc=True):
967 def head(self, wire, show_exc=True):
961 cache_on, context_uid, repo_id = self._cache_on(wire)
968 cache_on, context_uid, repo_id = self._cache_on(wire)
962 region = self._region(wire)
969 region = self._region(wire)
963
970
964 @region.conditional_cache_on_arguments(condition=cache_on)
971 @region.conditional_cache_on_arguments(condition=cache_on)
965 def _head(_context_uid, _repo_id, _show_exc):
972 def _head(_context_uid, _repo_id, _show_exc):
966 repo_init = self._factory.repo_libgit2(wire)
973 repo_init = self._factory.repo_libgit2(wire)
967 with repo_init as repo:
974 with repo_init as repo:
968 try:
975 try:
969 return repo.head.peel().hex
976 return repo.head.peel().hex
970 except Exception:
977 except Exception:
971 if show_exc:
978 if show_exc:
972 raise
979 raise
973 return _head(context_uid, repo_id, show_exc)
980 return _head(context_uid, repo_id, show_exc)
974
981
975 @reraise_safe_exceptions
982 @reraise_safe_exceptions
976 def init(self, wire):
983 def init(self, wire):
977 repo_path = safe_str(wire['path'])
984 repo_path = safe_str(wire['path'])
978 os.makedirs(repo_path, mode=0o755)
985 os.makedirs(repo_path, mode=0o755)
979 pygit2.init_repository(repo_path, bare=False)
986 pygit2.init_repository(repo_path, bare=False)
980
987
981 @reraise_safe_exceptions
988 @reraise_safe_exceptions
982 def init_bare(self, wire):
989 def init_bare(self, wire):
983 repo_path = safe_str(wire['path'])
990 repo_path = safe_str(wire['path'])
984 os.makedirs(repo_path, mode=0o755)
991 os.makedirs(repo_path, mode=0o755)
985 pygit2.init_repository(repo_path, bare=True)
992 pygit2.init_repository(repo_path, bare=True)
986
993
987 @reraise_safe_exceptions
994 @reraise_safe_exceptions
988 def revision(self, wire, rev):
995 def revision(self, wire, rev):
989
996
990 cache_on, context_uid, repo_id = self._cache_on(wire)
997 cache_on, context_uid, repo_id = self._cache_on(wire)
991 region = self._region(wire)
998 region = self._region(wire)
992
999
993 @region.conditional_cache_on_arguments(condition=cache_on)
1000 @region.conditional_cache_on_arguments(condition=cache_on)
994 def _revision(_context_uid, _repo_id, _rev):
1001 def _revision(_context_uid, _repo_id, _rev):
995 repo_init = self._factory.repo_libgit2(wire)
1002 repo_init = self._factory.repo_libgit2(wire)
996 with repo_init as repo:
1003 with repo_init as repo:
997 commit = repo[rev]
1004 commit = repo[rev]
998 obj_data = {
1005 obj_data = {
999 'id': commit.id.hex,
1006 'id': commit.id.hex,
1000 }
1007 }
1001 # tree objects itself don't have tree_id attribute
1008 # tree objects itself don't have tree_id attribute
1002 if hasattr(commit, 'tree_id'):
1009 if hasattr(commit, 'tree_id'):
1003 obj_data['tree'] = commit.tree_id.hex
1010 obj_data['tree'] = commit.tree_id.hex
1004
1011
1005 return obj_data
1012 return obj_data
1006 return _revision(context_uid, repo_id, rev)
1013 return _revision(context_uid, repo_id, rev)
1007
1014
1008 @reraise_safe_exceptions
1015 @reraise_safe_exceptions
1009 def date(self, wire, commit_id):
1016 def date(self, wire, commit_id):
1010 cache_on, context_uid, repo_id = self._cache_on(wire)
1017 cache_on, context_uid, repo_id = self._cache_on(wire)
1011 region = self._region(wire)
1018 region = self._region(wire)
1012
1019
1013 @region.conditional_cache_on_arguments(condition=cache_on)
1020 @region.conditional_cache_on_arguments(condition=cache_on)
1014 def _date(_repo_id, _commit_id):
1021 def _date(_repo_id, _commit_id):
1015 repo_init = self._factory.repo_libgit2(wire)
1022 repo_init = self._factory.repo_libgit2(wire)
1016 with repo_init as repo:
1023 with repo_init as repo:
1017 commit = repo[commit_id]
1024 commit = repo[commit_id]
1018
1025
1019 if hasattr(commit, 'commit_time'):
1026 if hasattr(commit, 'commit_time'):
1020 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
1027 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
1021 else:
1028 else:
1022 commit = commit.get_object()
1029 commit = commit.get_object()
1023 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
1030 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
1024
1031
1025 # TODO(marcink): check dulwich difference of offset vs timezone
1032 # TODO(marcink): check dulwich difference of offset vs timezone
1026 return [commit_time, commit_time_offset]
1033 return [commit_time, commit_time_offset]
1027 return _date(repo_id, commit_id)
1034 return _date(repo_id, commit_id)
1028
1035
1029 @reraise_safe_exceptions
1036 @reraise_safe_exceptions
1030 def author(self, wire, commit_id):
1037 def author(self, wire, commit_id):
1031 cache_on, context_uid, repo_id = self._cache_on(wire)
1038 cache_on, context_uid, repo_id = self._cache_on(wire)
1032 region = self._region(wire)
1039 region = self._region(wire)
1033
1040
1034 @region.conditional_cache_on_arguments(condition=cache_on)
1041 @region.conditional_cache_on_arguments(condition=cache_on)
1035 def _author(_repo_id, _commit_id):
1042 def _author(_repo_id, _commit_id):
1036 repo_init = self._factory.repo_libgit2(wire)
1043 repo_init = self._factory.repo_libgit2(wire)
1037 with repo_init as repo:
1044 with repo_init as repo:
1038 commit = repo[commit_id]
1045 commit = repo[commit_id]
1039
1046
1040 if hasattr(commit, 'author'):
1047 if hasattr(commit, 'author'):
1041 author = commit.author
1048 author = commit.author
1042 else:
1049 else:
1043 author = commit.get_object().author
1050 author = commit.get_object().author
1044
1051
1045 if author.email:
1052 if author.email:
1046 return f"{author.name} <{author.email}>"
1053 return f"{author.name} <{author.email}>"
1047
1054
1048 try:
1055 try:
1049 return f"{author.name}"
1056 return f"{author.name}"
1050 except Exception:
1057 except Exception:
1051 return f"{safe_str(author.raw_name)}"
1058 return f"{safe_str(author.raw_name)}"
1052
1059
1053 return _author(repo_id, commit_id)
1060 return _author(repo_id, commit_id)
1054
1061
1055 @reraise_safe_exceptions
1062 @reraise_safe_exceptions
1056 def message(self, wire, commit_id):
1063 def message(self, wire, commit_id):
1057 cache_on, context_uid, repo_id = self._cache_on(wire)
1064 cache_on, context_uid, repo_id = self._cache_on(wire)
1058 region = self._region(wire)
1065 region = self._region(wire)
1059
1066
1060 @region.conditional_cache_on_arguments(condition=cache_on)
1067 @region.conditional_cache_on_arguments(condition=cache_on)
1061 def _message(_repo_id, _commit_id):
1068 def _message(_repo_id, _commit_id):
1062 repo_init = self._factory.repo_libgit2(wire)
1069 repo_init = self._factory.repo_libgit2(wire)
1063 with repo_init as repo:
1070 with repo_init as repo:
1064 commit = repo[commit_id]
1071 commit = repo[commit_id]
1065 return commit.message
1072 return commit.message
1066 return _message(repo_id, commit_id)
1073 return _message(repo_id, commit_id)
1067
1074
1068 @reraise_safe_exceptions
1075 @reraise_safe_exceptions
1069 def parents(self, wire, commit_id):
1076 def parents(self, wire, commit_id):
1070 cache_on, context_uid, repo_id = self._cache_on(wire)
1077 cache_on, context_uid, repo_id = self._cache_on(wire)
1071 region = self._region(wire)
1078 region = self._region(wire)
1072
1079
1073 @region.conditional_cache_on_arguments(condition=cache_on)
1080 @region.conditional_cache_on_arguments(condition=cache_on)
1074 def _parents(_repo_id, _commit_id):
1081 def _parents(_repo_id, _commit_id):
1075 repo_init = self._factory.repo_libgit2(wire)
1082 repo_init = self._factory.repo_libgit2(wire)
1076 with repo_init as repo:
1083 with repo_init as repo:
1077 commit = repo[commit_id]
1084 commit = repo[commit_id]
1078 if hasattr(commit, 'parent_ids'):
1085 if hasattr(commit, 'parent_ids'):
1079 parent_ids = commit.parent_ids
1086 parent_ids = commit.parent_ids
1080 else:
1087 else:
1081 parent_ids = commit.get_object().parent_ids
1088 parent_ids = commit.get_object().parent_ids
1082
1089
1083 return [x.hex for x in parent_ids]
1090 return [x.hex for x in parent_ids]
1084 return _parents(repo_id, commit_id)
1091 return _parents(repo_id, commit_id)
1085
1092
1086 @reraise_safe_exceptions
1093 @reraise_safe_exceptions
1087 def children(self, wire, commit_id):
1094 def children(self, wire, commit_id):
1088 cache_on, context_uid, repo_id = self._cache_on(wire)
1095 cache_on, context_uid, repo_id = self._cache_on(wire)
1089 region = self._region(wire)
1096 region = self._region(wire)
1090
1097
1091 head = self.head(wire)
1098 head = self.head(wire)
1092
1099
1093 @region.conditional_cache_on_arguments(condition=cache_on)
1100 @region.conditional_cache_on_arguments(condition=cache_on)
1094 def _children(_repo_id, _commit_id):
1101 def _children(_repo_id, _commit_id):
1095
1102
1096 output, __ = self.run_git_command(
1103 output, __ = self.run_git_command(
1097 wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
1104 wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
1098
1105
1099 child_ids = []
1106 child_ids = []
1100 pat = re.compile(fr'^{commit_id}')
1107 pat = re.compile(fr'^{commit_id}')
1101 for line in output.splitlines():
1108 for line in output.splitlines():
1102 line = safe_str(line)
1109 line = safe_str(line)
1103 if pat.match(line):
1110 if pat.match(line):
1104 found_ids = line.split(' ')[1:]
1111 found_ids = line.split(' ')[1:]
1105 child_ids.extend(found_ids)
1112 child_ids.extend(found_ids)
1106 break
1113 break
1107
1114
1108 return child_ids
1115 return child_ids
1109 return _children(repo_id, commit_id)
1116 return _children(repo_id, commit_id)
1110
1117
1111 @reraise_safe_exceptions
1118 @reraise_safe_exceptions
1112 def set_refs(self, wire, key, value):
1119 def set_refs(self, wire, key, value):
1113 repo_init = self._factory.repo_libgit2(wire)
1120 repo_init = self._factory.repo_libgit2(wire)
1114 with repo_init as repo:
1121 with repo_init as repo:
1115 repo.references.create(key, value, force=True)
1122 repo.references.create(key, value, force=True)
1116
1123
1117 @reraise_safe_exceptions
1124 @reraise_safe_exceptions
1118 def update_refs(self, wire, key, value):
1125 def update_refs(self, wire, key, value):
1119 repo_init = self._factory.repo_libgit2(wire)
1126 repo_init = self._factory.repo_libgit2(wire)
1120 with repo_init as repo:
1127 with repo_init as repo:
1121 if key not in repo.references:
1128 if key not in repo.references:
1122 raise ValueError(f'Reference {key} not found in the repository')
1129 raise ValueError(f'Reference {key} not found in the repository')
1123 repo.references.create(key, value, force=True)
1130 repo.references.create(key, value, force=True)
1124
1131
1125 @reraise_safe_exceptions
1132 @reraise_safe_exceptions
1126 def create_branch(self, wire, branch_name, commit_id, force=False):
1133 def create_branch(self, wire, branch_name, commit_id, force=False):
1127 repo_init = self._factory.repo_libgit2(wire)
1134 repo_init = self._factory.repo_libgit2(wire)
1128 with repo_init as repo:
1135 with repo_init as repo:
1129 if commit_id:
1136 if commit_id:
1130 commit = repo[commit_id]
1137 commit = repo[commit_id]
1131 else:
1138 else:
1132 # if commit is not given just use the HEAD
1139 # if commit is not given just use the HEAD
1133 commit = repo.head()
1140 commit = repo.head()
1134
1141
1135 if force:
1142 if force:
1136 repo.branches.local.create(branch_name, commit, force=force)
1143 repo.branches.local.create(branch_name, commit, force=force)
1137 elif not repo.branches.get(branch_name):
1144 elif not repo.branches.get(branch_name):
1138 # create only if that branch isn't existing
1145 # create only if that branch isn't existing
1139 repo.branches.local.create(branch_name, commit, force=force)
1146 repo.branches.local.create(branch_name, commit, force=force)
1140
1147
1141 @reraise_safe_exceptions
1148 @reraise_safe_exceptions
1142 def remove_ref(self, wire, key):
1149 def remove_ref(self, wire, key):
1143 repo_init = self._factory.repo_libgit2(wire)
1150 repo_init = self._factory.repo_libgit2(wire)
1144 with repo_init as repo:
1151 with repo_init as repo:
1145 repo.references.delete(key)
1152 repo.references.delete(key)
1146
1153
1147 @reraise_safe_exceptions
1154 @reraise_safe_exceptions
1148 def tag_remove(self, wire, tag_name):
1155 def tag_remove(self, wire, tag_name):
1149 repo_init = self._factory.repo_libgit2(wire)
1156 repo_init = self._factory.repo_libgit2(wire)
1150 with repo_init as repo:
1157 with repo_init as repo:
1151 key = f'refs/tags/{tag_name}'
1158 key = f'refs/tags/{tag_name}'
1152 repo.references.delete(key)
1159 repo.references.delete(key)
1153
1160
1154 @reraise_safe_exceptions
1161 @reraise_safe_exceptions
1155 def tree_changes(self, wire, source_id, target_id):
1162 def tree_changes(self, wire, source_id, target_id):
1156 repo = self._factory.repo(wire)
1163 repo = self._factory.repo(wire)
1157 # source can be empty
1164 # source can be empty
1158 source_id = safe_bytes(source_id if source_id else b'')
1165 source_id = safe_bytes(source_id if source_id else b'')
1159 target_id = safe_bytes(target_id)
1166 target_id = safe_bytes(target_id)
1160
1167
1161 source = repo[source_id].tree if source_id else None
1168 source = repo[source_id].tree if source_id else None
1162 target = repo[target_id].tree
1169 target = repo[target_id].tree
1163 result = repo.object_store.tree_changes(source, target)
1170 result = repo.object_store.tree_changes(source, target)
1164
1171
1165 added = set()
1172 added = set()
1166 modified = set()
1173 modified = set()
1167 deleted = set()
1174 deleted = set()
1168 for (old_path, new_path), (_, _), (_, _) in list(result):
1175 for (old_path, new_path), (_, _), (_, _) in list(result):
1169 if new_path and old_path:
1176 if new_path and old_path:
1170 modified.add(new_path)
1177 modified.add(new_path)
1171 elif new_path and not old_path:
1178 elif new_path and not old_path:
1172 added.add(new_path)
1179 added.add(new_path)
1173 elif not new_path and old_path:
1180 elif not new_path and old_path:
1174 deleted.add(old_path)
1181 deleted.add(old_path)
1175
1182
1176 return list(added), list(modified), list(deleted)
1183 return list(added), list(modified), list(deleted)
1177
1184
1178 @reraise_safe_exceptions
1185 @reraise_safe_exceptions
1179 def tree_and_type_for_path(self, wire, commit_id, path):
1186 def tree_and_type_for_path(self, wire, commit_id, path):
1180
1187
1181 cache_on, context_uid, repo_id = self._cache_on(wire)
1188 cache_on, context_uid, repo_id = self._cache_on(wire)
1182 region = self._region(wire)
1189 region = self._region(wire)
1183
1190
1184 @region.conditional_cache_on_arguments(condition=cache_on)
1191 @region.conditional_cache_on_arguments(condition=cache_on)
1185 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
1192 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
1186 repo_init = self._factory.repo_libgit2(wire)
1193 repo_init = self._factory.repo_libgit2(wire)
1187
1194
1188 with repo_init as repo:
1195 with repo_init as repo:
1189 commit = repo[commit_id]
1196 commit = repo[commit_id]
1190 try:
1197 try:
1191 tree = commit.tree[path]
1198 tree = commit.tree[path]
1192 except KeyError:
1199 except KeyError:
1193 return None, None, None
1200 return None, None, None
1194
1201
1195 return tree.id.hex, tree.type_str, tree.filemode
1202 return tree.id.hex, tree.type_str, tree.filemode
1196 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1203 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1197
1204
1198 @reraise_safe_exceptions
1205 @reraise_safe_exceptions
1199 def tree_items(self, wire, tree_id):
1206 def tree_items(self, wire, tree_id):
1200 cache_on, context_uid, repo_id = self._cache_on(wire)
1207 cache_on, context_uid, repo_id = self._cache_on(wire)
1201 region = self._region(wire)
1208 region = self._region(wire)
1202
1209
1203 @region.conditional_cache_on_arguments(condition=cache_on)
1210 @region.conditional_cache_on_arguments(condition=cache_on)
1204 def _tree_items(_repo_id, _tree_id):
1211 def _tree_items(_repo_id, _tree_id):
1205
1212
1206 repo_init = self._factory.repo_libgit2(wire)
1213 repo_init = self._factory.repo_libgit2(wire)
1207 with repo_init as repo:
1214 with repo_init as repo:
1208 try:
1215 try:
1209 tree = repo[tree_id]
1216 tree = repo[tree_id]
1210 except KeyError:
1217 except KeyError:
1211 raise ObjectMissing(f'No tree with id: {tree_id}')
1218 raise ObjectMissing(f'No tree with id: {tree_id}')
1212
1219
1213 result = []
1220 result = []
1214 for item in tree:
1221 for item in tree:
1215 item_sha = item.hex
1222 item_sha = item.hex
1216 item_mode = item.filemode
1223 item_mode = item.filemode
1217 item_type = item.type_str
1224 item_type = item.type_str
1218
1225
1219 if item_type == 'commit':
1226 if item_type == 'commit':
1220 # NOTE(marcink): submodules we translate to 'link' for backward compat
1227 # NOTE(marcink): submodules we translate to 'link' for backward compat
1221 item_type = 'link'
1228 item_type = 'link'
1222
1229
1223 result.append((item.name, item_mode, item_sha, item_type))
1230 result.append((item.name, item_mode, item_sha, item_type))
1224 return result
1231 return result
1225 return _tree_items(repo_id, tree_id)
1232 return _tree_items(repo_id, tree_id)
1226
1233
1227 @reraise_safe_exceptions
1234 @reraise_safe_exceptions
1228 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1235 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1229 """
1236 """
1230 Old version that uses subprocess to call diff
1237 Old version that uses subprocess to call diff
1231 """
1238 """
1232
1239
1233 flags = [
1240 flags = [
1234 f'-U{context}', '--patch',
1241 f'-U{context}', '--patch',
1235 '--binary',
1242 '--binary',
1236 '--find-renames',
1243 '--find-renames',
1237 '--no-indent-heuristic',
1244 '--no-indent-heuristic',
1238 # '--indent-heuristic',
1245 # '--indent-heuristic',
1239 #'--full-index',
1246 #'--full-index',
1240 #'--abbrev=40'
1247 #'--abbrev=40'
1241 ]
1248 ]
1242
1249
1243 if opt_ignorews:
1250 if opt_ignorews:
1244 flags.append('--ignore-all-space')
1251 flags.append('--ignore-all-space')
1245
1252
1246 if commit_id_1 == self.EMPTY_COMMIT:
1253 if commit_id_1 == self.EMPTY_COMMIT:
1247 cmd = ['show'] + flags + [commit_id_2]
1254 cmd = ['show'] + flags + [commit_id_2]
1248 else:
1255 else:
1249 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1256 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1250
1257
1251 if file_filter:
1258 if file_filter:
1252 cmd.extend(['--', file_filter])
1259 cmd.extend(['--', file_filter])
1253
1260
1254 diff, __ = self.run_git_command(wire, cmd)
1261 diff, __ = self.run_git_command(wire, cmd)
1255 # If we used 'show' command, strip first few lines (until actual diff
1262 # If we used 'show' command, strip first few lines (until actual diff
1256 # starts)
1263 # starts)
1257 if commit_id_1 == self.EMPTY_COMMIT:
1264 if commit_id_1 == self.EMPTY_COMMIT:
1258 lines = diff.splitlines()
1265 lines = diff.splitlines()
1259 x = 0
1266 x = 0
1260 for line in lines:
1267 for line in lines:
1261 if line.startswith(b'diff'):
1268 if line.startswith(b'diff'):
1262 break
1269 break
1263 x += 1
1270 x += 1
1264 # Append new line just like 'diff' command do
1271 # Append new line just like 'diff' command do
1265 diff = '\n'.join(lines[x:]) + '\n'
1272 diff = '\n'.join(lines[x:]) + '\n'
1266 return diff
1273 return diff
1267
1274
1268 @reraise_safe_exceptions
1275 @reraise_safe_exceptions
1269 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1276 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1270 repo_init = self._factory.repo_libgit2(wire)
1277 repo_init = self._factory.repo_libgit2(wire)
1271
1278
1272 with repo_init as repo:
1279 with repo_init as repo:
1273 swap = True
1280 swap = True
1274 flags = 0
1281 flags = 0
1275 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1282 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1276
1283
1277 if opt_ignorews:
1284 if opt_ignorews:
1278 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1285 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1279
1286
1280 if commit_id_1 == self.EMPTY_COMMIT:
1287 if commit_id_1 == self.EMPTY_COMMIT:
1281 comm1 = repo[commit_id_2]
1288 comm1 = repo[commit_id_2]
1282 diff_obj = comm1.tree.diff_to_tree(
1289 diff_obj = comm1.tree.diff_to_tree(
1283 flags=flags, context_lines=context, swap=swap)
1290 flags=flags, context_lines=context, swap=swap)
1284
1291
1285 else:
1292 else:
1286 comm1 = repo[commit_id_2]
1293 comm1 = repo[commit_id_2]
1287 comm2 = repo[commit_id_1]
1294 comm2 = repo[commit_id_1]
1288 diff_obj = comm1.tree.diff_to_tree(
1295 diff_obj = comm1.tree.diff_to_tree(
1289 comm2.tree, flags=flags, context_lines=context, swap=swap)
1296 comm2.tree, flags=flags, context_lines=context, swap=swap)
1290 similar_flags = 0
1297 similar_flags = 0
1291 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1298 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1292 diff_obj.find_similar(flags=similar_flags)
1299 diff_obj.find_similar(flags=similar_flags)
1293
1300
1294 if file_filter:
1301 if file_filter:
1295 for p in diff_obj:
1302 for p in diff_obj:
1296 if p.delta.old_file.path == file_filter:
1303 if p.delta.old_file.path == file_filter:
1297 return BytesEnvelope(p.data) or BytesEnvelope(b'')
1304 return BytesEnvelope(p.data) or BytesEnvelope(b'')
1298 # fo matching path == no diff
1305 # fo matching path == no diff
1299 return BytesEnvelope(b'')
1306 return BytesEnvelope(b'')
1300
1307
1301 return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
1308 return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
1302
1309
1303 @reraise_safe_exceptions
1310 @reraise_safe_exceptions
1304 def node_history(self, wire, commit_id, path, limit):
1311 def node_history(self, wire, commit_id, path, limit):
1305 cache_on, context_uid, repo_id = self._cache_on(wire)
1312 cache_on, context_uid, repo_id = self._cache_on(wire)
1306 region = self._region(wire)
1313 region = self._region(wire)
1307
1314
1308 @region.conditional_cache_on_arguments(condition=cache_on)
1315 @region.conditional_cache_on_arguments(condition=cache_on)
1309 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1316 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1310 # optimize for n==1, rev-list is much faster for that use-case
1317 # optimize for n==1, rev-list is much faster for that use-case
1311 if limit == 1:
1318 if limit == 1:
1312 cmd = ['rev-list', '-1', commit_id, '--', path]
1319 cmd = ['rev-list', '-1', commit_id, '--', path]
1313 else:
1320 else:
1314 cmd = ['log']
1321 cmd = ['log']
1315 if limit:
1322 if limit:
1316 cmd.extend(['-n', str(safe_int(limit, 0))])
1323 cmd.extend(['-n', str(safe_int(limit, 0))])
1317 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1324 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1318
1325
1319 output, __ = self.run_git_command(wire, cmd)
1326 output, __ = self.run_git_command(wire, cmd)
1320 commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
1327 commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
1321
1328
1322 return [x for x in commit_ids]
1329 return [x for x in commit_ids]
1323 return _node_history(context_uid, repo_id, commit_id, path, limit)
1330 return _node_history(context_uid, repo_id, commit_id, path, limit)
1324
1331
1325 @reraise_safe_exceptions
1332 @reraise_safe_exceptions
1326 def node_annotate_legacy(self, wire, commit_id, path):
1333 def node_annotate_legacy(self, wire, commit_id, path):
1327 # note: replaced by pygit2 implementation
1334 # note: replaced by pygit2 implementation
1328 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1335 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1329 # -l ==> outputs long shas (and we need all 40 characters)
1336 # -l ==> outputs long shas (and we need all 40 characters)
1330 # --root ==> doesn't put '^' character for boundaries
1337 # --root ==> doesn't put '^' character for boundaries
1331 # -r commit_id ==> blames for the given commit
1338 # -r commit_id ==> blames for the given commit
1332 output, __ = self.run_git_command(wire, cmd)
1339 output, __ = self.run_git_command(wire, cmd)
1333
1340
1334 result = []
1341 result = []
1335 for i, blame_line in enumerate(output.splitlines()[:-1]):
1342 for i, blame_line in enumerate(output.splitlines()[:-1]):
1336 line_no = i + 1
1343 line_no = i + 1
1337 blame_commit_id, line = re.split(rb' ', blame_line, 1)
1344 blame_commit_id, line = re.split(rb' ', blame_line, 1)
1338 result.append((line_no, blame_commit_id, line))
1345 result.append((line_no, blame_commit_id, line))
1339
1346
1340 return result
1347 return result
1341
1348
1342 @reraise_safe_exceptions
1349 @reraise_safe_exceptions
1343 def node_annotate(self, wire, commit_id, path):
1350 def node_annotate(self, wire, commit_id, path):
1344
1351
1345 result_libgit = []
1352 result_libgit = []
1346 repo_init = self._factory.repo_libgit2(wire)
1353 repo_init = self._factory.repo_libgit2(wire)
1347 with repo_init as repo:
1354 with repo_init as repo:
1348 commit = repo[commit_id]
1355 commit = repo[commit_id]
1349 blame_obj = repo.blame(path, newest_commit=commit_id)
1356 blame_obj = repo.blame(path, newest_commit=commit_id)
1350 file_content = commit.tree[path].data
1357 file_content = commit.tree[path].data
1351 for i, line in enumerate(splitnewlines(file_content)):
1358 for i, line in enumerate(splitnewlines(file_content)):
1352 line_no = i + 1
1359 line_no = i + 1
1353 hunk = blame_obj.for_line(line_no)
1360 hunk = blame_obj.for_line(line_no)
1354 blame_commit_id = hunk.final_commit_id.hex
1361 blame_commit_id = hunk.final_commit_id.hex
1355
1362
1356 result_libgit.append((line_no, blame_commit_id, line))
1363 result_libgit.append((line_no, blame_commit_id, line))
1357
1364
1358 return BinaryEnvelope(result_libgit)
1365 return BinaryEnvelope(result_libgit)
1359
1366
1360 @reraise_safe_exceptions
1367 @reraise_safe_exceptions
1361 def update_server_info(self, wire, force=False):
1368 def update_server_info(self, wire, force=False):
1362 cmd = ['update-server-info']
1369 cmd = ['update-server-info']
1363 if force:
1370 if force:
1364 cmd += ['--force']
1371 cmd += ['--force']
1365 output, __ = self.run_git_command(wire, cmd)
1372 output, __ = self.run_git_command(wire, cmd)
1366 return output.splitlines()
1373 return output.splitlines()
1367
1374
1368 @reraise_safe_exceptions
1375 @reraise_safe_exceptions
1369 def get_all_commit_ids(self, wire):
1376 def get_all_commit_ids(self, wire):
1370
1377
1371 cache_on, context_uid, repo_id = self._cache_on(wire)
1378 cache_on, context_uid, repo_id = self._cache_on(wire)
1372 region = self._region(wire)
1379 region = self._region(wire)
1373
1380
1374 @region.conditional_cache_on_arguments(condition=cache_on)
1381 @region.conditional_cache_on_arguments(condition=cache_on)
1375 def _get_all_commit_ids(_context_uid, _repo_id):
1382 def _get_all_commit_ids(_context_uid, _repo_id):
1376
1383
1377 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1384 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1378 try:
1385 try:
1379 output, __ = self.run_git_command(wire, cmd)
1386 output, __ = self.run_git_command(wire, cmd)
1380 return output.splitlines()
1387 return output.splitlines()
1381 except Exception:
1388 except Exception:
1382 # Can be raised for empty repositories
1389 # Can be raised for empty repositories
1383 return []
1390 return []
1384
1391
1385 @region.conditional_cache_on_arguments(condition=cache_on)
1392 @region.conditional_cache_on_arguments(condition=cache_on)
1386 def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
1393 def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
1387 repo_init = self._factory.repo_libgit2(wire)
1394 repo_init = self._factory.repo_libgit2(wire)
1388 from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
1395 from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
1389 results = []
1396 results = []
1390 with repo_init as repo:
1397 with repo_init as repo:
1391 for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
1398 for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
1392 results.append(commit.id.hex)
1399 results.append(commit.id.hex)
1393
1400
1394 return _get_all_commit_ids(context_uid, repo_id)
1401 return _get_all_commit_ids(context_uid, repo_id)
1395
1402
1396 @reraise_safe_exceptions
1403 @reraise_safe_exceptions
1397 def run_git_command(self, wire, cmd, **opts):
1404 def run_git_command(self, wire, cmd, **opts):
1398 path = wire.get('path', None)
1405 path = wire.get('path', None)
1399 debug_mode = rhodecode.ConfigGet().get_bool('debug')
1406 debug_mode = rhodecode.ConfigGet().get_bool('debug')
1400
1407
1401 if path and os.path.isdir(path):
1408 if path and os.path.isdir(path):
1402 opts['cwd'] = path
1409 opts['cwd'] = path
1403
1410
1404 if '_bare' in opts:
1411 if '_bare' in opts:
1405 _copts = []
1412 _copts = []
1406 del opts['_bare']
1413 del opts['_bare']
1407 else:
1414 else:
1408 _copts = ['-c', 'core.quotepath=false', '-c', 'advice.diverging=false']
1415 _copts = ['-c', 'core.quotepath=false', '-c', 'advice.diverging=false']
1409 safe_call = False
1416 safe_call = False
1410 if '_safe' in opts:
1417 if '_safe' in opts:
1411 # no exc on failure
1418 # no exc on failure
1412 del opts['_safe']
1419 del opts['_safe']
1413 safe_call = True
1420 safe_call = True
1414
1421
1415 if '_copts' in opts:
1422 if '_copts' in opts:
1416 _copts.extend(opts['_copts'] or [])
1423 _copts.extend(opts['_copts'] or [])
1417 del opts['_copts']
1424 del opts['_copts']
1418
1425
1419 gitenv = os.environ.copy()
1426 gitenv = os.environ.copy()
1420 gitenv.update(opts.pop('extra_env', {}))
1427 gitenv.update(opts.pop('extra_env', {}))
1421 # need to clean fix GIT_DIR !
1428 # need to clean fix GIT_DIR !
1422 if 'GIT_DIR' in gitenv:
1429 if 'GIT_DIR' in gitenv:
1423 del gitenv['GIT_DIR']
1430 del gitenv['GIT_DIR']
1424 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1431 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1425 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1432 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1426
1433
1427 cmd = [settings.GIT_EXECUTABLE()] + _copts + cmd
1434 cmd = [settings.GIT_EXECUTABLE()] + _copts + cmd
1428 _opts = {'env': gitenv, 'shell': False}
1435 _opts = {'env': gitenv, 'shell': False}
1429
1436
1430 proc = None
1437 proc = None
1431 try:
1438 try:
1432 _opts.update(opts)
1439 _opts.update(opts)
1433 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1440 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1434
1441
1435 return b''.join(proc), b''.join(proc.stderr)
1442 return b''.join(proc), b''.join(proc.stderr)
1436 except OSError as err:
1443 except OSError as err:
1437 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
1444 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
1438 call_opts = {}
1445 call_opts = {}
1439 if debug_mode:
1446 if debug_mode:
1440 call_opts = _opts
1447 call_opts = _opts
1441
1448
1442 tb_err = ("Couldn't run git command ({}).\n"
1449 tb_err = ("Couldn't run git command ({}).\n"
1443 "Original error was:{}\n"
1450 "Original error was:{}\n"
1444 "Call options:{}\n"
1451 "Call options:{}\n"
1445 .format(cmd, err, call_opts))
1452 .format(cmd, err, call_opts))
1446 log.exception(tb_err)
1453 log.exception(tb_err)
1447 if safe_call:
1454 if safe_call:
1448 return '', err
1455 return '', err
1449 else:
1456 else:
1450 raise exceptions.VcsException()(tb_err)
1457 raise exceptions.VcsException()(tb_err)
1451 finally:
1458 finally:
1452 if proc:
1459 if proc:
1453 proc.close()
1460 proc.close()
1454
1461
1455 @reraise_safe_exceptions
1462 @reraise_safe_exceptions
1456 def install_hooks(self, wire, force=False):
1463 def install_hooks(self, wire, force=False):
1457 from vcsserver.hook_utils import install_git_hooks
1464 from vcsserver.hook_utils import install_git_hooks
1458 bare = self.bare(wire)
1465 bare = self.bare(wire)
1459 path = wire['path']
1466 path = wire['path']
1460 binary_dir = settings.BINARY_DIR
1467 binary_dir = settings.BINARY_DIR
1461 if binary_dir:
1468 if binary_dir:
1462 os.path.join(binary_dir, 'python3')
1469 os.path.join(binary_dir, 'python3')
1463 return install_git_hooks(path, bare, force_create=force)
1470 return install_git_hooks(path, bare, force_create=force)
1464
1471
1465 @reraise_safe_exceptions
1472 @reraise_safe_exceptions
1466 def get_hooks_info(self, wire):
1473 def get_hooks_info(self, wire):
1467 from vcsserver.hook_utils import (
1474 from vcsserver.hook_utils import (
1468 get_git_pre_hook_version, get_git_post_hook_version)
1475 get_git_pre_hook_version, get_git_post_hook_version)
1469 bare = self.bare(wire)
1476 bare = self.bare(wire)
1470 path = wire['path']
1477 path = wire['path']
1471 return {
1478 return {
1472 'pre_version': get_git_pre_hook_version(path, bare),
1479 'pre_version': get_git_pre_hook_version(path, bare),
1473 'post_version': get_git_post_hook_version(path, bare),
1480 'post_version': get_git_post_hook_version(path, bare),
1474 }
1481 }
1475
1482
1476 @reraise_safe_exceptions
1483 @reraise_safe_exceptions
1477 def set_head_ref(self, wire, head_name):
1484 def set_head_ref(self, wire, head_name):
1478 log.debug('Setting refs/head to `%s`', head_name)
1485 log.debug('Setting refs/head to `%s`', head_name)
1479 repo_init = self._factory.repo_libgit2(wire)
1486 repo_init = self._factory.repo_libgit2(wire)
1480 with repo_init as repo:
1487 with repo_init as repo:
1481 repo.set_head(f'refs/heads/{head_name}')
1488 repo.set_head(f'refs/heads/{head_name}')
1482
1489
1483 return [head_name] + [f'set HEAD to refs/heads/{head_name}']
1490 return [head_name] + [f'set HEAD to refs/heads/{head_name}']
1484
1491
1485 @reraise_safe_exceptions
1492 @reraise_safe_exceptions
1486 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1493 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1487 archive_dir_name, commit_id, cache_config):
1494 archive_dir_name, commit_id, cache_config):
1488
1495
1489 def file_walker(_commit_id, path):
1496 def file_walker(_commit_id, path):
1490 repo_init = self._factory.repo_libgit2(wire)
1497 repo_init = self._factory.repo_libgit2(wire)
1491
1498
1492 with repo_init as repo:
1499 with repo_init as repo:
1493 commit = repo[commit_id]
1500 commit = repo[commit_id]
1494
1501
1495 if path in ['', '/']:
1502 if path in ['', '/']:
1496 tree = commit.tree
1503 tree = commit.tree
1497 else:
1504 else:
1498 tree = commit.tree[path.rstrip('/')]
1505 tree = commit.tree[path.rstrip('/')]
1499 tree_id = tree.id.hex
1506 tree_id = tree.id.hex
1500 try:
1507 try:
1501 tree = repo[tree_id]
1508 tree = repo[tree_id]
1502 except KeyError:
1509 except KeyError:
1503 raise ObjectMissing(f'No tree with id: {tree_id}')
1510 raise ObjectMissing(f'No tree with id: {tree_id}')
1504
1511
1505 index = LibGit2Index.Index()
1512 index = LibGit2Index.Index()
1506 index.read_tree(tree)
1513 index.read_tree(tree)
1507 file_iter = index
1514 file_iter = index
1508
1515
1509 for file_node in file_iter:
1516 for file_node in file_iter:
1510 file_path = file_node.path
1517 file_path = file_node.path
1511 mode = file_node.mode
1518 mode = file_node.mode
1512 is_link = stat.S_ISLNK(mode)
1519 is_link = stat.S_ISLNK(mode)
1513 if mode == pygit2.GIT_FILEMODE_COMMIT:
1520 if mode == pygit2.GIT_FILEMODE_COMMIT:
1514 log.debug('Skipping path %s as a commit node', file_path)
1521 log.debug('Skipping path %s as a commit node', file_path)
1515 continue
1522 continue
1516 yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
1523 yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
1517
1524
1518 return store_archive_in_cache(
1525 return store_archive_in_cache(
1519 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
1526 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
@@ -1,1213 +1,1217 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import binascii
18 import binascii
19 import io
19 import io
20 import logging
20 import logging
21 import stat
21 import stat
22 import sys
22 import sys
23 import urllib.request
23 import urllib.request
24 import urllib.parse
24 import urllib.parse
25 import hashlib
25 import hashlib
26
26
27 from hgext import largefiles, rebase
27 from hgext import largefiles, rebase
28
28
29 from mercurial import commands
29 from mercurial import commands
30 from mercurial import unionrepo
30 from mercurial import unionrepo
31 from mercurial import verify
31 from mercurial import verify
32 from mercurial import repair
32 from mercurial import repair
33 from mercurial.error import AmbiguousPrefixLookupError
33 from mercurial.error import AmbiguousPrefixLookupError
34 from mercurial.utils.urlutil import path as hg_path
34
35
35 import vcsserver
36 import vcsserver
36 from vcsserver import exceptions
37 from vcsserver import exceptions
37 from vcsserver.base import (
38 from vcsserver.base import (
38 RepoFactory,
39 RepoFactory,
39 obfuscate_qs,
40 obfuscate_qs,
40 raise_from_original,
41 raise_from_original,
41 store_archive_in_cache,
42 store_archive_in_cache,
42 ArchiveNode,
43 ArchiveNode,
43 BytesEnvelope,
44 BytesEnvelope,
44 BinaryEnvelope,
45 BinaryEnvelope,
45 )
46 )
46 from vcsserver.hgcompat import (
47 from vcsserver.hgcompat import (
47 archival,
48 archival,
48 bin,
49 bin,
49 clone,
50 clone,
50 config as hgconfig,
51 config as hgconfig,
51 diffopts,
52 diffopts,
52 hex,
53 hex,
53 get_ctx,
54 get_ctx,
54 hg_url as url_parser,
55 hg_url as url_parser,
55 httpbasicauthhandler,
56 httpbasicauthhandler,
56 httpdigestauthhandler,
57 httpdigestauthhandler,
57 makepeer,
58 make_peer,
58 instance,
59 instance,
59 match,
60 match,
60 memctx,
61 memctx,
61 exchange,
62 exchange,
62 memfilectx,
63 memfilectx,
63 nullrev,
64 nullrev,
64 hg_merge,
65 hg_merge,
65 patch,
66 patch,
66 peer,
67 peer,
67 revrange,
68 revrange,
68 ui,
69 ui,
69 hg_tag,
70 hg_tag,
70 Abort,
71 Abort,
71 LookupError,
72 LookupError,
72 RepoError,
73 RepoError,
73 RepoLookupError,
74 RepoLookupError,
74 InterventionRequired,
75 InterventionRequired,
75 RequirementError,
76 RequirementError,
76 alwaysmatcher,
77 alwaysmatcher,
77 patternmatcher,
78 patternmatcher,
78 hgext_strip,
79 hgext_strip,
79 )
80 )
80 from vcsserver.str_utils import ascii_bytes, ascii_str, safe_str, safe_bytes, convert_to_str
81 from vcsserver.lib.str_utils import ascii_bytes, ascii_str, safe_str, safe_bytes, convert_to_str
81 from vcsserver.vcs_base import RemoteBase
82 from vcsserver.vcs_base import RemoteBase
82 from vcsserver.config import hooks as hooks_config
83 from vcsserver.config import hooks as hooks_config
83 from vcsserver.lib.exc_tracking import format_exc
84 from vcsserver.lib.exc_tracking import format_exc
84
85
85 log = logging.getLogger(__name__)
86 log = logging.getLogger(__name__)
86
87
87
88
88 def make_ui_from_config(repo_config):
89 def make_ui_from_config(repo_config, interactive=True):
89
90
90 class LoggingUI(ui.ui):
91 class LoggingUI(ui.ui):
91
92
92 def status(self, *msg, **opts):
93 def status(self, *msg, **opts):
93 str_msg = map(safe_str, msg)
94 str_msg = map(safe_str, msg)
94 log.info(' '.join(str_msg).rstrip('\n'))
95 log.info(' '.join(str_msg).rstrip('\n'))
95 #super(LoggingUI, self).status(*msg, **opts)
96 #super(LoggingUI, self).status(*msg, **opts)
96
97
97 def warn(self, *msg, **opts):
98 def warn(self, *msg, **opts):
98 str_msg = map(safe_str, msg)
99 str_msg = map(safe_str, msg)
99 log.warning('ui_logger:'+' '.join(str_msg).rstrip('\n'))
100 log.warning('ui_logger:'+' '.join(str_msg).rstrip('\n'))
100 #super(LoggingUI, self).warn(*msg, **opts)
101 #super(LoggingUI, self).warn(*msg, **opts)
101
102
102 def error(self, *msg, **opts):
103 def error(self, *msg, **opts):
103 str_msg = map(safe_str, msg)
104 str_msg = map(safe_str, msg)
104 log.error('ui_logger:'+' '.join(str_msg).rstrip('\n'))
105 log.error('ui_logger:'+' '.join(str_msg).rstrip('\n'))
105 #super(LoggingUI, self).error(*msg, **opts)
106 #super(LoggingUI, self).error(*msg, **opts)
106
107
107 def note(self, *msg, **opts):
108 def note(self, *msg, **opts):
108 str_msg = map(safe_str, msg)
109 str_msg = map(safe_str, msg)
109 log.info('ui_logger:'+' '.join(str_msg).rstrip('\n'))
110 log.info('ui_logger:'+' '.join(str_msg).rstrip('\n'))
110 #super(LoggingUI, self).note(*msg, **opts)
111 #super(LoggingUI, self).note(*msg, **opts)
111
112
112 def debug(self, *msg, **opts):
113 def debug(self, *msg, **opts):
113 str_msg = map(safe_str, msg)
114 str_msg = map(safe_str, msg)
114 log.debug('ui_logger:'+' '.join(str_msg).rstrip('\n'))
115 log.debug('ui_logger:'+' '.join(str_msg).rstrip('\n'))
115 #super(LoggingUI, self).debug(*msg, **opts)
116 #super(LoggingUI, self).debug(*msg, **opts)
116
117
117 baseui = LoggingUI()
118 baseui = LoggingUI()
118
119
119 # clean the baseui object
120 # clean the baseui object
120 baseui._ocfg = hgconfig.config()
121 baseui._ocfg = hgconfig.config()
121 baseui._ucfg = hgconfig.config()
122 baseui._ucfg = hgconfig.config()
122 baseui._tcfg = hgconfig.config()
123 baseui._tcfg = hgconfig.config()
123
124
124 for section, option, value in repo_config:
125 for section, option, value in repo_config:
125 baseui.setconfig(ascii_bytes(section), ascii_bytes(option), ascii_bytes(value))
126 baseui.setconfig(ascii_bytes(section), ascii_bytes(option), ascii_bytes(value))
126
127
127 # make our hgweb quiet so it doesn't print output
128 # make our hgweb quiet so it doesn't print output
128 baseui.setconfig(b'ui', b'quiet', b'true')
129 baseui.setconfig(b'ui', b'quiet', b'true')
129
130
130 baseui.setconfig(b'ui', b'paginate', b'never')
131 baseui.setconfig(b'ui', b'paginate', b'never')
131 # for better Error reporting of Mercurial
132 # for better Error reporting of Mercurial
132 baseui.setconfig(b'ui', b'message-output', b'stderr')
133 baseui.setconfig(b'ui', b'message-output', b'stderr')
133
134
134 # force mercurial to only use 1 thread, otherwise it may try to set a
135 # force mercurial to only use 1 thread, otherwise it may try to set a
135 # signal in a non-main thread, thus generating a ValueError.
136 # signal in a non-main thread, thus generating a ValueError.
136 baseui.setconfig(b'worker', b'numcpus', 1)
137 baseui.setconfig(b'worker', b'numcpus', 1)
137
138
138 # If there is no config for the largefiles extension, we explicitly disable
139 # If there is no config for the largefiles extension, we explicitly disable
139 # it here. This overrides settings from repositories hgrc file. Recent
140 # it here. This overrides settings from repositories hgrc file. Recent
140 # mercurial versions enable largefiles in hgrc on clone from largefile
141 # mercurial versions enable largefiles in hgrc on clone from largefile
141 # repo.
142 # repo.
142 if not baseui.hasconfig(b'extensions', b'largefiles'):
143 if not baseui.hasconfig(b'extensions', b'largefiles'):
143 log.debug('Explicitly disable largefiles extension for repo.')
144 log.debug('Explicitly disable largefiles extension for repo.')
144 baseui.setconfig(b'extensions', b'largefiles', b'!')
145 baseui.setconfig(b'extensions', b'largefiles', b'!')
145
146
147 baseui.setconfig(b'ui', b'interactive', b'true' if interactive else b'false')
146 return baseui
148 return baseui
147
149
148
150
149 def reraise_safe_exceptions(func):
151 def reraise_safe_exceptions(func):
150 """Decorator for converting mercurial exceptions to something neutral."""
152 """Decorator for converting mercurial exceptions to something neutral."""
151
153
152 def wrapper(*args, **kwargs):
154 def wrapper(*args, **kwargs):
153 try:
155 try:
154 return func(*args, **kwargs)
156 return func(*args, **kwargs)
155 except (Abort, InterventionRequired) as e:
157 except (Abort, InterventionRequired) as e:
156 raise_from_original(exceptions.AbortException(e), e)
158 raise_from_original(exceptions.AbortException(e), e)
157 except RepoLookupError as e:
159 except RepoLookupError as e:
158 raise_from_original(exceptions.LookupException(e), e)
160 raise_from_original(exceptions.LookupException(e), e)
159 except RequirementError as e:
161 except RequirementError as e:
160 raise_from_original(exceptions.RequirementException(e), e)
162 raise_from_original(exceptions.RequirementException(e), e)
161 except RepoError as e:
163 except RepoError as e:
162 raise_from_original(exceptions.VcsException(e), e)
164 raise_from_original(exceptions.VcsException(e), e)
163 except LookupError as e:
165 except LookupError as e:
164 raise_from_original(exceptions.LookupException(e), e)
166 raise_from_original(exceptions.LookupException(e), e)
165 except Exception as e:
167 except Exception as e:
166 if not hasattr(e, '_vcs_kind'):
168 if not hasattr(e, '_vcs_kind'):
167 log.exception("Unhandled exception in hg remote call")
169 log.exception("Unhandled exception in hg remote call")
168 raise_from_original(exceptions.UnhandledException(e), e)
170 raise_from_original(exceptions.UnhandledException(e), e)
169
171
170 raise
172 raise
171 return wrapper
173 return wrapper
172
174
173
175
174 class MercurialFactory(RepoFactory):
176 class MercurialFactory(RepoFactory):
175 repo_type = 'hg'
177 repo_type = 'hg'
176
178
177 def _create_config(self, config, hooks=True):
179 def _create_config(self, config, hooks=True):
178 if not hooks:
180 if not hooks:
179
181
180 hooks_to_clean = {
182 hooks_to_clean = {
181
183
182 hooks_config.HOOK_REPO_SIZE,
184 hooks_config.HOOK_REPO_SIZE,
183 hooks_config.HOOK_PRE_PULL,
185 hooks_config.HOOK_PRE_PULL,
184 hooks_config.HOOK_PULL,
186 hooks_config.HOOK_PULL,
185
187
186 hooks_config.HOOK_PRE_PUSH,
188 hooks_config.HOOK_PRE_PUSH,
187 # TODO: what about PRETXT, this was disabled in pre 5.0.0
189 # TODO: what about PRETXT, this was disabled in pre 5.0.0
188 hooks_config.HOOK_PRETX_PUSH,
190 hooks_config.HOOK_PRETX_PUSH,
189
191
190 }
192 }
191 new_config = []
193 new_config = []
192 for section, option, value in config:
194 for section, option, value in config:
193 if section == 'hooks' and option in hooks_to_clean:
195 if section == 'hooks' and option in hooks_to_clean:
194 continue
196 continue
195 new_config.append((section, option, value))
197 new_config.append((section, option, value))
196 config = new_config
198 config = new_config
197
199
198 baseui = make_ui_from_config(config)
200 baseui = make_ui_from_config(config)
199 return baseui
201 return baseui
200
202
201 def _create_repo(self, wire, create):
203 def _create_repo(self, wire, create):
202 baseui = self._create_config(wire["config"])
204 baseui = self._create_config(wire["config"])
203 repo = instance(baseui, safe_bytes(wire["path"]), create)
205 repo = instance(baseui, safe_bytes(wire["path"]), create)
204 log.debug('repository created: got HG object: %s', repo)
206 log.debug('repository created: got HG object: %s', repo)
205 return repo
207 return repo
206
208
207 def repo(self, wire, create=False):
209 def repo(self, wire, create=False):
208 """
210 """
209 Get a repository instance for the given path.
211 Get a repository instance for the given path.
210 """
212 """
211 return self._create_repo(wire, create)
213 return self._create_repo(wire, create)
212
214
213
215
214 def patch_ui_message_output(baseui):
216 def patch_ui_message_output(baseui):
215 baseui.setconfig(b'ui', b'quiet', b'false')
217 baseui.setconfig(b'ui', b'quiet', b'false')
216 output = io.BytesIO()
218 output = io.BytesIO()
217
219
218 def write(data, **unused_kwargs):
220 def write(data, **unused_kwargs):
219 output.write(data)
221 output.write(data)
220
222
221 baseui.status = write
223 baseui.status = write
222 baseui.write = write
224 baseui.write = write
223 baseui.warn = write
225 baseui.warn = write
224 baseui.debug = write
226 baseui.debug = write
225
227
226 return baseui, output
228 return baseui, output
227
229
228
230
229 def get_obfuscated_url(url_obj):
231 def get_obfuscated_url(url_obj):
230 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
232 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
231 url_obj.query = obfuscate_qs(url_obj.query)
233 url_obj.query = obfuscate_qs(url_obj.query)
232 obfuscated_uri = str(url_obj)
234 obfuscated_uri = str(url_obj)
233 return obfuscated_uri
235 return obfuscated_uri
234
236
235
237
236 def normalize_url_for_hg(url: str):
238 def normalize_url_for_hg(url: str):
237 _proto = None
239 _proto = None
238
240
239 if '+' in url[:url.find('://')]:
241 if '+' in url[:url.find('://')]:
240 _proto = url[0:url.find('+')]
242 _proto = url[0:url.find('+')]
241 url = url[url.find('+') + 1:]
243 url = url[url.find('+') + 1:]
242 return url, _proto
244 return url, _proto
243
245
244
246
245 class HgRemote(RemoteBase):
247 class HgRemote(RemoteBase):
246
248
247 def __init__(self, factory):
249 def __init__(self, factory):
248 self._factory = factory
250 self._factory = factory
249 self._bulk_methods = {
251 self._bulk_methods = {
250 "affected_files": self.ctx_files,
252 "affected_files": self.ctx_files,
251 "author": self.ctx_user,
253 "author": self.ctx_user,
252 "branch": self.ctx_branch,
254 "branch": self.ctx_branch,
253 "children": self.ctx_children,
255 "children": self.ctx_children,
254 "date": self.ctx_date,
256 "date": self.ctx_date,
255 "message": self.ctx_description,
257 "message": self.ctx_description,
256 "parents": self.ctx_parents,
258 "parents": self.ctx_parents,
257 "status": self.ctx_status,
259 "status": self.ctx_status,
258 "obsolete": self.ctx_obsolete,
260 "obsolete": self.ctx_obsolete,
259 "phase": self.ctx_phase,
261 "phase": self.ctx_phase,
260 "hidden": self.ctx_hidden,
262 "hidden": self.ctx_hidden,
261 "_file_paths": self.ctx_list,
263 "_file_paths": self.ctx_list,
262 }
264 }
263 self._bulk_file_methods = {
265 self._bulk_file_methods = {
264 "size": self.fctx_size,
266 "size": self.fctx_size,
265 "data": self.fctx_node_data,
267 "data": self.fctx_node_data,
266 "flags": self.fctx_flags,
268 "flags": self.fctx_flags,
267 "is_binary": self.is_binary,
269 "is_binary": self.is_binary,
268 "md5": self.md5_hash,
270 "md5": self.md5_hash,
269 }
271 }
270
272
271 def _get_ctx(self, repo, ref):
273 def _get_ctx(self, repo, ref):
272 return get_ctx(repo, ref)
274 return get_ctx(repo, ref)
273
275
274 @reraise_safe_exceptions
276 @reraise_safe_exceptions
275 def discover_hg_version(self):
277 def discover_hg_version(self):
276 from mercurial import util
278 from mercurial import util
277 return safe_str(util.version())
279 return safe_str(util.version())
278
280
279 @reraise_safe_exceptions
281 @reraise_safe_exceptions
280 def is_empty(self, wire):
282 def is_empty(self, wire):
281 repo = self._factory.repo(wire)
283 repo = self._factory.repo(wire)
282
284
283 try:
285 try:
284 return len(repo) == 0
286 return len(repo) == 0
285 except Exception:
287 except Exception:
286 log.exception("failed to read object_store")
288 log.exception("failed to read object_store")
287 return False
289 return False
288
290
289 @reraise_safe_exceptions
291 @reraise_safe_exceptions
290 def bookmarks(self, wire):
292 def bookmarks(self, wire):
291 cache_on, context_uid, repo_id = self._cache_on(wire)
293 cache_on, context_uid, repo_id = self._cache_on(wire)
292 region = self._region(wire)
294 region = self._region(wire)
293
295
294 @region.conditional_cache_on_arguments(condition=cache_on)
296 @region.conditional_cache_on_arguments(condition=cache_on)
295 def _bookmarks(_context_uid, _repo_id):
297 def _bookmarks(_context_uid, _repo_id):
296 repo = self._factory.repo(wire)
298 repo = self._factory.repo(wire)
297 return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo._bookmarks.items()}
299 return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo._bookmarks.items()}
298
300
299 return _bookmarks(context_uid, repo_id)
301 return _bookmarks(context_uid, repo_id)
300
302
301 @reraise_safe_exceptions
303 @reraise_safe_exceptions
302 def branches(self, wire, normal, closed):
304 def branches(self, wire, normal, closed):
303 cache_on, context_uid, repo_id = self._cache_on(wire)
305 cache_on, context_uid, repo_id = self._cache_on(wire)
304 region = self._region(wire)
306 region = self._region(wire)
305
307
306 @region.conditional_cache_on_arguments(condition=cache_on)
308 @region.conditional_cache_on_arguments(condition=cache_on)
307 def _branches(_context_uid, _repo_id, _normal, _closed):
309 def _branches(_context_uid, _repo_id, _normal, _closed):
308 repo = self._factory.repo(wire)
310 repo = self._factory.repo(wire)
309 iter_branches = repo.branchmap().iterbranches()
311 iter_branches = repo.branchmap().iterbranches()
310 bt = {}
312 bt = {}
311 for branch_name, _heads, tip_node, is_closed in iter_branches:
313 for branch_name, _heads, tip_node, is_closed in iter_branches:
312 if normal and not is_closed:
314 if normal and not is_closed:
313 bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
315 bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
314 if closed and is_closed:
316 if closed and is_closed:
315 bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
317 bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
316
318
317 return bt
319 return bt
318
320
319 return _branches(context_uid, repo_id, normal, closed)
321 return _branches(context_uid, repo_id, normal, closed)
320
322
321 @reraise_safe_exceptions
323 @reraise_safe_exceptions
322 def bulk_request(self, wire, commit_id, pre_load):
324 def bulk_request(self, wire, commit_id, pre_load):
323 cache_on, context_uid, repo_id = self._cache_on(wire)
325 cache_on, context_uid, repo_id = self._cache_on(wire)
324 region = self._region(wire)
326 region = self._region(wire)
325
327
326 @region.conditional_cache_on_arguments(condition=cache_on)
328 @region.conditional_cache_on_arguments(condition=cache_on)
327 def _bulk_request(_repo_id, _commit_id, _pre_load):
329 def _bulk_request(_repo_id, _commit_id, _pre_load):
328 result = {}
330 result = {}
329 for attr in pre_load:
331 for attr in pre_load:
330 try:
332 try:
331 method = self._bulk_methods[attr]
333 method = self._bulk_methods[attr]
332 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
334 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
333 result[attr] = method(wire, commit_id)
335 result[attr] = method(wire, commit_id)
334 except KeyError as e:
336 except KeyError as e:
335 raise exceptions.VcsException(e)(
337 raise exceptions.VcsException(e)(
336 f'Unknown bulk attribute: "{attr}"')
338 f'Unknown bulk attribute: "{attr}"')
337 return result
339 return result
338
340
339 return _bulk_request(repo_id, commit_id, sorted(pre_load))
341 return _bulk_request(repo_id, commit_id, sorted(pre_load))
340
342
341 @reraise_safe_exceptions
343 @reraise_safe_exceptions
342 def ctx_branch(self, wire, commit_id):
344 def ctx_branch(self, wire, commit_id):
343 cache_on, context_uid, repo_id = self._cache_on(wire)
345 cache_on, context_uid, repo_id = self._cache_on(wire)
344 region = self._region(wire)
346 region = self._region(wire)
345
347
346 @region.conditional_cache_on_arguments(condition=cache_on)
348 @region.conditional_cache_on_arguments(condition=cache_on)
347 def _ctx_branch(_repo_id, _commit_id):
349 def _ctx_branch(_repo_id, _commit_id):
348 repo = self._factory.repo(wire)
350 repo = self._factory.repo(wire)
349 ctx = self._get_ctx(repo, commit_id)
351 ctx = self._get_ctx(repo, commit_id)
350 return ctx.branch()
352 return ctx.branch()
351 return _ctx_branch(repo_id, commit_id)
353 return _ctx_branch(repo_id, commit_id)
352
354
353 @reraise_safe_exceptions
355 @reraise_safe_exceptions
354 def ctx_date(self, wire, commit_id):
356 def ctx_date(self, wire, commit_id):
355 cache_on, context_uid, repo_id = self._cache_on(wire)
357 cache_on, context_uid, repo_id = self._cache_on(wire)
356 region = self._region(wire)
358 region = self._region(wire)
357
359
358 @region.conditional_cache_on_arguments(condition=cache_on)
360 @region.conditional_cache_on_arguments(condition=cache_on)
359 def _ctx_date(_repo_id, _commit_id):
361 def _ctx_date(_repo_id, _commit_id):
360 repo = self._factory.repo(wire)
362 repo = self._factory.repo(wire)
361 ctx = self._get_ctx(repo, commit_id)
363 ctx = self._get_ctx(repo, commit_id)
362 return ctx.date()
364 return ctx.date()
363 return _ctx_date(repo_id, commit_id)
365 return _ctx_date(repo_id, commit_id)
364
366
365 @reraise_safe_exceptions
367 @reraise_safe_exceptions
366 def ctx_description(self, wire, revision):
368 def ctx_description(self, wire, revision):
367 repo = self._factory.repo(wire)
369 repo = self._factory.repo(wire)
368 ctx = self._get_ctx(repo, revision)
370 ctx = self._get_ctx(repo, revision)
369 return ctx.description()
371 return ctx.description()
370
372
371 @reraise_safe_exceptions
373 @reraise_safe_exceptions
372 def ctx_files(self, wire, commit_id):
374 def ctx_files(self, wire, commit_id):
373 cache_on, context_uid, repo_id = self._cache_on(wire)
375 cache_on, context_uid, repo_id = self._cache_on(wire)
374 region = self._region(wire)
376 region = self._region(wire)
375
377
376 @region.conditional_cache_on_arguments(condition=cache_on)
378 @region.conditional_cache_on_arguments(condition=cache_on)
377 def _ctx_files(_repo_id, _commit_id):
379 def _ctx_files(_repo_id, _commit_id):
378 repo = self._factory.repo(wire)
380 repo = self._factory.repo(wire)
379 ctx = self._get_ctx(repo, commit_id)
381 ctx = self._get_ctx(repo, commit_id)
380 return ctx.files()
382 return ctx.files()
381
383
382 return _ctx_files(repo_id, commit_id)
384 return _ctx_files(repo_id, commit_id)
383
385
384 @reraise_safe_exceptions
386 @reraise_safe_exceptions
385 def ctx_list(self, path, revision):
387 def ctx_list(self, path, revision):
386 repo = self._factory.repo(path)
388 repo = self._factory.repo(path)
387 ctx = self._get_ctx(repo, revision)
389 ctx = self._get_ctx(repo, revision)
388 return list(ctx)
390 return list(ctx)
389
391
390 @reraise_safe_exceptions
392 @reraise_safe_exceptions
391 def ctx_parents(self, wire, commit_id):
393 def ctx_parents(self, wire, commit_id):
392 cache_on, context_uid, repo_id = self._cache_on(wire)
394 cache_on, context_uid, repo_id = self._cache_on(wire)
393 region = self._region(wire)
395 region = self._region(wire)
394
396
395 @region.conditional_cache_on_arguments(condition=cache_on)
397 @region.conditional_cache_on_arguments(condition=cache_on)
396 def _ctx_parents(_repo_id, _commit_id):
398 def _ctx_parents(_repo_id, _commit_id):
397 repo = self._factory.repo(wire)
399 repo = self._factory.repo(wire)
398 ctx = self._get_ctx(repo, commit_id)
400 ctx = self._get_ctx(repo, commit_id)
399 return [parent.hex() for parent in ctx.parents()
401 return [parent.hex() for parent in ctx.parents()
400 if not (parent.hidden() or parent.obsolete())]
402 if not (parent.hidden() or parent.obsolete())]
401
403
402 return _ctx_parents(repo_id, commit_id)
404 return _ctx_parents(repo_id, commit_id)
403
405
404 @reraise_safe_exceptions
406 @reraise_safe_exceptions
405 def ctx_children(self, wire, commit_id):
407 def ctx_children(self, wire, commit_id):
406 cache_on, context_uid, repo_id = self._cache_on(wire)
408 cache_on, context_uid, repo_id = self._cache_on(wire)
407 region = self._region(wire)
409 region = self._region(wire)
408
410
409 @region.conditional_cache_on_arguments(condition=cache_on)
411 @region.conditional_cache_on_arguments(condition=cache_on)
410 def _ctx_children(_repo_id, _commit_id):
412 def _ctx_children(_repo_id, _commit_id):
411 repo = self._factory.repo(wire)
413 repo = self._factory.repo(wire)
412 ctx = self._get_ctx(repo, commit_id)
414 ctx = self._get_ctx(repo, commit_id)
413 return [child.hex() for child in ctx.children()
415 return [child.hex() for child in ctx.children()
414 if not (child.hidden() or child.obsolete())]
416 if not (child.hidden() or child.obsolete())]
415
417
416 return _ctx_children(repo_id, commit_id)
418 return _ctx_children(repo_id, commit_id)
417
419
418 @reraise_safe_exceptions
420 @reraise_safe_exceptions
419 def ctx_phase(self, wire, commit_id):
421 def ctx_phase(self, wire, commit_id):
420 cache_on, context_uid, repo_id = self._cache_on(wire)
422 cache_on, context_uid, repo_id = self._cache_on(wire)
421 region = self._region(wire)
423 region = self._region(wire)
422
424
423 @region.conditional_cache_on_arguments(condition=cache_on)
425 @region.conditional_cache_on_arguments(condition=cache_on)
424 def _ctx_phase(_context_uid, _repo_id, _commit_id):
426 def _ctx_phase(_context_uid, _repo_id, _commit_id):
425 repo = self._factory.repo(wire)
427 repo = self._factory.repo(wire)
426 ctx = self._get_ctx(repo, commit_id)
428 ctx = self._get_ctx(repo, commit_id)
427 # public=0, draft=1, secret=3
429 # public=0, draft=1, secret=3
428 return ctx.phase()
430 return ctx.phase()
429 return _ctx_phase(context_uid, repo_id, commit_id)
431 return _ctx_phase(context_uid, repo_id, commit_id)
430
432
431 @reraise_safe_exceptions
433 @reraise_safe_exceptions
432 def ctx_obsolete(self, wire, commit_id):
434 def ctx_obsolete(self, wire, commit_id):
433 cache_on, context_uid, repo_id = self._cache_on(wire)
435 cache_on, context_uid, repo_id = self._cache_on(wire)
434 region = self._region(wire)
436 region = self._region(wire)
435
437
436 @region.conditional_cache_on_arguments(condition=cache_on)
438 @region.conditional_cache_on_arguments(condition=cache_on)
437 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
439 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
438 repo = self._factory.repo(wire)
440 repo = self._factory.repo(wire)
439 ctx = self._get_ctx(repo, commit_id)
441 ctx = self._get_ctx(repo, commit_id)
440 return ctx.obsolete()
442 return ctx.obsolete()
441 return _ctx_obsolete(context_uid, repo_id, commit_id)
443 return _ctx_obsolete(context_uid, repo_id, commit_id)
442
444
443 @reraise_safe_exceptions
445 @reraise_safe_exceptions
444 def ctx_hidden(self, wire, commit_id):
446 def ctx_hidden(self, wire, commit_id):
445 cache_on, context_uid, repo_id = self._cache_on(wire)
447 cache_on, context_uid, repo_id = self._cache_on(wire)
446 region = self._region(wire)
448 region = self._region(wire)
447
449
448 @region.conditional_cache_on_arguments(condition=cache_on)
450 @region.conditional_cache_on_arguments(condition=cache_on)
449 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
451 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
450 repo = self._factory.repo(wire)
452 repo = self._factory.repo(wire)
451 ctx = self._get_ctx(repo, commit_id)
453 ctx = self._get_ctx(repo, commit_id)
452 return ctx.hidden()
454 return ctx.hidden()
453 return _ctx_hidden(context_uid, repo_id, commit_id)
455 return _ctx_hidden(context_uid, repo_id, commit_id)
454
456
455 @reraise_safe_exceptions
457 @reraise_safe_exceptions
456 def ctx_substate(self, wire, revision):
458 def ctx_substate(self, wire, revision):
457 repo = self._factory.repo(wire)
459 repo = self._factory.repo(wire)
458 ctx = self._get_ctx(repo, revision)
460 ctx = self._get_ctx(repo, revision)
459 return ctx.substate
461 return ctx.substate
460
462
461 @reraise_safe_exceptions
463 @reraise_safe_exceptions
462 def ctx_status(self, wire, revision):
464 def ctx_status(self, wire, revision):
463 repo = self._factory.repo(wire)
465 repo = self._factory.repo(wire)
464 ctx = self._get_ctx(repo, revision)
466 ctx = self._get_ctx(repo, revision)
465 status = repo[ctx.p1().node()].status(other=ctx.node())
467 status = repo[ctx.p1().node()].status(other=ctx.node())
466 # object of status (odd, custom named tuple in mercurial) is not
468 # object of status (odd, custom named tuple in mercurial) is not
467 # correctly serializable, we make it a list, as the underling
469 # correctly serializable, we make it a list, as the underling
468 # API expects this to be a list
470 # API expects this to be a list
469 return list(status)
471 return list(status)
470
472
471 @reraise_safe_exceptions
473 @reraise_safe_exceptions
472 def ctx_user(self, wire, revision):
474 def ctx_user(self, wire, revision):
473 repo = self._factory.repo(wire)
475 repo = self._factory.repo(wire)
474 ctx = self._get_ctx(repo, revision)
476 ctx = self._get_ctx(repo, revision)
475 return ctx.user()
477 return ctx.user()
476
478
477 @reraise_safe_exceptions
479 @reraise_safe_exceptions
478 def check_url(self, url, config):
480 def check_url(self, url, config):
479 url, _proto = normalize_url_for_hg(url)
481 url, _proto = normalize_url_for_hg(url)
480 url_obj = url_parser(safe_bytes(url))
482 url_obj = url_parser(safe_bytes(url))
481
483
482 test_uri = safe_str(url_obj.authinfo()[0])
484 test_uri = safe_str(url_obj.authinfo()[0])
483 authinfo = url_obj.authinfo()[1]
485 authinfo = url_obj.authinfo()[1]
484 obfuscated_uri = get_obfuscated_url(url_obj)
486 obfuscated_uri = get_obfuscated_url(url_obj)
485 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
487 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
486
488
487 handlers = []
489 handlers = []
488 if authinfo:
490 if authinfo:
489 # create a password manager
491 # create a password manager
490 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
492 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
491 passmgr.add_password(*convert_to_str(authinfo))
493 passmgr.add_password(*convert_to_str(authinfo))
492
494
493 handlers.extend((httpbasicauthhandler(passmgr),
495 handlers.extend((httpbasicauthhandler(passmgr),
494 httpdigestauthhandler(passmgr)))
496 httpdigestauthhandler(passmgr)))
495
497
496 o = urllib.request.build_opener(*handlers)
498 o = urllib.request.build_opener(*handlers)
497 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
499 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
498 ('Accept', 'application/mercurial-0.1')]
500 ('Accept', 'application/mercurial-0.1')]
499
501
500 q = {"cmd": 'between'}
502 q = {"cmd": 'between'}
501 q.update({'pairs': "{}-{}".format('0' * 40, '0' * 40)})
503 q.update({'pairs': "{}-{}".format('0' * 40, '0' * 40)})
502 qs = f'?{urllib.parse.urlencode(q)}'
504 qs = f'?{urllib.parse.urlencode(q)}'
503 cu = f"{test_uri}{qs}"
505 cu = f"{test_uri}{qs}"
504
506
505 try:
507 try:
506 req = urllib.request.Request(cu, None, {})
508 req = urllib.request.Request(cu, None, {})
507 log.debug("Trying to open URL %s", obfuscated_uri)
509 log.debug("Trying to open URL %s", obfuscated_uri)
508 resp = o.open(req)
510 resp = o.open(req)
509 if resp.code != 200:
511 if resp.code != 200:
510 raise exceptions.URLError()('Return Code is not 200')
512 raise exceptions.URLError()('Return Code is not 200')
511 except Exception as e:
513 except Exception as e:
512 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
514 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
513 # means it cannot be cloned
515 # means it cannot be cloned
514 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
516 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
515
517
516 # now check if it's a proper hg repo, but don't do it for svn
518 # now check if it's a proper hg repo, but don't do it for svn
517 try:
519 try:
518 if _proto == 'svn':
520 if _proto == 'svn':
519 pass
521 pass
520 else:
522 else:
521 # check for pure hg repos
523 # check for pure hg repos
522 log.debug(
524 log.debug(
523 "Verifying if URL is a Mercurial repository: %s", obfuscated_uri)
525 "Verifying if URL is a Mercurial repository: %s", obfuscated_uri)
524 ui = make_ui_from_config(config)
526 # Create repo path with custom mercurial path object
525 peer_checker = makepeer(ui, safe_bytes(url))
527 ui = make_ui_from_config(config, interactive=False)
528 repo_path = hg_path(ui=ui, rawloc=safe_bytes(url))
529 peer_checker = make_peer(ui, repo_path, False)
526 peer_checker.lookup(b'tip')
530 peer_checker.lookup(b'tip')
527 except Exception as e:
531 except Exception as e:
528 log.warning("URL is not a valid Mercurial repository: %s",
532 log.warning("URL is not a valid Mercurial repository: %s",
529 obfuscated_uri)
533 obfuscated_uri)
530 raise exceptions.URLError(e)(
534 raise exceptions.URLError(e)(
531 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
535 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
532
536
533 log.info("URL is a valid Mercurial repository: %s", obfuscated_uri)
537 log.info("URL is a valid Mercurial repository: %s", obfuscated_uri)
534 return True
538 return True
535
539
536 @reraise_safe_exceptions
540 @reraise_safe_exceptions
537 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_git, opt_ignorews, context):
541 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_git, opt_ignorews, context):
538 repo = self._factory.repo(wire)
542 repo = self._factory.repo(wire)
539
543
540 if file_filter:
544 if file_filter:
541 # unpack the file-filter
545 # unpack the file-filter
542 repo_path, node_path = file_filter
546 repo_path, node_path = file_filter
543 match_filter = match(safe_bytes(repo_path), b'', [safe_bytes(node_path)])
547 match_filter = match(safe_bytes(repo_path), b'', [safe_bytes(node_path)])
544 else:
548 else:
545 match_filter = file_filter
549 match_filter = file_filter
546 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context, showfunc=1)
550 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context, showfunc=1)
547
551
548 try:
552 try:
549 diff_iter = patch.diff(
553 diff_iter = patch.diff(
550 repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts)
554 repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts)
551 return BytesEnvelope(b"".join(diff_iter))
555 return BytesEnvelope(b"".join(diff_iter))
552 except RepoLookupError as e:
556 except RepoLookupError as e:
553 raise exceptions.LookupException(e)()
557 raise exceptions.LookupException(e)()
554
558
555 @reraise_safe_exceptions
559 @reraise_safe_exceptions
556 def node_history(self, wire, revision, path, limit):
560 def node_history(self, wire, revision, path, limit):
557 cache_on, context_uid, repo_id = self._cache_on(wire)
561 cache_on, context_uid, repo_id = self._cache_on(wire)
558 region = self._region(wire)
562 region = self._region(wire)
559
563
560 @region.conditional_cache_on_arguments(condition=cache_on)
564 @region.conditional_cache_on_arguments(condition=cache_on)
561 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
565 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
562 repo = self._factory.repo(wire)
566 repo = self._factory.repo(wire)
563
567
564 ctx = self._get_ctx(repo, revision)
568 ctx = self._get_ctx(repo, revision)
565 fctx = ctx.filectx(safe_bytes(path))
569 fctx = ctx.filectx(safe_bytes(path))
566
570
567 def history_iter():
571 def history_iter():
568 limit_rev = fctx.rev()
572 limit_rev = fctx.rev()
569
573
570 for fctx_candidate in reversed(list(fctx.filelog())):
574 for fctx_candidate in reversed(list(fctx.filelog())):
571 f_obj = fctx.filectx(fctx_candidate)
575 f_obj = fctx.filectx(fctx_candidate)
572
576
573 # NOTE: This can be problematic...we can hide ONLY history node resulting in empty history
577 # NOTE: This can be problematic...we can hide ONLY history node resulting in empty history
574 _ctx = f_obj.changectx()
578 _ctx = f_obj.changectx()
575 if _ctx.hidden() or _ctx.obsolete():
579 if _ctx.hidden() or _ctx.obsolete():
576 continue
580 continue
577
581
578 if limit_rev >= f_obj.rev():
582 if limit_rev >= f_obj.rev():
579 yield f_obj
583 yield f_obj
580
584
581 history = []
585 history = []
582 for cnt, obj in enumerate(history_iter()):
586 for cnt, obj in enumerate(history_iter()):
583 if limit and cnt >= limit:
587 if limit and cnt >= limit:
584 break
588 break
585 history.append(hex(obj.node()))
589 history.append(hex(obj.node()))
586
590
587 return [x for x in history]
591 return [x for x in history]
588 return _node_history(context_uid, repo_id, revision, path, limit)
592 return _node_history(context_uid, repo_id, revision, path, limit)
589
593
590 @reraise_safe_exceptions
594 @reraise_safe_exceptions
591 def node_history_until(self, wire, revision, path, limit):
595 def node_history_until(self, wire, revision, path, limit):
592 cache_on, context_uid, repo_id = self._cache_on(wire)
596 cache_on, context_uid, repo_id = self._cache_on(wire)
593 region = self._region(wire)
597 region = self._region(wire)
594
598
595 @region.conditional_cache_on_arguments(condition=cache_on)
599 @region.conditional_cache_on_arguments(condition=cache_on)
596 def _node_history_until(_context_uid, _repo_id):
600 def _node_history_until(_context_uid, _repo_id):
597 repo = self._factory.repo(wire)
601 repo = self._factory.repo(wire)
598 ctx = self._get_ctx(repo, revision)
602 ctx = self._get_ctx(repo, revision)
599 fctx = ctx.filectx(safe_bytes(path))
603 fctx = ctx.filectx(safe_bytes(path))
600
604
601 file_log = list(fctx.filelog())
605 file_log = list(fctx.filelog())
602 if limit:
606 if limit:
603 # Limit to the last n items
607 # Limit to the last n items
604 file_log = file_log[-limit:]
608 file_log = file_log[-limit:]
605
609
606 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
610 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
607 return _node_history_until(context_uid, repo_id, revision, path, limit)
611 return _node_history_until(context_uid, repo_id, revision, path, limit)
608
612
609 @reraise_safe_exceptions
613 @reraise_safe_exceptions
610 def bulk_file_request(self, wire, commit_id, path, pre_load):
614 def bulk_file_request(self, wire, commit_id, path, pre_load):
611 cache_on, context_uid, repo_id = self._cache_on(wire)
615 cache_on, context_uid, repo_id = self._cache_on(wire)
612 region = self._region(wire)
616 region = self._region(wire)
613
617
614 @region.conditional_cache_on_arguments(condition=cache_on)
618 @region.conditional_cache_on_arguments(condition=cache_on)
615 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
619 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
616 result = {}
620 result = {}
617 for attr in pre_load:
621 for attr in pre_load:
618 try:
622 try:
619 method = self._bulk_file_methods[attr]
623 method = self._bulk_file_methods[attr]
620 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
624 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
621 result[attr] = method(wire, _commit_id, _path)
625 result[attr] = method(wire, _commit_id, _path)
622 except KeyError as e:
626 except KeyError as e:
623 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
627 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
624 return result
628 return result
625
629
626 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
630 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
627
631
628 @reraise_safe_exceptions
632 @reraise_safe_exceptions
629 def fctx_annotate(self, wire, revision, path):
633 def fctx_annotate(self, wire, revision, path):
630 repo = self._factory.repo(wire)
634 repo = self._factory.repo(wire)
631 ctx = self._get_ctx(repo, revision)
635 ctx = self._get_ctx(repo, revision)
632 fctx = ctx.filectx(safe_bytes(path))
636 fctx = ctx.filectx(safe_bytes(path))
633
637
634 result = []
638 result = []
635 for i, annotate_obj in enumerate(fctx.annotate(), 1):
639 for i, annotate_obj in enumerate(fctx.annotate(), 1):
636 ln_no = i
640 ln_no = i
637 sha = hex(annotate_obj.fctx.node())
641 sha = hex(annotate_obj.fctx.node())
638 content = annotate_obj.text
642 content = annotate_obj.text
639 result.append((ln_no, ascii_str(sha), content))
643 result.append((ln_no, ascii_str(sha), content))
640 return BinaryEnvelope(result)
644 return BinaryEnvelope(result)
641
645
642 @reraise_safe_exceptions
646 @reraise_safe_exceptions
643 def fctx_node_data(self, wire, revision, path):
647 def fctx_node_data(self, wire, revision, path):
644 repo = self._factory.repo(wire)
648 repo = self._factory.repo(wire)
645 ctx = self._get_ctx(repo, revision)
649 ctx = self._get_ctx(repo, revision)
646 fctx = ctx.filectx(safe_bytes(path))
650 fctx = ctx.filectx(safe_bytes(path))
647 return BytesEnvelope(fctx.data())
651 return BytesEnvelope(fctx.data())
648
652
649 @reraise_safe_exceptions
653 @reraise_safe_exceptions
650 def fctx_flags(self, wire, commit_id, path):
654 def fctx_flags(self, wire, commit_id, path):
651 cache_on, context_uid, repo_id = self._cache_on(wire)
655 cache_on, context_uid, repo_id = self._cache_on(wire)
652 region = self._region(wire)
656 region = self._region(wire)
653
657
654 @region.conditional_cache_on_arguments(condition=cache_on)
658 @region.conditional_cache_on_arguments(condition=cache_on)
655 def _fctx_flags(_repo_id, _commit_id, _path):
659 def _fctx_flags(_repo_id, _commit_id, _path):
656 repo = self._factory.repo(wire)
660 repo = self._factory.repo(wire)
657 ctx = self._get_ctx(repo, commit_id)
661 ctx = self._get_ctx(repo, commit_id)
658 fctx = ctx.filectx(safe_bytes(path))
662 fctx = ctx.filectx(safe_bytes(path))
659 return fctx.flags()
663 return fctx.flags()
660
664
661 return _fctx_flags(repo_id, commit_id, path)
665 return _fctx_flags(repo_id, commit_id, path)
662
666
663 @reraise_safe_exceptions
667 @reraise_safe_exceptions
664 def fctx_size(self, wire, commit_id, path):
668 def fctx_size(self, wire, commit_id, path):
665 cache_on, context_uid, repo_id = self._cache_on(wire)
669 cache_on, context_uid, repo_id = self._cache_on(wire)
666 region = self._region(wire)
670 region = self._region(wire)
667
671
668 @region.conditional_cache_on_arguments(condition=cache_on)
672 @region.conditional_cache_on_arguments(condition=cache_on)
669 def _fctx_size(_repo_id, _revision, _path):
673 def _fctx_size(_repo_id, _revision, _path):
670 repo = self._factory.repo(wire)
674 repo = self._factory.repo(wire)
671 ctx = self._get_ctx(repo, commit_id)
675 ctx = self._get_ctx(repo, commit_id)
672 fctx = ctx.filectx(safe_bytes(path))
676 fctx = ctx.filectx(safe_bytes(path))
673 return fctx.size()
677 return fctx.size()
674 return _fctx_size(repo_id, commit_id, path)
678 return _fctx_size(repo_id, commit_id, path)
675
679
676 @reraise_safe_exceptions
680 @reraise_safe_exceptions
677 def get_all_commit_ids(self, wire, name):
681 def get_all_commit_ids(self, wire, name):
678 cache_on, context_uid, repo_id = self._cache_on(wire)
682 cache_on, context_uid, repo_id = self._cache_on(wire)
679 region = self._region(wire)
683 region = self._region(wire)
680
684
681 @region.conditional_cache_on_arguments(condition=cache_on)
685 @region.conditional_cache_on_arguments(condition=cache_on)
682 def _get_all_commit_ids(_context_uid, _repo_id, _name):
686 def _get_all_commit_ids(_context_uid, _repo_id, _name):
683 repo = self._factory.repo(wire)
687 repo = self._factory.repo(wire)
684 revs = [ascii_str(repo[x].hex()) for x in repo.filtered(b'visible').changelog.revs()]
688 revs = [ascii_str(repo[x].hex()) for x in repo.filtered(b'visible').changelog.revs()]
685 return revs
689 return revs
686 return _get_all_commit_ids(context_uid, repo_id, name)
690 return _get_all_commit_ids(context_uid, repo_id, name)
687
691
688 @reraise_safe_exceptions
692 @reraise_safe_exceptions
689 def get_config_value(self, wire, section, name, untrusted=False):
693 def get_config_value(self, wire, section, name, untrusted=False):
690 repo = self._factory.repo(wire)
694 repo = self._factory.repo(wire)
691 return repo.ui.config(ascii_bytes(section), ascii_bytes(name), untrusted=untrusted)
695 return repo.ui.config(ascii_bytes(section), ascii_bytes(name), untrusted=untrusted)
692
696
693 @reraise_safe_exceptions
697 @reraise_safe_exceptions
694 def is_large_file(self, wire, commit_id, path):
698 def is_large_file(self, wire, commit_id, path):
695 cache_on, context_uid, repo_id = self._cache_on(wire)
699 cache_on, context_uid, repo_id = self._cache_on(wire)
696 region = self._region(wire)
700 region = self._region(wire)
697
701
698 @region.conditional_cache_on_arguments(condition=cache_on)
702 @region.conditional_cache_on_arguments(condition=cache_on)
699 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
703 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
700 return largefiles.lfutil.isstandin(safe_bytes(path))
704 return largefiles.lfutil.isstandin(safe_bytes(path))
701
705
702 return _is_large_file(context_uid, repo_id, commit_id, path)
706 return _is_large_file(context_uid, repo_id, commit_id, path)
703
707
704 @reraise_safe_exceptions
708 @reraise_safe_exceptions
705 def is_binary(self, wire, revision, path):
709 def is_binary(self, wire, revision, path):
706 cache_on, context_uid, repo_id = self._cache_on(wire)
710 cache_on, context_uid, repo_id = self._cache_on(wire)
707 region = self._region(wire)
711 region = self._region(wire)
708
712
709 @region.conditional_cache_on_arguments(condition=cache_on)
713 @region.conditional_cache_on_arguments(condition=cache_on)
710 def _is_binary(_repo_id, _sha, _path):
714 def _is_binary(_repo_id, _sha, _path):
711 repo = self._factory.repo(wire)
715 repo = self._factory.repo(wire)
712 ctx = self._get_ctx(repo, revision)
716 ctx = self._get_ctx(repo, revision)
713 fctx = ctx.filectx(safe_bytes(path))
717 fctx = ctx.filectx(safe_bytes(path))
714 return fctx.isbinary()
718 return fctx.isbinary()
715
719
716 return _is_binary(repo_id, revision, path)
720 return _is_binary(repo_id, revision, path)
717
721
718 @reraise_safe_exceptions
722 @reraise_safe_exceptions
719 def md5_hash(self, wire, revision, path):
723 def md5_hash(self, wire, revision, path):
720 cache_on, context_uid, repo_id = self._cache_on(wire)
724 cache_on, context_uid, repo_id = self._cache_on(wire)
721 region = self._region(wire)
725 region = self._region(wire)
722
726
723 @region.conditional_cache_on_arguments(condition=cache_on)
727 @region.conditional_cache_on_arguments(condition=cache_on)
724 def _md5_hash(_repo_id, _sha, _path):
728 def _md5_hash(_repo_id, _sha, _path):
725 repo = self._factory.repo(wire)
729 repo = self._factory.repo(wire)
726 ctx = self._get_ctx(repo, revision)
730 ctx = self._get_ctx(repo, revision)
727 fctx = ctx.filectx(safe_bytes(path))
731 fctx = ctx.filectx(safe_bytes(path))
728 return hashlib.md5(fctx.data()).hexdigest()
732 return hashlib.md5(fctx.data()).hexdigest()
729
733
730 return _md5_hash(repo_id, revision, path)
734 return _md5_hash(repo_id, revision, path)
731
735
732 @reraise_safe_exceptions
736 @reraise_safe_exceptions
733 def in_largefiles_store(self, wire, sha):
737 def in_largefiles_store(self, wire, sha):
734 repo = self._factory.repo(wire)
738 repo = self._factory.repo(wire)
735 return largefiles.lfutil.instore(repo, sha)
739 return largefiles.lfutil.instore(repo, sha)
736
740
737 @reraise_safe_exceptions
741 @reraise_safe_exceptions
738 def in_user_cache(self, wire, sha):
742 def in_user_cache(self, wire, sha):
739 repo = self._factory.repo(wire)
743 repo = self._factory.repo(wire)
740 return largefiles.lfutil.inusercache(repo.ui, sha)
744 return largefiles.lfutil.inusercache(repo.ui, sha)
741
745
742 @reraise_safe_exceptions
746 @reraise_safe_exceptions
743 def store_path(self, wire, sha):
747 def store_path(self, wire, sha):
744 repo = self._factory.repo(wire)
748 repo = self._factory.repo(wire)
745 return largefiles.lfutil.storepath(repo, sha)
749 return largefiles.lfutil.storepath(repo, sha)
746
750
747 @reraise_safe_exceptions
751 @reraise_safe_exceptions
748 def link(self, wire, sha, path):
752 def link(self, wire, sha, path):
749 repo = self._factory.repo(wire)
753 repo = self._factory.repo(wire)
750 largefiles.lfutil.link(
754 largefiles.lfutil.link(
751 largefiles.lfutil.usercachepath(repo.ui, sha), path)
755 largefiles.lfutil.usercachepath(repo.ui, sha), path)
752
756
753 @reraise_safe_exceptions
757 @reraise_safe_exceptions
754 def localrepository(self, wire, create=False):
758 def localrepository(self, wire, create=False):
755 self._factory.repo(wire, create=create)
759 self._factory.repo(wire, create=create)
756
760
757 @reraise_safe_exceptions
761 @reraise_safe_exceptions
758 def lookup(self, wire, revision, both):
762 def lookup(self, wire, revision, both):
759 cache_on, context_uid, repo_id = self._cache_on(wire)
763 cache_on, context_uid, repo_id = self._cache_on(wire)
760 region = self._region(wire)
764 region = self._region(wire)
761
765
762 @region.conditional_cache_on_arguments(condition=cache_on)
766 @region.conditional_cache_on_arguments(condition=cache_on)
763 def _lookup(_context_uid, _repo_id, _revision, _both):
767 def _lookup(_context_uid, _repo_id, _revision, _both):
764 repo = self._factory.repo(wire)
768 repo = self._factory.repo(wire)
765 rev = _revision
769 rev = _revision
766 if isinstance(rev, int):
770 if isinstance(rev, int):
767 # NOTE(marcink):
771 # NOTE(marcink):
768 # since Mercurial doesn't support negative indexes properly
772 # since Mercurial doesn't support negative indexes properly
769 # we need to shift accordingly by one to get proper index, e.g
773 # we need to shift accordingly by one to get proper index, e.g
770 # repo[-1] => repo[-2]
774 # repo[-1] => repo[-2]
771 # repo[0] => repo[-1]
775 # repo[0] => repo[-1]
772 if rev <= 0:
776 if rev <= 0:
773 rev = rev + -1
777 rev = rev + -1
774 try:
778 try:
775 ctx = self._get_ctx(repo, rev)
779 ctx = self._get_ctx(repo, rev)
776 except AmbiguousPrefixLookupError:
780 except AmbiguousPrefixLookupError:
777 e = RepoLookupError(rev)
781 e = RepoLookupError(rev)
778 e._org_exc_tb = format_exc(sys.exc_info())
782 e._org_exc_tb = format_exc(sys.exc_info())
779 raise exceptions.LookupException(e)(rev)
783 raise exceptions.LookupException(e)(rev)
780 except (TypeError, RepoLookupError, binascii.Error) as e:
784 except (TypeError, RepoLookupError, binascii.Error) as e:
781 e._org_exc_tb = format_exc(sys.exc_info())
785 e._org_exc_tb = format_exc(sys.exc_info())
782 raise exceptions.LookupException(e)(rev)
786 raise exceptions.LookupException(e)(rev)
783 except LookupError as e:
787 except LookupError as e:
784 e._org_exc_tb = format_exc(sys.exc_info())
788 e._org_exc_tb = format_exc(sys.exc_info())
785 raise exceptions.LookupException(e)(e.name)
789 raise exceptions.LookupException(e)(e.name)
786
790
787 if not both:
791 if not both:
788 return ctx.hex()
792 return ctx.hex()
789
793
790 ctx = repo[ctx.hex()]
794 ctx = repo[ctx.hex()]
791 return ctx.hex(), ctx.rev()
795 return ctx.hex(), ctx.rev()
792
796
793 return _lookup(context_uid, repo_id, revision, both)
797 return _lookup(context_uid, repo_id, revision, both)
794
798
795 @reraise_safe_exceptions
799 @reraise_safe_exceptions
796 def sync_push(self, wire, url):
800 def sync_push(self, wire, url):
797 if not self.check_url(url, wire['config']):
801 if not self.check_url(url, wire['config']):
798 return
802 return
799
803
800 repo = self._factory.repo(wire)
804 repo = self._factory.repo(wire)
801
805
802 # Disable any prompts for this repo
806 # Disable any prompts for this repo
803 repo.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
807 repo.ui.setconfig(b'ui', b'interactive', b'false', b'-y')
804
808
805 bookmarks = list(dict(repo._bookmarks).keys())
809 bookmarks = list(dict(repo._bookmarks).keys())
806 remote = peer(repo, {}, safe_bytes(url))
810 remote = peer(repo, {}, safe_bytes(url))
807 # Disable any prompts for this remote
811 # Disable any prompts for this remote
808 remote.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
812 remote.ui.setconfig(b'ui', b'interactive', b'false', b'-y')
809
813
810 return exchange.push(
814 return exchange.push(
811 repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
815 repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
812
816
813 @reraise_safe_exceptions
817 @reraise_safe_exceptions
814 def revision(self, wire, rev):
818 def revision(self, wire, rev):
815 repo = self._factory.repo(wire)
819 repo = self._factory.repo(wire)
816 ctx = self._get_ctx(repo, rev)
820 ctx = self._get_ctx(repo, rev)
817 return ctx.rev()
821 return ctx.rev()
818
822
819 @reraise_safe_exceptions
823 @reraise_safe_exceptions
820 def rev_range(self, wire, commit_filter):
824 def rev_range(self, wire, commit_filter):
821 cache_on, context_uid, repo_id = self._cache_on(wire)
825 cache_on, context_uid, repo_id = self._cache_on(wire)
822 region = self._region(wire)
826 region = self._region(wire)
823
827
824 @region.conditional_cache_on_arguments(condition=cache_on)
828 @region.conditional_cache_on_arguments(condition=cache_on)
825 def _rev_range(_context_uid, _repo_id, _filter):
829 def _rev_range(_context_uid, _repo_id, _filter):
826 repo = self._factory.repo(wire)
830 repo = self._factory.repo(wire)
827 revisions = [
831 revisions = [
828 ascii_str(repo[rev].hex())
832 ascii_str(repo[rev].hex())
829 for rev in revrange(repo, list(map(ascii_bytes, commit_filter)))
833 for rev in revrange(repo, list(map(ascii_bytes, commit_filter)))
830 ]
834 ]
831 return revisions
835 return revisions
832
836
833 return _rev_range(context_uid, repo_id, sorted(commit_filter))
837 return _rev_range(context_uid, repo_id, sorted(commit_filter))
834
838
835 @reraise_safe_exceptions
839 @reraise_safe_exceptions
836 def rev_range_hash(self, wire, node):
840 def rev_range_hash(self, wire, node):
837 repo = self._factory.repo(wire)
841 repo = self._factory.repo(wire)
838
842
839 def get_revs(repo, rev_opt):
843 def get_revs(repo, rev_opt):
840 if rev_opt:
844 if rev_opt:
841 revs = revrange(repo, rev_opt)
845 revs = revrange(repo, rev_opt)
842 if len(revs) == 0:
846 if len(revs) == 0:
843 return (nullrev, nullrev)
847 return (nullrev, nullrev)
844 return max(revs), min(revs)
848 return max(revs), min(revs)
845 else:
849 else:
846 return len(repo) - 1, 0
850 return len(repo) - 1, 0
847
851
848 stop, start = get_revs(repo, [node + ':'])
852 stop, start = get_revs(repo, [node + ':'])
849 revs = [ascii_str(repo[r].hex()) for r in range(start, stop + 1)]
853 revs = [ascii_str(repo[r].hex()) for r in range(start, stop + 1)]
850 return revs
854 return revs
851
855
852 @reraise_safe_exceptions
856 @reraise_safe_exceptions
853 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
857 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
854 org_path = safe_bytes(wire["path"])
858 org_path = safe_bytes(wire["path"])
855 other_path = safe_bytes(kwargs.pop('other_path', ''))
859 other_path = safe_bytes(kwargs.pop('other_path', ''))
856
860
857 # case when we want to compare two independent repositories
861 # case when we want to compare two independent repositories
858 if other_path and other_path != wire["path"]:
862 if other_path and other_path != wire["path"]:
859 baseui = self._factory._create_config(wire["config"])
863 baseui = self._factory._create_config(wire["config"])
860 repo = unionrepo.makeunionrepository(baseui, other_path, org_path)
864 repo = unionrepo.makeunionrepository(baseui, other_path, org_path)
861 else:
865 else:
862 repo = self._factory.repo(wire)
866 repo = self._factory.repo(wire)
863 return list(repo.revs(rev_spec, *args))
867 return list(repo.revs(rev_spec, *args))
864
868
865 @reraise_safe_exceptions
869 @reraise_safe_exceptions
866 def verify(self, wire,):
870 def verify(self, wire,):
867 repo = self._factory.repo(wire)
871 repo = self._factory.repo(wire)
868 baseui = self._factory._create_config(wire['config'])
872 baseui = self._factory._create_config(wire['config'])
869
873
870 baseui, output = patch_ui_message_output(baseui)
874 baseui, output = patch_ui_message_output(baseui)
871
875
872 repo.ui = baseui
876 repo.ui = baseui
873 verify.verify(repo)
877 verify.verify(repo)
874 return output.getvalue()
878 return output.getvalue()
875
879
876 @reraise_safe_exceptions
880 @reraise_safe_exceptions
877 def hg_update_cache(self, wire,):
881 def hg_update_cache(self, wire,):
878 repo = self._factory.repo(wire)
882 repo = self._factory.repo(wire)
879 baseui = self._factory._create_config(wire['config'])
883 baseui = self._factory._create_config(wire['config'])
880 baseui, output = patch_ui_message_output(baseui)
884 baseui, output = patch_ui_message_output(baseui)
881
885
882 repo.ui = baseui
886 repo.ui = baseui
883 with repo.wlock(), repo.lock():
887 with repo.wlock(), repo.lock():
884 repo.updatecaches(full=True)
888 repo.updatecaches(full=True)
885
889
886 return output.getvalue()
890 return output.getvalue()
887
891
888 @reraise_safe_exceptions
892 @reraise_safe_exceptions
889 def hg_rebuild_fn_cache(self, wire,):
893 def hg_rebuild_fn_cache(self, wire,):
890 repo = self._factory.repo(wire)
894 repo = self._factory.repo(wire)
891 baseui = self._factory._create_config(wire['config'])
895 baseui = self._factory._create_config(wire['config'])
892 baseui, output = patch_ui_message_output(baseui)
896 baseui, output = patch_ui_message_output(baseui)
893
897
894 repo.ui = baseui
898 repo.ui = baseui
895
899
896 repair.rebuildfncache(baseui, repo)
900 repair.rebuildfncache(baseui, repo)
897
901
898 return output.getvalue()
902 return output.getvalue()
899
903
900 @reraise_safe_exceptions
904 @reraise_safe_exceptions
901 def tags(self, wire):
905 def tags(self, wire):
902 cache_on, context_uid, repo_id = self._cache_on(wire)
906 cache_on, context_uid, repo_id = self._cache_on(wire)
903 region = self._region(wire)
907 region = self._region(wire)
904
908
905 @region.conditional_cache_on_arguments(condition=cache_on)
909 @region.conditional_cache_on_arguments(condition=cache_on)
906 def _tags(_context_uid, _repo_id):
910 def _tags(_context_uid, _repo_id):
907 repo = self._factory.repo(wire)
911 repo = self._factory.repo(wire)
908 return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo.tags().items()}
912 return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo.tags().items()}
909
913
910 return _tags(context_uid, repo_id)
914 return _tags(context_uid, repo_id)
911
915
912 @reraise_safe_exceptions
916 @reraise_safe_exceptions
913 def update(self, wire, node='', clean=False):
917 def update(self, wire, node='', clean=False):
914 repo = self._factory.repo(wire)
918 repo = self._factory.repo(wire)
915 baseui = self._factory._create_config(wire['config'])
919 baseui = self._factory._create_config(wire['config'])
916 node = safe_bytes(node)
920 node = safe_bytes(node)
917
921
918 commands.update(baseui, repo, node=node, clean=clean)
922 commands.update(baseui, repo, node=node, clean=clean)
919
923
920 @reraise_safe_exceptions
924 @reraise_safe_exceptions
921 def identify(self, wire):
925 def identify(self, wire):
922 repo = self._factory.repo(wire)
926 repo = self._factory.repo(wire)
923 baseui = self._factory._create_config(wire['config'])
927 baseui = self._factory._create_config(wire['config'])
924 output = io.BytesIO()
928 output = io.BytesIO()
925 baseui.write = output.write
929 baseui.write = output.write
926 # This is required to get a full node id
930 # This is required to get a full node id
927 baseui.debugflag = True
931 baseui.debugflag = True
928 commands.identify(baseui, repo, id=True)
932 commands.identify(baseui, repo, id=True)
929
933
930 return output.getvalue()
934 return output.getvalue()
931
935
932 @reraise_safe_exceptions
936 @reraise_safe_exceptions
933 def heads(self, wire, branch=None):
937 def heads(self, wire, branch=None):
934 repo = self._factory.repo(wire)
938 repo = self._factory.repo(wire)
935 baseui = self._factory._create_config(wire['config'])
939 baseui = self._factory._create_config(wire['config'])
936 output = io.BytesIO()
940 output = io.BytesIO()
937
941
938 def write(data, **unused_kwargs):
942 def write(data, **unused_kwargs):
939 output.write(data)
943 output.write(data)
940
944
941 baseui.write = write
945 baseui.write = write
942 if branch:
946 if branch:
943 args = [safe_bytes(branch)]
947 args = [safe_bytes(branch)]
944 else:
948 else:
945 args = []
949 args = []
946 commands.heads(baseui, repo, template=b'{node} ', *args)
950 commands.heads(baseui, repo, template=b'{node} ', *args)
947
951
948 return output.getvalue()
952 return output.getvalue()
949
953
950 @reraise_safe_exceptions
954 @reraise_safe_exceptions
951 def ancestor(self, wire, revision1, revision2):
955 def ancestor(self, wire, revision1, revision2):
952 repo = self._factory.repo(wire)
956 repo = self._factory.repo(wire)
953 changelog = repo.changelog
957 changelog = repo.changelog
954 lookup = repo.lookup
958 lookup = repo.lookup
955 a = changelog.ancestor(lookup(safe_bytes(revision1)), lookup(safe_bytes(revision2)))
959 a = changelog.ancestor(lookup(safe_bytes(revision1)), lookup(safe_bytes(revision2)))
956 return hex(a)
960 return hex(a)
957
961
958 @reraise_safe_exceptions
962 @reraise_safe_exceptions
959 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
963 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
960 baseui = self._factory._create_config(wire["config"], hooks=hooks)
964 baseui = self._factory._create_config(wire["config"], hooks=hooks)
961 clone(baseui, safe_bytes(source), safe_bytes(dest), noupdate=not update_after_clone)
965 clone(baseui, safe_bytes(source), safe_bytes(dest), noupdate=not update_after_clone)
962
966
963 @reraise_safe_exceptions
967 @reraise_safe_exceptions
964 def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
968 def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
965
969
966 repo = self._factory.repo(wire)
970 repo = self._factory.repo(wire)
967 baseui = self._factory._create_config(wire['config'])
971 baseui = self._factory._create_config(wire['config'])
968 publishing = baseui.configbool(b'phases', b'publish')
972 publishing = baseui.configbool(b'phases', b'publish')
969
973
970 def _filectxfn(_repo, ctx, path: bytes):
974 def _filectxfn(_repo, ctx, path: bytes):
971 """
975 """
972 Marks given path as added/changed/removed in a given _repo. This is
976 Marks given path as added/changed/removed in a given _repo. This is
973 for internal mercurial commit function.
977 for internal mercurial commit function.
974 """
978 """
975
979
976 # check if this path is removed
980 # check if this path is removed
977 if safe_str(path) in removed:
981 if safe_str(path) in removed:
978 # returning None is a way to mark node for removal
982 # returning None is a way to mark node for removal
979 return None
983 return None
980
984
981 # check if this path is added
985 # check if this path is added
982 for node in updated:
986 for node in updated:
983 if safe_bytes(node['path']) == path:
987 if safe_bytes(node['path']) == path:
984 return memfilectx(
988 return memfilectx(
985 _repo,
989 _repo,
986 changectx=ctx,
990 changectx=ctx,
987 path=safe_bytes(node['path']),
991 path=safe_bytes(node['path']),
988 data=safe_bytes(node['content']),
992 data=safe_bytes(node['content']),
989 islink=False,
993 islink=False,
990 isexec=bool(node['mode'] & stat.S_IXUSR),
994 isexec=bool(node['mode'] & stat.S_IXUSR),
991 copysource=False)
995 copysource=False)
992 abort_exc = exceptions.AbortException()
996 abort_exc = exceptions.AbortException()
993 raise abort_exc(f"Given path haven't been marked as added, changed or removed ({path})")
997 raise abort_exc(f"Given path haven't been marked as added, changed or removed ({path})")
994
998
995 if publishing:
999 if publishing:
996 new_commit_phase = b'public'
1000 new_commit_phase = b'public'
997 else:
1001 else:
998 new_commit_phase = b'draft'
1002 new_commit_phase = b'draft'
999 with repo.ui.configoverride({(b'phases', b'new-commit'): new_commit_phase}):
1003 with repo.ui.configoverride({(b'phases', b'new-commit'): new_commit_phase}):
1000 kwargs = {safe_bytes(k): safe_bytes(v) for k, v in extra.items()}
1004 kwargs = {safe_bytes(k): safe_bytes(v) for k, v in extra.items()}
1001 commit_ctx = memctx(
1005 commit_ctx = memctx(
1002 repo=repo,
1006 repo=repo,
1003 parents=parents,
1007 parents=parents,
1004 text=safe_bytes(message),
1008 text=safe_bytes(message),
1005 files=[safe_bytes(x) for x in files],
1009 files=[safe_bytes(x) for x in files],
1006 filectxfn=_filectxfn,
1010 filectxfn=_filectxfn,
1007 user=safe_bytes(user),
1011 user=safe_bytes(user),
1008 date=(commit_time, commit_timezone),
1012 date=(commit_time, commit_timezone),
1009 extra=kwargs)
1013 extra=kwargs)
1010
1014
1011 n = repo.commitctx(commit_ctx)
1015 n = repo.commitctx(commit_ctx)
1012 new_id = hex(n)
1016 new_id = hex(n)
1013
1017
1014 return new_id
1018 return new_id
1015
1019
1016 @reraise_safe_exceptions
1020 @reraise_safe_exceptions
1017 def pull(self, wire, url, commit_ids=None):
1021 def pull(self, wire, url, commit_ids=None):
1018 repo = self._factory.repo(wire)
1022 repo = self._factory.repo(wire)
1019 # Disable any prompts for this repo
1023 # Disable any prompts for this repo
1020 repo.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
1024 repo.ui.setconfig(b'ui', b'interactive', b'false', b'-y')
1021
1025
1022 remote = peer(repo, {}, safe_bytes(url))
1026 remote = peer(repo, {}, safe_bytes(url))
1023 # Disable any prompts for this remote
1027 # Disable any prompts for this remote
1024 remote.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
1028 remote.ui.setconfig(b'ui', b'interactive', b'false', b'-y')
1025
1029
1026 if commit_ids:
1030 if commit_ids:
1027 commit_ids = [bin(commit_id) for commit_id in commit_ids]
1031 commit_ids = [bin(commit_id) for commit_id in commit_ids]
1028
1032
1029 return exchange.pull(
1033 return exchange.pull(
1030 repo, remote, heads=commit_ids, force=None).cgresult
1034 repo, remote, heads=commit_ids, force=None).cgresult
1031
1035
1032 @reraise_safe_exceptions
1036 @reraise_safe_exceptions
1033 def pull_cmd(self, wire, source, bookmark='', branch='', revision='', hooks=True):
1037 def pull_cmd(self, wire, source, bookmark='', branch='', revision='', hooks=True):
1034 repo = self._factory.repo(wire)
1038 repo = self._factory.repo(wire)
1035 baseui = self._factory._create_config(wire['config'], hooks=hooks)
1039 baseui = self._factory._create_config(wire['config'], hooks=hooks)
1036
1040
1037 source = safe_bytes(source)
1041 source = safe_bytes(source)
1038
1042
1039 # Mercurial internally has a lot of logic that checks ONLY if
1043 # Mercurial internally has a lot of logic that checks ONLY if
1040 # option is defined, we just pass those if they are defined then
1044 # option is defined, we just pass those if they are defined then
1041 opts = {}
1045 opts = {"remote_hidden": False}
1042
1046
1043 if bookmark:
1047 if bookmark:
1044 opts['bookmark'] = [safe_bytes(x) for x in bookmark] \
1048 opts['bookmark'] = [safe_bytes(x) for x in bookmark] \
1045 if isinstance(bookmark, list) else safe_bytes(bookmark)
1049 if isinstance(bookmark, list) else safe_bytes(bookmark)
1046
1050
1047 if branch:
1051 if branch:
1048 opts['branch'] = [safe_bytes(x) for x in branch] \
1052 opts['branch'] = [safe_bytes(x) for x in branch] \
1049 if isinstance(branch, list) else safe_bytes(branch)
1053 if isinstance(branch, list) else safe_bytes(branch)
1050
1054
1051 if revision:
1055 if revision:
1052 opts['rev'] = [safe_bytes(x) for x in revision] \
1056 opts['rev'] = [safe_bytes(x) for x in revision] \
1053 if isinstance(revision, list) else safe_bytes(revision)
1057 if isinstance(revision, list) else safe_bytes(revision)
1054
1058
1055 commands.pull(baseui, repo, source, **opts)
1059 commands.pull(baseui, repo, source, **opts)
1056
1060
1057 @reraise_safe_exceptions
1061 @reraise_safe_exceptions
1058 def push(self, wire, revisions, dest_path, hooks: bool = True, push_branches: bool = False):
1062 def push(self, wire, revisions, dest_path, hooks: bool = True, push_branches: bool = False):
1059 repo = self._factory.repo(wire)
1063 repo = self._factory.repo(wire)
1060 baseui = self._factory._create_config(wire['config'], hooks=hooks)
1064 baseui = self._factory._create_config(wire['config'], hooks=hooks)
1061
1065
1062 revisions = [safe_bytes(x) for x in revisions] \
1066 revisions = [safe_bytes(x) for x in revisions] \
1063 if isinstance(revisions, list) else safe_bytes(revisions)
1067 if isinstance(revisions, list) else safe_bytes(revisions)
1064
1068
1065 commands.push(baseui, repo, safe_bytes(dest_path),
1069 commands.push(baseui, repo, safe_bytes(dest_path),
1066 rev=revisions,
1070 rev=revisions,
1067 new_branch=push_branches)
1071 new_branch=push_branches)
1068
1072
1069 @reraise_safe_exceptions
1073 @reraise_safe_exceptions
1070 def strip(self, wire, revision, update, backup):
1074 def strip(self, wire, revision, update, backup):
1071 repo = self._factory.repo(wire)
1075 repo = self._factory.repo(wire)
1072 ctx = self._get_ctx(repo, revision)
1076 ctx = self._get_ctx(repo, revision)
1073 hgext_strip.strip(
1077 hgext_strip.strip(
1074 repo.baseui, repo, ctx.node(), update=update, backup=backup)
1078 repo.baseui, repo, ctx.node(), update=update, backup=backup)
1075
1079
1076 @reraise_safe_exceptions
1080 @reraise_safe_exceptions
1077 def get_unresolved_files(self, wire):
1081 def get_unresolved_files(self, wire):
1078 repo = self._factory.repo(wire)
1082 repo = self._factory.repo(wire)
1079
1083
1080 log.debug('Calculating unresolved files for repo: %s', repo)
1084 log.debug('Calculating unresolved files for repo: %s', repo)
1081 output = io.BytesIO()
1085 output = io.BytesIO()
1082
1086
1083 def write(data, **unused_kwargs):
1087 def write(data, **unused_kwargs):
1084 output.write(data)
1088 output.write(data)
1085
1089
1086 baseui = self._factory._create_config(wire['config'])
1090 baseui = self._factory._create_config(wire['config'])
1087 baseui.write = write
1091 baseui.write = write
1088
1092
1089 commands.resolve(baseui, repo, list=True)
1093 commands.resolve(baseui, repo, list=True)
1090 unresolved = output.getvalue().splitlines(0)
1094 unresolved = output.getvalue().splitlines(0)
1091 return unresolved
1095 return unresolved
1092
1096
1093 @reraise_safe_exceptions
1097 @reraise_safe_exceptions
1094 def merge(self, wire, revision):
1098 def merge(self, wire, revision):
1095 repo = self._factory.repo(wire)
1099 repo = self._factory.repo(wire)
1096 baseui = self._factory._create_config(wire['config'])
1100 baseui = self._factory._create_config(wire['config'])
1097 repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
1101 repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
1098
1102
1099 # In case of sub repositories are used mercurial prompts the user in
1103 # In case of sub repositories are used mercurial prompts the user in
1100 # case of merge conflicts or different sub repository sources. By
1104 # case of merge conflicts or different sub repository sources. By
1101 # setting the interactive flag to `False` mercurial doesn't prompt the
1105 # setting the interactive flag to `False` mercurial doesn't prompt the
1102 # used but instead uses a default value.
1106 # used but instead uses a default value.
1103 repo.ui.setconfig(b'ui', b'interactive', False)
1107 repo.ui.setconfig(b'ui', b'interactive', b'false')
1104 commands.merge(baseui, repo, rev=safe_bytes(revision))
1108 commands.merge(baseui, repo, rev=safe_bytes(revision))
1105
1109
1106 @reraise_safe_exceptions
1110 @reraise_safe_exceptions
1107 def merge_state(self, wire):
1111 def merge_state(self, wire):
1108 repo = self._factory.repo(wire)
1112 repo = self._factory.repo(wire)
1109 repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
1113 repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
1110
1114
1111 # In case of sub repositories are used mercurial prompts the user in
1115 # In case of sub repositories are used mercurial prompts the user in
1112 # case of merge conflicts or different sub repository sources. By
1116 # case of merge conflicts or different sub repository sources. By
1113 # setting the interactive flag to `False` mercurial doesn't prompt the
1117 # setting the interactive flag to `False` mercurial doesn't prompt the
1114 # used but instead uses a default value.
1118 # used but instead uses a default value.
1115 repo.ui.setconfig(b'ui', b'interactive', False)
1119 repo.ui.setconfig(b'ui', b'interactive', b'false')
1116 ms = hg_merge.mergestate(repo)
1120 ms = hg_merge.mergestate(repo)
1117 return [x for x in ms.unresolved()]
1121 return [x for x in ms.unresolved()]
1118
1122
1119 @reraise_safe_exceptions
1123 @reraise_safe_exceptions
1120 def commit(self, wire, message, username, close_branch=False):
1124 def commit(self, wire, message, username, close_branch=False):
1121 repo = self._factory.repo(wire)
1125 repo = self._factory.repo(wire)
1122 baseui = self._factory._create_config(wire['config'])
1126 baseui = self._factory._create_config(wire['config'])
1123 repo.ui.setconfig(b'ui', b'username', safe_bytes(username))
1127 repo.ui.setconfig(b'ui', b'username', safe_bytes(username))
1124 commands.commit(baseui, repo, message=safe_bytes(message), close_branch=close_branch)
1128 commands.commit(baseui, repo, message=safe_bytes(message), close_branch=close_branch)
1125
1129
1126 @reraise_safe_exceptions
1130 @reraise_safe_exceptions
1127 def rebase(self, wire, source='', dest='', abort=False):
1131 def rebase(self, wire, source='', dest='', abort=False):
1128
1132
1129 repo = self._factory.repo(wire)
1133 repo = self._factory.repo(wire)
1130 baseui = self._factory._create_config(wire['config'])
1134 baseui = self._factory._create_config(wire['config'])
1131 repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
1135 repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
1132 # In case of sub repositories are used mercurial prompts the user in
1136 # In case of sub repositories are used mercurial prompts the user in
1133 # case of merge conflicts or different sub repository sources. By
1137 # case of merge conflicts or different sub repository sources. By
1134 # setting the interactive flag to `False` mercurial doesn't prompt the
1138 # setting the interactive flag to `False` mercurial doesn't prompt the
1135 # used but instead uses a default value.
1139 # used but instead uses a default value.
1136 repo.ui.setconfig(b'ui', b'interactive', False)
1140 repo.ui.setconfig(b'ui', b'interactive', b'false')
1137
1141
1138 rebase_kws = dict(
1142 rebase_kws = dict(
1139 keep=not abort,
1143 keep=not abort,
1140 abort=abort
1144 abort=abort
1141 )
1145 )
1142
1146
1143 if source:
1147 if source:
1144 source = repo[source]
1148 source = repo[source]
1145 rebase_kws['base'] = [source.hex()]
1149 rebase_kws['base'] = [source.hex()]
1146 if dest:
1150 if dest:
1147 dest = repo[dest]
1151 dest = repo[dest]
1148 rebase_kws['dest'] = dest.hex()
1152 rebase_kws['dest'] = dest.hex()
1149
1153
1150 rebase.rebase(baseui, repo, **rebase_kws)
1154 rebase.rebase(baseui, repo, **rebase_kws)
1151
1155
1152 @reraise_safe_exceptions
1156 @reraise_safe_exceptions
1153 def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
1157 def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
1154 repo = self._factory.repo(wire)
1158 repo = self._factory.repo(wire)
1155 ctx = self._get_ctx(repo, revision)
1159 ctx = self._get_ctx(repo, revision)
1156 node = ctx.node()
1160 node = ctx.node()
1157
1161
1158 date = (tag_time, tag_timezone)
1162 date = (tag_time, tag_timezone)
1159 try:
1163 try:
1160 hg_tag.tag(repo, safe_bytes(name), node, safe_bytes(message), local, safe_bytes(user), date)
1164 hg_tag.tag(repo, safe_bytes(name), node, safe_bytes(message), local, safe_bytes(user), date)
1161 except Abort as e:
1165 except Abort as e:
1162 log.exception("Tag operation aborted")
1166 log.exception("Tag operation aborted")
1163 # Exception can contain unicode which we convert
1167 # Exception can contain unicode which we convert
1164 raise exceptions.AbortException(e)(repr(e))
1168 raise exceptions.AbortException(e)(repr(e))
1165
1169
1166 @reraise_safe_exceptions
1170 @reraise_safe_exceptions
1167 def bookmark(self, wire, bookmark, revision=''):
1171 def bookmark(self, wire, bookmark, revision=''):
1168 repo = self._factory.repo(wire)
1172 repo = self._factory.repo(wire)
1169 baseui = self._factory._create_config(wire['config'])
1173 baseui = self._factory._create_config(wire['config'])
1170 revision = revision or ''
1174 revision = revision or ''
1171 commands.bookmark(baseui, repo, safe_bytes(bookmark), rev=safe_bytes(revision), force=True)
1175 commands.bookmark(baseui, repo, safe_bytes(bookmark), rev=safe_bytes(revision), force=True)
1172
1176
1173 @reraise_safe_exceptions
1177 @reraise_safe_exceptions
1174 def install_hooks(self, wire, force=False):
1178 def install_hooks(self, wire, force=False):
1175 # we don't need any special hooks for Mercurial
1179 # we don't need any special hooks for Mercurial
1176 pass
1180 pass
1177
1181
1178 @reraise_safe_exceptions
1182 @reraise_safe_exceptions
1179 def get_hooks_info(self, wire):
1183 def get_hooks_info(self, wire):
1180 return {
1184 return {
1181 'pre_version': vcsserver.get_version(),
1185 'pre_version': vcsserver.get_version(),
1182 'post_version': vcsserver.get_version(),
1186 'post_version': vcsserver.get_version(),
1183 }
1187 }
1184
1188
1185 @reraise_safe_exceptions
1189 @reraise_safe_exceptions
1186 def set_head_ref(self, wire, head_name):
1190 def set_head_ref(self, wire, head_name):
1187 pass
1191 pass
1188
1192
1189 @reraise_safe_exceptions
1193 @reraise_safe_exceptions
1190 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1194 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1191 archive_dir_name, commit_id, cache_config):
1195 archive_dir_name, commit_id, cache_config):
1192
1196
1193 def file_walker(_commit_id, path):
1197 def file_walker(_commit_id, path):
1194 repo = self._factory.repo(wire)
1198 repo = self._factory.repo(wire)
1195 ctx = repo[_commit_id]
1199 ctx = repo[_commit_id]
1196 is_root = path in ['', '/']
1200 is_root = path in ['', '/']
1197 if is_root:
1201 if is_root:
1198 matcher = alwaysmatcher(badfn=None)
1202 matcher = alwaysmatcher(badfn=None)
1199 else:
1203 else:
1200 matcher = patternmatcher('', [(b'glob', safe_bytes(path)+b'/**', b'')], badfn=None)
1204 matcher = patternmatcher('', [(b'glob', safe_bytes(path)+b'/**', b'')], badfn=None)
1201 file_iter = ctx.manifest().walk(matcher)
1205 file_iter = ctx.manifest().walk(matcher)
1202
1206
1203 for fn in file_iter:
1207 for fn in file_iter:
1204 file_path = fn
1208 file_path = fn
1205 flags = ctx.flags(fn)
1209 flags = ctx.flags(fn)
1206 mode = b'x' in flags and 0o755 or 0o644
1210 mode = b'x' in flags and 0o755 or 0o644
1207 is_link = b'l' in flags
1211 is_link = b'l' in flags
1208
1212
1209 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
1213 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
1210
1214
1211 return store_archive_in_cache(
1215 return store_archive_in_cache(
1212 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
1216 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
1213
1217
@@ -1,954 +1,959 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18
19 import os
19 import os
20 import subprocess
20 import subprocess
21 from urllib.error import URLError
21 from urllib.error import URLError
22 import urllib.parse
22 import urllib.parse
23 import logging
23 import logging
24 import posixpath as vcspath
24 import posixpath as vcspath
25 import io
25 import io
26 import urllib.request
26 import urllib.request
27 import urllib.parse
27 import urllib.parse
28 import urllib.error
28 import urllib.error
29 import traceback
29 import traceback
30
30
31
32 import svn.client # noqa
31 import svn.client # noqa
33 import svn.core # noqa
32 import svn.core # noqa
34 import svn.delta # noqa
33 import svn.delta # noqa
35 import svn.diff # noqa
34 import svn.diff # noqa
36 import svn.fs # noqa
35 import svn.fs # noqa
37 import svn.repos # noqa
36 import svn.repos # noqa
38
37
39 import rhodecode
38 import rhodecode
40 from vcsserver import svn_diff, exceptions, subprocessio, settings
39 from vcsserver import svn_diff, exceptions, subprocessio, settings
41 from vcsserver.base import (
40 from vcsserver.base import (
42 RepoFactory,
41 RepoFactory,
43 raise_from_original,
42 raise_from_original,
44 ArchiveNode,
43 ArchiveNode,
45 store_archive_in_cache,
44 store_archive_in_cache,
46 BytesEnvelope,
45 BytesEnvelope,
47 BinaryEnvelope,
46 BinaryEnvelope,
48 )
47 )
49 from vcsserver.exceptions import NoContentException
48 from vcsserver.exceptions import NoContentException
50 from vcsserver.str_utils import safe_str, safe_bytes
51 from vcsserver.type_utils import assert_bytes
52 from vcsserver.vcs_base import RemoteBase
49 from vcsserver.vcs_base import RemoteBase
50 from vcsserver.lib.str_utils import safe_str, safe_bytes
51 from vcsserver.lib.type_utils import assert_bytes
53 from vcsserver.lib.svnremoterepo import svnremoterepo
52 from vcsserver.lib.svnremoterepo import svnremoterepo
53 from vcsserver.lib.svn_txn_utils import store_txn_id_data
54
54
55 log = logging.getLogger(__name__)
55 log = logging.getLogger(__name__)
56
56
57
57
58 svn_compatible_versions_map = {
58 svn_compatible_versions_map = {
59 'pre-1.4-compatible': '1.3',
59 'pre-1.4-compatible': '1.3',
60 'pre-1.5-compatible': '1.4',
60 'pre-1.5-compatible': '1.4',
61 'pre-1.6-compatible': '1.5',
61 'pre-1.6-compatible': '1.5',
62 'pre-1.8-compatible': '1.7',
62 'pre-1.8-compatible': '1.7',
63 'pre-1.9-compatible': '1.8',
63 'pre-1.9-compatible': '1.8',
64 }
64 }
65
65
66 current_compatible_version = '1.14'
66 current_compatible_version = '1.14'
67
67
68
68
69 def reraise_safe_exceptions(func):
69 def reraise_safe_exceptions(func):
70 """Decorator for converting svn exceptions to something neutral."""
70 """Decorator for converting svn exceptions to something neutral."""
71 def wrapper(*args, **kwargs):
71 def wrapper(*args, **kwargs):
72 try:
72 try:
73 return func(*args, **kwargs)
73 return func(*args, **kwargs)
74 except Exception as e:
74 except Exception as e:
75 if not hasattr(e, '_vcs_kind'):
75 if not hasattr(e, '_vcs_kind'):
76 log.exception("Unhandled exception in svn remote call")
76 log.exception("Unhandled exception in svn remote call")
77 raise_from_original(exceptions.UnhandledException(e), e)
77 raise_from_original(exceptions.UnhandledException(e), e)
78 raise
78 raise
79 return wrapper
79 return wrapper
80
80
81
81
82 class SubversionFactory(RepoFactory):
82 class SubversionFactory(RepoFactory):
83 repo_type = 'svn'
83 repo_type = 'svn'
84
84
85 def _create_repo(self, wire, create, compatible_version):
85 def _create_repo(self, wire, create, compatible_version):
86 path = svn.core.svn_path_canonicalize(wire['path'])
86 path = svn.core.svn_path_canonicalize(wire['path'])
87 if create:
87 if create:
88 fs_config = {'compatible-version': current_compatible_version}
88 fs_config = {'compatible-version': current_compatible_version}
89 if compatible_version:
89 if compatible_version:
90
90
91 compatible_version_string = \
91 compatible_version_string = \
92 svn_compatible_versions_map.get(compatible_version) \
92 svn_compatible_versions_map.get(compatible_version) \
93 or compatible_version
93 or compatible_version
94 fs_config['compatible-version'] = compatible_version_string
94 fs_config['compatible-version'] = compatible_version_string
95
95
96 log.debug('Create SVN repo with config `%s`', fs_config)
96 log.debug('Create SVN repo with config `%s`', fs_config)
97 repo = svn.repos.create(path, "", "", None, fs_config)
97 repo = svn.repos.create(path, "", "", None, fs_config)
98 else:
98 else:
99 repo = svn.repos.open(path)
99 repo = svn.repos.open(path)
100
100
101 log.debug('repository created: got SVN object: %s', repo)
101 log.debug('repository created: got SVN object: %s', repo)
102 return repo
102 return repo
103
103
104 def repo(self, wire, create=False, compatible_version=None):
104 def repo(self, wire, create=False, compatible_version=None):
105 """
105 """
106 Get a repository instance for the given path.
106 Get a repository instance for the given path.
107 """
107 """
108 return self._create_repo(wire, create, compatible_version)
108 return self._create_repo(wire, create, compatible_version)
109
109
110
110
111 NODE_TYPE_MAPPING = {
111 NODE_TYPE_MAPPING = {
112 svn.core.svn_node_file: 'file',
112 svn.core.svn_node_file: 'file',
113 svn.core.svn_node_dir: 'dir',
113 svn.core.svn_node_dir: 'dir',
114 }
114 }
115
115
116
116
117 class SvnRemote(RemoteBase):
117 class SvnRemote(RemoteBase):
118
118
119 def __init__(self, factory, hg_factory=None):
119 def __init__(self, factory, hg_factory=None):
120 self._factory = factory
120 self._factory = factory
121
121
122 self._bulk_methods = {
122 self._bulk_methods = {
123 # NOT supported in SVN ATM...
123 # NOT supported in SVN ATM...
124 }
124 }
125 self._bulk_file_methods = {
125 self._bulk_file_methods = {
126 "size": self.get_file_size,
126 "size": self.get_file_size,
127 "data": self.get_file_content,
127 "data": self.get_file_content,
128 "flags": self.get_node_type,
128 "flags": self.get_node_type,
129 "is_binary": self.is_binary,
129 "is_binary": self.is_binary,
130 "md5": self.md5_hash
130 "md5": self.md5_hash
131 }
131 }
132
132
133 @reraise_safe_exceptions
133 @reraise_safe_exceptions
134 def bulk_file_request(self, wire, commit_id, path, pre_load):
134 def bulk_file_request(self, wire, commit_id, path, pre_load):
135 cache_on, context_uid, repo_id = self._cache_on(wire)
135 cache_on, context_uid, repo_id = self._cache_on(wire)
136 region = self._region(wire)
136 region = self._region(wire)
137
137
138 # since we use unified API, we need to cast from str to in for SVN
138 # since we use unified API, we need to cast from str to in for SVN
139 commit_id = int(commit_id)
139 commit_id = int(commit_id)
140
140
141 @region.conditional_cache_on_arguments(condition=cache_on)
141 @region.conditional_cache_on_arguments(condition=cache_on)
142 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
142 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
143 result = {}
143 result = {}
144 for attr in pre_load:
144 for attr in pre_load:
145 try:
145 try:
146 method = self._bulk_file_methods[attr]
146 method = self._bulk_file_methods[attr]
147 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
147 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
148 result[attr] = method(wire, _commit_id, _path)
148 result[attr] = method(wire, _commit_id, _path)
149 except KeyError as e:
149 except KeyError as e:
150 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
150 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
151 return result
151 return result
152
152
153 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
153 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
154
154
155 @reraise_safe_exceptions
155 @reraise_safe_exceptions
156 def discover_svn_version(self):
156 def discover_svn_version(self):
157 try:
157 try:
158 import svn.core
158 import svn.core
159 svn_ver = svn.core.SVN_VERSION
159 svn_ver = svn.core.SVN_VERSION
160 except ImportError:
160 except ImportError:
161 svn_ver = None
161 svn_ver = None
162 return safe_str(svn_ver)
162 return safe_str(svn_ver)
163
163
164 @reraise_safe_exceptions
164 @reraise_safe_exceptions
165 def is_empty(self, wire):
165 def is_empty(self, wire):
166 try:
166 try:
167 return self.lookup(wire, -1) == 0
167 return self.lookup(wire, -1) == 0
168 except Exception:
168 except Exception:
169 log.exception("failed to read object_store")
169 log.exception("failed to read object_store")
170 return False
170 return False
171
171
172 def check_url(self, url, config):
172 def check_url(self, url, config):
173
173
174 # uuid function gets only valid UUID from proper repo, else
174 # uuid function gets only valid UUID from proper repo, else
175 # throws exception
175 # throws exception
176 username, password, src_url = self.get_url_and_credentials(url)
176 username, password, src_url = self.get_url_and_credentials(url)
177 try:
177 try:
178 svnremoterepo(safe_bytes(username), safe_bytes(password), safe_bytes(src_url)).svn().uuid
178 svnremoterepo(safe_bytes(username), safe_bytes(password), safe_bytes(src_url)).svn().uuid
179 except Exception:
179 except Exception:
180 tb = traceback.format_exc()
180 tb = traceback.format_exc()
181 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
181 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
182 raise URLError(f'"{url}" is not a valid Subversion source url.')
182 raise URLError(f'"{url}" is not a valid Subversion source url.')
183 return True
183 return True
184
184
185 def is_path_valid_repository(self, wire, path):
185 def is_path_valid_repository(self, wire, path):
186 # NOTE(marcink): short circuit the check for SVN repo
186 # NOTE(marcink): short circuit the check for SVN repo
187 # the repos.open might be expensive to check, but we have one cheap
187 # the repos.open might be expensive to check, but we have one cheap
188 # pre-condition that we can use, to check for 'format' file
188 # pre-condition that we can use, to check for 'format' file
189 if not os.path.isfile(os.path.join(path, 'format')):
189 if not os.path.isfile(os.path.join(path, 'format')):
190 return False
190 return False
191
191
192 cache_on, context_uid, repo_id = self._cache_on(wire)
192 cache_on, context_uid, repo_id = self._cache_on(wire)
193 region = self._region(wire)
193 region = self._region(wire)
194
194
195 @region.conditional_cache_on_arguments(condition=cache_on)
195 @region.conditional_cache_on_arguments(condition=cache_on)
196 def _assert_correct_path(_context_uid, _repo_id, fast_check):
196 def _assert_correct_path(_context_uid, _repo_id, fast_check):
197
197
198 try:
198 try:
199 svn.repos.open(path)
199 svn.repos.open(path)
200 except svn.core.SubversionException:
200 except svn.core.SubversionException:
201 tb = traceback.format_exc()
201 tb = traceback.format_exc()
202 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
202 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
203 return False
203 return False
204 return True
204 return True
205
205
206 return _assert_correct_path(context_uid, repo_id, True)
206 return _assert_correct_path(context_uid, repo_id, True)
207
207
208 @reraise_safe_exceptions
208 @reraise_safe_exceptions
209 def verify(self, wire,):
209 def verify(self, wire,):
210 repo_path = wire['path']
210 repo_path = wire['path']
211 if not self.is_path_valid_repository(wire, repo_path):
211 if not self.is_path_valid_repository(wire, repo_path):
212 raise Exception(
212 raise Exception(
213 f"Path {repo_path} is not a valid Subversion repository.")
213 f"Path {repo_path} is not a valid Subversion repository.")
214
214
215 cmd = ['svnadmin', 'info', repo_path]
215 cmd = ['svnadmin', 'info', repo_path]
216 stdout, stderr = subprocessio.run_command(cmd)
216 stdout, stderr = subprocessio.run_command(cmd)
217 return stdout
217 return stdout
218
218
219 @reraise_safe_exceptions
219 @reraise_safe_exceptions
220 def lookup(self, wire, revision):
220 def lookup(self, wire, revision):
221 if revision not in [-1, None, 'HEAD']:
221 if revision not in [-1, None, 'HEAD']:
222 raise NotImplementedError
222 raise NotImplementedError
223 repo = self._factory.repo(wire)
223 repo = self._factory.repo(wire)
224 fs_ptr = svn.repos.fs(repo)
224 fs_ptr = svn.repos.fs(repo)
225 head = svn.fs.youngest_rev(fs_ptr)
225 head = svn.fs.youngest_rev(fs_ptr)
226 return head
226 return head
227
227
228 @reraise_safe_exceptions
228 @reraise_safe_exceptions
229 def lookup_interval(self, wire, start_ts, end_ts):
229 def lookup_interval(self, wire, start_ts, end_ts):
230 repo = self._factory.repo(wire)
230 repo = self._factory.repo(wire)
231 fsobj = svn.repos.fs(repo)
231 fsobj = svn.repos.fs(repo)
232 start_rev = None
232 start_rev = None
233 end_rev = None
233 end_rev = None
234 if start_ts:
234 if start_ts:
235 start_ts_svn = apr_time_t(start_ts)
235 start_ts_svn = apr_time_t(start_ts)
236 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
236 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
237 else:
237 else:
238 start_rev = 1
238 start_rev = 1
239 if end_ts:
239 if end_ts:
240 end_ts_svn = apr_time_t(end_ts)
240 end_ts_svn = apr_time_t(end_ts)
241 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
241 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
242 else:
242 else:
243 end_rev = svn.fs.youngest_rev(fsobj)
243 end_rev = svn.fs.youngest_rev(fsobj)
244 return start_rev, end_rev
244 return start_rev, end_rev
245
245
246 @reraise_safe_exceptions
246 @reraise_safe_exceptions
247 def revision_properties(self, wire, revision):
247 def revision_properties(self, wire, revision):
248
248
249 cache_on, context_uid, repo_id = self._cache_on(wire)
249 cache_on, context_uid, repo_id = self._cache_on(wire)
250 region = self._region(wire)
250 region = self._region(wire)
251
251
252 @region.conditional_cache_on_arguments(condition=cache_on)
252 @region.conditional_cache_on_arguments(condition=cache_on)
253 def _revision_properties(_repo_id, _revision):
253 def _revision_properties(_repo_id, _revision):
254 repo = self._factory.repo(wire)
254 repo = self._factory.repo(wire)
255 fs_ptr = svn.repos.fs(repo)
255 fs_ptr = svn.repos.fs(repo)
256 return svn.fs.revision_proplist(fs_ptr, revision)
256 return svn.fs.revision_proplist(fs_ptr, revision)
257 return _revision_properties(repo_id, revision)
257 return _revision_properties(repo_id, revision)
258
258
259 def revision_changes(self, wire, revision):
259 def revision_changes(self, wire, revision):
260
260
261 repo = self._factory.repo(wire)
261 repo = self._factory.repo(wire)
262 fsobj = svn.repos.fs(repo)
262 fsobj = svn.repos.fs(repo)
263 rev_root = svn.fs.revision_root(fsobj, revision)
263 rev_root = svn.fs.revision_root(fsobj, revision)
264
264
265 editor = svn.repos.ChangeCollector(fsobj, rev_root)
265 editor = svn.repos.ChangeCollector(fsobj, rev_root)
266 editor_ptr, editor_baton = svn.delta.make_editor(editor)
266 editor_ptr, editor_baton = svn.delta.make_editor(editor)
267 base_dir = ""
267 base_dir = ""
268 send_deltas = False
268 send_deltas = False
269 svn.repos.replay2(
269 svn.repos.replay2(
270 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
270 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
271 editor_ptr, editor_baton, None)
271 editor_ptr, editor_baton, None)
272
272
273 added = []
273 added = []
274 changed = []
274 changed = []
275 removed = []
275 removed = []
276
276
277 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
277 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
278 for path, change in editor.changes.items():
278 for path, change in editor.changes.items():
279 # TODO: Decide what to do with directory nodes. Subversion can add
279 # TODO: Decide what to do with directory nodes. Subversion can add
280 # empty directories.
280 # empty directories.
281
281
282 if change.item_kind == svn.core.svn_node_dir:
282 if change.item_kind == svn.core.svn_node_dir:
283 continue
283 continue
284 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
284 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
285 added.append(path)
285 added.append(path)
286 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
286 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
287 svn.repos.CHANGE_ACTION_REPLACE]:
287 svn.repos.CHANGE_ACTION_REPLACE]:
288 changed.append(path)
288 changed.append(path)
289 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
289 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
290 removed.append(path)
290 removed.append(path)
291 else:
291 else:
292 raise NotImplementedError(
292 raise NotImplementedError(
293 "Action {} not supported on path {}".format(
293 "Action {} not supported on path {}".format(
294 change.action, path))
294 change.action, path))
295
295
296 changes = {
296 changes = {
297 'added': added,
297 'added': added,
298 'changed': changed,
298 'changed': changed,
299 'removed': removed,
299 'removed': removed,
300 }
300 }
301 return changes
301 return changes
302
302
303 @reraise_safe_exceptions
303 @reraise_safe_exceptions
304 def node_history(self, wire, path, revision, limit):
304 def node_history(self, wire, path, revision, limit):
305 cache_on, context_uid, repo_id = self._cache_on(wire)
305 cache_on, context_uid, repo_id = self._cache_on(wire)
306 region = self._region(wire)
306 region = self._region(wire)
307
307
308 @region.conditional_cache_on_arguments(condition=cache_on)
308 @region.conditional_cache_on_arguments(condition=cache_on)
309 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
309 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
310 cross_copies = False
310 cross_copies = False
311 repo = self._factory.repo(wire)
311 repo = self._factory.repo(wire)
312 fsobj = svn.repos.fs(repo)
312 fsobj = svn.repos.fs(repo)
313 rev_root = svn.fs.revision_root(fsobj, revision)
313 rev_root = svn.fs.revision_root(fsobj, revision)
314
314
315 history_revisions = []
315 history_revisions = []
316 history = svn.fs.node_history(rev_root, path)
316 history = svn.fs.node_history(rev_root, path)
317 history = svn.fs.history_prev(history, cross_copies)
317 history = svn.fs.history_prev(history, cross_copies)
318 while history:
318 while history:
319 __, node_revision = svn.fs.history_location(history)
319 __, node_revision = svn.fs.history_location(history)
320 history_revisions.append(node_revision)
320 history_revisions.append(node_revision)
321 if limit and len(history_revisions) >= limit:
321 if limit and len(history_revisions) >= limit:
322 break
322 break
323 history = svn.fs.history_prev(history, cross_copies)
323 history = svn.fs.history_prev(history, cross_copies)
324 return history_revisions
324 return history_revisions
325 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
325 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
326
326
327 @reraise_safe_exceptions
327 @reraise_safe_exceptions
328 def node_properties(self, wire, path, revision):
328 def node_properties(self, wire, path, revision):
329 cache_on, context_uid, repo_id = self._cache_on(wire)
329 cache_on, context_uid, repo_id = self._cache_on(wire)
330 region = self._region(wire)
330 region = self._region(wire)
331
331
332 @region.conditional_cache_on_arguments(condition=cache_on)
332 @region.conditional_cache_on_arguments(condition=cache_on)
333 def _node_properties(_repo_id, _path, _revision):
333 def _node_properties(_repo_id, _path, _revision):
334 repo = self._factory.repo(wire)
334 repo = self._factory.repo(wire)
335 fsobj = svn.repos.fs(repo)
335 fsobj = svn.repos.fs(repo)
336 rev_root = svn.fs.revision_root(fsobj, revision)
336 rev_root = svn.fs.revision_root(fsobj, revision)
337 return svn.fs.node_proplist(rev_root, path)
337 return svn.fs.node_proplist(rev_root, path)
338 return _node_properties(repo_id, path, revision)
338 return _node_properties(repo_id, path, revision)
339
339
340 def file_annotate(self, wire, path, revision):
340 def file_annotate(self, wire, path, revision):
341 abs_path = 'file://' + urllib.request.pathname2url(
341 abs_path = 'file://' + urllib.request.pathname2url(
342 vcspath.join(wire['path'], path))
342 vcspath.join(wire['path'], path))
343 file_uri = svn.core.svn_path_canonicalize(abs_path)
343 file_uri = svn.core.svn_path_canonicalize(abs_path)
344
344
345 start_rev = svn_opt_revision_value_t(0)
345 start_rev = svn_opt_revision_value_t(0)
346 peg_rev = svn_opt_revision_value_t(revision)
346 peg_rev = svn_opt_revision_value_t(revision)
347 end_rev = peg_rev
347 end_rev = peg_rev
348
348
349 annotations = []
349 annotations = []
350
350
351 def receiver(line_no, revision, author, date, line, pool):
351 def receiver(line_no, revision, author, date, line, pool):
352 annotations.append((line_no, revision, line))
352 annotations.append((line_no, revision, line))
353
353
354 # TODO: Cannot use blame5, missing typemap function in the swig code
354 # TODO: Cannot use blame5, missing typemap function in the swig code
355 try:
355 try:
356 svn.client.blame2(
356 svn.client.blame2(
357 file_uri, peg_rev, start_rev, end_rev,
357 file_uri, peg_rev, start_rev, end_rev,
358 receiver, svn.client.create_context())
358 receiver, svn.client.create_context())
359 except svn.core.SubversionException as exc:
359 except svn.core.SubversionException as exc:
360 log.exception("Error during blame operation.")
360 log.exception("Error during blame operation.")
361 raise Exception(
361 raise Exception(
362 f"Blame not supported or file does not exist at path {path}. "
362 f"Blame not supported or file does not exist at path {path}. "
363 f"Error {exc}.")
363 f"Error {exc}.")
364
364
365 return BinaryEnvelope(annotations)
365 return BinaryEnvelope(annotations)
366
366
367 @reraise_safe_exceptions
367 @reraise_safe_exceptions
368 def get_node_type(self, wire, revision=None, path=''):
368 def get_node_type(self, wire, revision=None, path=''):
369
369
370 cache_on, context_uid, repo_id = self._cache_on(wire)
370 cache_on, context_uid, repo_id = self._cache_on(wire)
371 region = self._region(wire)
371 region = self._region(wire)
372
372
373 @region.conditional_cache_on_arguments(condition=cache_on)
373 @region.conditional_cache_on_arguments(condition=cache_on)
374 def _get_node_type(_repo_id, _revision, _path):
374 def _get_node_type(_repo_id, _revision, _path):
375 repo = self._factory.repo(wire)
375 repo = self._factory.repo(wire)
376 fs_ptr = svn.repos.fs(repo)
376 fs_ptr = svn.repos.fs(repo)
377 if _revision is None:
377 if _revision is None:
378 _revision = svn.fs.youngest_rev(fs_ptr)
378 _revision = svn.fs.youngest_rev(fs_ptr)
379 root = svn.fs.revision_root(fs_ptr, _revision)
379 root = svn.fs.revision_root(fs_ptr, _revision)
380 node = svn.fs.check_path(root, path)
380 node = svn.fs.check_path(root, path)
381 return NODE_TYPE_MAPPING.get(node, None)
381 return NODE_TYPE_MAPPING.get(node, None)
382 return _get_node_type(repo_id, revision, path)
382 return _get_node_type(repo_id, revision, path)
383
383
384 @reraise_safe_exceptions
384 @reraise_safe_exceptions
385 def get_nodes(self, wire, revision=None, path=''):
385 def get_nodes(self, wire, revision=None, path=''):
386
386
387 cache_on, context_uid, repo_id = self._cache_on(wire)
387 cache_on, context_uid, repo_id = self._cache_on(wire)
388 region = self._region(wire)
388 region = self._region(wire)
389
389
390 @region.conditional_cache_on_arguments(condition=cache_on)
390 @region.conditional_cache_on_arguments(condition=cache_on)
391 def _get_nodes(_repo_id, _path, _revision):
391 def _get_nodes(_repo_id, _path, _revision):
392 repo = self._factory.repo(wire)
392 repo = self._factory.repo(wire)
393 fsobj = svn.repos.fs(repo)
393 fsobj = svn.repos.fs(repo)
394 if _revision is None:
394 if _revision is None:
395 _revision = svn.fs.youngest_rev(fsobj)
395 _revision = svn.fs.youngest_rev(fsobj)
396 root = svn.fs.revision_root(fsobj, _revision)
396 root = svn.fs.revision_root(fsobj, _revision)
397 entries = svn.fs.dir_entries(root, path)
397 entries = svn.fs.dir_entries(root, path)
398 result = []
398 result = []
399 for entry_path, entry_info in entries.items():
399 for entry_path, entry_info in entries.items():
400 result.append(
400 result.append(
401 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
401 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
402 return result
402 return result
403 return _get_nodes(repo_id, path, revision)
403 return _get_nodes(repo_id, path, revision)
404
404
405 @reraise_safe_exceptions
405 @reraise_safe_exceptions
406 def get_file_content(self, wire, rev=None, path=''):
406 def get_file_content(self, wire, rev=None, path=''):
407 repo = self._factory.repo(wire)
407 repo = self._factory.repo(wire)
408 fsobj = svn.repos.fs(repo)
408 fsobj = svn.repos.fs(repo)
409
409
410 if rev is None:
410 if rev is None:
411 rev = svn.fs.youngest_rev(fsobj)
411 rev = svn.fs.youngest_rev(fsobj)
412
412
413 root = svn.fs.revision_root(fsobj, rev)
413 root = svn.fs.revision_root(fsobj, rev)
414 content = svn.core.Stream(svn.fs.file_contents(root, path))
414 content = svn.core.Stream(svn.fs.file_contents(root, path))
415 return BytesEnvelope(content.read())
415 return BytesEnvelope(content.read())
416
416
417 @reraise_safe_exceptions
417 @reraise_safe_exceptions
418 def get_file_size(self, wire, revision=None, path=''):
418 def get_file_size(self, wire, revision=None, path=''):
419
419
420 cache_on, context_uid, repo_id = self._cache_on(wire)
420 cache_on, context_uid, repo_id = self._cache_on(wire)
421 region = self._region(wire)
421 region = self._region(wire)
422
422
423 @region.conditional_cache_on_arguments(condition=cache_on)
423 @region.conditional_cache_on_arguments(condition=cache_on)
424 def _get_file_size(_repo_id, _revision, _path):
424 def _get_file_size(_repo_id, _revision, _path):
425 repo = self._factory.repo(wire)
425 repo = self._factory.repo(wire)
426 fsobj = svn.repos.fs(repo)
426 fsobj = svn.repos.fs(repo)
427 if _revision is None:
427 if _revision is None:
428 _revision = svn.fs.youngest_revision(fsobj)
428 _revision = svn.fs.youngest_revision(fsobj)
429 root = svn.fs.revision_root(fsobj, _revision)
429 root = svn.fs.revision_root(fsobj, _revision)
430 size = svn.fs.file_length(root, path)
430 size = svn.fs.file_length(root, path)
431 return size
431 return size
432 return _get_file_size(repo_id, revision, path)
432 return _get_file_size(repo_id, revision, path)
433
433
434 def create_repository(self, wire, compatible_version=None):
434 def create_repository(self, wire, compatible_version=None):
435 log.info('Creating Subversion repository in path "%s"', wire['path'])
435 log.info('Creating Subversion repository in path "%s"', wire['path'])
436 self._factory.repo(wire, create=True,
436 self._factory.repo(wire, create=True,
437 compatible_version=compatible_version)
437 compatible_version=compatible_version)
438
438
439 def get_url_and_credentials(self, src_url) -> tuple[str, str, str]:
439 def get_url_and_credentials(self, src_url) -> tuple[str, str, str]:
440 obj = urllib.parse.urlparse(src_url)
440 obj = urllib.parse.urlparse(src_url)
441 username = obj.username or ''
441 username = obj.username or ''
442 password = obj.password or ''
442 password = obj.password or ''
443 return username, password, src_url
443 return username, password, src_url
444
444
445 def import_remote_repository(self, wire, src_url):
445 def import_remote_repository(self, wire, src_url):
446 repo_path = wire['path']
446 repo_path = wire['path']
447 if not self.is_path_valid_repository(wire, repo_path):
447 if not self.is_path_valid_repository(wire, repo_path):
448 raise Exception(
448 raise Exception(
449 f"Path {repo_path} is not a valid Subversion repository.")
449 f"Path {repo_path} is not a valid Subversion repository.")
450
450
451 username, password, src_url = self.get_url_and_credentials(src_url)
451 username, password, src_url = self.get_url_and_credentials(src_url)
452 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
452 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
453 '--trust-server-cert-failures=unknown-ca']
453 '--trust-server-cert-failures=unknown-ca']
454 if username and password:
454 if username and password:
455 rdump_cmd += ['--username', username, '--password', password]
455 rdump_cmd += ['--username', username, '--password', password]
456 rdump_cmd += [src_url]
456 rdump_cmd += [src_url]
457
457
458 rdump = subprocess.Popen(
458 rdump = subprocess.Popen(
459 rdump_cmd,
459 rdump_cmd,
460 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
460 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
461 load = subprocess.Popen(
461 load = subprocess.Popen(
462 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
462 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
463
463
464 # TODO: johbo: This can be a very long operation, might be better
464 # TODO: johbo: This can be a very long operation, might be better
465 # to track some kind of status and provide an api to check if the
465 # to track some kind of status and provide an api to check if the
466 # import is done.
466 # import is done.
467 rdump.wait()
467 rdump.wait()
468 load.wait()
468 load.wait()
469
469
470 log.debug('Return process ended with code: %s', rdump.returncode)
470 log.debug('Return process ended with code: %s', rdump.returncode)
471 if rdump.returncode != 0:
471 if rdump.returncode != 0:
472 errors = rdump.stderr.read()
472 errors = rdump.stderr.read()
473 log.error('svnrdump dump failed: statuscode %s: message: %s', rdump.returncode, errors)
473 log.error('svnrdump dump failed: statuscode %s: message: %s', rdump.returncode, errors)
474
474
475 reason = 'UNKNOWN'
475 reason = 'UNKNOWN'
476 if b'svnrdump: E230001:' in errors:
476 if b'svnrdump: E230001:' in errors:
477 reason = 'INVALID_CERTIFICATE'
477 reason = 'INVALID_CERTIFICATE'
478
478
479 if reason == 'UNKNOWN':
479 if reason == 'UNKNOWN':
480 reason = f'UNKNOWN:{safe_str(errors)}'
480 reason = f'UNKNOWN:{safe_str(errors)}'
481
481
482 raise Exception(
482 raise Exception(
483 'Failed to dump the remote repository from {}. Reason:{}'.format(
483 'Failed to dump the remote repository from {}. Reason:{}'.format(
484 src_url, reason))
484 src_url, reason))
485 if load.returncode != 0:
485 if load.returncode != 0:
486 raise Exception(
486 raise Exception(
487 f'Failed to load the dump of remote repository from {src_url}.')
487 f'Failed to load the dump of remote repository from {src_url}.')
488
488
489 def commit(self, wire, message, author, timestamp, updated, removed):
489 def commit(self, wire, message, author, timestamp, updated, removed):
490
490
491 message = safe_bytes(message)
491 message = safe_bytes(message)
492 author = safe_bytes(author)
492 author = safe_bytes(author)
493
493
494 repo = self._factory.repo(wire)
494 repo = self._factory.repo(wire)
495 fsobj = svn.repos.fs(repo)
495 fsobj = svn.repos.fs(repo)
496
496
497 rev = svn.fs.youngest_rev(fsobj)
497 rev = svn.fs.youngest_rev(fsobj)
498 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
498 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
499 txn_root = svn.fs.txn_root(txn)
499 txn_root = svn.fs.txn_root(txn)
500
500
501 for node in updated:
501 for node in updated:
502 TxnNodeProcessor(node, txn_root).update()
502 TxnNodeProcessor(node, txn_root).update()
503 for node in removed:
503 for node in removed:
504 TxnNodeProcessor(node, txn_root).remove()
504 TxnNodeProcessor(node, txn_root).remove()
505
505
506 svn_txn_id = safe_str(svn.fs.svn_fs_txn_name(txn))
507 full_repo_path = wire['path']
508 txn_id_data = {'svn_txn_id': svn_txn_id, 'rc_internal_commit': True}
509
510 store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
506 commit_id = svn.repos.fs_commit_txn(repo, txn)
511 commit_id = svn.repos.fs_commit_txn(repo, txn)
507
512
508 if timestamp:
513 if timestamp:
509 apr_time = apr_time_t(timestamp)
514 apr_time = apr_time_t(timestamp)
510 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
515 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
511 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
516 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
512
517
513 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
518 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
514 return commit_id
519 return commit_id
515
520
516 @reraise_safe_exceptions
521 @reraise_safe_exceptions
517 def diff(self, wire, rev1, rev2, path1=None, path2=None,
522 def diff(self, wire, rev1, rev2, path1=None, path2=None,
518 ignore_whitespace=False, context=3):
523 ignore_whitespace=False, context=3):
519
524
520 wire.update(cache=False)
525 wire.update(cache=False)
521 repo = self._factory.repo(wire)
526 repo = self._factory.repo(wire)
522 diff_creator = SvnDiffer(
527 diff_creator = SvnDiffer(
523 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
528 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
524 try:
529 try:
525 return BytesEnvelope(diff_creator.generate_diff())
530 return BytesEnvelope(diff_creator.generate_diff())
526 except svn.core.SubversionException as e:
531 except svn.core.SubversionException as e:
527 log.exception(
532 log.exception(
528 "Error during diff operation operation. "
533 "Error during diff operation operation. "
529 "Path might not exist %s, %s", path1, path2)
534 "Path might not exist %s, %s", path1, path2)
530 return BytesEnvelope(b'')
535 return BytesEnvelope(b'')
531
536
532 @reraise_safe_exceptions
537 @reraise_safe_exceptions
533 def is_large_file(self, wire, path):
538 def is_large_file(self, wire, path):
534 return False
539 return False
535
540
536 @reraise_safe_exceptions
541 @reraise_safe_exceptions
537 def is_binary(self, wire, rev, path):
542 def is_binary(self, wire, rev, path):
538 cache_on, context_uid, repo_id = self._cache_on(wire)
543 cache_on, context_uid, repo_id = self._cache_on(wire)
539 region = self._region(wire)
544 region = self._region(wire)
540
545
541 @region.conditional_cache_on_arguments(condition=cache_on)
546 @region.conditional_cache_on_arguments(condition=cache_on)
542 def _is_binary(_repo_id, _rev, _path):
547 def _is_binary(_repo_id, _rev, _path):
543 raw_bytes = self.get_file_content(wire, rev, path)
548 raw_bytes = self.get_file_content(wire, rev, path)
544 if not raw_bytes:
549 if not raw_bytes:
545 return False
550 return False
546 return b'\0' in raw_bytes
551 return b'\0' in raw_bytes
547
552
548 return _is_binary(repo_id, rev, path)
553 return _is_binary(repo_id, rev, path)
549
554
550 @reraise_safe_exceptions
555 @reraise_safe_exceptions
551 def md5_hash(self, wire, rev, path):
556 def md5_hash(self, wire, rev, path):
552 cache_on, context_uid, repo_id = self._cache_on(wire)
557 cache_on, context_uid, repo_id = self._cache_on(wire)
553 region = self._region(wire)
558 region = self._region(wire)
554
559
555 @region.conditional_cache_on_arguments(condition=cache_on)
560 @region.conditional_cache_on_arguments(condition=cache_on)
556 def _md5_hash(_repo_id, _rev, _path):
561 def _md5_hash(_repo_id, _rev, _path):
557 return ''
562 return ''
558
563
559 return _md5_hash(repo_id, rev, path)
564 return _md5_hash(repo_id, rev, path)
560
565
561 @reraise_safe_exceptions
566 @reraise_safe_exceptions
562 def run_svn_command(self, wire, cmd, **opts):
567 def run_svn_command(self, wire, cmd, **opts):
563 path = wire.get('path', None)
568 path = wire.get('path', None)
564 debug_mode = rhodecode.ConfigGet().get_bool('debug')
569 debug_mode = rhodecode.ConfigGet().get_bool('debug')
565
570
566 if path and os.path.isdir(path):
571 if path and os.path.isdir(path):
567 opts['cwd'] = path
572 opts['cwd'] = path
568
573
569 safe_call = opts.pop('_safe', False)
574 safe_call = opts.pop('_safe', False)
570
575
571 svnenv = os.environ.copy()
576 svnenv = os.environ.copy()
572 svnenv.update(opts.pop('extra_env', {}))
577 svnenv.update(opts.pop('extra_env', {}))
573
578
574 _opts = {'env': svnenv, 'shell': False}
579 _opts = {'env': svnenv, 'shell': False}
575
580
576 try:
581 try:
577 _opts.update(opts)
582 _opts.update(opts)
578 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
583 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
579
584
580 return b''.join(proc), b''.join(proc.stderr)
585 return b''.join(proc), b''.join(proc.stderr)
581 except OSError as err:
586 except OSError as err:
582 if safe_call:
587 if safe_call:
583 return '', safe_str(err).strip()
588 return '', safe_str(err).strip()
584 else:
589 else:
585 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
590 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
586 call_opts = {}
591 call_opts = {}
587 if debug_mode:
592 if debug_mode:
588 call_opts = _opts
593 call_opts = _opts
589
594
590 tb_err = ("Couldn't run svn command ({}).\n"
595 tb_err = ("Couldn't run svn command ({}).\n"
591 "Original error was:{}\n"
596 "Original error was:{}\n"
592 "Call options:{}\n"
597 "Call options:{}\n"
593 .format(cmd, err, call_opts))
598 .format(cmd, err, call_opts))
594 log.exception(tb_err)
599 log.exception(tb_err)
595 raise exceptions.VcsException()(tb_err)
600 raise exceptions.VcsException()(tb_err)
596
601
597 @reraise_safe_exceptions
602 @reraise_safe_exceptions
598 def install_hooks(self, wire, force=False):
603 def install_hooks(self, wire, force=False):
599 from vcsserver.hook_utils import install_svn_hooks
604 from vcsserver.hook_utils import install_svn_hooks
600 repo_path = wire['path']
605 repo_path = wire['path']
601 binary_dir = settings.BINARY_DIR
606 binary_dir = settings.BINARY_DIR
602 executable = None
607 executable = None
603 if binary_dir:
608 if binary_dir:
604 executable = os.path.join(binary_dir, 'python3')
609 executable = os.path.join(binary_dir, 'python3')
605 return install_svn_hooks(repo_path, force_create=force)
610 return install_svn_hooks(repo_path, force_create=force)
606
611
607 @reraise_safe_exceptions
612 @reraise_safe_exceptions
608 def get_hooks_info(self, wire):
613 def get_hooks_info(self, wire):
609 from vcsserver.hook_utils import (
614 from vcsserver.hook_utils import (
610 get_svn_pre_hook_version, get_svn_post_hook_version)
615 get_svn_pre_hook_version, get_svn_post_hook_version)
611 repo_path = wire['path']
616 repo_path = wire['path']
612 return {
617 return {
613 'pre_version': get_svn_pre_hook_version(repo_path),
618 'pre_version': get_svn_pre_hook_version(repo_path),
614 'post_version': get_svn_post_hook_version(repo_path),
619 'post_version': get_svn_post_hook_version(repo_path),
615 }
620 }
616
621
617 @reraise_safe_exceptions
622 @reraise_safe_exceptions
618 def set_head_ref(self, wire, head_name):
623 def set_head_ref(self, wire, head_name):
619 pass
624 pass
620
625
621 @reraise_safe_exceptions
626 @reraise_safe_exceptions
622 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
627 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
623 archive_dir_name, commit_id, cache_config):
628 archive_dir_name, commit_id, cache_config):
624
629
625 def walk_tree(root, root_dir, _commit_id):
630 def walk_tree(root, root_dir, _commit_id):
626 """
631 """
627 Special recursive svn repo walker
632 Special recursive svn repo walker
628 """
633 """
629 root_dir = safe_bytes(root_dir)
634 root_dir = safe_bytes(root_dir)
630
635
631 filemode_default = 0o100644
636 filemode_default = 0o100644
632 filemode_executable = 0o100755
637 filemode_executable = 0o100755
633
638
634 file_iter = svn.fs.dir_entries(root, root_dir)
639 file_iter = svn.fs.dir_entries(root, root_dir)
635 for f_name in file_iter:
640 for f_name in file_iter:
636 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
641 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
637
642
638 if f_type == 'dir':
643 if f_type == 'dir':
639 # return only DIR, and then all entries in that dir
644 # return only DIR, and then all entries in that dir
640 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
645 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
641 new_root = os.path.join(root_dir, f_name)
646 new_root = os.path.join(root_dir, f_name)
642 yield from walk_tree(root, new_root, _commit_id)
647 yield from walk_tree(root, new_root, _commit_id)
643 else:
648 else:
644
649
645 f_path = os.path.join(root_dir, f_name).rstrip(b'/')
650 f_path = os.path.join(root_dir, f_name).rstrip(b'/')
646 prop_list = svn.fs.node_proplist(root, f_path)
651 prop_list = svn.fs.node_proplist(root, f_path)
647
652
648 f_mode = filemode_default
653 f_mode = filemode_default
649 if prop_list.get('svn:executable'):
654 if prop_list.get('svn:executable'):
650 f_mode = filemode_executable
655 f_mode = filemode_executable
651
656
652 f_is_link = False
657 f_is_link = False
653 if prop_list.get('svn:special'):
658 if prop_list.get('svn:special'):
654 f_is_link = True
659 f_is_link = True
655
660
656 data = {
661 data = {
657 'is_link': f_is_link,
662 'is_link': f_is_link,
658 'mode': f_mode,
663 'mode': f_mode,
659 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
664 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
660 }
665 }
661
666
662 yield f_path, data, f_type
667 yield f_path, data, f_type
663
668
664 def file_walker(_commit_id, path):
669 def file_walker(_commit_id, path):
665 repo = self._factory.repo(wire)
670 repo = self._factory.repo(wire)
666 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
671 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
667
672
668 def no_content():
673 def no_content():
669 raise NoContentException()
674 raise NoContentException()
670
675
671 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
676 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
672 file_path = f_name
677 file_path = f_name
673
678
674 if f_type == 'dir':
679 if f_type == 'dir':
675 mode = f_data['mode']
680 mode = f_data['mode']
676 yield ArchiveNode(file_path, mode, False, no_content)
681 yield ArchiveNode(file_path, mode, False, no_content)
677 else:
682 else:
678 mode = f_data['mode']
683 mode = f_data['mode']
679 is_link = f_data['is_link']
684 is_link = f_data['is_link']
680 data_stream = f_data['content_stream']
685 data_stream = f_data['content_stream']
681 yield ArchiveNode(file_path, mode, is_link, data_stream)
686 yield ArchiveNode(file_path, mode, is_link, data_stream)
682
687
683 return store_archive_in_cache(
688 return store_archive_in_cache(
684 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
689 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
685
690
686
691
687 class SvnDiffer:
692 class SvnDiffer:
688 """
693 """
689 Utility to create diffs based on difflib and the Subversion api
694 Utility to create diffs based on difflib and the Subversion api
690 """
695 """
691
696
692 binary_content = False
697 binary_content = False
693
698
694 def __init__(
699 def __init__(
695 self, repo, src_rev, src_path, tgt_rev, tgt_path,
700 self, repo, src_rev, src_path, tgt_rev, tgt_path,
696 ignore_whitespace, context):
701 ignore_whitespace, context):
697 self.repo = repo
702 self.repo = repo
698 self.ignore_whitespace = ignore_whitespace
703 self.ignore_whitespace = ignore_whitespace
699 self.context = context
704 self.context = context
700
705
701 fsobj = svn.repos.fs(repo)
706 fsobj = svn.repos.fs(repo)
702
707
703 self.tgt_rev = tgt_rev
708 self.tgt_rev = tgt_rev
704 self.tgt_path = tgt_path or ''
709 self.tgt_path = tgt_path or ''
705 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
710 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
706 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
711 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
707
712
708 self.src_rev = src_rev
713 self.src_rev = src_rev
709 self.src_path = src_path or self.tgt_path
714 self.src_path = src_path or self.tgt_path
710 self.src_root = svn.fs.revision_root(fsobj, src_rev)
715 self.src_root = svn.fs.revision_root(fsobj, src_rev)
711 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
716 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
712
717
713 self._validate()
718 self._validate()
714
719
715 def _validate(self):
720 def _validate(self):
716 if (self.tgt_kind != svn.core.svn_node_none and
721 if (self.tgt_kind != svn.core.svn_node_none and
717 self.src_kind != svn.core.svn_node_none and
722 self.src_kind != svn.core.svn_node_none and
718 self.src_kind != self.tgt_kind):
723 self.src_kind != self.tgt_kind):
719 # TODO: johbo: proper error handling
724 # TODO: johbo: proper error handling
720 raise Exception(
725 raise Exception(
721 "Source and target are not compatible for diff generation. "
726 "Source and target are not compatible for diff generation. "
722 "Source type: %s, target type: %s" %
727 "Source type: %s, target type: %s" %
723 (self.src_kind, self.tgt_kind))
728 (self.src_kind, self.tgt_kind))
724
729
725 def generate_diff(self) -> bytes:
730 def generate_diff(self) -> bytes:
726 buf = io.BytesIO()
731 buf = io.BytesIO()
727 if self.tgt_kind == svn.core.svn_node_dir:
732 if self.tgt_kind == svn.core.svn_node_dir:
728 self._generate_dir_diff(buf)
733 self._generate_dir_diff(buf)
729 else:
734 else:
730 self._generate_file_diff(buf)
735 self._generate_file_diff(buf)
731 return buf.getvalue()
736 return buf.getvalue()
732
737
733 def _generate_dir_diff(self, buf: io.BytesIO):
738 def _generate_dir_diff(self, buf: io.BytesIO):
734 editor = DiffChangeEditor()
739 editor = DiffChangeEditor()
735 editor_ptr, editor_baton = svn.delta.make_editor(editor)
740 editor_ptr, editor_baton = svn.delta.make_editor(editor)
736 svn.repos.dir_delta2(
741 svn.repos.dir_delta2(
737 self.src_root,
742 self.src_root,
738 self.src_path,
743 self.src_path,
739 '', # src_entry
744 '', # src_entry
740 self.tgt_root,
745 self.tgt_root,
741 self.tgt_path,
746 self.tgt_path,
742 editor_ptr, editor_baton,
747 editor_ptr, editor_baton,
743 authorization_callback_allow_all,
748 authorization_callback_allow_all,
744 False, # text_deltas
749 False, # text_deltas
745 svn.core.svn_depth_infinity, # depth
750 svn.core.svn_depth_infinity, # depth
746 False, # entry_props
751 False, # entry_props
747 False, # ignore_ancestry
752 False, # ignore_ancestry
748 )
753 )
749
754
750 for path, __, change in sorted(editor.changes):
755 for path, __, change in sorted(editor.changes):
751 self._generate_node_diff(
756 self._generate_node_diff(
752 buf, change, path, self.tgt_path, path, self.src_path)
757 buf, change, path, self.tgt_path, path, self.src_path)
753
758
754 def _generate_file_diff(self, buf: io.BytesIO):
759 def _generate_file_diff(self, buf: io.BytesIO):
755 change = None
760 change = None
756 if self.src_kind == svn.core.svn_node_none:
761 if self.src_kind == svn.core.svn_node_none:
757 change = "add"
762 change = "add"
758 elif self.tgt_kind == svn.core.svn_node_none:
763 elif self.tgt_kind == svn.core.svn_node_none:
759 change = "delete"
764 change = "delete"
760 tgt_base, tgt_path = vcspath.split(self.tgt_path)
765 tgt_base, tgt_path = vcspath.split(self.tgt_path)
761 src_base, src_path = vcspath.split(self.src_path)
766 src_base, src_path = vcspath.split(self.src_path)
762 self._generate_node_diff(
767 self._generate_node_diff(
763 buf, change, tgt_path, tgt_base, src_path, src_base)
768 buf, change, tgt_path, tgt_base, src_path, src_base)
764
769
765 def _generate_node_diff(
770 def _generate_node_diff(
766 self, buf: io.BytesIO, change, tgt_path, tgt_base, src_path, src_base):
771 self, buf: io.BytesIO, change, tgt_path, tgt_base, src_path, src_base):
767
772
768 tgt_path_bytes = safe_bytes(tgt_path)
773 tgt_path_bytes = safe_bytes(tgt_path)
769 tgt_path = safe_str(tgt_path)
774 tgt_path = safe_str(tgt_path)
770
775
771 src_path_bytes = safe_bytes(src_path)
776 src_path_bytes = safe_bytes(src_path)
772 src_path = safe_str(src_path)
777 src_path = safe_str(src_path)
773
778
774 if self.src_rev == self.tgt_rev and tgt_base == src_base:
779 if self.src_rev == self.tgt_rev and tgt_base == src_base:
775 # makes consistent behaviour with git/hg to return empty diff if
780 # makes consistent behaviour with git/hg to return empty diff if
776 # we compare same revisions
781 # we compare same revisions
777 return
782 return
778
783
779 tgt_full_path = vcspath.join(tgt_base, tgt_path)
784 tgt_full_path = vcspath.join(tgt_base, tgt_path)
780 src_full_path = vcspath.join(src_base, src_path)
785 src_full_path = vcspath.join(src_base, src_path)
781
786
782 self.binary_content = False
787 self.binary_content = False
783 mime_type = self._get_mime_type(tgt_full_path)
788 mime_type = self._get_mime_type(tgt_full_path)
784
789
785 if mime_type and not mime_type.startswith(b'text'):
790 if mime_type and not mime_type.startswith(b'text'):
786 self.binary_content = True
791 self.binary_content = True
787 buf.write(b"=" * 67 + b'\n')
792 buf.write(b"=" * 67 + b'\n')
788 buf.write(b"Cannot display: file marked as a binary type.\n")
793 buf.write(b"Cannot display: file marked as a binary type.\n")
789 buf.write(b"svn:mime-type = %s\n" % mime_type)
794 buf.write(b"svn:mime-type = %s\n" % mime_type)
790 buf.write(b"Index: %b\n" % tgt_path_bytes)
795 buf.write(b"Index: %b\n" % tgt_path_bytes)
791 buf.write(b"=" * 67 + b'\n')
796 buf.write(b"=" * 67 + b'\n')
792 buf.write(b"diff --git a/%b b/%b\n" % (tgt_path_bytes, tgt_path_bytes))
797 buf.write(b"diff --git a/%b b/%b\n" % (tgt_path_bytes, tgt_path_bytes))
793
798
794 if change == 'add':
799 if change == 'add':
795 # TODO: johbo: SVN is missing a zero here compared to git
800 # TODO: johbo: SVN is missing a zero here compared to git
796 buf.write(b"new file mode 10644\n")
801 buf.write(b"new file mode 10644\n")
797
802
798 # TODO(marcink): intro to binary detection of svn patches
803 # TODO(marcink): intro to binary detection of svn patches
799 # if self.binary_content:
804 # if self.binary_content:
800 # buf.write(b'GIT binary patch\n')
805 # buf.write(b'GIT binary patch\n')
801
806
802 buf.write(b"--- /dev/null\t(revision 0)\n")
807 buf.write(b"--- /dev/null\t(revision 0)\n")
803 src_lines = []
808 src_lines = []
804 else:
809 else:
805 if change == 'delete':
810 if change == 'delete':
806 buf.write(b"deleted file mode 10644\n")
811 buf.write(b"deleted file mode 10644\n")
807
812
808 # TODO(marcink): intro to binary detection of svn patches
813 # TODO(marcink): intro to binary detection of svn patches
809 # if self.binary_content:
814 # if self.binary_content:
810 # buf.write('GIT binary patch\n')
815 # buf.write('GIT binary patch\n')
811
816
812 buf.write(b"--- a/%b\t(revision %d)\n" % (src_path_bytes, self.src_rev))
817 buf.write(b"--- a/%b\t(revision %d)\n" % (src_path_bytes, self.src_rev))
813 src_lines = self._svn_readlines(self.src_root, src_full_path)
818 src_lines = self._svn_readlines(self.src_root, src_full_path)
814
819
815 if change == 'delete':
820 if change == 'delete':
816 buf.write(b"+++ /dev/null\t(revision %d)\n" % self.tgt_rev)
821 buf.write(b"+++ /dev/null\t(revision %d)\n" % self.tgt_rev)
817 tgt_lines = []
822 tgt_lines = []
818 else:
823 else:
819 buf.write(b"+++ b/%b\t(revision %d)\n" % (tgt_path_bytes, self.tgt_rev))
824 buf.write(b"+++ b/%b\t(revision %d)\n" % (tgt_path_bytes, self.tgt_rev))
820 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
825 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
821
826
822 # we made our diff header, time to generate the diff content into our buffer
827 # we made our diff header, time to generate the diff content into our buffer
823
828
824 if not self.binary_content:
829 if not self.binary_content:
825 udiff = svn_diff.unified_diff(
830 udiff = svn_diff.unified_diff(
826 src_lines, tgt_lines, context=self.context,
831 src_lines, tgt_lines, context=self.context,
827 ignore_blank_lines=self.ignore_whitespace,
832 ignore_blank_lines=self.ignore_whitespace,
828 ignore_case=False,
833 ignore_case=False,
829 ignore_space_changes=self.ignore_whitespace)
834 ignore_space_changes=self.ignore_whitespace)
830
835
831 buf.writelines(udiff)
836 buf.writelines(udiff)
832
837
833 def _get_mime_type(self, path) -> bytes:
838 def _get_mime_type(self, path) -> bytes:
834 try:
839 try:
835 mime_type = svn.fs.node_prop(
840 mime_type = svn.fs.node_prop(
836 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
841 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
837 except svn.core.SubversionException:
842 except svn.core.SubversionException:
838 mime_type = svn.fs.node_prop(
843 mime_type = svn.fs.node_prop(
839 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
844 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
840 return mime_type
845 return mime_type
841
846
842 def _svn_readlines(self, fs_root, node_path):
847 def _svn_readlines(self, fs_root, node_path):
843 if self.binary_content:
848 if self.binary_content:
844 return []
849 return []
845 node_kind = svn.fs.check_path(fs_root, node_path)
850 node_kind = svn.fs.check_path(fs_root, node_path)
846 if node_kind not in (
851 if node_kind not in (
847 svn.core.svn_node_file, svn.core.svn_node_symlink):
852 svn.core.svn_node_file, svn.core.svn_node_symlink):
848 return []
853 return []
849 content = svn.core.Stream(
854 content = svn.core.Stream(
850 svn.fs.file_contents(fs_root, node_path)).read()
855 svn.fs.file_contents(fs_root, node_path)).read()
851
856
852 return content.splitlines(True)
857 return content.splitlines(True)
853
858
854
859
855 class DiffChangeEditor(svn.delta.Editor):
860 class DiffChangeEditor(svn.delta.Editor):
856 """
861 """
857 Records changes between two given revisions
862 Records changes between two given revisions
858 """
863 """
859
864
860 def __init__(self):
865 def __init__(self):
861 self.changes = []
866 self.changes = []
862
867
863 def delete_entry(self, path, revision, parent_baton, pool=None):
868 def delete_entry(self, path, revision, parent_baton, pool=None):
864 self.changes.append((path, None, 'delete'))
869 self.changes.append((path, None, 'delete'))
865
870
866 def add_file(
871 def add_file(
867 self, path, parent_baton, copyfrom_path, copyfrom_revision,
872 self, path, parent_baton, copyfrom_path, copyfrom_revision,
868 file_pool=None):
873 file_pool=None):
869 self.changes.append((path, 'file', 'add'))
874 self.changes.append((path, 'file', 'add'))
870
875
871 def open_file(self, path, parent_baton, base_revision, file_pool=None):
876 def open_file(self, path, parent_baton, base_revision, file_pool=None):
872 self.changes.append((path, 'file', 'change'))
877 self.changes.append((path, 'file', 'change'))
873
878
874
879
875 def authorization_callback_allow_all(root, path, pool):
880 def authorization_callback_allow_all(root, path, pool):
876 return True
881 return True
877
882
878
883
879 class TxnNodeProcessor:
884 class TxnNodeProcessor:
880 """
885 """
881 Utility to process the change of one node within a transaction root.
886 Utility to process the change of one node within a transaction root.
882
887
883 It encapsulates the knowledge of how to add, update or remove
888 It encapsulates the knowledge of how to add, update or remove
884 a node for a given transaction root. The purpose is to support the method
889 a node for a given transaction root. The purpose is to support the method
885 `SvnRemote.commit`.
890 `SvnRemote.commit`.
886 """
891 """
887
892
888 def __init__(self, node, txn_root):
893 def __init__(self, node, txn_root):
889 assert_bytes(node['path'])
894 assert_bytes(node['path'])
890
895
891 self.node = node
896 self.node = node
892 self.txn_root = txn_root
897 self.txn_root = txn_root
893
898
894 def update(self):
899 def update(self):
895 self._ensure_parent_dirs()
900 self._ensure_parent_dirs()
896 self._add_file_if_node_does_not_exist()
901 self._add_file_if_node_does_not_exist()
897 self._update_file_content()
902 self._update_file_content()
898 self._update_file_properties()
903 self._update_file_properties()
899
904
900 def remove(self):
905 def remove(self):
901 svn.fs.delete(self.txn_root, self.node['path'])
906 svn.fs.delete(self.txn_root, self.node['path'])
902 # TODO: Clean up directory if empty
907 # TODO: Clean up directory if empty
903
908
904 def _ensure_parent_dirs(self):
909 def _ensure_parent_dirs(self):
905 curdir = vcspath.dirname(self.node['path'])
910 curdir = vcspath.dirname(self.node['path'])
906 dirs_to_create = []
911 dirs_to_create = []
907 while not self._svn_path_exists(curdir):
912 while not self._svn_path_exists(curdir):
908 dirs_to_create.append(curdir)
913 dirs_to_create.append(curdir)
909 curdir = vcspath.dirname(curdir)
914 curdir = vcspath.dirname(curdir)
910
915
911 for curdir in reversed(dirs_to_create):
916 for curdir in reversed(dirs_to_create):
912 log.debug('Creating missing directory "%s"', curdir)
917 log.debug('Creating missing directory "%s"', curdir)
913 svn.fs.make_dir(self.txn_root, curdir)
918 svn.fs.make_dir(self.txn_root, curdir)
914
919
915 def _svn_path_exists(self, path):
920 def _svn_path_exists(self, path):
916 path_status = svn.fs.check_path(self.txn_root, path)
921 path_status = svn.fs.check_path(self.txn_root, path)
917 return path_status != svn.core.svn_node_none
922 return path_status != svn.core.svn_node_none
918
923
919 def _add_file_if_node_does_not_exist(self):
924 def _add_file_if_node_does_not_exist(self):
920 kind = svn.fs.check_path(self.txn_root, self.node['path'])
925 kind = svn.fs.check_path(self.txn_root, self.node['path'])
921 if kind == svn.core.svn_node_none:
926 if kind == svn.core.svn_node_none:
922 svn.fs.make_file(self.txn_root, self.node['path'])
927 svn.fs.make_file(self.txn_root, self.node['path'])
923
928
924 def _update_file_content(self):
929 def _update_file_content(self):
925 assert_bytes(self.node['content'])
930 assert_bytes(self.node['content'])
926
931
927 handler, baton = svn.fs.apply_textdelta(
932 handler, baton = svn.fs.apply_textdelta(
928 self.txn_root, self.node['path'], None, None)
933 self.txn_root, self.node['path'], None, None)
929 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
934 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
930
935
931 def _update_file_properties(self):
936 def _update_file_properties(self):
932 properties = self.node.get('properties', {})
937 properties = self.node.get('properties', {})
933 for key, value in properties.items():
938 for key, value in properties.items():
934 svn.fs.change_node_prop(
939 svn.fs.change_node_prop(
935 self.txn_root, self.node['path'], safe_bytes(key), safe_bytes(value))
940 self.txn_root, self.node['path'], safe_bytes(key), safe_bytes(value))
936
941
937
942
938 def apr_time_t(timestamp):
943 def apr_time_t(timestamp):
939 """
944 """
940 Convert a Python timestamp into APR timestamp type apr_time_t
945 Convert a Python timestamp into APR timestamp type apr_time_t
941 """
946 """
942 return int(timestamp * 1E6)
947 return int(timestamp * 1E6)
943
948
944
949
945 def svn_opt_revision_value_t(num):
950 def svn_opt_revision_value_t(num):
946 """
951 """
947 Put `num` into a `svn_opt_revision_value_t` structure.
952 Put `num` into a `svn_opt_revision_value_t` structure.
948 """
953 """
949 value = svn.core.svn_opt_revision_value_t()
954 value = svn.core.svn_opt_revision_value_t()
950 value.number = num
955 value.number = num
951 revision = svn.core.svn_opt_revision_t()
956 revision = svn.core.svn_opt_revision_t()
952 revision.kind = svn.core.svn_opt_revision_number
957 revision.kind = svn.core.svn_opt_revision_number
953 revision.value = value
958 revision.value = value
954 return revision
959 return revision
@@ -1,255 +1,258 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import logging
19 import logging
20 import itertools
20 import itertools
21
21
22 import mercurial
22 import mercurial
23 import mercurial.error
23 import mercurial.error
24 import mercurial.wireprotoserver
24 import mercurial.wireprotoserver
25 import mercurial.hgweb.common
25 import mercurial.hgweb.common
26 import mercurial.hgweb.hgweb_mod
26 import mercurial.hgweb.hgweb_mod
27 import webob.exc
27 import webob.exc
28
28
29 from vcsserver import pygrack, exceptions, settings, git_lfs
29 from vcsserver import pygrack, exceptions, settings, git_lfs
30 from vcsserver.str_utils import ascii_bytes, safe_bytes
30 from vcsserver.lib.str_utils import ascii_bytes, safe_bytes
31
31
32 log = logging.getLogger(__name__)
32 log = logging.getLogger(__name__)
33
33
34
34
35 # propagated from mercurial documentation
35 # propagated from mercurial documentation
36 HG_UI_SECTIONS = [
36 HG_UI_SECTIONS = [
37 'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions',
37 'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions',
38 'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp',
38 'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp',
39 'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web',
39 'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web',
40 ]
40 ]
41
41
42
42
43 class HgWeb(mercurial.hgweb.hgweb_mod.hgweb):
43 class HgWeb(mercurial.hgweb.hgweb_mod.hgweb):
44 """Extension of hgweb that simplifies some functions."""
44 """Extension of hgweb that simplifies some functions."""
45
45
46 def _get_view(self, repo):
46 def _get_view(self, repo):
47 """Views are not supported."""
47 """Views are not supported."""
48 return repo
48 return repo
49
49
50 def loadsubweb(self):
50 def loadsubweb(self):
51 """The result is only used in the templater method which is not used."""
51 """The result is only used in the templater method which is not used."""
52 return None
52 return None
53
53
54 def run(self):
54 def run(self):
55 """Unused function so raise an exception if accidentally called."""
55 """Unused function so raise an exception if accidentally called."""
56 raise NotImplementedError
56 raise NotImplementedError
57
57
58 def templater(self, req):
58 def templater(self, req):
59 """Function used in an unreachable code path.
59 """Function used in an unreachable code path.
60
60
61 This code is unreachable because we guarantee that the HTTP request,
61 This code is unreachable because we guarantee that the HTTP request,
62 corresponds to a Mercurial command. See the is_hg method. So, we are
62 corresponds to a Mercurial command. See the is_hg method. So, we are
63 never going to get a user-visible url.
63 never going to get a user-visible url.
64 """
64 """
65 raise NotImplementedError
65 raise NotImplementedError
66
66
67 def archivelist(self, nodeid):
67 def archivelist(self, nodeid):
68 """Unused function so raise an exception if accidentally called."""
68 """Unused function so raise an exception if accidentally called."""
69 raise NotImplementedError
69 raise NotImplementedError
70
70
71 def __call__(self, environ, start_response):
71 def __call__(self, environ, start_response):
72 """Run the WSGI application.
72 """Run the WSGI application.
73
73
74 This may be called by multiple threads.
74 This may be called by multiple threads.
75 """
75 """
76 from mercurial.hgweb import request as requestmod
76 from mercurial.hgweb import request as requestmod
77 req = requestmod.parserequestfromenv(environ)
77 req = requestmod.parserequestfromenv(environ)
78 res = requestmod.wsgiresponse(req, start_response)
78 res = requestmod.wsgiresponse(req, start_response)
79 gen = self.run_wsgi(req, res)
79 gen = self.run_wsgi(req, res)
80
80
81 first_chunk = None
81 first_chunk = None
82
82
83 try:
83 try:
84 data = next(gen)
84 data = next(gen)
85
85
86 def first_chunk():
86 def first_chunk():
87 yield data
87 yield data
88 except StopIteration:
88 except StopIteration:
89 pass
89 pass
90
90
91 if first_chunk:
91 if first_chunk:
92 return itertools.chain(first_chunk(), gen)
92 return itertools.chain(first_chunk(), gen)
93 return gen
93 return gen
94
94
95 def _runwsgi(self, req, res, repo):
95 def _runwsgi(self, req, res, repo):
96
96
97 cmd = req.qsparams.get(b'cmd', '')
97 cmd = req.qsparams.get(b'cmd', '')
98 if not mercurial.wireprotoserver.iscmd(cmd):
98 if not mercurial.wireprotoserver.iscmd(cmd):
99 # NOTE(marcink): for unsupported commands, we return bad request
99 # NOTE(marcink): for unsupported commands, we return bad request
100 # internally from HG
100 # internally from HG
101 log.warning('cmd: `%s` is not supported by the mercurial wireprotocol v1', cmd)
101 log.warning('cmd: `%s` is not supported by the mercurial wireprotocol v1', cmd)
102 from mercurial.hgweb.common import statusmessage
102 from mercurial.hgweb.common import statusmessage
103 res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST)
103 res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST)
104 res.setbodybytes(b'')
104 res.setbodybytes(b'')
105 return res.sendresponse()
105 return res.sendresponse()
106
106
107 return super()._runwsgi(req, res, repo)
107 return super()._runwsgi(req, res, repo)
108
108
109
109
110 def sanitize_hg_ui(baseui):
110 def sanitize_hg_ui(baseui):
111 # NOTE(marcink): since python3 hgsubversion is deprecated.
111 # NOTE(marcink): since python3 hgsubversion is deprecated.
112 # From old installations we might still have this set enabled
112 # From old installations we might still have this set enabled
113 # we explicitly remove this now here to make sure it wont propagate further
113 # we explicitly remove this now here to make sure it wont propagate further
114
114
115 if baseui.config(b'extensions', b'hgsubversion') is not None:
115 if baseui.config(b'extensions', b'hgsubversion') is not None:
116 for cfg in (baseui._ocfg, baseui._tcfg, baseui._ucfg):
116 for cfg in (baseui._ocfg, baseui._tcfg, baseui._ucfg):
117 if b'extensions' in cfg:
117 if b'extensions' in cfg:
118 if b'hgsubversion' in cfg[b'extensions']:
118 if b'hgsubversion' in cfg[b'extensions']:
119 del cfg[b'extensions'][b'hgsubversion']
119 del cfg[b'extensions'][b'hgsubversion']
120
120
121
121
122 def make_hg_ui_from_config(repo_config):
122 def make_hg_ui_from_config(repo_config):
123 baseui = mercurial.ui.ui()
123 baseui = mercurial.ui.ui()
124
124
125 # clean the baseui object
125 # clean the baseui object
126 baseui._ocfg = mercurial.config.config()
126 baseui._ocfg = mercurial.config.config()
127 baseui._ucfg = mercurial.config.config()
127 baseui._ucfg = mercurial.config.config()
128 baseui._tcfg = mercurial.config.config()
128 baseui._tcfg = mercurial.config.config()
129
129
130 for section, option, value in repo_config:
130 for section, option, value in repo_config:
131 baseui.setconfig(
131 baseui.setconfig(
132 ascii_bytes(section, allow_bytes=True),
132 ascii_bytes(section, allow_bytes=True),
133 ascii_bytes(option, allow_bytes=True),
133 ascii_bytes(option, allow_bytes=True),
134 ascii_bytes(value, allow_bytes=True))
134 ascii_bytes(value, allow_bytes=True))
135
135
136 # make our hgweb quiet so it doesn't print output
136 # make our hgweb quiet so it doesn't print output
137 baseui.setconfig(b'ui', b'quiet', b'true')
137 baseui.setconfig(b'ui', b'quiet', b'true')
138
138
139 # use POST requests with args instead of GET with headers - fixes issues with big repos with lots of branches
140 baseui.setconfig(b'experimental', b'httppostargs', b'true')
141
139 return baseui
142 return baseui
140
143
141
144
142 def update_hg_ui_from_hgrc(baseui, repo_path):
145 def update_hg_ui_from_hgrc(baseui, repo_path):
143 path = os.path.join(repo_path, '.hg', 'hgrc')
146 path = os.path.join(repo_path, '.hg', 'hgrc')
144
147
145 if not os.path.isfile(path):
148 if not os.path.isfile(path):
146 log.debug('hgrc file is not present at %s, skipping...', path)
149 log.debug('hgrc file is not present at %s, skipping...', path)
147 return
150 return
148 log.debug('reading hgrc from %s', path)
151 log.debug('reading hgrc from %s', path)
149 cfg = mercurial.config.config()
152 cfg = mercurial.config.config()
150 cfg.read(ascii_bytes(path))
153 cfg.read(ascii_bytes(path))
151 for section in HG_UI_SECTIONS:
154 for section in HG_UI_SECTIONS:
152 for k, v in cfg.items(section):
155 for k, v in cfg.items(section):
153 log.debug('settings ui from file: [%s] %s=%s', section, k, v)
156 log.debug('settings ui from file: [%s] %s=%s', section, k, v)
154 baseui.setconfig(
157 baseui.setconfig(
155 ascii_bytes(section, allow_bytes=True),
158 ascii_bytes(section, allow_bytes=True),
156 ascii_bytes(k, allow_bytes=True),
159 ascii_bytes(k, allow_bytes=True),
157 ascii_bytes(v, allow_bytes=True))
160 ascii_bytes(v, allow_bytes=True))
158
161
159
162
160 def create_hg_wsgi_app(repo_path, repo_name, config):
163 def create_hg_wsgi_app(repo_path, repo_name, config):
161 """
164 """
162 Prepares a WSGI application to handle Mercurial requests.
165 Prepares a WSGI application to handle Mercurial requests.
163
166
164 :param config: is a list of 3-item tuples representing a ConfigObject
167 :param config: is a list of 3-item tuples representing a ConfigObject
165 (it is the serialized version of the config object).
168 (it is the serialized version of the config object).
166 """
169 """
167 log.debug("Creating Mercurial WSGI application")
170 log.debug("Creating Mercurial WSGI application")
168
171
169 baseui = make_hg_ui_from_config(config)
172 baseui = make_hg_ui_from_config(config)
170 update_hg_ui_from_hgrc(baseui, repo_path)
173 update_hg_ui_from_hgrc(baseui, repo_path)
171 sanitize_hg_ui(baseui)
174 sanitize_hg_ui(baseui)
172
175
173 try:
176 try:
174 return HgWeb(safe_bytes(repo_path), name=safe_bytes(repo_name), baseui=baseui)
177 return HgWeb(safe_bytes(repo_path), name=safe_bytes(repo_name), baseui=baseui)
175 except mercurial.error.RequirementError as e:
178 except mercurial.error.RequirementError as e:
176 raise exceptions.RequirementException(e)(e)
179 raise exceptions.RequirementException(e)(e)
177
180
178
181
179 class GitHandler:
182 class GitHandler:
180 """
183 """
181 Handler for Git operations like push/pull etc
184 Handler for Git operations like push/pull etc
182 """
185 """
183 def __init__(self, repo_location, repo_name, git_path, update_server_info,
186 def __init__(self, repo_location, repo_name, git_path, update_server_info,
184 extras):
187 extras):
185 if not os.path.isdir(repo_location):
188 if not os.path.isdir(repo_location):
186 raise OSError(repo_location)
189 raise OSError(repo_location)
187 self.content_path = repo_location
190 self.content_path = repo_location
188 self.repo_name = repo_name
191 self.repo_name = repo_name
189 self.repo_location = repo_location
192 self.repo_location = repo_location
190 self.extras = extras
193 self.extras = extras
191 self.git_path = git_path
194 self.git_path = git_path
192 self.update_server_info = update_server_info
195 self.update_server_info = update_server_info
193
196
194 def __call__(self, environ, start_response):
197 def __call__(self, environ, start_response):
195 app = webob.exc.HTTPNotFound()
198 app = webob.exc.HTTPNotFound()
196 candidate_paths = (
199 candidate_paths = (
197 self.content_path, os.path.join(self.content_path, '.git'))
200 self.content_path, os.path.join(self.content_path, '.git'))
198
201
199 for content_path in candidate_paths:
202 for content_path in candidate_paths:
200 try:
203 try:
201 app = pygrack.GitRepository(
204 app = pygrack.GitRepository(
202 self.repo_name, content_path, self.git_path,
205 self.repo_name, content_path, self.git_path,
203 self.update_server_info, self.extras)
206 self.update_server_info, self.extras)
204 break
207 break
205 except OSError:
208 except OSError:
206 continue
209 continue
207
210
208 return app(environ, start_response)
211 return app(environ, start_response)
209
212
210
213
211 def create_git_wsgi_app(repo_path, repo_name, config):
214 def create_git_wsgi_app(repo_path, repo_name, config):
212 """
215 """
213 Creates a WSGI application to handle Git requests.
216 Creates a WSGI application to handle Git requests.
214
217
215 :param config: is a dictionary holding the extras.
218 :param config: is a dictionary holding the extras.
216 """
219 """
217 git_path = settings.GIT_EXECUTABLE()
220 git_path = settings.GIT_EXECUTABLE()
218 update_server_info = config.pop('git_update_server_info')
221 update_server_info = config.pop('git_update_server_info')
219 app = GitHandler(
222 app = GitHandler(
220 repo_path, repo_name, git_path, update_server_info, config)
223 repo_path, repo_name, git_path, update_server_info, config)
221
224
222 return app
225 return app
223
226
224
227
225 class GitLFSHandler:
228 class GitLFSHandler:
226 """
229 """
227 Handler for Git LFS operations
230 Handler for Git LFS operations
228 """
231 """
229
232
230 def __init__(self, repo_location, repo_name, git_path, update_server_info,
233 def __init__(self, repo_location, repo_name, git_path, update_server_info,
231 extras):
234 extras):
232 if not os.path.isdir(repo_location):
235 if not os.path.isdir(repo_location):
233 raise OSError(repo_location)
236 raise OSError(repo_location)
234 self.content_path = repo_location
237 self.content_path = repo_location
235 self.repo_name = repo_name
238 self.repo_name = repo_name
236 self.repo_location = repo_location
239 self.repo_location = repo_location
237 self.extras = extras
240 self.extras = extras
238 self.git_path = git_path
241 self.git_path = git_path
239 self.update_server_info = update_server_info
242 self.update_server_info = update_server_info
240
243
241 def get_app(self, git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
244 def get_app(self, git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
242 app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme)
245 app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme)
243 return app
246 return app
244
247
245
248
246 def create_git_lfs_wsgi_app(repo_path, repo_name, config):
249 def create_git_lfs_wsgi_app(repo_path, repo_name, config):
247 git_path = settings.GIT_EXECUTABLE()
250 git_path = settings.GIT_EXECUTABLE()
248 update_server_info = config.pop('git_update_server_info')
251 update_server_info = config.pop('git_update_server_info')
249 git_lfs_enabled = config.pop('git_lfs_enabled')
252 git_lfs_enabled = config.pop('git_lfs_enabled')
250 git_lfs_store_path = config.pop('git_lfs_store_path')
253 git_lfs_store_path = config.pop('git_lfs_store_path')
251 git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http')
254 git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http')
252 app = GitLFSHandler(
255 app = GitLFSHandler(
253 repo_path, repo_name, git_path, update_server_info, config)
256 repo_path, repo_name, git_path, update_server_info, config)
254
257
255 return app.get_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme)
258 return app.get_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme)
@@ -1,563 +1,563 b''
1 """
1 """
2 Module provides a class allowing to wrap communication over subprocess.Popen
2 Module provides a class allowing to wrap communication over subprocess.Popen
3 input, output, error streams into a meaningfull, non-blocking, concurrent
3 input, output, error streams into a meaningfull, non-blocking, concurrent
4 stream processor exposing the output data as an iterator fitting to be a
4 stream processor exposing the output data as an iterator fitting to be a
5 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
5 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
6
6
7 Copyright (c) 2011 Daniel Dotsenko <dotsa[at]hotmail.com>
7 Copyright (c) 2011 Daniel Dotsenko <dotsa[at]hotmail.com>
8
8
9 This file is part of git_http_backend.py Project.
9 This file is part of git_http_backend.py Project.
10
10
11 git_http_backend.py Project is free software: you can redistribute it and/or
11 git_http_backend.py Project is free software: you can redistribute it and/or
12 modify it under the terms of the GNU Lesser General Public License as
12 modify it under the terms of the GNU Lesser General Public License as
13 published by the Free Software Foundation, either version 2.1 of the License,
13 published by the Free Software Foundation, either version 2.1 of the License,
14 or (at your option) any later version.
14 or (at your option) any later version.
15
15
16 git_http_backend.py Project is distributed in the hope that it will be useful,
16 git_http_backend.py Project is distributed in the hope that it will be useful,
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 GNU Lesser General Public License for more details.
19 GNU Lesser General Public License for more details.
20
20
21 You should have received a copy of the GNU Lesser General Public License
21 You should have received a copy of the GNU Lesser General Public License
22 along with git_http_backend.py Project.
22 along with git_http_backend.py Project.
23 If not, see <http://www.gnu.org/licenses/>.
23 If not, see <http://www.gnu.org/licenses/>.
24 """
24 """
25 import os
25 import os
26 import collections
26 import collections
27 import logging
27 import logging
28 import subprocess
28 import subprocess
29 import threading
29 import threading
30
30
31 from vcsserver.str_utils import safe_str
31 from vcsserver.lib.str_utils import safe_str
32
32
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 class StreamFeeder(threading.Thread):
36 class StreamFeeder(threading.Thread):
37 """
37 """
38 Normal writing into pipe-like is blocking once the buffer is filled.
38 Normal writing into pipe-like is blocking once the buffer is filled.
39 This thread allows a thread to seep data from a file-like into a pipe
39 This thread allows a thread to seep data from a file-like into a pipe
40 without blocking the main thread.
40 without blocking the main thread.
41 We close inpipe once the end of the source stream is reached.
41 We close inpipe once the end of the source stream is reached.
42 """
42 """
43
43
44 def __init__(self, source):
44 def __init__(self, source):
45 super().__init__()
45 super().__init__()
46 self.daemon = True
46 self.daemon = True
47 filelike = False
47 filelike = False
48 self.bytes = b''
48 self.bytes = b''
49 if type(source) in (str, bytes, bytearray): # string-like
49 if type(source) in (str, bytes, bytearray): # string-like
50 self.bytes = bytes(source)
50 self.bytes = bytes(source)
51 else: # can be either file pointer or file-like
51 else: # can be either file pointer or file-like
52 if isinstance(source, int): # file pointer it is
52 if isinstance(source, int): # file pointer it is
53 # converting file descriptor (int) stdin into file-like
53 # converting file descriptor (int) stdin into file-like
54 source = os.fdopen(source, 'rb', 16384)
54 source = os.fdopen(source, 'rb', 16384)
55 # let's see if source is file-like by now
55 # let's see if source is file-like by now
56 filelike = hasattr(source, 'read')
56 filelike = hasattr(source, 'read')
57 if not filelike and not self.bytes:
57 if not filelike and not self.bytes:
58 raise TypeError("StreamFeeder's source object must be a readable "
58 raise TypeError("StreamFeeder's source object must be a readable "
59 "file-like, a file descriptor, or a string-like.")
59 "file-like, a file descriptor, or a string-like.")
60 self.source = source
60 self.source = source
61 self.readiface, self.writeiface = os.pipe()
61 self.readiface, self.writeiface = os.pipe()
62
62
63 def run(self):
63 def run(self):
64 writer = self.writeiface
64 writer = self.writeiface
65 try:
65 try:
66 if self.bytes:
66 if self.bytes:
67 os.write(writer, self.bytes)
67 os.write(writer, self.bytes)
68 else:
68 else:
69 s = self.source
69 s = self.source
70
70
71 while 1:
71 while 1:
72 _bytes = s.read(4096)
72 _bytes = s.read(4096)
73 if not _bytes:
73 if not _bytes:
74 break
74 break
75 os.write(writer, _bytes)
75 os.write(writer, _bytes)
76
76
77 finally:
77 finally:
78 os.close(writer)
78 os.close(writer)
79
79
80 @property
80 @property
81 def output(self):
81 def output(self):
82 return self.readiface
82 return self.readiface
83
83
84
84
85 class InputStreamChunker(threading.Thread):
85 class InputStreamChunker(threading.Thread):
86 def __init__(self, source, target, buffer_size, chunk_size):
86 def __init__(self, source, target, buffer_size, chunk_size):
87
87
88 super().__init__()
88 super().__init__()
89
89
90 self.daemon = True # die die die.
90 self.daemon = True # die die die.
91
91
92 self.source = source
92 self.source = source
93 self.target = target
93 self.target = target
94 self.chunk_count_max = int(buffer_size / chunk_size) + 1
94 self.chunk_count_max = int(buffer_size / chunk_size) + 1
95 self.chunk_size = chunk_size
95 self.chunk_size = chunk_size
96
96
97 self.data_added = threading.Event()
97 self.data_added = threading.Event()
98 self.data_added.clear()
98 self.data_added.clear()
99
99
100 self.keep_reading = threading.Event()
100 self.keep_reading = threading.Event()
101 self.keep_reading.set()
101 self.keep_reading.set()
102
102
103 self.EOF = threading.Event()
103 self.EOF = threading.Event()
104 self.EOF.clear()
104 self.EOF.clear()
105
105
106 self.go = threading.Event()
106 self.go = threading.Event()
107 self.go.set()
107 self.go.set()
108
108
109 def stop(self):
109 def stop(self):
110 self.go.clear()
110 self.go.clear()
111 self.EOF.set()
111 self.EOF.set()
112 try:
112 try:
113 # this is not proper, but is done to force the reader thread let
113 # this is not proper, but is done to force the reader thread let
114 # go of the input because, if successful, .close() will send EOF
114 # go of the input because, if successful, .close() will send EOF
115 # down the pipe.
115 # down the pipe.
116 self.source.close()
116 self.source.close()
117 except Exception:
117 except Exception:
118 pass
118 pass
119
119
120 def run(self):
120 def run(self):
121 s = self.source
121 s = self.source
122 t = self.target
122 t = self.target
123 cs = self.chunk_size
123 cs = self.chunk_size
124 chunk_count_max = self.chunk_count_max
124 chunk_count_max = self.chunk_count_max
125 keep_reading = self.keep_reading
125 keep_reading = self.keep_reading
126 da = self.data_added
126 da = self.data_added
127 go = self.go
127 go = self.go
128
128
129 try:
129 try:
130 b = s.read(cs)
130 b = s.read(cs)
131 except ValueError:
131 except ValueError:
132 b = ''
132 b = ''
133
133
134 timeout_input = 20
134 timeout_input = 20
135 while b and go.is_set():
135 while b and go.is_set():
136 if len(t) > chunk_count_max:
136 if len(t) > chunk_count_max:
137 keep_reading.clear()
137 keep_reading.clear()
138 keep_reading.wait(timeout_input)
138 keep_reading.wait(timeout_input)
139 if len(t) > chunk_count_max + timeout_input:
139 if len(t) > chunk_count_max + timeout_input:
140 log.error("Timed out while waiting for input from subprocess.")
140 log.error("Timed out while waiting for input from subprocess.")
141 os._exit(-1) # this will cause the worker to recycle itself
141 os._exit(-1) # this will cause the worker to recycle itself
142
142
143 t.append(b)
143 t.append(b)
144 da.set()
144 da.set()
145
145
146 try:
146 try:
147 b = s.read(cs)
147 b = s.read(cs)
148 except ValueError: # probably "I/O operation on closed file"
148 except ValueError: # probably "I/O operation on closed file"
149 b = ''
149 b = ''
150
150
151 self.EOF.set()
151 self.EOF.set()
152 da.set() # for cases when done but there was no input.
152 da.set() # for cases when done but there was no input.
153
153
154
154
155 class BufferedGenerator:
155 class BufferedGenerator:
156 """
156 """
157 Class behaves as a non-blocking, buffered pipe reader.
157 Class behaves as a non-blocking, buffered pipe reader.
158 Reads chunks of data (through a thread)
158 Reads chunks of data (through a thread)
159 from a blocking pipe, and attaches these to an array (Deque) of chunks.
159 from a blocking pipe, and attaches these to an array (Deque) of chunks.
160 Reading is halted in the thread when max chunks is internally buffered.
160 Reading is halted in the thread when max chunks is internally buffered.
161 The .next() may operate in blocking or non-blocking fashion by yielding
161 The .next() may operate in blocking or non-blocking fashion by yielding
162 '' if no data is ready
162 '' if no data is ready
163 to be sent or by not returning until there is some data to send
163 to be sent or by not returning until there is some data to send
164 When we get EOF from underlying source pipe we raise the marker to raise
164 When we get EOF from underlying source pipe we raise the marker to raise
165 StopIteration after the last chunk of data is yielded.
165 StopIteration after the last chunk of data is yielded.
166 """
166 """
167
167
168 def __init__(self, name, source, buffer_size=65536, chunk_size=4096,
168 def __init__(self, name, source, buffer_size=65536, chunk_size=4096,
169 starting_values=None, bottomless=False):
169 starting_values=None, bottomless=False):
170 starting_values = starting_values or []
170 starting_values = starting_values or []
171 self.name = name
171 self.name = name
172 self.buffer_size = buffer_size
172 self.buffer_size = buffer_size
173 self.chunk_size = chunk_size
173 self.chunk_size = chunk_size
174
174
175 if bottomless:
175 if bottomless:
176 maxlen = int(buffer_size / chunk_size)
176 maxlen = int(buffer_size / chunk_size)
177 else:
177 else:
178 maxlen = None
178 maxlen = None
179
179
180 self.data_queue = collections.deque(starting_values, maxlen)
180 self.data_queue = collections.deque(starting_values, maxlen)
181 self.worker = InputStreamChunker(source, self.data_queue, buffer_size, chunk_size)
181 self.worker = InputStreamChunker(source, self.data_queue, buffer_size, chunk_size)
182 if starting_values:
182 if starting_values:
183 self.worker.data_added.set()
183 self.worker.data_added.set()
184 self.worker.start()
184 self.worker.start()
185
185
186 ####################
186 ####################
187 # Generator's methods
187 # Generator's methods
188 ####################
188 ####################
189 def __str__(self):
189 def __str__(self):
190 return f'BufferedGenerator(name={self.name} chunk: {self.chunk_size} on buffer: {self.buffer_size})'
190 return f'BufferedGenerator(name={self.name} chunk: {self.chunk_size} on buffer: {self.buffer_size})'
191
191
192 def __iter__(self):
192 def __iter__(self):
193 return self
193 return self
194
194
195 def __next__(self):
195 def __next__(self):
196
196
197 while not self.length and not self.worker.EOF.is_set():
197 while not self.length and not self.worker.EOF.is_set():
198 self.worker.data_added.clear()
198 self.worker.data_added.clear()
199 self.worker.data_added.wait(0.2)
199 self.worker.data_added.wait(0.2)
200
200
201 if self.length:
201 if self.length:
202 self.worker.keep_reading.set()
202 self.worker.keep_reading.set()
203 return bytes(self.data_queue.popleft())
203 return bytes(self.data_queue.popleft())
204 elif self.worker.EOF.is_set():
204 elif self.worker.EOF.is_set():
205 raise StopIteration
205 raise StopIteration
206
206
207 def throw(self, exc_type, value=None, traceback=None):
207 def throw(self, exc_type, value=None, traceback=None):
208 if not self.worker.EOF.is_set():
208 if not self.worker.EOF.is_set():
209 raise exc_type(value)
209 raise exc_type(value)
210
210
211 def start(self):
211 def start(self):
212 self.worker.start()
212 self.worker.start()
213
213
214 def stop(self):
214 def stop(self):
215 self.worker.stop()
215 self.worker.stop()
216
216
217 def close(self):
217 def close(self):
218 try:
218 try:
219 self.worker.stop()
219 self.worker.stop()
220 self.throw(GeneratorExit)
220 self.throw(GeneratorExit)
221 except (GeneratorExit, StopIteration):
221 except (GeneratorExit, StopIteration):
222 pass
222 pass
223
223
224 ####################
224 ####################
225 # Threaded reader's infrastructure.
225 # Threaded reader's infrastructure.
226 ####################
226 ####################
227 @property
227 @property
228 def input(self):
228 def input(self):
229 return self.worker.w
229 return self.worker.w
230
230
231 @property
231 @property
232 def data_added_event(self):
232 def data_added_event(self):
233 return self.worker.data_added
233 return self.worker.data_added
234
234
235 @property
235 @property
236 def data_added(self):
236 def data_added(self):
237 return self.worker.data_added.is_set()
237 return self.worker.data_added.is_set()
238
238
239 @property
239 @property
240 def reading_paused(self):
240 def reading_paused(self):
241 return not self.worker.keep_reading.is_set()
241 return not self.worker.keep_reading.is_set()
242
242
243 @property
243 @property
244 def done_reading_event(self):
244 def done_reading_event(self):
245 """
245 """
246 Done_reding does not mean that the iterator's buffer is empty.
246 Done_reding does not mean that the iterator's buffer is empty.
247 Iterator might have done reading from underlying source, but the read
247 Iterator might have done reading from underlying source, but the read
248 chunks might still be available for serving through .next() method.
248 chunks might still be available for serving through .next() method.
249
249
250 :returns: An Event class instance.
250 :returns: An Event class instance.
251 """
251 """
252 return self.worker.EOF
252 return self.worker.EOF
253
253
254 @property
254 @property
255 def done_reading(self):
255 def done_reading(self):
256 """
256 """
257 Done_reading does not mean that the iterator's buffer is empty.
257 Done_reading does not mean that the iterator's buffer is empty.
258 Iterator might have done reading from underlying source, but the read
258 Iterator might have done reading from underlying source, but the read
259 chunks might still be available for serving through .next() method.
259 chunks might still be available for serving through .next() method.
260
260
261 :returns: An Bool value.
261 :returns: An Bool value.
262 """
262 """
263 return self.worker.EOF.is_set()
263 return self.worker.EOF.is_set()
264
264
265 @property
265 @property
266 def length(self):
266 def length(self):
267 """
267 """
268 returns int.
268 returns int.
269
269
270 This is the length of the queue of chunks, not the length of
270 This is the length of the queue of chunks, not the length of
271 the combined contents in those chunks.
271 the combined contents in those chunks.
272
272
273 __len__() cannot be meaningfully implemented because this
273 __len__() cannot be meaningfully implemented because this
274 reader is just flying through a bottomless pit content and
274 reader is just flying through a bottomless pit content and
275 can only know the length of what it already saw.
275 can only know the length of what it already saw.
276
276
277 If __len__() on WSGI server per PEP 3333 returns a value,
277 If __len__() on WSGI server per PEP 3333 returns a value,
278 the response's length will be set to that. In order not to
278 the response's length will be set to that. In order not to
279 confuse WSGI PEP3333 servers, we will not implement __len__
279 confuse WSGI PEP3333 servers, we will not implement __len__
280 at all.
280 at all.
281 """
281 """
282 return len(self.data_queue)
282 return len(self.data_queue)
283
283
284 def prepend(self, x):
284 def prepend(self, x):
285 self.data_queue.appendleft(x)
285 self.data_queue.appendleft(x)
286
286
287 def append(self, x):
287 def append(self, x):
288 self.data_queue.append(x)
288 self.data_queue.append(x)
289
289
290 def extend(self, o):
290 def extend(self, o):
291 self.data_queue.extend(o)
291 self.data_queue.extend(o)
292
292
293 def __getitem__(self, i):
293 def __getitem__(self, i):
294 return self.data_queue[i]
294 return self.data_queue[i]
295
295
296
296
297 class SubprocessIOChunker:
297 class SubprocessIOChunker:
298 """
298 """
299 Processor class wrapping handling of subprocess IO.
299 Processor class wrapping handling of subprocess IO.
300
300
301 .. important::
301 .. important::
302
302
303 Watch out for the method `__del__` on this class. If this object
303 Watch out for the method `__del__` on this class. If this object
304 is deleted, it will kill the subprocess, so avoid to
304 is deleted, it will kill the subprocess, so avoid to
305 return the `output` attribute or usage of it like in the following
305 return the `output` attribute or usage of it like in the following
306 example::
306 example::
307
307
308 # `args` expected to run a program that produces a lot of output
308 # `args` expected to run a program that produces a lot of output
309 output = ''.join(SubprocessIOChunker(
309 output = ''.join(SubprocessIOChunker(
310 args, shell=False, inputstream=inputstream, env=environ).output)
310 args, shell=False, inputstream=inputstream, env=environ).output)
311
311
312 # `output` will not contain all the data, because the __del__ method
312 # `output` will not contain all the data, because the __del__ method
313 # has already killed the subprocess in this case before all output
313 # has already killed the subprocess in this case before all output
314 # has been consumed.
314 # has been consumed.
315
315
316
316
317
317
318 In a way, this is a "communicate()" replacement with a twist.
318 In a way, this is a "communicate()" replacement with a twist.
319
319
320 - We are multithreaded. Writing in and reading out, err are all sep threads.
320 - We are multithreaded. Writing in and reading out, err are all sep threads.
321 - We support concurrent (in and out) stream processing.
321 - We support concurrent (in and out) stream processing.
322 - The output is not a stream. It's a queue of read string (bytes, not str)
322 - The output is not a stream. It's a queue of read string (bytes, not str)
323 chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
323 chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
324 - We are non-blocking in more respects than communicate()
324 - We are non-blocking in more respects than communicate()
325 (reading from subprocess out pauses when internal buffer is full, but
325 (reading from subprocess out pauses when internal buffer is full, but
326 does not block the parent calling code. On the flip side, reading from
326 does not block the parent calling code. On the flip side, reading from
327 slow-yielding subprocess may block the iteration until data shows up. This
327 slow-yielding subprocess may block the iteration until data shows up. This
328 does not block the parallel inpipe reading occurring parallel thread.)
328 does not block the parallel inpipe reading occurring parallel thread.)
329
329
330 The purpose of the object is to allow us to wrap subprocess interactions into
330 The purpose of the object is to allow us to wrap subprocess interactions into
331 an iterable that can be passed to a WSGI server as the application's return
331 an iterable that can be passed to a WSGI server as the application's return
332 value. Because of stream-processing-ability, WSGI does not have to read ALL
332 value. Because of stream-processing-ability, WSGI does not have to read ALL
333 of the subprocess's output and buffer it, before handing it to WSGI server for
333 of the subprocess's output and buffer it, before handing it to WSGI server for
334 HTTP response. Instead, the class initializer reads just a bit of the stream
334 HTTP response. Instead, the class initializer reads just a bit of the stream
335 to figure out if error occurred or likely to occur and if not, just hands the
335 to figure out if error occurred or likely to occur and if not, just hands the
336 further iteration over subprocess output to the server for completion of HTTP
336 further iteration over subprocess output to the server for completion of HTTP
337 response.
337 response.
338
338
339 The real or perceived subprocess error is trapped and raised as one of
339 The real or perceived subprocess error is trapped and raised as one of
340 OSError family of exceptions
340 OSError family of exceptions
341
341
342 Example usage:
342 Example usage:
343 # try:
343 # try:
344 # answer = SubprocessIOChunker(
344 # answer = SubprocessIOChunker(
345 # cmd,
345 # cmd,
346 # input,
346 # input,
347 # buffer_size = 65536,
347 # buffer_size = 65536,
348 # chunk_size = 4096
348 # chunk_size = 4096
349 # )
349 # )
350 # except (OSError) as e:
350 # except (OSError) as e:
351 # print str(e)
351 # print str(e)
352 # raise e
352 # raise e
353 #
353 #
354 # return answer
354 # return answer
355
355
356
356
357 """
357 """
358
358
359 # TODO: johbo: This is used to make sure that the open end of the PIPE
359 # TODO: johbo: This is used to make sure that the open end of the PIPE
360 # is closed in the end. It would be way better to wrap this into an
360 # is closed in the end. It would be way better to wrap this into an
361 # object, so that it is closed automatically once it is consumed or
361 # object, so that it is closed automatically once it is consumed or
362 # something similar.
362 # something similar.
363 _close_input_fd = None
363 _close_input_fd = None
364
364
365 _closed = False
365 _closed = False
366 _stdout = None
366 _stdout = None
367 _stderr = None
367 _stderr = None
368
368
369 def __init__(self, cmd, input_stream=None, buffer_size=65536,
369 def __init__(self, cmd, input_stream=None, buffer_size=65536,
370 chunk_size=4096, starting_values=None, fail_on_stderr=True,
370 chunk_size=4096, starting_values=None, fail_on_stderr=True,
371 fail_on_return_code=True, **kwargs):
371 fail_on_return_code=True, **kwargs):
372 """
372 """
373 Initializes SubprocessIOChunker
373 Initializes SubprocessIOChunker
374
374
375 :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
375 :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
376 :param input_stream: (Default: None) A file-like, string, or file pointer.
376 :param input_stream: (Default: None) A file-like, string, or file pointer.
377 :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
377 :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
378 :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
378 :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
379 :param starting_values: (Default: []) An array of strings to put in front of output que.
379 :param starting_values: (Default: []) An array of strings to put in front of output que.
380 :param fail_on_stderr: (Default: True) Whether to raise an exception in
380 :param fail_on_stderr: (Default: True) Whether to raise an exception in
381 case something is written to stderr.
381 case something is written to stderr.
382 :param fail_on_return_code: (Default: True) Whether to raise an
382 :param fail_on_return_code: (Default: True) Whether to raise an
383 exception if the return code is not 0.
383 exception if the return code is not 0.
384 """
384 """
385
385
386 kwargs['shell'] = kwargs.get('shell', True)
386 kwargs['shell'] = kwargs.get('shell', True)
387
387
388 starting_values = starting_values or []
388 starting_values = starting_values or []
389 if input_stream:
389 if input_stream:
390 input_streamer = StreamFeeder(input_stream)
390 input_streamer = StreamFeeder(input_stream)
391 input_streamer.start()
391 input_streamer.start()
392 input_stream = input_streamer.output
392 input_stream = input_streamer.output
393 self._close_input_fd = input_stream
393 self._close_input_fd = input_stream
394
394
395 self._fail_on_stderr = fail_on_stderr
395 self._fail_on_stderr = fail_on_stderr
396 self._fail_on_return_code = fail_on_return_code
396 self._fail_on_return_code = fail_on_return_code
397 self.cmd = cmd
397 self.cmd = cmd
398
398
399 _p = subprocess.Popen(cmd, bufsize=-1, stdin=input_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
399 _p = subprocess.Popen(cmd, bufsize=-1, stdin=input_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
400 **kwargs)
400 **kwargs)
401 self.process = _p
401 self.process = _p
402
402
403 bg_out = BufferedGenerator('stdout', _p.stdout, buffer_size, chunk_size, starting_values)
403 bg_out = BufferedGenerator('stdout', _p.stdout, buffer_size, chunk_size, starting_values)
404 bg_err = BufferedGenerator('stderr', _p.stderr, 10240, 1, bottomless=True)
404 bg_err = BufferedGenerator('stderr', _p.stderr, 10240, 1, bottomless=True)
405
405
406 while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
406 while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
407 # doing this until we reach either end of file, or end of buffer.
407 # doing this until we reach either end of file, or end of buffer.
408 bg_out.data_added_event.wait(0.2)
408 bg_out.data_added_event.wait(0.2)
409 bg_out.data_added_event.clear()
409 bg_out.data_added_event.clear()
410
410
411 # at this point it's still ambiguous if we are done reading or just full buffer.
411 # at this point it's still ambiguous if we are done reading or just full buffer.
412 # Either way, if error (returned by ended process, or implied based on
412 # Either way, if error (returned by ended process, or implied based on
413 # presence of stuff in stderr output) we error out.
413 # presence of stuff in stderr output) we error out.
414 # Else, we are happy.
414 # Else, we are happy.
415 return_code = _p.poll()
415 return_code = _p.poll()
416 ret_code_ok = return_code in [None, 0]
416 ret_code_ok = return_code in [None, 0]
417 ret_code_fail = return_code is not None and return_code != 0
417 ret_code_fail = return_code is not None and return_code != 0
418 if (
418 if (
419 (ret_code_fail and fail_on_return_code) or
419 (ret_code_fail and fail_on_return_code) or
420 (ret_code_ok and fail_on_stderr and bg_err.length)
420 (ret_code_ok and fail_on_stderr and bg_err.length)
421 ):
421 ):
422
422
423 try:
423 try:
424 _p.terminate()
424 _p.terminate()
425 except Exception:
425 except Exception:
426 pass
426 pass
427
427
428 bg_out.stop()
428 bg_out.stop()
429 out = b''.join(bg_out)
429 out = b''.join(bg_out)
430 self._stdout = out
430 self._stdout = out
431
431
432 bg_err.stop()
432 bg_err.stop()
433 err = b''.join(bg_err)
433 err = b''.join(bg_err)
434 self._stderr = err
434 self._stderr = err
435
435
436 # code from https://github.com/schacon/grack/pull/7
436 # code from https://github.com/schacon/grack/pull/7
437 if err.strip() == b'fatal: The remote end hung up unexpectedly' and out.startswith(b'0034shallow '):
437 if err.strip() == b'fatal: The remote end hung up unexpectedly' and out.startswith(b'0034shallow '):
438 bg_out = iter([out])
438 bg_out = iter([out])
439 _p = None
439 _p = None
440 elif err and fail_on_stderr:
440 elif err and fail_on_stderr:
441 text_err = err.decode()
441 text_err = err.decode()
442 raise OSError(
442 raise OSError(
443 f"Subprocess exited due to an error:\n{text_err}")
443 f"Subprocess exited due to an error:\n{text_err}")
444
444
445 if ret_code_fail and fail_on_return_code:
445 if ret_code_fail and fail_on_return_code:
446 text_err = err.decode()
446 text_err = err.decode()
447 if not err:
447 if not err:
448 # maybe get empty stderr, try stdout instead
448 # maybe get empty stderr, try stdout instead
449 # in many cases git reports the errors on stdout too
449 # in many cases git reports the errors on stdout too
450 text_err = out.decode()
450 text_err = out.decode()
451 raise OSError(
451 raise OSError(
452 f"Subprocess exited with non 0 ret code:{return_code}: stderr:{text_err}")
452 f"Subprocess exited with non 0 ret code:{return_code}: stderr:{text_err}")
453
453
454 self.stdout = bg_out
454 self.stdout = bg_out
455 self.stderr = bg_err
455 self.stderr = bg_err
456 self.inputstream = input_stream
456 self.inputstream = input_stream
457
457
458 def __str__(self):
458 def __str__(self):
459 proc = getattr(self, 'process', 'NO_PROCESS')
459 proc = getattr(self, 'process', 'NO_PROCESS')
460 return f'SubprocessIOChunker: {proc}'
460 return f'SubprocessIOChunker: {proc}'
461
461
462 def __iter__(self):
462 def __iter__(self):
463 return self
463 return self
464
464
465 def __next__(self):
465 def __next__(self):
466 # Note: mikhail: We need to be sure that we are checking the return
466 # Note: mikhail: We need to be sure that we are checking the return
467 # code after the stdout stream is closed. Some processes, e.g. git
467 # code after the stdout stream is closed. Some processes, e.g. git
468 # are doing some magic in between closing stdout and terminating the
468 # are doing some magic in between closing stdout and terminating the
469 # process and, as a result, we are not getting return code on "slow"
469 # process and, as a result, we are not getting return code on "slow"
470 # systems.
470 # systems.
471 result = None
471 result = None
472 stop_iteration = None
472 stop_iteration = None
473 try:
473 try:
474 result = next(self.stdout)
474 result = next(self.stdout)
475 except StopIteration as e:
475 except StopIteration as e:
476 stop_iteration = e
476 stop_iteration = e
477
477
478 if self.process:
478 if self.process:
479 return_code = self.process.poll()
479 return_code = self.process.poll()
480 ret_code_fail = return_code is not None and return_code != 0
480 ret_code_fail = return_code is not None and return_code != 0
481 if ret_code_fail and self._fail_on_return_code:
481 if ret_code_fail and self._fail_on_return_code:
482 self.stop_streams()
482 self.stop_streams()
483 err = self.get_stderr()
483 err = self.get_stderr()
484 raise OSError(
484 raise OSError(
485 f"Subprocess exited (exit_code:{return_code}) due to an error during iteration:\n{err}")
485 f"Subprocess exited (exit_code:{return_code}) due to an error during iteration:\n{err}")
486
486
487 if stop_iteration:
487 if stop_iteration:
488 raise stop_iteration
488 raise stop_iteration
489 return result
489 return result
490
490
491 def throw(self, exc_type, value=None, traceback=None):
491 def throw(self, exc_type, value=None, traceback=None):
492 if self.stdout.length or not self.stdout.done_reading:
492 if self.stdout.length or not self.stdout.done_reading:
493 raise exc_type(value)
493 raise exc_type(value)
494
494
495 def close(self):
495 def close(self):
496 if self._closed:
496 if self._closed:
497 return
497 return
498
498
499 try:
499 try:
500 self.process.terminate()
500 self.process.terminate()
501 except Exception:
501 except Exception:
502 pass
502 pass
503 if self._close_input_fd:
503 if self._close_input_fd:
504 os.close(self._close_input_fd)
504 os.close(self._close_input_fd)
505 try:
505 try:
506 self.stdout.close()
506 self.stdout.close()
507 except Exception:
507 except Exception:
508 pass
508 pass
509 try:
509 try:
510 self.stderr.close()
510 self.stderr.close()
511 except Exception:
511 except Exception:
512 pass
512 pass
513 try:
513 try:
514 os.close(self.inputstream)
514 os.close(self.inputstream)
515 except Exception:
515 except Exception:
516 pass
516 pass
517
517
518 self._closed = True
518 self._closed = True
519
519
520 def stop_streams(self):
520 def stop_streams(self):
521 getattr(self.stdout, 'stop', lambda: None)()
521 getattr(self.stdout, 'stop', lambda: None)()
522 getattr(self.stderr, 'stop', lambda: None)()
522 getattr(self.stderr, 'stop', lambda: None)()
523
523
524 def get_stdout(self):
524 def get_stdout(self):
525 if self._stdout:
525 if self._stdout:
526 return self._stdout
526 return self._stdout
527 else:
527 else:
528 return b''.join(self.stdout)
528 return b''.join(self.stdout)
529
529
530 def get_stderr(self):
530 def get_stderr(self):
531 if self._stderr:
531 if self._stderr:
532 return self._stderr
532 return self._stderr
533 else:
533 else:
534 return b''.join(self.stderr)
534 return b''.join(self.stderr)
535
535
536
536
537 def run_command(arguments, env=None):
537 def run_command(arguments, env=None):
538 """
538 """
539 Run the specified command and return the stdout.
539 Run the specified command and return the stdout.
540
540
541 :param arguments: sequence of program arguments (including the program name)
541 :param arguments: sequence of program arguments (including the program name)
542 :type arguments: list[str]
542 :type arguments: list[str]
543 """
543 """
544
544
545 cmd = arguments
545 cmd = arguments
546 log.debug('Running subprocessio command %s', cmd)
546 log.debug('Running subprocessio command %s', cmd)
547 proc = None
547 proc = None
548 try:
548 try:
549 _opts = {'shell': False, 'fail_on_stderr': False}
549 _opts = {'shell': False, 'fail_on_stderr': False}
550 if env:
550 if env:
551 _opts.update({'env': env})
551 _opts.update({'env': env})
552 proc = SubprocessIOChunker(cmd, **_opts)
552 proc = SubprocessIOChunker(cmd, **_opts)
553 return b''.join(proc), b''.join(proc.stderr)
553 return b''.join(proc), b''.join(proc.stderr)
554 except OSError as err:
554 except OSError as err:
555 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
555 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
556 tb_err = ("Couldn't run subprocessio command (%s).\n"
556 tb_err = ("Couldn't run subprocessio command (%s).\n"
557 "Original error was:%s\n" % (cmd, err))
557 "Original error was:%s\n" % (cmd, err))
558 log.exception(tb_err)
558 log.exception(tb_err)
559 raise Exception(tb_err)
559 raise Exception(tb_err)
560 finally:
560 finally:
561 if proc:
561 if proc:
562 proc.close()
562 proc.close()
563
563
@@ -1,257 +1,257 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import threading
18 import threading
19 import msgpack
19 import msgpack
20
20
21 from http.server import BaseHTTPRequestHandler
21 from http.server import BaseHTTPRequestHandler
22 from socketserver import TCPServer
22 from socketserver import TCPServer
23
23
24 import mercurial.ui
24 import mercurial.ui
25 import mock
25 import mock
26 import pytest
26 import pytest
27
27
28 from vcsserver.hooks import HooksHttpClient
28 from vcsserver.hooks import HooksHttpClient
29 from vcsserver.lib.rc_json import json
29 from vcsserver.lib.ext_json import json
30 from vcsserver import hooks
30 from vcsserver import hooks
31
31
32
32
33 def get_hg_ui(extras=None):
33 def get_hg_ui(extras=None):
34 """Create a Config object with a valid RC_SCM_DATA entry."""
34 """Create a Config object with a valid RC_SCM_DATA entry."""
35 extras = extras or {}
35 extras = extras or {}
36 required_extras = {
36 required_extras = {
37 'username': '',
37 'username': '',
38 'repository': '',
38 'repository': '',
39 'locked_by': '',
39 'locked_by': '',
40 'scm': '',
40 'scm': '',
41 'make_lock': '',
41 'make_lock': '',
42 'action': '',
42 'action': '',
43 'ip': '',
43 'ip': '',
44 'hooks_uri': 'fake_hooks_uri',
44 'hooks_uri': 'fake_hooks_uri',
45 }
45 }
46 required_extras.update(extras)
46 required_extras.update(extras)
47 hg_ui = mercurial.ui.ui()
47 hg_ui = mercurial.ui.ui()
48 hg_ui.setconfig(b'rhodecode', b'RC_SCM_DATA', json.dumps(required_extras))
48 hg_ui.setconfig(b'rhodecode', b'RC_SCM_DATA', json.dumps(required_extras))
49
49
50 return hg_ui
50 return hg_ui
51
51
52
52
53 def test_git_pre_receive_is_disabled():
53 def test_git_pre_receive_is_disabled():
54 extras = {'hooks': ['pull']}
54 extras = {'hooks': ['pull']}
55 response = hooks.git_pre_receive(None, None,
55 response = hooks.git_pre_receive(None, None,
56 {'RC_SCM_DATA': json.dumps(extras)})
56 {'RC_SCM_DATA': json.dumps(extras)})
57
57
58 assert response == 0
58 assert response == 0
59
59
60
60
61 def test_git_post_receive_is_disabled():
61 def test_git_post_receive_is_disabled():
62 extras = {'hooks': ['pull']}
62 extras = {'hooks': ['pull']}
63 response = hooks.git_post_receive(None, '',
63 response = hooks.git_post_receive(None, '',
64 {'RC_SCM_DATA': json.dumps(extras)})
64 {'RC_SCM_DATA': json.dumps(extras)})
65
65
66 assert response == 0
66 assert response == 0
67
67
68
68
69 def test_git_post_receive_calls_repo_size():
69 def test_git_post_receive_calls_repo_size():
70 extras = {'hooks': ['push', 'repo_size']}
70 extras = {'hooks': ['push', 'repo_size']}
71
71
72 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
72 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
73 hooks.git_post_receive(
73 hooks.git_post_receive(
74 None, '', {'RC_SCM_DATA': json.dumps(extras)})
74 None, '', {'RC_SCM_DATA': json.dumps(extras)})
75 extras.update({'commit_ids': [], 'hook_type': 'post_receive',
75 extras.update({'commit_ids': [], 'hook_type': 'post_receive',
76 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
76 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
77 expected_calls = [
77 expected_calls = [
78 mock.call('repo_size', extras, mock.ANY),
78 mock.call('repo_size', extras, mock.ANY),
79 mock.call('post_push', extras, mock.ANY),
79 mock.call('post_push', extras, mock.ANY),
80 ]
80 ]
81 assert call_hook_mock.call_args_list == expected_calls
81 assert call_hook_mock.call_args_list == expected_calls
82
82
83
83
84 def test_git_post_receive_does_not_call_disabled_repo_size():
84 def test_git_post_receive_does_not_call_disabled_repo_size():
85 extras = {'hooks': ['push']}
85 extras = {'hooks': ['push']}
86
86
87 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
87 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
88 hooks.git_post_receive(
88 hooks.git_post_receive(
89 None, '', {'RC_SCM_DATA': json.dumps(extras)})
89 None, '', {'RC_SCM_DATA': json.dumps(extras)})
90 extras.update({'commit_ids': [], 'hook_type': 'post_receive',
90 extras.update({'commit_ids': [], 'hook_type': 'post_receive',
91 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
91 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
92 expected_calls = [
92 expected_calls = [
93 mock.call('post_push', extras, mock.ANY)
93 mock.call('post_push', extras, mock.ANY)
94 ]
94 ]
95 assert call_hook_mock.call_args_list == expected_calls
95 assert call_hook_mock.call_args_list == expected_calls
96
96
97
97
98 def test_repo_size_exception_does_not_affect_git_post_receive():
98 def test_repo_size_exception_does_not_affect_git_post_receive():
99 extras = {'hooks': ['push', 'repo_size']}
99 extras = {'hooks': ['push', 'repo_size']}
100 status = 0
100 status = 0
101
101
102 def side_effect(name, *args, **kwargs):
102 def side_effect(name, *args, **kwargs):
103 if name == 'repo_size':
103 if name == 'repo_size':
104 raise Exception('Fake exception')
104 raise Exception('Fake exception')
105 else:
105 else:
106 return status
106 return status
107
107
108 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
108 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
109 call_hook_mock.side_effect = side_effect
109 call_hook_mock.side_effect = side_effect
110 result = hooks.git_post_receive(
110 result = hooks.git_post_receive(
111 None, '', {'RC_SCM_DATA': json.dumps(extras)})
111 None, '', {'RC_SCM_DATA': json.dumps(extras)})
112 assert result == status
112 assert result == status
113
113
114
114
115 def test_git_pre_pull_is_disabled():
115 def test_git_pre_pull_is_disabled():
116 assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')
116 assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')
117
117
118
118
119 def test_git_post_pull_is_disabled():
119 def test_git_post_pull_is_disabled():
120 assert (
120 assert (
121 hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, ''))
121 hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, ''))
122
122
123
123
124 class TestGetHooksClient:
124 class TestGetHooksClient:
125
125
126 def test_returns_http_client_when_protocol_matches(self):
126 def test_returns_http_client_when_protocol_matches(self):
127 hooks_uri = 'localhost:8000'
127 hooks_uri = 'localhost:8000'
128 result = hooks._get_hooks_client({
128 result = hooks._get_hooks_client({
129 'hooks_uri': hooks_uri,
129 'hooks_uri': hooks_uri,
130 'hooks_protocol': 'http'
130 'hooks_protocol': 'http'
131 })
131 })
132 assert isinstance(result, hooks.HooksHttpClient)
132 assert isinstance(result, hooks.HooksHttpClient)
133 assert result.hooks_uri == hooks_uri
133 assert result.hooks_uri == hooks_uri
134
134
135 def test_return_celery_client_when_queue_and_backend_provided(self):
135 def test_return_celery_client_when_queue_and_backend_provided(self):
136 task_queue = 'redis://task_queue:0'
136 task_queue = 'redis://task_queue:0'
137 task_backend = task_queue
137 task_backend = task_queue
138 result = hooks._get_hooks_client({
138 result = hooks._get_hooks_client({
139 'task_queue': task_queue,
139 'task_queue': task_queue,
140 'task_backend': task_backend
140 'task_backend': task_backend
141 })
141 })
142 assert isinstance(result, hooks.HooksCeleryClient)
142 assert isinstance(result, hooks.HooksCeleryClient)
143
143
144
144
145 class TestHooksHttpClient:
145 class TestHooksHttpClient:
146 def test_init_sets_hooks_uri(self):
146 def test_init_sets_hooks_uri(self):
147 uri = 'localhost:3000'
147 uri = 'localhost:3000'
148 client = hooks.HooksHttpClient(uri)
148 client = hooks.HooksHttpClient(uri)
149 assert client.hooks_uri == uri
149 assert client.hooks_uri == uri
150
150
151 def test_serialize_returns_serialized_string(self):
151 def test_serialize_returns_serialized_string(self):
152 client = hooks.HooksHttpClient('localhost:3000')
152 client = hooks.HooksHttpClient('localhost:3000')
153 hook_name = 'test'
153 hook_name = 'test'
154 extras = {
154 extras = {
155 'first': 1,
155 'first': 1,
156 'second': 'two'
156 'second': 'two'
157 }
157 }
158 hooks_proto, result = client._serialize(hook_name, extras)
158 hooks_proto, result = client._serialize(hook_name, extras)
159 expected_result = msgpack.packb({
159 expected_result = msgpack.packb({
160 'method': hook_name,
160 'method': hook_name,
161 'extras': extras,
161 'extras': extras,
162 })
162 })
163 assert hooks_proto == {'rc-hooks-protocol': 'msgpack.v1', 'Connection': 'keep-alive'}
163 assert hooks_proto == {'rc-hooks-protocol': 'msgpack.v1', 'Connection': 'keep-alive'}
164 assert result == expected_result
164 assert result == expected_result
165
165
166 def test_call_queries_http_server(self, http_mirror):
166 def test_call_queries_http_server(self, http_mirror):
167 client = hooks.HooksHttpClient(http_mirror.uri)
167 client = hooks.HooksHttpClient(http_mirror.uri)
168 hook_name = 'test'
168 hook_name = 'test'
169 extras = {
169 extras = {
170 'first': 1,
170 'first': 1,
171 'second': 'two'
171 'second': 'two'
172 }
172 }
173 result = client(hook_name, extras)
173 result = client(hook_name, extras)
174 expected_result = msgpack.unpackb(msgpack.packb({
174 expected_result = msgpack.unpackb(msgpack.packb({
175 'method': hook_name,
175 'method': hook_name,
176 'extras': extras
176 'extras': extras
177 }), raw=False)
177 }), raw=False)
178 assert result == expected_result
178 assert result == expected_result
179
179
180
180
181 @pytest.fixture
181 @pytest.fixture
182 def http_mirror(request):
182 def http_mirror(request):
183 server = MirrorHttpServer()
183 server = MirrorHttpServer()
184 request.addfinalizer(server.stop)
184 request.addfinalizer(server.stop)
185 return server
185 return server
186
186
187
187
188 class MirrorHttpHandler(BaseHTTPRequestHandler):
188 class MirrorHttpHandler(BaseHTTPRequestHandler):
189
189
190 def do_POST(self):
190 def do_POST(self):
191 length = int(self.headers['Content-Length'])
191 length = int(self.headers['Content-Length'])
192 body = self.rfile.read(length)
192 body = self.rfile.read(length)
193 self.send_response(200)
193 self.send_response(200)
194 self.end_headers()
194 self.end_headers()
195 self.wfile.write(body)
195 self.wfile.write(body)
196
196
197
197
198 class MirrorHttpServer:
198 class MirrorHttpServer:
199 ip_address = '127.0.0.1'
199 ip_address = '127.0.0.1'
200 port = 0
200 port = 0
201
201
202 def __init__(self):
202 def __init__(self):
203 self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler)
203 self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler)
204 _, self.port = self._daemon.server_address
204 _, self.port = self._daemon.server_address
205 self._thread = threading.Thread(target=self._daemon.serve_forever)
205 self._thread = threading.Thread(target=self._daemon.serve_forever)
206 self._thread.daemon = True
206 self._thread.daemon = True
207 self._thread.start()
207 self._thread.start()
208
208
209 def stop(self):
209 def stop(self):
210 self._daemon.shutdown()
210 self._daemon.shutdown()
211 self._thread.join()
211 self._thread.join()
212 self._daemon = None
212 self._daemon = None
213 self._thread = None
213 self._thread = None
214
214
215 @property
215 @property
216 def uri(self):
216 def uri(self):
217 return '{}:{}'.format(self.ip_address, self.port)
217 return '{}:{}'.format(self.ip_address, self.port)
218
218
219
219
220 def test_hooks_http_client_init():
220 def test_hooks_http_client_init():
221 hooks_uri = 'http://localhost:8000'
221 hooks_uri = 'http://localhost:8000'
222 client = HooksHttpClient(hooks_uri)
222 client = HooksHttpClient(hooks_uri)
223 assert client.hooks_uri == hooks_uri
223 assert client.hooks_uri == hooks_uri
224
224
225
225
226 def test_hooks_http_client_call():
226 def test_hooks_http_client_call():
227 hooks_uri = 'http://localhost:8000'
227 hooks_uri = 'http://localhost:8000'
228
228
229 method = 'test_method'
229 method = 'test_method'
230 extras = {'key': 'value'}
230 extras = {'key': 'value'}
231
231
232 with \
232 with \
233 mock.patch('http.client.HTTPConnection') as mock_connection,\
233 mock.patch('http.client.HTTPConnection') as mock_connection,\
234 mock.patch('msgpack.load') as mock_load:
234 mock.patch('msgpack.load') as mock_load:
235
235
236 client = HooksHttpClient(hooks_uri)
236 client = HooksHttpClient(hooks_uri)
237
237
238 mock_load.return_value = {'result': 'success'}
238 mock_load.return_value = {'result': 'success'}
239 response = mock.MagicMock()
239 response = mock.MagicMock()
240 response.status = 200
240 response.status = 200
241 mock_connection.request.side_effect = None
241 mock_connection.request.side_effect = None
242 mock_connection.getresponse.return_value = response
242 mock_connection.getresponse.return_value = response
243
243
244 result = client(method, extras)
244 result = client(method, extras)
245
245
246 mock_connection.assert_called_with(hooks_uri)
246 mock_connection.assert_called_with(hooks_uri)
247 mock_connection.return_value.request.assert_called_once()
247 mock_connection.return_value.request.assert_called_once()
248 assert result == {'result': 'success'}
248 assert result == {'result': 'success'}
249
249
250
250
251 def test_hooks_http_client_serialize():
251 def test_hooks_http_client_serialize():
252 method = 'test_method'
252 method = 'test_method'
253 extras = {'key': 'value'}
253 extras = {'key': 'value'}
254 headers, body = HooksHttpClient._serialize(method, extras)
254 headers, body = HooksHttpClient._serialize(method, extras)
255
255
256 assert headers == {'rc-hooks-protocol': HooksHttpClient.proto, 'Connection': 'keep-alive'}
256 assert headers == {'rc-hooks-protocol': HooksHttpClient.proto, 'Connection': 'keep-alive'}
257 assert msgpack.unpackb(body) == {'method': method, 'extras': extras}
257 assert msgpack.unpackb(body) == {'method': method, 'extras': extras}
@@ -1,289 +1,289 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import sys
19 import sys
20 import stat
20 import stat
21 import pytest
21 import pytest
22 import vcsserver
22 import vcsserver
23 import tempfile
23 import tempfile
24 from vcsserver import hook_utils
24 from vcsserver import hook_utils
25 from vcsserver.hook_utils import set_permissions_if_needed, HOOKS_DIR_MODE, HOOKS_FILE_MODE
25 from vcsserver.hook_utils import set_permissions_if_needed, HOOKS_DIR_MODE, HOOKS_FILE_MODE
26 from vcsserver.tests.fixture import no_newline_id_generator
26 from vcsserver.tests.fixture import no_newline_id_generator
27 from vcsserver.str_utils import safe_bytes
27 from vcsserver.lib.str_utils import safe_bytes
28 from vcsserver.utils import AttributeDict
28 from vcsserver.utils import AttributeDict
29
29
30
30
31 class TestCheckRhodecodeHook:
31 class TestCheckRhodecodeHook:
32
32
33 def test_returns_false_when_hook_file_is_wrong_found(self, tmpdir):
33 def test_returns_false_when_hook_file_is_wrong_found(self, tmpdir):
34 hook = os.path.join(str(tmpdir), 'fake_hook_file.py')
34 hook = os.path.join(str(tmpdir), 'fake_hook_file.py')
35 with open(hook, 'wb') as f:
35 with open(hook, 'wb') as f:
36 f.write(b'dummy test')
36 f.write(b'dummy test')
37 result = hook_utils.check_rhodecode_hook(hook)
37 result = hook_utils.check_rhodecode_hook(hook)
38 assert result is False
38 assert result is False
39
39
40 def test_returns_true_when_no_hook_file_found(self, tmpdir):
40 def test_returns_true_when_no_hook_file_found(self, tmpdir):
41 hook = os.path.join(str(tmpdir), 'fake_hook_file_not_existing.py')
41 hook = os.path.join(str(tmpdir), 'fake_hook_file_not_existing.py')
42 result = hook_utils.check_rhodecode_hook(hook)
42 result = hook_utils.check_rhodecode_hook(hook)
43 assert result
43 assert result
44
44
45 @pytest.mark.parametrize("file_content, expected_result", [
45 @pytest.mark.parametrize("file_content, expected_result", [
46 ("RC_HOOK_VER = '3.3.3'\n", True),
46 ("RC_HOOK_VER = '3.3.3'\n", True),
47 ("RC_HOOK = '3.3.3'\n", False),
47 ("RC_HOOK = '3.3.3'\n", False),
48 ], ids=no_newline_id_generator)
48 ], ids=no_newline_id_generator)
49 def test_signatures(self, file_content, expected_result, tmpdir):
49 def test_signatures(self, file_content, expected_result, tmpdir):
50 hook = os.path.join(str(tmpdir), 'fake_hook_file_1.py')
50 hook = os.path.join(str(tmpdir), 'fake_hook_file_1.py')
51 with open(hook, 'wb') as f:
51 with open(hook, 'wb') as f:
52 f.write(safe_bytes(file_content))
52 f.write(safe_bytes(file_content))
53
53
54 result = hook_utils.check_rhodecode_hook(hook)
54 result = hook_utils.check_rhodecode_hook(hook)
55
55
56 assert result is expected_result
56 assert result is expected_result
57
57
58
58
59 class BaseInstallHooks:
59 class BaseInstallHooks:
60 HOOK_FILES = ()
60 HOOK_FILES = ()
61
61
62 def _check_hook_file_dir_mode(self, file_path):
62 def _check_hook_file_dir_mode(self, file_path):
63 dir_path = os.path.dirname(file_path)
63 dir_path = os.path.dirname(file_path)
64 assert os.path.exists(dir_path), f'dir {file_path} missing'
64 assert os.path.exists(dir_path), f'dir {file_path} missing'
65 stat_info = os.stat(dir_path)
65 stat_info = os.stat(dir_path)
66
66
67 file_mode = stat.S_IMODE(stat_info.st_mode)
67 file_mode = stat.S_IMODE(stat_info.st_mode)
68 expected_mode = int(HOOKS_DIR_MODE)
68 expected_mode = int(HOOKS_DIR_MODE)
69 assert expected_mode == file_mode, f'expected mode: {oct(expected_mode)} got: {oct(file_mode)} for {dir_path}'
69 assert expected_mode == file_mode, f'expected mode: {oct(expected_mode)} got: {oct(file_mode)} for {dir_path}'
70
70
71 def _check_hook_file_mode(self, file_path):
71 def _check_hook_file_mode(self, file_path):
72 assert os.path.exists(file_path), f'path {file_path} missing'
72 assert os.path.exists(file_path), f'path {file_path} missing'
73 stat_info = os.stat(file_path)
73 stat_info = os.stat(file_path)
74
74
75 file_mode = stat.S_IMODE(stat_info.st_mode)
75 file_mode = stat.S_IMODE(stat_info.st_mode)
76 expected_mode = int(HOOKS_FILE_MODE)
76 expected_mode = int(HOOKS_FILE_MODE)
77 assert expected_mode == file_mode, f'expected mode: {oct(expected_mode)} got: {oct(file_mode)} for {file_path}'
77 assert expected_mode == file_mode, f'expected mode: {oct(expected_mode)} got: {oct(file_mode)} for {file_path}'
78
78
79 def _check_hook_file_content(self, file_path, executable):
79 def _check_hook_file_content(self, file_path, executable):
80 executable = executable or sys.executable
80 executable = executable or sys.executable
81 with open(file_path, 'rt') as hook_file:
81 with open(file_path, 'rt') as hook_file:
82 content = hook_file.read()
82 content = hook_file.read()
83
83
84 expected_env = '#!{}'.format(executable)
84 expected_env = '#!{}'.format(executable)
85 expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(vcsserver.get_version())
85 expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(vcsserver.get_version())
86 assert content.strip().startswith(expected_env)
86 assert content.strip().startswith(expected_env)
87 assert expected_rc_version in content
87 assert expected_rc_version in content
88
88
89 def _create_fake_hook(self, file_path, content):
89 def _create_fake_hook(self, file_path, content):
90 with open(file_path, 'w') as hook_file:
90 with open(file_path, 'w') as hook_file:
91 hook_file.write(content)
91 hook_file.write(content)
92
92
93 def create_dummy_repo(self, repo_type):
93 def create_dummy_repo(self, repo_type):
94 tmpdir = tempfile.mkdtemp()
94 tmpdir = tempfile.mkdtemp()
95 repo = AttributeDict()
95 repo = AttributeDict()
96 if repo_type == 'git':
96 if repo_type == 'git':
97 repo.path = os.path.join(tmpdir, 'test_git_hooks_installation_repo')
97 repo.path = os.path.join(tmpdir, 'test_git_hooks_installation_repo')
98 os.makedirs(repo.path)
98 os.makedirs(repo.path)
99 os.makedirs(os.path.join(repo.path, 'hooks'))
99 os.makedirs(os.path.join(repo.path, 'hooks'))
100 repo.bare = True
100 repo.bare = True
101
101
102 elif repo_type == 'svn':
102 elif repo_type == 'svn':
103 repo.path = os.path.join(tmpdir, 'test_svn_hooks_installation_repo')
103 repo.path = os.path.join(tmpdir, 'test_svn_hooks_installation_repo')
104 os.makedirs(repo.path)
104 os.makedirs(repo.path)
105 os.makedirs(os.path.join(repo.path, 'hooks'))
105 os.makedirs(os.path.join(repo.path, 'hooks'))
106
106
107 return repo
107 return repo
108
108
109 def check_hooks(self, repo_path, repo_bare=True):
109 def check_hooks(self, repo_path, repo_bare=True):
110 for file_name in self.HOOK_FILES:
110 for file_name in self.HOOK_FILES:
111 if repo_bare:
111 if repo_bare:
112 file_path = os.path.join(repo_path, 'hooks', file_name)
112 file_path = os.path.join(repo_path, 'hooks', file_name)
113 else:
113 else:
114 file_path = os.path.join(repo_path, '.git', 'hooks', file_name)
114 file_path = os.path.join(repo_path, '.git', 'hooks', file_name)
115
115
116 self._check_hook_file_dir_mode(file_path)
116 self._check_hook_file_dir_mode(file_path)
117 self._check_hook_file_mode(file_path)
117 self._check_hook_file_mode(file_path)
118 self._check_hook_file_content(file_path, sys.executable)
118 self._check_hook_file_content(file_path, sys.executable)
119
119
120
120
121 class TestInstallGitHooks(BaseInstallHooks):
121 class TestInstallGitHooks(BaseInstallHooks):
122 HOOK_FILES = ('pre-receive', 'post-receive')
122 HOOK_FILES = ('pre-receive', 'post-receive')
123
123
124 def test_hooks_are_installed(self):
124 def test_hooks_are_installed(self):
125 repo = self.create_dummy_repo('git')
125 repo = self.create_dummy_repo('git')
126 result = hook_utils.install_git_hooks(repo.path, repo.bare)
126 result = hook_utils.install_git_hooks(repo.path, repo.bare)
127 assert result
127 assert result
128 self.check_hooks(repo.path, repo.bare)
128 self.check_hooks(repo.path, repo.bare)
129
129
130 def test_hooks_are_replaced(self):
130 def test_hooks_are_replaced(self):
131 repo = self.create_dummy_repo('git')
131 repo = self.create_dummy_repo('git')
132 hooks_path = os.path.join(repo.path, 'hooks')
132 hooks_path = os.path.join(repo.path, 'hooks')
133 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
133 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
134 self._create_fake_hook(
134 self._create_fake_hook(
135 file_path, content="RC_HOOK_VER = 'abcde'\n")
135 file_path, content="RC_HOOK_VER = 'abcde'\n")
136
136
137 result = hook_utils.install_git_hooks(repo.path, repo.bare)
137 result = hook_utils.install_git_hooks(repo.path, repo.bare)
138 assert result
138 assert result
139 self.check_hooks(repo.path, repo.bare)
139 self.check_hooks(repo.path, repo.bare)
140
140
141 def test_non_rc_hooks_are_not_replaced(self):
141 def test_non_rc_hooks_are_not_replaced(self):
142 repo = self.create_dummy_repo('git')
142 repo = self.create_dummy_repo('git')
143 hooks_path = os.path.join(repo.path, 'hooks')
143 hooks_path = os.path.join(repo.path, 'hooks')
144 non_rc_content = 'echo "non rc hook"\n'
144 non_rc_content = 'echo "non rc hook"\n'
145 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
145 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
146 self._create_fake_hook(
146 self._create_fake_hook(
147 file_path, content=non_rc_content)
147 file_path, content=non_rc_content)
148
148
149 result = hook_utils.install_git_hooks(repo.path, repo.bare)
149 result = hook_utils.install_git_hooks(repo.path, repo.bare)
150 assert result
150 assert result
151
151
152 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
152 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
153 with open(file_path, 'rt') as hook_file:
153 with open(file_path, 'rt') as hook_file:
154 content = hook_file.read()
154 content = hook_file.read()
155 assert content == non_rc_content
155 assert content == non_rc_content
156
156
157 def test_non_rc_hooks_are_replaced_with_force_flag(self):
157 def test_non_rc_hooks_are_replaced_with_force_flag(self):
158 repo = self.create_dummy_repo('git')
158 repo = self.create_dummy_repo('git')
159 hooks_path = os.path.join(repo.path, 'hooks')
159 hooks_path = os.path.join(repo.path, 'hooks')
160 non_rc_content = 'echo "non rc hook"\n'
160 non_rc_content = 'echo "non rc hook"\n'
161 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
161 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
162 self._create_fake_hook(
162 self._create_fake_hook(
163 file_path, content=non_rc_content)
163 file_path, content=non_rc_content)
164
164
165 result = hook_utils.install_git_hooks(
165 result = hook_utils.install_git_hooks(
166 repo.path, repo.bare, force_create=True)
166 repo.path, repo.bare, force_create=True)
167 assert result
167 assert result
168 self.check_hooks(repo.path, repo.bare)
168 self.check_hooks(repo.path, repo.bare)
169
169
170
170
171 class TestInstallSvnHooks(BaseInstallHooks):
171 class TestInstallSvnHooks(BaseInstallHooks):
172 HOOK_FILES = ('pre-commit', 'post-commit')
172 HOOK_FILES = ('pre-commit', 'post-commit')
173
173
174 def test_hooks_are_installed(self):
174 def test_hooks_are_installed(self):
175 repo = self.create_dummy_repo('svn')
175 repo = self.create_dummy_repo('svn')
176 result = hook_utils.install_svn_hooks(repo.path)
176 result = hook_utils.install_svn_hooks(repo.path)
177 assert result
177 assert result
178 self.check_hooks(repo.path)
178 self.check_hooks(repo.path)
179
179
180 def test_hooks_are_replaced(self):
180 def test_hooks_are_replaced(self):
181 repo = self.create_dummy_repo('svn')
181 repo = self.create_dummy_repo('svn')
182 hooks_path = os.path.join(repo.path, 'hooks')
182 hooks_path = os.path.join(repo.path, 'hooks')
183 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
183 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
184 self._create_fake_hook(
184 self._create_fake_hook(
185 file_path, content="RC_HOOK_VER = 'abcde'\n")
185 file_path, content="RC_HOOK_VER = 'abcde'\n")
186
186
187 result = hook_utils.install_svn_hooks(repo.path)
187 result = hook_utils.install_svn_hooks(repo.path)
188 assert result
188 assert result
189 self.check_hooks(repo.path)
189 self.check_hooks(repo.path)
190
190
191 def test_non_rc_hooks_are_not_replaced(self):
191 def test_non_rc_hooks_are_not_replaced(self):
192 repo = self.create_dummy_repo('svn')
192 repo = self.create_dummy_repo('svn')
193 hooks_path = os.path.join(repo.path, 'hooks')
193 hooks_path = os.path.join(repo.path, 'hooks')
194 non_rc_content = 'echo "non rc hook"\n'
194 non_rc_content = 'echo "non rc hook"\n'
195 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
195 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
196 self._create_fake_hook(
196 self._create_fake_hook(
197 file_path, content=non_rc_content)
197 file_path, content=non_rc_content)
198
198
199 result = hook_utils.install_svn_hooks(repo.path)
199 result = hook_utils.install_svn_hooks(repo.path)
200 assert result
200 assert result
201
201
202 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
202 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
203 with open(file_path, 'rt') as hook_file:
203 with open(file_path, 'rt') as hook_file:
204 content = hook_file.read()
204 content = hook_file.read()
205 assert content == non_rc_content
205 assert content == non_rc_content
206
206
207 def test_non_rc_hooks_are_replaced_with_force_flag(self):
207 def test_non_rc_hooks_are_replaced_with_force_flag(self):
208 repo = self.create_dummy_repo('svn')
208 repo = self.create_dummy_repo('svn')
209 hooks_path = os.path.join(repo.path, 'hooks')
209 hooks_path = os.path.join(repo.path, 'hooks')
210 non_rc_content = 'echo "non rc hook"\n'
210 non_rc_content = 'echo "non rc hook"\n'
211 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
211 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
212 self._create_fake_hook(
212 self._create_fake_hook(
213 file_path, content=non_rc_content)
213 file_path, content=non_rc_content)
214
214
215 result = hook_utils.install_svn_hooks(
215 result = hook_utils.install_svn_hooks(
216 repo.path, force_create=True)
216 repo.path, force_create=True)
217 assert result
217 assert result
218 self.check_hooks(repo.path, )
218 self.check_hooks(repo.path, )
219
219
220
220
221 def create_test_file(filename):
221 def create_test_file(filename):
222 """Utility function to create a test file."""
222 """Utility function to create a test file."""
223 with open(filename, 'w') as f:
223 with open(filename, 'w') as f:
224 f.write("Test file")
224 f.write("Test file")
225
225
226
226
227 def remove_test_file(filename):
227 def remove_test_file(filename):
228 """Utility function to remove a test file."""
228 """Utility function to remove a test file."""
229 if os.path.exists(filename):
229 if os.path.exists(filename):
230 os.remove(filename)
230 os.remove(filename)
231
231
232
232
233 @pytest.fixture
233 @pytest.fixture
234 def test_file():
234 def test_file():
235 filename = 'test_file.txt'
235 filename = 'test_file.txt'
236 create_test_file(filename)
236 create_test_file(filename)
237 yield filename
237 yield filename
238 remove_test_file(filename)
238 remove_test_file(filename)
239
239
240
240
241 def test_increase_permissions(test_file):
241 def test_increase_permissions(test_file):
242 # Set initial lower permissions
242 # Set initial lower permissions
243 initial_perms = 0o644
243 initial_perms = 0o644
244 os.chmod(test_file, initial_perms)
244 os.chmod(test_file, initial_perms)
245
245
246 # Set higher permissions
246 # Set higher permissions
247 new_perms = 0o666
247 new_perms = 0o666
248 set_permissions_if_needed(test_file, new_perms)
248 set_permissions_if_needed(test_file, new_perms)
249
249
250 # Check if permissions were updated
250 # Check if permissions were updated
251 assert (os.stat(test_file).st_mode & 0o777) == new_perms
251 assert (os.stat(test_file).st_mode & 0o777) == new_perms
252
252
253
253
254 def test_no_permission_change_needed(test_file):
254 def test_no_permission_change_needed(test_file):
255 # Set initial permissions
255 # Set initial permissions
256 initial_perms = 0o666
256 initial_perms = 0o666
257 os.chmod(test_file, initial_perms)
257 os.chmod(test_file, initial_perms)
258
258
259 # Attempt to set the same permissions
259 # Attempt to set the same permissions
260 set_permissions_if_needed(test_file, initial_perms)
260 set_permissions_if_needed(test_file, initial_perms)
261
261
262 # Check if permissions were unchanged
262 # Check if permissions were unchanged
263 assert (os.stat(test_file).st_mode & 0o777) == initial_perms
263 assert (os.stat(test_file).st_mode & 0o777) == initial_perms
264
264
265
265
266 def test_no_permission_reduction(test_file):
266 def test_no_permission_reduction(test_file):
267 # Set initial higher permissions
267 # Set initial higher permissions
268 initial_perms = 0o666
268 initial_perms = 0o666
269 os.chmod(test_file, initial_perms)
269 os.chmod(test_file, initial_perms)
270
270
271 # Attempt to set lower permissions
271 # Attempt to set lower permissions
272 lower_perms = 0o644
272 lower_perms = 0o644
273 set_permissions_if_needed(test_file, lower_perms)
273 set_permissions_if_needed(test_file, lower_perms)
274
274
275 # Check if permissions were not reduced
275 # Check if permissions were not reduced
276 assert (os.stat(test_file).st_mode & 0o777) == initial_perms
276 assert (os.stat(test_file).st_mode & 0o777) == initial_perms
277
277
278
278
279 def test_no_permission_reduction_when_on_777(test_file):
279 def test_no_permission_reduction_when_on_777(test_file):
280 # Set initial higher permissions
280 # Set initial higher permissions
281 initial_perms = 0o777
281 initial_perms = 0o777
282 os.chmod(test_file, initial_perms)
282 os.chmod(test_file, initial_perms)
283
283
284 # Attempt to set lower permissions
284 # Attempt to set lower permissions
285 lower_perms = 0o755
285 lower_perms = 0o755
286 set_permissions_if_needed(test_file, lower_perms)
286 set_permissions_if_needed(test_file, lower_perms)
287
287
288 # Check if permissions were not reduced
288 # Check if permissions were not reduced
289 assert (os.stat(test_file).st_mode & 0o777) == initial_perms
289 assert (os.stat(test_file).st_mode & 0o777) == initial_perms
@@ -1,295 +1,295 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import more_itertools
19 import more_itertools
20
20
21 import dulwich.protocol
21 import dulwich.protocol
22 import mock
22 import mock
23 import pytest
23 import pytest
24 import webob
24 import webob
25 import webtest
25 import webtest
26
26
27 from vcsserver import hooks, pygrack
27 from vcsserver import hooks, pygrack
28
28
29 from vcsserver.str_utils import ascii_bytes
29 from vcsserver.lib.str_utils import ascii_bytes
30
30
31
31
32 @pytest.fixture()
32 @pytest.fixture()
33 def pygrack_instance(tmpdir):
33 def pygrack_instance(tmpdir):
34 """
34 """
35 Creates a pygrack app instance.
35 Creates a pygrack app instance.
36
36
37 Right now, it does not much helpful regarding the passed directory.
37 Right now, it does not much helpful regarding the passed directory.
38 It just contains the required folders to pass the signature test.
38 It just contains the required folders to pass the signature test.
39 """
39 """
40 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
40 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
41 tmpdir.mkdir(dir_name)
41 tmpdir.mkdir(dir_name)
42
42
43 return pygrack.GitRepository('repo_name', str(tmpdir), 'git', False, {})
43 return pygrack.GitRepository('repo_name', str(tmpdir), 'git', False, {})
44
44
45
45
46 @pytest.fixture()
46 @pytest.fixture()
47 def pygrack_app(pygrack_instance):
47 def pygrack_app(pygrack_instance):
48 """
48 """
49 Creates a pygrack app wrapped in webtest.TestApp.
49 Creates a pygrack app wrapped in webtest.TestApp.
50 """
50 """
51 return webtest.TestApp(pygrack_instance)
51 return webtest.TestApp(pygrack_instance)
52
52
53
53
54 def test_invalid_service_info_refs_returns_403(pygrack_app):
54 def test_invalid_service_info_refs_returns_403(pygrack_app):
55 response = pygrack_app.get('/info/refs?service=git-upload-packs',
55 response = pygrack_app.get('/info/refs?service=git-upload-packs',
56 expect_errors=True)
56 expect_errors=True)
57
57
58 assert response.status_int == 403
58 assert response.status_int == 403
59
59
60
60
61 def test_invalid_endpoint_returns_403(pygrack_app):
61 def test_invalid_endpoint_returns_403(pygrack_app):
62 response = pygrack_app.post('/git-upload-packs', expect_errors=True)
62 response = pygrack_app.post('/git-upload-packs', expect_errors=True)
63
63
64 assert response.status_int == 403
64 assert response.status_int == 403
65
65
66
66
67 @pytest.mark.parametrize('sideband', [
67 @pytest.mark.parametrize('sideband', [
68 'side-band-64k',
68 'side-band-64k',
69 'side-band',
69 'side-band',
70 'side-band no-progress',
70 'side-band no-progress',
71 ])
71 ])
72 def test_pre_pull_hook_fails_with_sideband(pygrack_app, sideband):
72 def test_pre_pull_hook_fails_with_sideband(pygrack_app, sideband):
73 request = ''.join([
73 request = ''.join([
74 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ',
74 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ',
75 f'multi_ack {sideband} ofs-delta\n',
75 f'multi_ack {sideband} ofs-delta\n',
76 '0000',
76 '0000',
77 '0009done\n',
77 '0009done\n',
78 ])
78 ])
79 with mock.patch('vcsserver.hooks.git_pre_pull', return_value=hooks.HookResponse(1, 'foo')):
79 with mock.patch('vcsserver.hooks.git_pre_pull', return_value=hooks.HookResponse(1, 'foo')):
80 response = pygrack_app.post(
80 response = pygrack_app.post(
81 '/git-upload-pack', params=request,
81 '/git-upload-pack', params=request,
82 content_type='application/x-git-upload-pack')
82 content_type='application/x-git-upload-pack')
83
83
84 data = io.BytesIO(response.body)
84 data = io.BytesIO(response.body)
85 proto = dulwich.protocol.Protocol(data.read, None)
85 proto = dulwich.protocol.Protocol(data.read, None)
86 packets = list(proto.read_pkt_seq())
86 packets = list(proto.read_pkt_seq())
87
87
88 expected_packets = [
88 expected_packets = [
89 b'NAK\n', b'\x02foo', b'\x02Pre pull hook failed: aborting\n',
89 b'NAK\n', b'\x02foo', b'\x02Pre pull hook failed: aborting\n',
90 b'\x01' + pygrack.GitRepository.EMPTY_PACK,
90 b'\x01' + pygrack.GitRepository.EMPTY_PACK,
91 ]
91 ]
92 assert packets == expected_packets
92 assert packets == expected_packets
93
93
94
94
95 def test_pre_pull_hook_fails_no_sideband(pygrack_app):
95 def test_pre_pull_hook_fails_no_sideband(pygrack_app):
96 request = ''.join([
96 request = ''.join([
97 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
97 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
98 'multi_ack ofs-delta\n'
98 'multi_ack ofs-delta\n'
99 '0000',
99 '0000',
100 '0009done\n',
100 '0009done\n',
101 ])
101 ])
102 with mock.patch('vcsserver.hooks.git_pre_pull',
102 with mock.patch('vcsserver.hooks.git_pre_pull',
103 return_value=hooks.HookResponse(1, 'foo')):
103 return_value=hooks.HookResponse(1, 'foo')):
104 response = pygrack_app.post(
104 response = pygrack_app.post(
105 '/git-upload-pack', params=request,
105 '/git-upload-pack', params=request,
106 content_type='application/x-git-upload-pack')
106 content_type='application/x-git-upload-pack')
107
107
108 assert response.body == pygrack.GitRepository.EMPTY_PACK
108 assert response.body == pygrack.GitRepository.EMPTY_PACK
109
109
110
110
111 def test_pull_has_hook_messages(pygrack_app):
111 def test_pull_has_hook_messages(pygrack_app):
112 request = ''.join([
112 request = ''.join([
113 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
113 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
114 'multi_ack side-band-64k ofs-delta\n'
114 'multi_ack side-band-64k ofs-delta\n'
115 '0000',
115 '0000',
116 '0009done\n',
116 '0009done\n',
117 ])
117 ])
118
118
119 pre_pull = 'pre_pull_output'
119 pre_pull = 'pre_pull_output'
120 post_pull = 'post_pull_output'
120 post_pull = 'post_pull_output'
121
121
122 with mock.patch('vcsserver.hooks.git_pre_pull',
122 with mock.patch('vcsserver.hooks.git_pre_pull',
123 return_value=hooks.HookResponse(0, pre_pull)):
123 return_value=hooks.HookResponse(0, pre_pull)):
124 with mock.patch('vcsserver.hooks.git_post_pull',
124 with mock.patch('vcsserver.hooks.git_post_pull',
125 return_value=hooks.HookResponse(1, post_pull)):
125 return_value=hooks.HookResponse(1, post_pull)):
126 with mock.patch('vcsserver.subprocessio.SubprocessIOChunker',
126 with mock.patch('vcsserver.subprocessio.SubprocessIOChunker',
127 return_value=more_itertools.always_iterable([b'0008NAK\n0009subp\n0000'])):
127 return_value=more_itertools.always_iterable([b'0008NAK\n0009subp\n0000'])):
128 response = pygrack_app.post(
128 response = pygrack_app.post(
129 '/git-upload-pack', params=request,
129 '/git-upload-pack', params=request,
130 content_type='application/x-git-upload-pack')
130 content_type='application/x-git-upload-pack')
131
131
132 data = io.BytesIO(response.body)
132 data = io.BytesIO(response.body)
133 proto = dulwich.protocol.Protocol(data.read, None)
133 proto = dulwich.protocol.Protocol(data.read, None)
134 packets = list(proto.read_pkt_seq())
134 packets = list(proto.read_pkt_seq())
135
135
136 assert packets == [b'NAK\n',
136 assert packets == [b'NAK\n',
137 # pre-pull only outputs if IT FAILS as in != 0 ret code
137 # pre-pull only outputs if IT FAILS as in != 0 ret code
138 #b'\x02pre_pull_output',
138 #b'\x02pre_pull_output',
139 b'subp\n',
139 b'subp\n',
140 b'\x02post_pull_output']
140 b'\x02post_pull_output']
141
141
142
142
143 def test_get_want_capabilities(pygrack_instance):
143 def test_get_want_capabilities(pygrack_instance):
144 data = io.BytesIO(
144 data = io.BytesIO(
145 b'0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
145 b'0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
146 b'multi_ack side-band-64k ofs-delta\n00000009done\n')
146 b'multi_ack side-band-64k ofs-delta\n00000009done\n')
147
147
148 request = webob.Request({
148 request = webob.Request({
149 'wsgi.input': data,
149 'wsgi.input': data,
150 'REQUEST_METHOD': 'POST',
150 'REQUEST_METHOD': 'POST',
151 'webob.is_body_seekable': True
151 'webob.is_body_seekable': True
152 })
152 })
153
153
154 capabilities = pygrack_instance._get_want_capabilities(request)
154 capabilities = pygrack_instance._get_want_capabilities(request)
155
155
156 assert capabilities == frozenset(
156 assert capabilities == frozenset(
157 (b'ofs-delta', b'multi_ack', b'side-band-64k'))
157 (b'ofs-delta', b'multi_ack', b'side-band-64k'))
158 assert data.tell() == 0
158 assert data.tell() == 0
159
159
160
160
161 @pytest.mark.parametrize('data,capabilities,expected', [
161 @pytest.mark.parametrize('data,capabilities,expected', [
162 ('foo', [], []),
162 ('foo', [], []),
163 ('', [pygrack.CAPABILITY_SIDE_BAND_64K], []),
163 ('', [pygrack.CAPABILITY_SIDE_BAND_64K], []),
164 ('', [pygrack.CAPABILITY_SIDE_BAND], []),
164 ('', [pygrack.CAPABILITY_SIDE_BAND], []),
165 ('foo', [pygrack.CAPABILITY_SIDE_BAND_64K], [b'0008\x02foo']),
165 ('foo', [pygrack.CAPABILITY_SIDE_BAND_64K], [b'0008\x02foo']),
166 ('foo', [pygrack.CAPABILITY_SIDE_BAND], [b'0008\x02foo']),
166 ('foo', [pygrack.CAPABILITY_SIDE_BAND], [b'0008\x02foo']),
167 ('f'*1000, [pygrack.CAPABILITY_SIDE_BAND_64K], [b'03ed\x02' + b'f' * 1000]),
167 ('f'*1000, [pygrack.CAPABILITY_SIDE_BAND_64K], [b'03ed\x02' + b'f' * 1000]),
168 ('f'*1000, [pygrack.CAPABILITY_SIDE_BAND], [b'03e8\x02' + b'f' * 995, b'000a\x02fffff']),
168 ('f'*1000, [pygrack.CAPABILITY_SIDE_BAND], [b'03e8\x02' + b'f' * 995, b'000a\x02fffff']),
169 ('f'*65520, [pygrack.CAPABILITY_SIDE_BAND_64K], [b'fff0\x02' + b'f' * 65515, b'000a\x02fffff']),
169 ('f'*65520, [pygrack.CAPABILITY_SIDE_BAND_64K], [b'fff0\x02' + b'f' * 65515, b'000a\x02fffff']),
170 ('f'*65520, [pygrack.CAPABILITY_SIDE_BAND], [b'03e8\x02' + b'f' * 995] * 65 + [b'0352\x02' + b'f' * 845]),
170 ('f'*65520, [pygrack.CAPABILITY_SIDE_BAND], [b'03e8\x02' + b'f' * 995] * 65 + [b'0352\x02' + b'f' * 845]),
171 ], ids=[
171 ], ids=[
172 'foo-empty',
172 'foo-empty',
173 'empty-64k', 'empty',
173 'empty-64k', 'empty',
174 'foo-64k', 'foo',
174 'foo-64k', 'foo',
175 'f-1000-64k', 'f-1000',
175 'f-1000-64k', 'f-1000',
176 'f-65520-64k', 'f-65520'])
176 'f-65520-64k', 'f-65520'])
177 def test_get_messages(pygrack_instance, data, capabilities, expected):
177 def test_get_messages(pygrack_instance, data, capabilities, expected):
178 messages = pygrack_instance._get_messages(data, capabilities)
178 messages = pygrack_instance._get_messages(data, capabilities)
179
179
180 assert messages == expected
180 assert messages == expected
181
181
182
182
183 @pytest.mark.parametrize('response,capabilities,pre_pull_messages,post_pull_messages', [
183 @pytest.mark.parametrize('response,capabilities,pre_pull_messages,post_pull_messages', [
184 # Unexpected response
184 # Unexpected response
185 ([b'unexpected_response[no_initial_header]'], [pygrack.CAPABILITY_SIDE_BAND_64K], 'foo', 'bar'),
185 ([b'unexpected_response[no_initial_header]'], [pygrack.CAPABILITY_SIDE_BAND_64K], 'foo', 'bar'),
186 # No sideband
186 # No sideband
187 ([b'no-sideband'], [], 'foo', 'bar'),
187 ([b'no-sideband'], [], 'foo', 'bar'),
188 # No messages
188 # No messages
189 ([b'no-messages'], [pygrack.CAPABILITY_SIDE_BAND_64K], '', ''),
189 ([b'no-messages'], [pygrack.CAPABILITY_SIDE_BAND_64K], '', ''),
190 ])
190 ])
191 def test_inject_messages_to_response_nothing_to_do(
191 def test_inject_messages_to_response_nothing_to_do(
192 pygrack_instance, response, capabilities, pre_pull_messages, post_pull_messages):
192 pygrack_instance, response, capabilities, pre_pull_messages, post_pull_messages):
193
193
194 new_response = pygrack_instance._build_post_pull_response(
194 new_response = pygrack_instance._build_post_pull_response(
195 more_itertools.always_iterable(response), capabilities, pre_pull_messages, post_pull_messages)
195 more_itertools.always_iterable(response), capabilities, pre_pull_messages, post_pull_messages)
196
196
197 assert list(new_response) == response
197 assert list(new_response) == response
198
198
199
199
200 @pytest.mark.parametrize('capabilities', [
200 @pytest.mark.parametrize('capabilities', [
201 [pygrack.CAPABILITY_SIDE_BAND],
201 [pygrack.CAPABILITY_SIDE_BAND],
202 [pygrack.CAPABILITY_SIDE_BAND_64K],
202 [pygrack.CAPABILITY_SIDE_BAND_64K],
203 ])
203 ])
204 def test_inject_messages_to_response_single_element(pygrack_instance, capabilities):
204 def test_inject_messages_to_response_single_element(pygrack_instance, capabilities):
205 response = [b'0008NAK\n0009subp\n0000']
205 response = [b'0008NAK\n0009subp\n0000']
206 new_response = pygrack_instance._build_post_pull_response(
206 new_response = pygrack_instance._build_post_pull_response(
207 more_itertools.always_iterable(response), capabilities, 'foo', 'bar')
207 more_itertools.always_iterable(response), capabilities, 'foo', 'bar')
208
208
209 expected_response = b''.join([
209 expected_response = b''.join([
210 b'0008NAK\n',
210 b'0008NAK\n',
211 b'0008\x02foo',
211 b'0008\x02foo',
212 b'0009subp\n',
212 b'0009subp\n',
213 b'0008\x02bar',
213 b'0008\x02bar',
214 b'0000'])
214 b'0000'])
215
215
216 assert b''.join(new_response) == expected_response
216 assert b''.join(new_response) == expected_response
217
217
218
218
219 @pytest.mark.parametrize('capabilities', [
219 @pytest.mark.parametrize('capabilities', [
220 [pygrack.CAPABILITY_SIDE_BAND],
220 [pygrack.CAPABILITY_SIDE_BAND],
221 [pygrack.CAPABILITY_SIDE_BAND_64K],
221 [pygrack.CAPABILITY_SIDE_BAND_64K],
222 ])
222 ])
223 def test_inject_messages_to_response_multi_element(pygrack_instance, capabilities):
223 def test_inject_messages_to_response_multi_element(pygrack_instance, capabilities):
224 response = more_itertools.always_iterable([
224 response = more_itertools.always_iterable([
225 b'0008NAK\n000asubp1\n', b'000asubp2\n', b'000asubp3\n', b'000asubp4\n0000'
225 b'0008NAK\n000asubp1\n', b'000asubp2\n', b'000asubp3\n', b'000asubp4\n0000'
226 ])
226 ])
227 new_response = pygrack_instance._build_post_pull_response(response, capabilities, 'foo', 'bar')
227 new_response = pygrack_instance._build_post_pull_response(response, capabilities, 'foo', 'bar')
228
228
229 expected_response = b''.join([
229 expected_response = b''.join([
230 b'0008NAK\n',
230 b'0008NAK\n',
231 b'0008\x02foo',
231 b'0008\x02foo',
232 b'000asubp1\n', b'000asubp2\n', b'000asubp3\n', b'000asubp4\n',
232 b'000asubp1\n', b'000asubp2\n', b'000asubp3\n', b'000asubp4\n',
233 b'0008\x02bar',
233 b'0008\x02bar',
234 b'0000'
234 b'0000'
235 ])
235 ])
236
236
237 assert b''.join(new_response) == expected_response
237 assert b''.join(new_response) == expected_response
238
238
239
239
240 def test_build_failed_pre_pull_response_no_sideband(pygrack_instance):
240 def test_build_failed_pre_pull_response_no_sideband(pygrack_instance):
241 response = pygrack_instance._build_failed_pre_pull_response([], 'foo')
241 response = pygrack_instance._build_failed_pre_pull_response([], 'foo')
242
242
243 assert response == [pygrack.GitRepository.EMPTY_PACK]
243 assert response == [pygrack.GitRepository.EMPTY_PACK]
244
244
245
245
246 @pytest.mark.parametrize('capabilities', [
246 @pytest.mark.parametrize('capabilities', [
247 [pygrack.CAPABILITY_SIDE_BAND],
247 [pygrack.CAPABILITY_SIDE_BAND],
248 [pygrack.CAPABILITY_SIDE_BAND_64K],
248 [pygrack.CAPABILITY_SIDE_BAND_64K],
249 [pygrack.CAPABILITY_SIDE_BAND_64K, b'no-progress'],
249 [pygrack.CAPABILITY_SIDE_BAND_64K, b'no-progress'],
250 ])
250 ])
251 def test_build_failed_pre_pull_response(pygrack_instance, capabilities):
251 def test_build_failed_pre_pull_response(pygrack_instance, capabilities):
252 response = pygrack_instance._build_failed_pre_pull_response(capabilities, 'foo')
252 response = pygrack_instance._build_failed_pre_pull_response(capabilities, 'foo')
253
253
254 expected_response = [
254 expected_response = [
255 b'0008NAK\n', b'0008\x02foo', b'0024\x02Pre pull hook failed: aborting\n',
255 b'0008NAK\n', b'0008\x02foo', b'0024\x02Pre pull hook failed: aborting\n',
256 b'%04x\x01%s' % (len(pygrack.GitRepository.EMPTY_PACK) + 5, pygrack.GitRepository.EMPTY_PACK),
256 b'%04x\x01%s' % (len(pygrack.GitRepository.EMPTY_PACK) + 5, pygrack.GitRepository.EMPTY_PACK),
257 pygrack.GitRepository.FLUSH_PACKET,
257 pygrack.GitRepository.FLUSH_PACKET,
258 ]
258 ]
259
259
260 assert response == expected_response
260 assert response == expected_response
261
261
262
262
263 def test_inject_messages_to_response_generator(pygrack_instance):
263 def test_inject_messages_to_response_generator(pygrack_instance):
264
264
265 def response_generator():
265 def response_generator():
266 response = [
266 response = [
267 # protocol start
267 # protocol start
268 b'0008NAK\n',
268 b'0008NAK\n',
269 ]
269 ]
270 response += [ascii_bytes(f'000asubp{x}\n') for x in range(1000)]
270 response += [ascii_bytes(f'000asubp{x}\n') for x in range(1000)]
271 response += [
271 response += [
272 # protocol end
272 # protocol end
273 pygrack.GitRepository.FLUSH_PACKET
273 pygrack.GitRepository.FLUSH_PACKET
274 ]
274 ]
275 for elem in response:
275 for elem in response:
276 yield elem
276 yield elem
277
277
278 new_response = pygrack_instance._build_post_pull_response(
278 new_response = pygrack_instance._build_post_pull_response(
279 response_generator(), [pygrack.CAPABILITY_SIDE_BAND_64K, b'no-progress'], 'PRE_PULL_MSG\n', 'POST_PULL_MSG\n')
279 response_generator(), [pygrack.CAPABILITY_SIDE_BAND_64K, b'no-progress'], 'PRE_PULL_MSG\n', 'POST_PULL_MSG\n')
280
280
281 assert iter(new_response)
281 assert iter(new_response)
282
282
283 expected_response = b''.join([
283 expected_response = b''.join([
284 # start
284 # start
285 b'0008NAK\n0012\x02PRE_PULL_MSG\n',
285 b'0008NAK\n0012\x02PRE_PULL_MSG\n',
286 ] + [
286 ] + [
287 # ... rest
287 # ... rest
288 ascii_bytes(f'000asubp{x}\n') for x in range(1000)
288 ascii_bytes(f'000asubp{x}\n') for x in range(1000)
289 ] + [
289 ] + [
290 # final message,
290 # final message,
291 b'0013\x02POST_PULL_MSG\n0000',
291 b'0013\x02POST_PULL_MSG\n0000',
292
292
293 ])
293 ])
294
294
295 assert b''.join(new_response) == expected_response
295 assert b''.join(new_response) == expected_response
@@ -1,87 +1,87 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19
19
20 import mercurial.hg
20 import mercurial.hg
21 import mercurial.ui
21 import mercurial.ui
22 import mercurial.error
22 import mercurial.error
23 import mock
23 import mock
24 import pytest
24 import pytest
25 import webtest
25 import webtest
26
26
27 from vcsserver import scm_app
27 from vcsserver import scm_app
28 from vcsserver.str_utils import ascii_bytes
28 from vcsserver.lib.str_utils import ascii_bytes
29
29
30
30
31 def test_hg_does_not_accept_invalid_cmd(tmpdir):
31 def test_hg_does_not_accept_invalid_cmd(tmpdir):
32 repo = mercurial.hg.repository(mercurial.ui.ui(), ascii_bytes(str(tmpdir)), create=True)
32 repo = mercurial.hg.repository(mercurial.ui.ui(), ascii_bytes(str(tmpdir)), create=True)
33 app = webtest.TestApp(scm_app.HgWeb(repo))
33 app = webtest.TestApp(scm_app.HgWeb(repo))
34
34
35 response = app.get('/repo?cmd=invalidcmd', expect_errors=True)
35 response = app.get('/repo?cmd=invalidcmd', expect_errors=True)
36
36
37 assert response.status_int == 400
37 assert response.status_int == 400
38
38
39
39
40 def test_create_hg_wsgi_app_requirement_error(tmpdir):
40 def test_create_hg_wsgi_app_requirement_error(tmpdir):
41 repo = mercurial.hg.repository(mercurial.ui.ui(), ascii_bytes(str(tmpdir)), create=True)
41 repo = mercurial.hg.repository(mercurial.ui.ui(), ascii_bytes(str(tmpdir)), create=True)
42 config = (
42 config = (
43 ('paths', 'default', ''),
43 ('paths', 'default', ''),
44 )
44 )
45 with mock.patch('vcsserver.scm_app.HgWeb') as hgweb_mock:
45 with mock.patch('vcsserver.scm_app.HgWeb') as hgweb_mock:
46 hgweb_mock.side_effect = mercurial.error.RequirementError()
46 hgweb_mock.side_effect = mercurial.error.RequirementError()
47 with pytest.raises(Exception):
47 with pytest.raises(Exception):
48 scm_app.create_hg_wsgi_app(str(tmpdir), repo, config)
48 scm_app.create_hg_wsgi_app(str(tmpdir), repo, config)
49
49
50
50
51 def test_git_returns_not_found(tmpdir):
51 def test_git_returns_not_found(tmpdir):
52 app = webtest.TestApp(
52 app = webtest.TestApp(
53 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
53 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
54
54
55 response = app.get('/repo_name/inforefs?service=git-upload-pack',
55 response = app.get('/repo_name/inforefs?service=git-upload-pack',
56 expect_errors=True)
56 expect_errors=True)
57
57
58 assert response.status_int == 404
58 assert response.status_int == 404
59
59
60
60
61 def test_git(tmpdir):
61 def test_git(tmpdir):
62 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
62 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
63 tmpdir.mkdir(dir_name)
63 tmpdir.mkdir(dir_name)
64
64
65 app = webtest.TestApp(
65 app = webtest.TestApp(
66 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
66 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
67
67
68 # We set service to git-upload-packs to trigger a 403
68 # We set service to git-upload-packs to trigger a 403
69 response = app.get('/repo_name/inforefs?service=git-upload-packs',
69 response = app.get('/repo_name/inforefs?service=git-upload-packs',
70 expect_errors=True)
70 expect_errors=True)
71
71
72 assert response.status_int == 403
72 assert response.status_int == 403
73
73
74
74
75 def test_git_fallbacks_to_git_folder(tmpdir):
75 def test_git_fallbacks_to_git_folder(tmpdir):
76 tmpdir.mkdir('.git')
76 tmpdir.mkdir('.git')
77 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
77 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
78 tmpdir.mkdir(os.path.join('.git', dir_name))
78 tmpdir.mkdir(os.path.join('.git', dir_name))
79
79
80 app = webtest.TestApp(
80 app = webtest.TestApp(
81 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
81 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
82
82
83 # We set service to git-upload-packs to trigger a 403
83 # We set service to git-upload-packs to trigger a 403
84 response = app.get('/repo_name/inforefs?service=git-upload-packs',
84 response = app.get('/repo_name/inforefs?service=git-upload-packs',
85 expect_errors=True)
85 expect_errors=True)
86
86
87 assert response.status_int == 403
87 assert response.status_int == 403
@@ -1,155 +1,155 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import sys
20 import sys
21
21
22 import pytest
22 import pytest
23
23
24 from vcsserver import subprocessio
24 from vcsserver import subprocessio
25 from vcsserver.str_utils import ascii_bytes
25 from vcsserver.lib.str_utils import ascii_bytes
26
26
27
27
28 class FileLikeObj: # pragma: no cover
28 class FileLikeObj: # pragma: no cover
29
29
30 def __init__(self, data: bytes, size):
30 def __init__(self, data: bytes, size):
31 chunks = size // len(data)
31 chunks = size // len(data)
32
32
33 self.stream = self._get_stream(data, chunks)
33 self.stream = self._get_stream(data, chunks)
34
34
35 def _get_stream(self, data, chunks):
35 def _get_stream(self, data, chunks):
36 for x in range(chunks):
36 for x in range(chunks):
37 yield data
37 yield data
38
38
39 def read(self, n):
39 def read(self, n):
40
40
41 buffer_stream = b''
41 buffer_stream = b''
42 for chunk in self.stream:
42 for chunk in self.stream:
43 buffer_stream += chunk
43 buffer_stream += chunk
44 if len(buffer_stream) >= n:
44 if len(buffer_stream) >= n:
45 break
45 break
46
46
47 # self.stream = self.bytes[n:]
47 # self.stream = self.bytes[n:]
48 return buffer_stream
48 return buffer_stream
49
49
50
50
51 @pytest.fixture(scope='module')
51 @pytest.fixture(scope='module')
52 def environ():
52 def environ():
53 """Delete coverage variables, as they make the tests fail."""
53 """Delete coverage variables, as they make the tests fail."""
54 env = dict(os.environ)
54 env = dict(os.environ)
55 for key in list(env.keys()):
55 for key in list(env.keys()):
56 if key.startswith('COV_CORE_'):
56 if key.startswith('COV_CORE_'):
57 del env[key]
57 del env[key]
58
58
59 return env
59 return env
60
60
61
61
62 def _get_python_args(script):
62 def _get_python_args(script):
63 return [sys.executable, '-c', 'import sys; import time; import shutil; ' + script]
63 return [sys.executable, '-c', 'import sys; import time; import shutil; ' + script]
64
64
65
65
66 def test_raise_exception_on_non_zero_return_code(environ):
66 def test_raise_exception_on_non_zero_return_code(environ):
67 call_args = _get_python_args('raise ValueError("fail")')
67 call_args = _get_python_args('raise ValueError("fail")')
68 with pytest.raises(OSError):
68 with pytest.raises(OSError):
69 b''.join(subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ))
69 b''.join(subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ))
70
70
71
71
72 def test_does_not_fail_on_non_zero_return_code(environ):
72 def test_does_not_fail_on_non_zero_return_code(environ):
73 call_args = _get_python_args('sys.stdout.write("hello"); sys.exit(1)')
73 call_args = _get_python_args('sys.stdout.write("hello"); sys.exit(1)')
74 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_return_code=False, env=environ)
74 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_return_code=False, env=environ)
75 output = b''.join(proc)
75 output = b''.join(proc)
76
76
77 assert output == b'hello'
77 assert output == b'hello'
78
78
79
79
80 def test_raise_exception_on_stderr(environ):
80 def test_raise_exception_on_stderr(environ):
81 call_args = _get_python_args('sys.stderr.write("WRITE_TO_STDERR"); time.sleep(1);')
81 call_args = _get_python_args('sys.stderr.write("WRITE_TO_STDERR"); time.sleep(1);')
82
82
83 with pytest.raises(OSError) as excinfo:
83 with pytest.raises(OSError) as excinfo:
84 b''.join(subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ))
84 b''.join(subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ))
85
85
86 assert 'exited due to an error:\nWRITE_TO_STDERR' in str(excinfo.value)
86 assert 'exited due to an error:\nWRITE_TO_STDERR' in str(excinfo.value)
87
87
88
88
89 def test_does_not_fail_on_stderr(environ):
89 def test_does_not_fail_on_stderr(environ):
90 call_args = _get_python_args('sys.stderr.write("WRITE_TO_STDERR"); sys.stderr.flush; time.sleep(2);')
90 call_args = _get_python_args('sys.stderr.write("WRITE_TO_STDERR"); sys.stderr.flush; time.sleep(2);')
91 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_stderr=False, env=environ)
91 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_stderr=False, env=environ)
92 output = b''.join(proc)
92 output = b''.join(proc)
93
93
94 assert output == b''
94 assert output == b''
95
95
96
96
97 @pytest.mark.parametrize('size', [
97 @pytest.mark.parametrize('size', [
98 1,
98 1,
99 10 ** 5
99 10 ** 5
100 ])
100 ])
101 def test_output_with_no_input(size, environ):
101 def test_output_with_no_input(size, environ):
102 call_args = _get_python_args(f'sys.stdout.write("X" * {size});')
102 call_args = _get_python_args(f'sys.stdout.write("X" * {size});')
103 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ)
103 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ)
104 output = b''.join(proc)
104 output = b''.join(proc)
105
105
106 assert output == ascii_bytes("X" * size)
106 assert output == ascii_bytes("X" * size)
107
107
108
108
109 @pytest.mark.parametrize('size', [
109 @pytest.mark.parametrize('size', [
110 1,
110 1,
111 10 ** 5
111 10 ** 5
112 ])
112 ])
113 def test_output_with_no_input_does_not_fail(size, environ):
113 def test_output_with_no_input_does_not_fail(size, environ):
114
114
115 call_args = _get_python_args(f'sys.stdout.write("X" * {size}); sys.exit(1)')
115 call_args = _get_python_args(f'sys.stdout.write("X" * {size}); sys.exit(1)')
116 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_return_code=False, env=environ)
116 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_return_code=False, env=environ)
117 output = b''.join(proc)
117 output = b''.join(proc)
118
118
119 assert output == ascii_bytes("X" * size)
119 assert output == ascii_bytes("X" * size)
120
120
121
121
122 @pytest.mark.parametrize('size', [
122 @pytest.mark.parametrize('size', [
123 1,
123 1,
124 10 ** 5
124 10 ** 5
125 ])
125 ])
126 def test_output_with_input(size, environ):
126 def test_output_with_input(size, environ):
127 data_len = size
127 data_len = size
128 inputstream = FileLikeObj(b'X', size)
128 inputstream = FileLikeObj(b'X', size)
129
129
130 # This acts like the cat command.
130 # This acts like the cat command.
131 call_args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
131 call_args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
132 # note: in this tests we explicitly don't assign chunker to a variable and let it stream directly
132 # note: in this tests we explicitly don't assign chunker to a variable and let it stream directly
133 output = b''.join(
133 output = b''.join(
134 subprocessio.SubprocessIOChunker(call_args, shell=False, input_stream=inputstream, env=environ)
134 subprocessio.SubprocessIOChunker(call_args, shell=False, input_stream=inputstream, env=environ)
135 )
135 )
136
136
137 assert len(output) == data_len
137 assert len(output) == data_len
138
138
139
139
140 @pytest.mark.parametrize('size', [
140 @pytest.mark.parametrize('size', [
141 1,
141 1,
142 10 ** 5
142 10 ** 5
143 ])
143 ])
144 def test_output_with_input_skipping_iterator(size, environ):
144 def test_output_with_input_skipping_iterator(size, environ):
145 data_len = size
145 data_len = size
146 inputstream = FileLikeObj(b'X', size)
146 inputstream = FileLikeObj(b'X', size)
147
147
148 # This acts like the cat command.
148 # This acts like the cat command.
149 call_args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
149 call_args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
150
150
151 # Note: assigning the chunker makes sure that it is not deleted too early
151 # Note: assigning the chunker makes sure that it is not deleted too early
152 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, input_stream=inputstream, env=environ)
152 proc = subprocessio.SubprocessIOChunker(call_args, shell=False, input_stream=inputstream, env=environ)
153 output = b''.join(proc.stdout)
153 output = b''.join(proc.stdout)
154
154
155 assert len(output) == data_len
155 assert len(output) == data_len
@@ -1,103 +1,103 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import mock
19 import mock
20 import pytest
20 import pytest
21 import sys
21 import sys
22
22
23 from vcsserver.str_utils import ascii_bytes
23 from vcsserver.lib.str_utils import ascii_bytes
24
24
25
25
26 class MockPopen:
26 class MockPopen:
27 def __init__(self, stderr):
27 def __init__(self, stderr):
28 self.stdout = io.BytesIO(b'')
28 self.stdout = io.BytesIO(b'')
29 self.stderr = io.BytesIO(stderr)
29 self.stderr = io.BytesIO(stderr)
30 self.returncode = 1
30 self.returncode = 1
31
31
32 def wait(self):
32 def wait(self):
33 pass
33 pass
34
34
35
35
36 INVALID_CERTIFICATE_STDERR = '\n'.join([
36 INVALID_CERTIFICATE_STDERR = '\n'.join([
37 'svnrdump: E230001: Unable to connect to a repository at URL url',
37 'svnrdump: E230001: Unable to connect to a repository at URL url',
38 'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
38 'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
39 ])
39 ])
40
40
41
41
42 @pytest.mark.parametrize('stderr,expected_reason', [
42 @pytest.mark.parametrize('stderr,expected_reason', [
43 (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
43 (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
44 ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
44 ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
45 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
45 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
46 @pytest.mark.xfail(sys.platform == "cygwin",
46 @pytest.mark.xfail(sys.platform == "cygwin",
47 reason="SVN not packaged for Cygwin")
47 reason="SVN not packaged for Cygwin")
48 def test_import_remote_repository_certificate_error(stderr, expected_reason):
48 def test_import_remote_repository_certificate_error(stderr, expected_reason):
49 from vcsserver.remote import svn_remote
49 from vcsserver.remote import svn_remote
50 factory = mock.Mock()
50 factory = mock.Mock()
51 factory.repo = mock.Mock(return_value=mock.Mock())
51 factory.repo = mock.Mock(return_value=mock.Mock())
52
52
53 remote = svn_remote.SvnRemote(factory)
53 remote = svn_remote.SvnRemote(factory)
54 remote.is_path_valid_repository = lambda wire, path: True
54 remote.is_path_valid_repository = lambda wire, path: True
55
55
56 with mock.patch('subprocess.Popen',
56 with mock.patch('subprocess.Popen',
57 return_value=MockPopen(ascii_bytes(stderr))):
57 return_value=MockPopen(ascii_bytes(stderr))):
58 with pytest.raises(Exception) as excinfo:
58 with pytest.raises(Exception) as excinfo:
59 remote.import_remote_repository({'path': 'path'}, 'url')
59 remote.import_remote_repository({'path': 'path'}, 'url')
60
60
61 expected_error_args = 'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason)
61 expected_error_args = 'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason)
62
62
63 assert excinfo.value.args[0] == expected_error_args
63 assert excinfo.value.args[0] == expected_error_args
64
64
65
65
66 def test_svn_libraries_can_be_imported():
66 def test_svn_libraries_can_be_imported():
67 import svn.client # noqa
67 import svn.client # noqa
68 assert svn.client is not None
68 assert svn.client is not None
69
69
70
70
71 @pytest.mark.parametrize('example_url, parts', [
71 @pytest.mark.parametrize('example_url, parts', [
72 ('http://server.com', ('', '', 'http://server.com')),
72 ('http://server.com', ('', '', 'http://server.com')),
73 ('http://user@server.com', ('user', '', 'http://user@server.com')),
73 ('http://user@server.com', ('user', '', 'http://user@server.com')),
74 ('http://user:pass@server.com', ('user', 'pass', 'http://user:pass@server.com')),
74 ('http://user:pass@server.com', ('user', 'pass', 'http://user:pass@server.com')),
75 ('<script>', ('', '', '<script>')),
75 ('<script>', ('', '', '<script>')),
76 ('http://', ('', '', 'http://')),
76 ('http://', ('', '', 'http://')),
77 ])
77 ])
78 def test_username_password_extraction_from_url(example_url, parts):
78 def test_username_password_extraction_from_url(example_url, parts):
79 from vcsserver.remote import svn_remote
79 from vcsserver.remote import svn_remote
80
80
81 factory = mock.Mock()
81 factory = mock.Mock()
82 factory.repo = mock.Mock(return_value=mock.Mock())
82 factory.repo = mock.Mock(return_value=mock.Mock())
83
83
84 remote = svn_remote.SvnRemote(factory)
84 remote = svn_remote.SvnRemote(factory)
85 remote.is_path_valid_repository = lambda wire, path: True
85 remote.is_path_valid_repository = lambda wire, path: True
86
86
87 assert remote.get_url_and_credentials(example_url) == parts
87 assert remote.get_url_and_credentials(example_url) == parts
88
88
89
89
90 @pytest.mark.parametrize('call_url', [
90 @pytest.mark.parametrize('call_url', [
91 b'https://svn.code.sf.net/p/svnbook/source/trunk/',
91 b'https://svn.code.sf.net/p/svnbook/source/trunk/',
92 b'https://marcink@svn.code.sf.net/p/svnbook/source/trunk/',
92 b'https://marcink@svn.code.sf.net/p/svnbook/source/trunk/',
93 b'https://marcink:qweqwe@svn.code.sf.net/p/svnbook/source/trunk/',
93 b'https://marcink:qweqwe@svn.code.sf.net/p/svnbook/source/trunk/',
94 ])
94 ])
95 def test_check_url(call_url):
95 def test_check_url(call_url):
96 from vcsserver.remote import svn_remote
96 from vcsserver.remote import svn_remote
97 factory = mock.Mock()
97 factory = mock.Mock()
98 factory.repo = mock.Mock(return_value=mock.Mock())
98 factory.repo = mock.Mock(return_value=mock.Mock())
99
99
100 remote = svn_remote.SvnRemote(factory)
100 remote = svn_remote.SvnRemote(factory)
101 remote.is_path_valid_repository = lambda wire, path: True
101 remote.is_path_valid_repository = lambda wire, path: True
102 assert remote.check_url(call_url, {'dummy': 'config'})
102 assert remote.check_url(call_url, {'dummy': 'config'})
103
103
@@ -1,69 +1,69 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import pytest
18 import pytest
19 from vcsserver.str_utils import ascii_bytes, ascii_str, convert_to_str
19 from vcsserver.lib.str_utils import ascii_bytes, ascii_str, convert_to_str
20
20
21
21
22 @pytest.mark.parametrize('given, expected', [
22 @pytest.mark.parametrize('given, expected', [
23 ('a', b'a'),
23 ('a', b'a'),
24 ('a', b'a'),
24 ('a', b'a'),
25 ])
25 ])
26 def test_ascii_bytes(given, expected):
26 def test_ascii_bytes(given, expected):
27 assert ascii_bytes(given) == expected
27 assert ascii_bytes(given) == expected
28
28
29
29
30 @pytest.mark.parametrize('given', [
30 @pytest.mark.parametrize('given', [
31 'Γ₯',
31 'Γ₯',
32 'Γ₯'.encode('utf8')
32 'Γ₯'.encode('utf8')
33 ])
33 ])
34 def test_ascii_bytes_raises(given):
34 def test_ascii_bytes_raises(given):
35 with pytest.raises(ValueError):
35 with pytest.raises(ValueError):
36 ascii_bytes(given)
36 ascii_bytes(given)
37
37
38
38
39 @pytest.mark.parametrize('given, expected', [
39 @pytest.mark.parametrize('given, expected', [
40 (b'a', 'a'),
40 (b'a', 'a'),
41 ])
41 ])
42 def test_ascii_str(given, expected):
42 def test_ascii_str(given, expected):
43 assert ascii_str(given) == expected
43 assert ascii_str(given) == expected
44
44
45
45
46 @pytest.mark.parametrize('given', [
46 @pytest.mark.parametrize('given', [
47 'a',
47 'a',
48 'Γ₯'.encode('utf8'),
48 'Γ₯'.encode('utf8'),
49 'Γ₯'
49 'Γ₯'
50 ])
50 ])
51 def test_ascii_str_raises(given):
51 def test_ascii_str_raises(given):
52 with pytest.raises(ValueError):
52 with pytest.raises(ValueError):
53 ascii_str(given)
53 ascii_str(given)
54
54
55
55
56 @pytest.mark.parametrize('given, expected', [
56 @pytest.mark.parametrize('given, expected', [
57 ('a', 'a'),
57 ('a', 'a'),
58 (b'a', 'a'),
58 (b'a', 'a'),
59 # tuple
59 # tuple
60 (('a', b'b', b'c'), ('a', 'b', 'c')),
60 (('a', b'b', b'c'), ('a', 'b', 'c')),
61 # nested tuple
61 # nested tuple
62 (('a', b'b', (b'd', b'e')), ('a', 'b', ('d', 'e'))),
62 (('a', b'b', (b'd', b'e')), ('a', 'b', ('d', 'e'))),
63 # list
63 # list
64 (['a', b'b', b'c'], ['a', 'b', 'c']),
64 (['a', b'b', b'c'], ['a', 'b', 'c']),
65 # mixed
65 # mixed
66 (['a', b'b', b'c', (b'b1', b'b2')], ['a', 'b', 'c', ('b1', 'b2')])
66 (['a', b'b', b'c', (b'b1', b'b2')], ['a', 'b', 'c', ('b1', 'b2')])
67 ])
67 ])
68 def test_convert_to_str(given, expected):
68 def test_convert_to_str(given, expected):
69 assert convert_to_str(given) == expected
69 assert convert_to_str(given) == expected
@@ -1,98 +1,98 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import wsgiref.simple_server
18 import wsgiref.simple_server
19 import wsgiref.validate
19 import wsgiref.validate
20
20
21 from vcsserver import wsgi_app_caller
21 from vcsserver import wsgi_app_caller
22 from vcsserver.str_utils import ascii_bytes, safe_str
22 from vcsserver.lib.str_utils import ascii_bytes, safe_str
23
23
24
24
25 @wsgiref.validate.validator
25 @wsgiref.validate.validator
26 def demo_app(environ, start_response):
26 def demo_app(environ, start_response):
27 """WSGI app used for testing."""
27 """WSGI app used for testing."""
28
28
29 input_data = safe_str(environ['wsgi.input'].read(1024))
29 input_data = safe_str(environ['wsgi.input'].read(1024))
30
30
31 data = [
31 data = [
32 'Hello World!\n',
32 'Hello World!\n',
33 f'input_data={input_data}\n',
33 f'input_data={input_data}\n',
34 ]
34 ]
35 for key, value in sorted(environ.items()):
35 for key, value in sorted(environ.items()):
36 data.append(f'{key}={value}\n')
36 data.append(f'{key}={value}\n')
37
37
38 write = start_response("200 OK", [('Content-Type', 'text/plain')])
38 write = start_response("200 OK", [('Content-Type', 'text/plain')])
39 write(b'Old school write method\n')
39 write(b'Old school write method\n')
40 write(b'***********************\n')
40 write(b'***********************\n')
41 return list(map(ascii_bytes, data))
41 return list(map(ascii_bytes, data))
42
42
43
43
44 BASE_ENVIRON = {
44 BASE_ENVIRON = {
45 'REQUEST_METHOD': 'GET',
45 'REQUEST_METHOD': 'GET',
46 'SERVER_NAME': 'localhost',
46 'SERVER_NAME': 'localhost',
47 'SERVER_PORT': '80',
47 'SERVER_PORT': '80',
48 'SCRIPT_NAME': '',
48 'SCRIPT_NAME': '',
49 'PATH_INFO': '/',
49 'PATH_INFO': '/',
50 'QUERY_STRING': '',
50 'QUERY_STRING': '',
51 'foo.var': 'bla',
51 'foo.var': 'bla',
52 }
52 }
53
53
54
54
55 def test_complete_environ():
55 def test_complete_environ():
56 environ = dict(BASE_ENVIRON)
56 environ = dict(BASE_ENVIRON)
57 data = b"data"
57 data = b"data"
58 wsgi_app_caller._complete_environ(environ, data)
58 wsgi_app_caller._complete_environ(environ, data)
59 wsgiref.validate.check_environ(environ)
59 wsgiref.validate.check_environ(environ)
60
60
61 assert data == environ['wsgi.input'].read(1024)
61 assert data == environ['wsgi.input'].read(1024)
62
62
63
63
64 def test_start_response():
64 def test_start_response():
65 start_response = wsgi_app_caller._StartResponse()
65 start_response = wsgi_app_caller._StartResponse()
66 status = '200 OK'
66 status = '200 OK'
67 headers = [('Content-Type', 'text/plain')]
67 headers = [('Content-Type', 'text/plain')]
68 start_response(status, headers)
68 start_response(status, headers)
69
69
70 assert status == start_response.status
70 assert status == start_response.status
71 assert headers == start_response.headers
71 assert headers == start_response.headers
72
72
73
73
74 def test_start_response_with_error():
74 def test_start_response_with_error():
75 start_response = wsgi_app_caller._StartResponse()
75 start_response = wsgi_app_caller._StartResponse()
76 status = '500 Internal Server Error'
76 status = '500 Internal Server Error'
77 headers = [('Content-Type', 'text/plain')]
77 headers = [('Content-Type', 'text/plain')]
78 start_response(status, headers, (None, None, None))
78 start_response(status, headers, (None, None, None))
79
79
80 assert status == start_response.status
80 assert status == start_response.status
81 assert headers == start_response.headers
81 assert headers == start_response.headers
82
82
83
83
84 def test_wsgi_app_caller():
84 def test_wsgi_app_caller():
85 environ = dict(BASE_ENVIRON)
85 environ = dict(BASE_ENVIRON)
86 input_data = 'some text'
86 input_data = 'some text'
87
87
88 caller = wsgi_app_caller.WSGIAppCaller(demo_app)
88 caller = wsgi_app_caller.WSGIAppCaller(demo_app)
89 responses, status, headers = caller.handle(environ, input_data)
89 responses, status, headers = caller.handle(environ, input_data)
90 response = b''.join(responses)
90 response = b''.join(responses)
91
91
92 assert status == '200 OK'
92 assert status == '200 OK'
93 assert headers == [('Content-Type', 'text/plain')]
93 assert headers == [('Content-Type', 'text/plain')]
94 assert response.startswith(b'Old school write method\n***********************\n')
94 assert response.startswith(b'Old school write method\n***********************\n')
95 assert b'Hello World!\n' in response
95 assert b'Hello World!\n' in response
96 assert b'foo.var=bla\n' in response
96 assert b'foo.var=bla\n' in response
97
97
98 assert ascii_bytes(f'input_data={input_data}\n') in response
98 assert ascii_bytes(f'input_data={input_data}\n') in response
@@ -1,123 +1,123 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import base64
17 import base64
18 import logging
18 import logging
19 import time
19 import time
20
20
21 import msgpack
21 import msgpack
22
22
23 import vcsserver
23 import vcsserver
24 from vcsserver.str_utils import safe_str
24 from vcsserver.lib.str_utils import safe_str
25
25
26 log = logging.getLogger(__name__)
26 log = logging.getLogger(__name__)
27
27
28
28
29 def get_access_path(environ):
29 def get_access_path(environ):
30 path = environ.get('PATH_INFO')
30 path = environ.get('PATH_INFO')
31 return path
31 return path
32
32
33
33
34 def get_user_agent(environ):
34 def get_user_agent(environ):
35 return environ.get('HTTP_USER_AGENT')
35 return environ.get('HTTP_USER_AGENT')
36
36
37
37
38 def get_call_context(request) -> dict:
38 def get_call_context(request) -> dict:
39 cc = {}
39 cc = {}
40 registry = request.registry
40 registry = request.registry
41 if hasattr(registry, 'vcs_call_context'):
41 if hasattr(registry, 'vcs_call_context'):
42 cc.update({
42 cc.update({
43 'X-RC-Method': registry.vcs_call_context.get('method'),
43 'X-RC-Method': registry.vcs_call_context.get('method'),
44 'X-RC-Repo-Name': registry.vcs_call_context.get('repo_name')
44 'X-RC-Repo-Name': registry.vcs_call_context.get('repo_name')
45 })
45 })
46
46
47 return cc
47 return cc
48
48
49
49
50 def get_headers_call_context(environ, strict=True):
50 def get_headers_call_context(environ, strict=True):
51 if 'HTTP_X_RC_VCS_STREAM_CALL_CONTEXT' in environ:
51 if 'HTTP_X_RC_VCS_STREAM_CALL_CONTEXT' in environ:
52 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
52 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
53 return msgpack.unpackb(packed_cc)
53 return msgpack.unpackb(packed_cc)
54 elif strict:
54 elif strict:
55 raise ValueError('Expected header HTTP_X_RC_VCS_STREAM_CALL_CONTEXT not found')
55 raise ValueError('Expected header HTTP_X_RC_VCS_STREAM_CALL_CONTEXT not found')
56
56
57
57
58 class RequestWrapperTween:
58 class RequestWrapperTween:
59 def __init__(self, handler, registry):
59 def __init__(self, handler, registry):
60 self.handler = handler
60 self.handler = handler
61 self.registry = registry
61 self.registry = registry
62
62
63 # one-time configuration code goes here
63 # one-time configuration code goes here
64
64
65 def __call__(self, request):
65 def __call__(self, request):
66 start = time.time()
66 start = time.time()
67 log.debug('Starting request time measurement')
67 log.debug('Starting request time measurement')
68 response = None
68 response = None
69
69
70 try:
70 try:
71 response = self.handler(request)
71 response = self.handler(request)
72 finally:
72 finally:
73 ua = get_user_agent(request.environ)
73 ua = get_user_agent(request.environ)
74 call_context = get_call_context(request)
74 call_context = get_call_context(request)
75 vcs_method = call_context.get('X-RC-Method', '_NO_VCS_METHOD')
75 vcs_method = call_context.get('X-RC-Method', '_NO_VCS_METHOD')
76 repo_name = call_context.get('X-RC-Repo-Name', '')
76 repo_name = call_context.get('X-RC-Repo-Name', '')
77
77
78 count = request.request_count()
78 count = request.request_count()
79 _ver_ = vcsserver.get_version()
79 _ver_ = vcsserver.get_version()
80 _path = safe_str(get_access_path(request.environ))
80 _path = safe_str(get_access_path(request.environ))
81
81
82 ip = '127.0.0.1'
82 ip = '127.0.0.1'
83 match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
83 match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
84 resp_code = getattr(response, 'status_code', 'UNDEFINED')
84 resp_code = getattr(response, 'status_code', 'UNDEFINED')
85
85
86 _view_path = f"{repo_name}@{_path}/{vcs_method}"
86 _view_path = f"{repo_name}@{_path}/{vcs_method}"
87
87
88 total = time.time() - start
88 total = time.time() - start
89
89
90 log.info(
90 log.info(
91 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
91 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
92 count, ip, request.environ.get('REQUEST_METHOD'),
92 count, ip, request.environ.get('REQUEST_METHOD'),
93 _view_path, total, ua, _ver_,
93 _view_path, total, ua, _ver_,
94 extra={"time": total, "ver": _ver_, "code": resp_code,
94 extra={"time": total, "ver": _ver_, "code": resp_code,
95 "path": _path, "view_name": match_route, "user_agent": ua,
95 "path": _path, "view_name": match_route, "user_agent": ua,
96 "vcs_method": vcs_method, "repo_name": repo_name}
96 "vcs_method": vcs_method, "repo_name": repo_name}
97 )
97 )
98
98
99 statsd = request.registry.statsd
99 statsd = request.registry.statsd
100 if statsd:
100 if statsd:
101 match_route = request.matched_route.name if request.matched_route else _path
101 match_route = request.matched_route.name if request.matched_route else _path
102 elapsed_time_ms = round(1000.0 * total) # use ms only
102 elapsed_time_ms = round(1000.0 * total) # use ms only
103 statsd.timing(
103 statsd.timing(
104 "vcsserver_req_timing.histogram", elapsed_time_ms,
104 "vcsserver_req_timing.histogram", elapsed_time_ms,
105 tags=[
105 tags=[
106 f"view_name:{match_route}",
106 f"view_name:{match_route}",
107 f"code:{resp_code}"
107 f"code:{resp_code}"
108 ],
108 ],
109 use_decimals=False
109 use_decimals=False
110 )
110 )
111 statsd.incr(
111 statsd.incr(
112 "vcsserver_req_total", tags=[
112 "vcsserver_req_total", tags=[
113 f"view_name:{match_route}",
113 f"view_name:{match_route}",
114 f"code:{resp_code}"
114 f"code:{resp_code}"
115 ])
115 ])
116
116
117 return response
117 return response
118
118
119
119
120 def includeme(config):
120 def includeme(config):
121 config.add_tween(
121 config.add_tween(
122 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
122 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
123 )
123 )
@@ -1,116 +1,116 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """Extract the responses of a WSGI app."""
18 """Extract the responses of a WSGI app."""
19
19
20 __all__ = ('WSGIAppCaller',)
20 __all__ = ('WSGIAppCaller',)
21
21
22 import io
22 import io
23 import logging
23 import logging
24 import os
24 import os
25
25
26 from vcsserver.str_utils import ascii_bytes
26 from vcsserver.lib.str_utils import ascii_bytes
27
27
28 log = logging.getLogger(__name__)
28 log = logging.getLogger(__name__)
29
29
30 DEV_NULL = open(os.devnull)
30 DEV_NULL = open(os.devnull)
31
31
32
32
33 def _complete_environ(environ, input_data: bytes):
33 def _complete_environ(environ, input_data: bytes):
34 """Update the missing wsgi.* variables of a WSGI environment.
34 """Update the missing wsgi.* variables of a WSGI environment.
35
35
36 :param environ: WSGI environment to update
36 :param environ: WSGI environment to update
37 :type environ: dict
37 :type environ: dict
38 :param input_data: data to be read by the app
38 :param input_data: data to be read by the app
39 :type input_data: bytes
39 :type input_data: bytes
40 """
40 """
41 environ.update({
41 environ.update({
42 'wsgi.version': (1, 0),
42 'wsgi.version': (1, 0),
43 'wsgi.url_scheme': 'http',
43 'wsgi.url_scheme': 'http',
44 'wsgi.multithread': True,
44 'wsgi.multithread': True,
45 'wsgi.multiprocess': True,
45 'wsgi.multiprocess': True,
46 'wsgi.run_once': False,
46 'wsgi.run_once': False,
47 'wsgi.input': io.BytesIO(input_data),
47 'wsgi.input': io.BytesIO(input_data),
48 'wsgi.errors': DEV_NULL,
48 'wsgi.errors': DEV_NULL,
49 })
49 })
50
50
51
51
52 # pylint: disable=too-few-public-methods
52 # pylint: disable=too-few-public-methods
53 class _StartResponse:
53 class _StartResponse:
54 """Save the arguments of a start_response call."""
54 """Save the arguments of a start_response call."""
55
55
56 __slots__ = ['status', 'headers', 'content']
56 __slots__ = ['status', 'headers', 'content']
57
57
58 def __init__(self):
58 def __init__(self):
59 self.status = None
59 self.status = None
60 self.headers = None
60 self.headers = None
61 self.content = []
61 self.content = []
62
62
63 def __call__(self, status, headers, exc_info=None):
63 def __call__(self, status, headers, exc_info=None):
64 # TODO(skreft): do something meaningful with the exc_info
64 # TODO(skreft): do something meaningful with the exc_info
65 exc_info = None # avoid dangling circular reference
65 exc_info = None # avoid dangling circular reference
66 self.status = status
66 self.status = status
67 self.headers = headers
67 self.headers = headers
68
68
69 return self.write
69 return self.write
70
70
71 def write(self, content):
71 def write(self, content):
72 """Write method returning when calling this object.
72 """Write method returning when calling this object.
73
73
74 All the data written is then available in content.
74 All the data written is then available in content.
75 """
75 """
76 self.content.append(content)
76 self.content.append(content)
77
77
78
78
79 class WSGIAppCaller:
79 class WSGIAppCaller:
80 """Calls a WSGI app."""
80 """Calls a WSGI app."""
81
81
82 def __init__(self, app):
82 def __init__(self, app):
83 """
83 """
84 :param app: WSGI app to call
84 :param app: WSGI app to call
85 """
85 """
86 self.app = app
86 self.app = app
87
87
88 def handle(self, environ, input_data):
88 def handle(self, environ, input_data):
89 """Process a request with the WSGI app.
89 """Process a request with the WSGI app.
90
90
91 The returned data of the app is fully consumed into a list.
91 The returned data of the app is fully consumed into a list.
92
92
93 :param environ: WSGI environment to update
93 :param environ: WSGI environment to update
94 :type environ: dict
94 :type environ: dict
95 :param input_data: data to be read by the app
95 :param input_data: data to be read by the app
96 :type input_data: str/bytes
96 :type input_data: str/bytes
97
97
98 :returns: a tuple with the contents, status and headers
98 :returns: a tuple with the contents, status and headers
99 :rtype: (list<str>, str, list<(str, str)>)
99 :rtype: (list<str>, str, list<(str, str)>)
100 """
100 """
101 _complete_environ(environ, ascii_bytes(input_data, allow_bytes=True))
101 _complete_environ(environ, ascii_bytes(input_data, allow_bytes=True))
102 start_response = _StartResponse()
102 start_response = _StartResponse()
103 log.debug("Calling wrapped WSGI application")
103 log.debug("Calling wrapped WSGI application")
104 responses = self.app(environ, start_response)
104 responses = self.app(environ, start_response)
105 responses_list = list(responses)
105 responses_list = list(responses)
106 existing_responses = start_response.content
106 existing_responses = start_response.content
107 if existing_responses:
107 if existing_responses:
108 log.debug("Adding returned response to response written via write()")
108 log.debug("Adding returned response to response written via write()")
109 existing_responses.extend(responses_list)
109 existing_responses.extend(responses_list)
110 responses_list = existing_responses
110 responses_list = existing_responses
111 if hasattr(responses, 'close'):
111 if hasattr(responses, 'close'):
112 log.debug("Closing iterator from WSGI application")
112 log.debug("Closing iterator from WSGI application")
113 responses.close()
113 responses.close()
114
114
115 log.debug("Handling of WSGI request done, returning response")
115 log.debug("Handling of WSGI request done, returning response")
116 return responses_list, start_response.status, start_response.headers
116 return responses_list, start_response.status, start_response.headers
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now