##// END OF EJS Templates
merged default branch into stable
super-admin -
r1219:e9ee7632 merge stable
parent child Browse files
Show More

The requested changes are too big and content was truncated. Show full diff

@@ -0,0 +1,53 b''
1 ; #####################
2 ; LOGGING CONFIGURATION
3 ; #####################
4 ; Logging template, used for configure the logging
5 ; some variables here are replaced by RhodeCode to default values
6
7 [loggers]
8 keys = root, vcsserver
9
10 [handlers]
11 keys = console
12
13 [formatters]
14 keys = generic, json
15
16 ; #######
17 ; LOGGERS
18 ; #######
19 [logger_root]
20 level = NOTSET
21 handlers = console
22
23 [logger_vcsserver]
24 level = $RC_LOGGING_LEVEL
25 handlers =
26 qualname = vcsserver
27 propagate = 1
28
29 ; ########
30 ; HANDLERS
31 ; ########
32
33 [handler_console]
34 class = StreamHandler
35 args = (sys.stderr, )
36 level = $RC_LOGGING_LEVEL
37 ; To enable JSON formatted logs replace generic with json
38 ; This allows sending properly formatted logs to grafana loki or elasticsearch
39 #formatter = json
40 #formatter = generic
41 formatter = $RC_LOGGING_FORMATTER
42
43 ; ##########
44 ; FORMATTERS
45 ; ##########
46
47 [formatter_generic]
48 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
49 datefmt = %Y-%m-%d %H:%M:%S
50
51 [formatter_json]
52 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
53 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -0,0 +1,73 b''
1 [build-system]
2 requires = ["setuptools>=61.0.0", "wheel"]
3 build-backend = "setuptools.build_meta"
4
5 [project]
6 name = "rhodecode-vcsserver"
7 description = "Version Control System Server for RhodeCode"
8 authors = [
9 {name = "RhodeCode GmbH", email = "support@rhodecode.com"},
10 ]
11
12 license = {text = "GPL V3"}
13 requires-python = ">=3.10"
14 dynamic = ["version", "readme", "dependencies", "optional-dependencies"]
15 classifiers = [
16 'Development Status :: 6 - Mature',
17 'Intended Audience :: Developers',
18 'Operating System :: OS Independent',
19 'Topic :: Software Development :: Version Control',
20 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
21 'Programming Language :: Python :: 3.10',
22 ]
23
24 [project.entry-points."paste.app_factory"]
25 main = "vcsserver.http_main:main"
26
27
28 [tool.setuptools]
29 packages = ["vcsserver"]
30
31 [tool.setuptools.dynamic]
32 readme = {file = ["README.rst"], content-type = "text/rst"}
33 version = {file = "vcsserver/VERSION"}
34 dependencies = {file = ["requirements.txt"]}
35 optional-dependencies.tests = {file = ["requirements_test.txt"]}
36
37 [tool.ruff]
38
39 select = [
40 # Pyflakes
41 "F",
42 # Pycodestyle
43 "E",
44 "W",
45 # isort
46 "I001"
47 ]
48
49 ignore = [
50 "E501", # line too long, handled by black
51 ]
52
53 # Same as Black.
54 line-length = 120
55
56 [tool.ruff.isort]
57
58 known-first-party = ["vcsserver"]
59
60 [tool.ruff.format]
61
62 # Like Black, use double quotes for strings.
63 quote-style = "double"
64
65 # Like Black, indent with spaces, rather than tabs.
66 indent-style = "space"
67
68 # Like Black, respect magic trailing commas.
69 skip-magic-trailing-comma = false
70
71 # Like Black, automatically detect the appropriate line ending.
72 line-ending = "auto"
73
@@ -0,0 +1,1 b''
1 # Copyright (C) 2014-2023 RhodeCode GmbH
@@ -0,0 +1,27 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 HOOK_REPO_SIZE = 'changegroup.repo_size'
20
21 # HG
22 HOOK_PRE_PULL = 'preoutgoing.pre_pull'
23 HOOK_PULL = 'outgoing.pull_logger'
24 HOOK_PRE_PUSH = 'prechangegroup.pre_push'
25 HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push'
26 HOOK_PUSH = 'changegroup.push_logger'
27 HOOK_PUSH_KEY = 'pushkey.key_push'
@@ -0,0 +1,168 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import os
20 import textwrap
21 import string
22 import functools
23 import logging
24 import tempfile
25 import logging.config
26
27 from vcsserver.type_utils import str2bool, aslist
28
29 log = logging.getLogger(__name__)
30
31 # skip keys, that are set here, so we don't double process those
32 set_keys = {
33 '__file__': ''
34 }
35
36
37 class SettingsMaker:
38
39 def __init__(self, app_settings):
40 self.settings = app_settings
41
42 @classmethod
43 def _bool_func(cls, input_val):
44 if isinstance(input_val, bytes):
45 # decode to str
46 input_val = input_val.decode('utf8')
47 return str2bool(input_val)
48
49 @classmethod
50 def _int_func(cls, input_val):
51 return int(input_val)
52
53 @classmethod
54 def _list_func(cls, input_val, sep=','):
55 return aslist(input_val, sep=sep)
56
57 @classmethod
58 def _string_func(cls, input_val, lower=True):
59 if lower:
60 input_val = input_val.lower()
61 return input_val
62
63 @classmethod
64 def _float_func(cls, input_val):
65 return float(input_val)
66
67 @classmethod
68 def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
69
70 # ensure we have our dir created
71 if not os.path.isdir(input_val) and ensure_dir:
72 os.makedirs(input_val, mode=mode, exist_ok=True)
73
74 if not os.path.isdir(input_val):
75 raise Exception(f'Dir at {input_val} does not exist')
76 return input_val
77
78 @classmethod
79 def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
80 dirname = os.path.dirname(input_val)
81 cls._dir_func(dirname, ensure_dir=ensure_dir)
82 return input_val
83
84 @classmethod
85 def _key_transformator(cls, key):
86 return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
87
88 def maybe_env_key(self, key):
89 # now maybe we have this KEY in env, search and use the value with higher priority.
90 transformed_key = self._key_transformator(key)
91 envvar_value = os.environ.get(transformed_key)
92 if envvar_value:
93 log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
94
95 return envvar_value
96
97 def env_expand(self):
98 replaced = {}
99 for k, v in self.settings.items():
100 if k not in set_keys:
101 envvar_value = self.maybe_env_key(k)
102 if envvar_value:
103 replaced[k] = envvar_value
104 set_keys[k] = envvar_value
105
106 # replace ALL keys updated
107 self.settings.update(replaced)
108
109 def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
110 """
111 Helper to enable debug on running instance
112 :return:
113 """
114
115 if not str2bool(self.settings.get('logging.autoconfigure')):
116 log.info('logging configuration based on main .ini file')
117 return
118
119 if logging_conf is None:
120 logging_conf = self.settings.get('logging.logging_conf_file') or ''
121
122 if not os.path.isfile(logging_conf):
123 log.error('Unable to setup logging based on %s, '
124 'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
125 return
126
127 with open(logging_conf, 'rt') as f:
128 ini_template = textwrap.dedent(f.read())
129 ini_template = string.Template(ini_template).safe_substitute(
130 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
131 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
132 )
133
134 with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
135 log.info('Saved Temporary LOGGING config at %s', f.name)
136 f.write(ini_template)
137
138 logging.config.fileConfig(f.name)
139 os.remove(f.name)
140
141 def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
142 input_val = self.settings.get(key, default)
143
144 if default_when_empty and not input_val:
145 # use default value when value is set in the config but it is empty
146 input_val = default
147
148 parser_func = {
149 'bool': self._bool_func,
150 'int': self._int_func,
151 'list': self._list_func,
152 'list:newline': functools.partial(self._list_func, sep='/n'),
153 'list:spacesep': functools.partial(self._list_func, sep=' '),
154 'string': functools.partial(self._string_func, lower=lower),
155 'dir': self._dir_func,
156 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
157 'file': self._file_path_func,
158 'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
159 None: lambda i: i
160 }[parser]
161
162 envvar_value = self.maybe_env_key(key)
163 if envvar_value:
164 input_val = envvar_value
165 set_keys[key] = input_val
166
167 self.settings[key] = parser_func(input_val)
168 return self.settings[key]
@@ -0,0 +1,243 b''
1 '''
2 This library is provided to allow standard python logging
3 to output log data as JSON formatted strings
4 '''
5 import logging
6 import json
7 import re
8 from datetime import date, datetime, time, tzinfo, timedelta
9 import traceback
10 import importlib
11
12 from inspect import istraceback
13
14 from collections import OrderedDict
15
16
17 def _inject_req_id(record, *args, **kwargs):
18 return record
19
20
21 ExceptionAwareFormatter = logging.Formatter
22
23
24 ZERO = timedelta(0)
25 HOUR = timedelta(hours=1)
26
27
28 class UTC(tzinfo):
29 """UTC"""
30
31 def utcoffset(self, dt):
32 return ZERO
33
34 def tzname(self, dt):
35 return "UTC"
36
37 def dst(self, dt):
38 return ZERO
39
40 utc = UTC()
41
42
43 # skip natural LogRecord attributes
44 # http://docs.python.org/library/logging.html#logrecord-attributes
45 RESERVED_ATTRS = (
46 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
47 'funcName', 'levelname', 'levelno', 'lineno', 'module',
48 'msecs', 'message', 'msg', 'name', 'pathname', 'process',
49 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName')
50
51
52 def merge_record_extra(record, target, reserved):
53 """
54 Merges extra attributes from LogRecord object into target dictionary
55
56 :param record: logging.LogRecord
57 :param target: dict to update
58 :param reserved: dict or list with reserved keys to skip
59 """
60 for key, value in record.__dict__.items():
61 # this allows to have numeric keys
62 if (key not in reserved
63 and not (hasattr(key, "startswith")
64 and key.startswith('_'))):
65 target[key] = value
66 return target
67
68
69 class JsonEncoder(json.JSONEncoder):
70 """
71 A custom encoder extending the default JSONEncoder
72 """
73
74 def default(self, obj):
75 if isinstance(obj, (date, datetime, time)):
76 return self.format_datetime_obj(obj)
77
78 elif istraceback(obj):
79 return ''.join(traceback.format_tb(obj)).strip()
80
81 elif type(obj) == Exception \
82 or isinstance(obj, Exception) \
83 or type(obj) == type:
84 return str(obj)
85
86 try:
87 return super().default(obj)
88
89 except TypeError:
90 try:
91 return str(obj)
92
93 except Exception:
94 return None
95
96 def format_datetime_obj(self, obj):
97 return obj.isoformat()
98
99
100 class JsonFormatter(ExceptionAwareFormatter):
101 """
102 A custom formatter to format logging records as json strings.
103 Extra values will be formatted as str() if not supported by
104 json default encoder
105 """
106
107 def __init__(self, *args, **kwargs):
108 """
109 :param json_default: a function for encoding non-standard objects
110 as outlined in http://docs.python.org/2/library/json.html
111 :param json_encoder: optional custom encoder
112 :param json_serializer: a :meth:`json.dumps`-compatible callable
113 that will be used to serialize the log record.
114 :param json_indent: an optional :meth:`json.dumps`-compatible numeric value
115 that will be used to customize the indent of the output json.
116 :param prefix: an optional string prefix added at the beginning of
117 the formatted string
118 :param json_indent: indent parameter for json.dumps
119 :param json_ensure_ascii: ensure_ascii parameter for json.dumps
120 :param reserved_attrs: an optional list of fields that will be skipped when
121 outputting json log record. Defaults to all log record attributes:
122 http://docs.python.org/library/logging.html#logrecord-attributes
123 :param timestamp: an optional string/boolean field to add a timestamp when
124 outputting the json log record. If string is passed, timestamp will be added
125 to log record using string as key. If True boolean is passed, timestamp key
126 will be "timestamp". Defaults to False/off.
127 """
128 self.json_default = self._str_to_fn(kwargs.pop("json_default", None))
129 self.json_encoder = self._str_to_fn(kwargs.pop("json_encoder", None))
130 self.json_serializer = self._str_to_fn(kwargs.pop("json_serializer", json.dumps))
131 self.json_indent = kwargs.pop("json_indent", None)
132 self.json_ensure_ascii = kwargs.pop("json_ensure_ascii", True)
133 self.prefix = kwargs.pop("prefix", "")
134 reserved_attrs = kwargs.pop("reserved_attrs", RESERVED_ATTRS)
135 self.reserved_attrs = dict(list(zip(reserved_attrs, reserved_attrs)))
136 self.timestamp = kwargs.pop("timestamp", True)
137
138 # super(JsonFormatter, self).__init__(*args, **kwargs)
139 logging.Formatter.__init__(self, *args, **kwargs)
140 if not self.json_encoder and not self.json_default:
141 self.json_encoder = JsonEncoder
142
143 self._required_fields = self.parse()
144 self._skip_fields = dict(list(zip(self._required_fields,
145 self._required_fields)))
146 self._skip_fields.update(self.reserved_attrs)
147
148 def _str_to_fn(self, fn_as_str):
149 """
150 If the argument is not a string, return whatever was passed in.
151 Parses a string such as package.module.function, imports the module
152 and returns the function.
153
154 :param fn_as_str: The string to parse. If not a string, return it.
155 """
156 if not isinstance(fn_as_str, str):
157 return fn_as_str
158
159 path, _, function = fn_as_str.rpartition('.')
160 module = importlib.import_module(path)
161 return getattr(module, function)
162
163 def parse(self):
164 """
165 Parses format string looking for substitutions
166
167 This method is responsible for returning a list of fields (as strings)
168 to include in all log messages.
169 """
170 standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE)
171 return standard_formatters.findall(self._fmt)
172
173 def add_fields(self, log_record, record, message_dict):
174 """
175 Override this method to implement custom logic for adding fields.
176 """
177 for field in self._required_fields:
178 log_record[field] = record.__dict__.get(field)
179 log_record.update(message_dict)
180 merge_record_extra(record, log_record, reserved=self._skip_fields)
181
182 if self.timestamp:
183 key = self.timestamp if type(self.timestamp) == str else 'timestamp'
184 log_record[key] = datetime.fromtimestamp(record.created, tz=utc)
185
186 def process_log_record(self, log_record):
187 """
188 Override this method to implement custom logic
189 on the possibly ordered dictionary.
190 """
191 return log_record
192
193 def jsonify_log_record(self, log_record):
194 """Returns a json string of the log record."""
195 return self.json_serializer(log_record,
196 default=self.json_default,
197 cls=self.json_encoder,
198 indent=self.json_indent,
199 ensure_ascii=self.json_ensure_ascii)
200
201 def serialize_log_record(self, log_record):
202 """Returns the final representation of the log record."""
203 return "{}{}".format(self.prefix, self.jsonify_log_record(log_record))
204
205 def format(self, record):
206 """Formats a log record and serializes to json"""
207 message_dict = {}
208 # FIXME: logging.LogRecord.msg and logging.LogRecord.message in typeshed
209 # are always type of str. We shouldn't need to override that.
210 if isinstance(record.msg, dict):
211 message_dict = record.msg
212 record.message = None
213 else:
214 record.message = record.getMessage()
215 # only format time if needed
216 if "asctime" in self._required_fields:
217 record.asctime = self.formatTime(record, self.datefmt)
218
219 # Display formatted exception, but allow overriding it in the
220 # user-supplied dict.
221 if record.exc_info and not message_dict.get('exc_info'):
222 message_dict['exc_info'] = self.formatException(record.exc_info)
223 if not message_dict.get('exc_info') and record.exc_text:
224 message_dict['exc_info'] = record.exc_text
225 # Display formatted record of stack frames
226 # default format is a string returned from :func:`traceback.print_stack`
227 try:
228 if record.stack_info and not message_dict.get('stack_info'):
229 message_dict['stack_info'] = self.formatStack(record.stack_info)
230 except AttributeError:
231 # Python2.7 doesn't have stack_info.
232 pass
233
234 try:
235 log_record = OrderedDict()
236 except NameError:
237 log_record = {}
238
239 _inject_req_id(record, with_prefix=False)
240 self.add_fields(log_record, record, message_dict)
241 log_record = self.process_log_record(log_record)
242
243 return self.serialize_log_record(log_record)
@@ -0,0 +1,53 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import sys
20 import logging
21
22
23 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
24
25 # Sequences
26 RESET_SEQ = "\033[0m"
27 COLOR_SEQ = "\033[0;%dm"
28 BOLD_SEQ = "\033[1m"
29
30 COLORS = {
31 'CRITICAL': MAGENTA,
32 'ERROR': RED,
33 'WARNING': CYAN,
34 'INFO': GREEN,
35 'DEBUG': BLUE,
36 'SQL': YELLOW
37 }
38
39
40 class ColorFormatter(logging.Formatter):
41
42 def format(self, record):
43 """
44 Change record's levelname to use with COLORS enum
45 """
46 def_record = super().format(record)
47
48 levelname = record.levelname
49 start = COLOR_SEQ % (COLORS[levelname])
50 end = RESET_SEQ
51
52 colored_record = ''.join([start, def_record, end])
53 return colored_record
@@ -0,0 +1,87 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import logging
19 import os
20 import diskcache
21 from diskcache import RLock
22
23 log = logging.getLogger(__name__)
24
25 cache_meta = None
26
27
28 class ReentrantLock(RLock):
29 def __enter__(self):
30 reentrant_lock_key = self._key
31
32 log.debug('Acquire ReentrantLock(key=%s) for archive cache generation...', reentrant_lock_key)
33 #self.acquire()
34 log.debug('Lock for key=%s acquired', reentrant_lock_key)
35
36 def __exit__(self, *exc_info):
37 #self.release()
38 pass
39
40
41 def get_archival_config(config):
42
43 final_config = {
44 'archive_cache.eviction_policy': 'least-frequently-used'
45 }
46
47 for k, v in config.items():
48 if k.startswith('archive_cache'):
49 final_config[k] = v
50
51 return final_config
52
53
54 def get_archival_cache_store(config):
55
56 global cache_meta
57 if cache_meta is not None:
58 return cache_meta
59
60 config = get_archival_config(config)
61
62 archive_cache_dir = config['archive_cache.store_dir']
63 archive_cache_size_gb = config['archive_cache.cache_size_gb']
64 archive_cache_shards = config['archive_cache.cache_shards']
65 archive_cache_eviction_policy = config['archive_cache.eviction_policy']
66
67 log.debug('Initializing archival cache instance under %s', archive_cache_dir)
68
69 # check if it's ok to write, and re-create the archive cache
70 if not os.path.isdir(archive_cache_dir):
71 os.makedirs(archive_cache_dir, exist_ok=True)
72
73 d_cache = diskcache.FanoutCache(
74 archive_cache_dir, shards=archive_cache_shards,
75 cull_limit=0, # manual eviction required
76 size_limit=archive_cache_size_gb * 1024 * 1024 * 1024,
77 eviction_policy=archive_cache_eviction_policy,
78 timeout=30
79 )
80 cache_meta = d_cache
81 return cache_meta
82
83
84 def includeme(config):
85 # init our cache at start, for vcsserver we don't init at runtime
86 # because our cache config is sent via wire on make archive call, this call just lazy-enables the client
87 return
@@ -0,0 +1,2 b''
1 # use orjson by default
2 import orjson as json
@@ -0,0 +1,70 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 from vcsserver.lib._vendor.statsd import client_from_config
19
20
21 class StatsdClientNotInitialised(Exception):
22 pass
23
24
25 class _Singleton(type):
26 """A metaclass that creates a Singleton base class when called."""
27
28 _instances = {}
29
30 def __call__(cls, *args, **kwargs):
31 if cls not in cls._instances:
32 cls._instances[cls] = super().__call__(*args, **kwargs)
33 return cls._instances[cls]
34
35
36 class Singleton(_Singleton("SingletonMeta", (object,), {})):
37 pass
38
39
40 class StatsdClientClass(Singleton):
41 setup_run = False
42 statsd_client = None
43 statsd = None
44 strict_mode_init = False
45
46 def __getattribute__(self, name):
47
48 if name.startswith("statsd"):
49 if self.setup_run:
50 return super().__getattribute__(name)
51 else:
52 if self.strict_mode_init:
53 raise StatsdClientNotInitialised(f"requested key was {name}")
54 return None
55
56 return super().__getattribute__(name)
57
58 def setup(self, settings):
59 """
60 Initialize the client
61 """
62 strict_init_mode = settings.pop('statsd_strict_init', False)
63
64 statsd = client_from_config(settings)
65 self.statsd = statsd
66 self.statsd_client = statsd
67 self.setup_run = True
68
69
70 StatsdClient = StatsdClientClass()
@@ -0,0 +1,160 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import os
19 import tempfile
20
21 from svn import client
22 from svn import core
23 from svn import ra
24
25 from mercurial import error
26
27 from vcsserver.str_utils import safe_bytes
28
29 core.svn_config_ensure(None)
30 svn_config = core.svn_config_get_config(None)
31
32
33 class RaCallbacks(ra.Callbacks):
34 @staticmethod
35 def open_tmp_file(pool): # pragma: no cover
36 (fd, fn) = tempfile.mkstemp()
37 os.close(fd)
38 return fn
39
40 @staticmethod
41 def get_client_string(pool):
42 return b'RhodeCode-subversion-url-checker'
43
44
45 class SubversionException(Exception):
46 pass
47
48
49 class SubversionConnectionException(SubversionException):
50 """Exception raised when a generic error occurs when connecting to a repository."""
51
52
53 def normalize_url(url):
54 if not url:
55 return url
56 if url.startswith(b'svn+http://') or url.startswith(b'svn+https://'):
57 url = url[4:]
58 url = url.rstrip(b'/')
59 return url
60
61
62 def _create_auth_baton(pool):
63 """Create a Subversion authentication baton. """
64 # Give the client context baton a suite of authentication
65 # providers.h
66 platform_specific = [
67 'svn_auth_get_gnome_keyring_simple_provider',
68 'svn_auth_get_gnome_keyring_ssl_client_cert_pw_provider',
69 'svn_auth_get_keychain_simple_provider',
70 'svn_auth_get_keychain_ssl_client_cert_pw_provider',
71 'svn_auth_get_kwallet_simple_provider',
72 'svn_auth_get_kwallet_ssl_client_cert_pw_provider',
73 'svn_auth_get_ssl_client_cert_file_provider',
74 'svn_auth_get_windows_simple_provider',
75 'svn_auth_get_windows_ssl_server_trust_provider',
76 ]
77
78 providers = []
79
80 for p in platform_specific:
81 if getattr(core, p, None) is not None:
82 try:
83 providers.append(getattr(core, p)())
84 except RuntimeError:
85 pass
86
87 providers += [
88 client.get_simple_provider(),
89 client.get_username_provider(),
90 client.get_ssl_client_cert_file_provider(),
91 client.get_ssl_client_cert_pw_file_provider(),
92 client.get_ssl_server_trust_file_provider(),
93 ]
94
95 return core.svn_auth_open(providers, pool)
96
97
98 class SubversionRepo:
99 """Wrapper for a Subversion repository.
100
101 It uses the SWIG Python bindings, see above for requirements.
102 """
103 def __init__(self, svn_url: bytes = b'', username: bytes = b'', password: bytes = b''):
104
105 self.username = username
106 self.password = password
107 self.svn_url = core.svn_path_canonicalize(svn_url)
108
109 self.auth_baton_pool = core.Pool()
110 self.auth_baton = _create_auth_baton(self.auth_baton_pool)
111 # self.init_ra_and_client() assumes that a pool already exists
112 self.pool = core.Pool()
113
114 self.ra = self.init_ra_and_client()
115 self.uuid = ra.get_uuid(self.ra, self.pool)
116
117 def init_ra_and_client(self):
118 """Initializes the RA and client layers, because sometimes getting
119 unified diffs runs the remote server out of open files.
120 """
121
122 if self.username:
123 core.svn_auth_set_parameter(self.auth_baton,
124 core.SVN_AUTH_PARAM_DEFAULT_USERNAME,
125 self.username)
126 if self.password:
127 core.svn_auth_set_parameter(self.auth_baton,
128 core.SVN_AUTH_PARAM_DEFAULT_PASSWORD,
129 self.password)
130
131 callbacks = RaCallbacks()
132 callbacks.auth_baton = self.auth_baton
133
134 try:
135 return ra.open2(self.svn_url, callbacks, svn_config, self.pool)
136 except SubversionException as e:
137 # e.child contains a detailed error messages
138 msglist = []
139 svn_exc = e
140 while svn_exc:
141 if svn_exc.args[0]:
142 msglist.append(svn_exc.args[0])
143 svn_exc = svn_exc.child
144 msg = '\n'.join(msglist)
145 raise SubversionConnectionException(msg)
146
147
148 class svnremoterepo:
149 """ the dumb wrapper for actual Subversion repositories """
150
151 def __init__(self, username: bytes = b'', password: bytes = b'', svn_url: bytes = b''):
152 self.username = username or b''
153 self.password = password or b''
154 self.path = normalize_url(svn_url)
155
156 def svn(self):
157 try:
158 return SubversionRepo(self.path, self.username, self.password)
159 except SubversionConnectionException as e:
160 raise error.Abort(safe_bytes(e))
@@ -0,0 +1,17 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
1 NO CONTENT: new file 100644
NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: new file 100644
NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: new file 100644
NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff
@@ -1,6 +1,5 b''
1 [bumpversion]
1 [bumpversion]
2 current_version = 4.27.1
2 current_version = 5.0.0
3 message = release: Bump version {current_version} to {new_version}
3 message = release: Bump version {current_version} to {new_version}
4
4
5 [bumpversion:file:vcsserver/VERSION]
5 [bumpversion:file:vcsserver/VERSION]
6
@@ -1,37 +1,41 b''
1 syntax: glob
1 syntax: glob
2
2 *.orig
3 *.orig
3 *.pyc
4 *.pyc
4 *.swp
5 *.swp
5 *.sqlite
6 *.sqlite
6 *.tox
7 *.tox
7 *.egg-info
8 *.egg-info
8 *.egg
9 *.egg
9 *.eggs
10 *.eggs
10 *.idea
11 *.idea
11 .DS_Store*
12 .DS_Store*
12
13
13
14
14 syntax: regexp
15 syntax: regexp
15
16
16 #.filename
17 #.filename
17 ^\.settings$
18 ^\.settings$
18 ^\.project$
19 ^\.project$
19 ^\.pydevproject$
20 ^\.pydevproject$
20 ^\.coverage$
21 ^\.coverage$
21 ^\.cache.*$
22 ^\.cache.*$
23 ^\.venv.*$
24 ^\.ruff_cache.*$
22 ^\.rhodecode$
25 ^\.rhodecode$
23
26
27
24 ^.dev
28 ^.dev
25 ^build/
29 ^build/
26 ^coverage\.xml$
30 ^coverage\.xml$
27 ^data$
31 ^data$
28 ^dev.ini$
32 ^dev.ini$
29 ^acceptance_tests/dev.*\.ini$
33 ^acceptance_tests/dev.*\.ini$
30 ^dist/
34 ^dist/
31 ^fabfile.py
35 ^fabfile.py
32 ^htmlcov
36 ^htmlcov
33 ^junit\.xml$
37 ^junit\.xml$
34 ^node_modules/
38 ^node_modules/
35 ^pylint.log$
39 ^pylint.log$
36 ^build$
40 ^build$
37 ^result$
41 ^result$
@@ -1,16 +1,23 b''
1 # top level files
1 # top level files
2 include *.rst
2 include *.rst
3 include *.txt
3 include *.txt
4
4
5 # package extras
5 # package extras
6 include vcsserver/VERSION
6 include vcsserver/VERSION
7
7
8 # all python files inside vcsserver
9 graft vcsserver
10
8 # all config files
11 # all config files
9 recursive-include configs *
12 recursive-include configs *
10
13
11 # hook templates
14 # hook templates
12 recursive-include vcsserver/hook_utils/hook_templates *
15 recursive-include vcsserver/hook_utils/hook_templates *
13
16
14 # skip any tests files
17 # skip any tests files
15 recursive-exclude vcsserver/tests *
18 recursive-exclude vcsserver/tests *
16
19
20 recursive-exclude docs/_build *
21 recursive-exclude * __pycache__
22 recursive-exclude * *.py[co]
23 recursive-exclude * .*.sw[a-z]
@@ -1,45 +1,139 b''
1 .DEFAULT_GOAL := help
1 # required for pushd to work..
2 SHELL = /bin/bash
3
2
4
3 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
5 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
4 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
6 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
5
7
6 .PHONY: clean
8 .PHONY: clean
7 clean: ## full clean
9 ## Cleanup compiled and cache py files
10 clean:
8 make test-clean
11 make test-clean
9 find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
12 find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
13 find . -type d -name "build" -prune -exec rm -rf '{}' ';'
10
14
11
15
12 .PHONY: test
16 .PHONY: test
13 test: ## run test-clean and tests
17 ## run test-clean and tests
18 test:
14 make test-clean
19 make test-clean
15 make test-only
20 make test-only
16
21
17
22
18 .PHONY:test-clean
23 .PHONY: test-clean
19 test-clean: ## run test-clean and tests
24 ## run test-clean and tests
25 test-clean:
20 rm -rf coverage.xml htmlcov junit.xml pylint.log result
26 rm -rf coverage.xml htmlcov junit.xml pylint.log result
21 find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
27 find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
22 find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
28 find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
23
29
24
30
25 .PHONY: test-only
31 .PHONY: test-only
26 test-only: ## run tests
32 ## Run tests only without cleanup
33 test-only:
27 PYTHONHASHSEED=random \
34 PYTHONHASHSEED=random \
28 py.test -x -vv -r xw -p no:sugar \
35 py.test -x -vv -r xw -p no:sugar \
29 --cov=vcsserver --cov-report=term-missing --cov-report=html \
36 --cov-report=term-missing --cov-report=html \
30 vcsserver
37 --cov=vcsserver vcsserver
31
38
32
39
33 .PHONY: generate-pkgs
40 .PHONY: ruff-check
34 generate-pkgs: ## generate new python packages
41 ## run a ruff analysis
35 nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
42 ruff-check:
43 ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev .
36
44
37
45
38 .PHONY: pip-packages
46 .PHONY: pip-packages
39 pip-packages: ## show outdated packages
47 ## Show outdated packages
48 pip-packages:
40 python ${OUTDATED_PACKAGES}
49 python ${OUTDATED_PACKAGES}
41
50
42
51
43 .PHONY: help
52 .PHONY: build
44 help:
53 ## Build sdist/egg
45 @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}'
54 build:
55 python -m build
56
57
58 .PHONY: dev-sh
59 ## make dev-sh
60 dev-sh:
61 sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
62 sudo apt-get update
63 sudo apt-get install -y zsh carapace-bin
64 rm -rf /home/rhodecode/.oh-my-zsh
65 curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
66 echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
67 PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
68
69
70 .PHONY: dev-env
71 ## make dev-env based on the requirements files and install develop of packages
72 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
73 dev-env:
74 pip install build virtualenv
75 pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
76 pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
77 pip install -e .
78
79
80 .PHONY: sh
81 ## shortcut for make dev-sh dev-env
82 sh:
83 make dev-env
84 make dev-sh
85
86
87 .PHONY: dev-srv
88 ## run develop server instance, docker exec -it $(docker ps -q --filter 'name=dev-enterprise-ce') /bin/bash
89 dev-srv:
90 pserve --reload .dev/dev.ini
91
92
93 .PHONY: dev-srv-g
94 ## run gunicorn multi process workers
95 dev-srv-g:
96 gunicorn --workers=4 --paste .dev/dev.ini --bind=0.0.0.0:10010 --config=.dev/gunicorn_config.py
97
98
99 # Default command on calling make
100 .DEFAULT_GOAL := show-help
101
102 .PHONY: show-help
103 show-help:
104 @echo "$$(tput bold)Available rules:$$(tput sgr0)"
105 @echo
106 @sed -n -e "/^## / { \
107 h; \
108 s/.*//; \
109 :doc" \
110 -e "H; \
111 n; \
112 s/^## //; \
113 t doc" \
114 -e "s/:.*//; \
115 G; \
116 s/\\n## /---/; \
117 s/\\n/ /g; \
118 p; \
119 }" ${MAKEFILE_LIST} \
120 | LC_ALL='C' sort --ignore-case \
121 | awk -F '---' \
122 -v ncol=$$(tput cols) \
123 -v indent=19 \
124 -v col_on="$$(tput setaf 6)" \
125 -v col_off="$$(tput sgr0)" \
126 '{ \
127 printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
128 n = split($$2, words, " "); \
129 line_length = ncol - indent; \
130 for (i = 1; i <= n; i++) { \
131 line_length -= length(words[i]) + 1; \
132 if (line_length <= 0) { \
133 line_length = ncol - indent - length(words[i]) - 1; \
134 printf "\n%*s ", -indent, " "; \
135 } \
136 printf "%s ", words[i]; \
137 } \
138 printf "\n"; \
139 }'
@@ -1,246 +1,204 b''
1 ## -*- coding: utf-8 -*-
1 #
2
2
3 ; #################################
3 ; #################################
4 ; RHODECODE VCSSERVER CONFIGURATION
4 ; RHODECODE VCSSERVER CONFIGURATION
5 ; #################################
5 ; #################################
6
6
7 [server:main]
7 [server:main]
8 ; COMMON HOST/IP CONFIG
8 ; COMMON HOST/IP CONFIG
9 host = 0.0.0.0
9 host = 0.0.0.0
10 port = 9900
10 port = 10010
11
11
12 ; ##################################################
12 ; ##################################################
13 ; WAITRESS WSGI SERVER - Recommended for Development
13 ; WAITRESS WSGI SERVER - Recommended for Development
14 ; ##################################################
14 ; ##################################################
15
15
16 ; use server type
16 ; use server type
17 use = egg:waitress#main
17 use = egg:waitress#main
18
18
19 ; number of worker threads
19 ; number of worker threads
20 threads = 5
20 threads = 5
21
21
22 ; MAX BODY SIZE 100GB
22 ; MAX BODY SIZE 100GB
23 max_request_body_size = 107374182400
23 max_request_body_size = 107374182400
24
24
25 ; Use poll instead of select, fixes file descriptors limits problems.
25 ; Use poll instead of select, fixes file descriptors limits problems.
26 ; May not work on old windows systems.
26 ; May not work on old windows systems.
27 asyncore_use_poll = true
27 asyncore_use_poll = true
28
28
29
29
30 ; ###########################
30 ; ###########################
31 ; GUNICORN APPLICATION SERVER
31 ; GUNICORN APPLICATION SERVER
32 ; ###########################
32 ; ###########################
33
33
34 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
34 ; run with gunicorn --paste rhodecode.ini
35
35
36 ; Module to use, this setting shouldn't be changed
36 ; Module to use, this setting shouldn't be changed
37 #use = egg:gunicorn#main
37 #use = egg:gunicorn#main
38
38
39 ; Sets the number of process workers. More workers means more concurrent connections
40 ; RhodeCode can handle at the same time. Each additional worker also it increases
41 ; memory usage as each has it's own set of caches.
42 ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
43 ; than 8-10 unless for really big deployments .e.g 700-1000 users.
44 ; `instance_id = *` must be set in the [app:main] section below (which is the default)
45 ; when using more than 1 worker.
46 #workers = 2
47
48 ; Gunicorn access log level
49 #loglevel = info
50
51 ; Process name visible in process list
52 #proc_name = rhodecode_vcsserver
53
54 ; Type of worker class, one of `sync`, `gevent`
55 ; currently `sync` is the only option allowed.
56 #worker_class = sync
57
58 ; The maximum number of simultaneous clients. Valid only for gevent
59 #worker_connections = 10
60
61 ; Max number of requests that worker will handle before being gracefully restarted.
62 ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
63 #max_requests = 1000
64 #max_requests_jitter = 30
65
66 ; Amount of time a worker can spend with handling a request before it
67 ; gets killed and restarted. By default set to 21600 (6hrs)
68 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
69 #timeout = 21600
70
71 ; The maximum size of HTTP request line in bytes.
72 ; 0 for unlimited
73 #limit_request_line = 0
74
75 ; Limit the number of HTTP headers fields in a request.
76 ; By default this value is 100 and can't be larger than 32768.
77 #limit_request_fields = 32768
78
79 ; Limit the allowed size of an HTTP request header field.
80 ; Value is a positive number or 0.
81 ; Setting it to 0 will allow unlimited header field sizes.
82 #limit_request_field_size = 0
83
84 ; Timeout for graceful workers restart.
85 ; After receiving a restart signal, workers have this much time to finish
86 ; serving requests. Workers still alive after the timeout (starting from the
87 ; receipt of the restart signal) are force killed.
88 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
89 #graceful_timeout = 3600
90
91 # The number of seconds to wait for requests on a Keep-Alive connection.
92 # Generally set in the 1-5 seconds range.
93 #keepalive = 2
94
95 ; Maximum memory usage that each worker can use before it will receive a
96 ; graceful restart signal 0 = memory monitoring is disabled
97 ; Examples: 268435456 (256MB), 536870912 (512MB)
98 ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
99 #memory_max_usage = 0
100
101 ; How often in seconds to check for memory usage for each gunicorn worker
102 #memory_usage_check_interval = 60
103
104 ; Threshold value for which we don't recycle worker if GarbageCollection
105 ; frees up enough resources. Before each restart we try to run GC on worker
106 ; in case we get enough free memory after that, restart will not happen.
107 #memory_usage_recovery_threshold = 0.8
108
109
110 [app:main]
39 [app:main]
111 ; The %(here)s variable will be replaced with the absolute path of parent directory
40 ; The %(here)s variable will be replaced with the absolute path of parent directory
112 ; of this file
41 ; of this file
42 ; Each option in the app:main can be override by an environmental variable
43 ;
44 ;To override an option:
45 ;
46 ;RC_<KeyName>
47 ;Everything should be uppercase, . and - should be replaced by _.
48 ;For example, if you have these configuration settings:
49 ;rc_cache.repo_object.backend = foo
50 ;can be overridden by
51 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
52
113 use = egg:rhodecode-vcsserver
53 use = egg:rhodecode-vcsserver
114
54
115
55
116 ; #############
56 ; #############
117 ; DEBUG OPTIONS
57 ; DEBUG OPTIONS
118 ; #############
58 ; #############
119
59
120 # During development the we want to have the debug toolbar enabled
60 # During development the we want to have the debug toolbar enabled
121 pyramid.includes =
61 pyramid.includes =
122 pyramid_debugtoolbar
62 pyramid_debugtoolbar
123
63
124 debugtoolbar.hosts = 0.0.0.0/0
64 debugtoolbar.hosts = 0.0.0.0/0
125 debugtoolbar.exclude_prefixes =
65 debugtoolbar.exclude_prefixes =
126 /css
66 /css
127 /fonts
67 /fonts
128 /images
68 /images
129 /js
69 /js
130
70
131 ; #################
71 ; #################
132 ; END DEBUG OPTIONS
72 ; END DEBUG OPTIONS
133 ; #################
73 ; #################
134
74
135 ; Pyramid default locales, we need this to be set
75 ; Pyramid default locales, we need this to be set
136 pyramid.default_locale_name = en
76 #pyramid.default_locale_name = en
137
77
138 ; default locale used by VCS systems
78 ; default locale used by VCS systems
139 locale = en_US.UTF-8
79 #locale = en_US.UTF-8
140
80
141 ; path to binaries for vcsserver, it should be set by the installer
81 ; path to binaries for vcsserver, it should be set by the installer
142 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
82 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
143 ; it can also be a path to nix-build output in case of development
83 ; it can also be a path to nix-build output in case of development
144 core.binary_dir = ""
84 core.binary_dir = ""
145
85
146 ; Custom exception store path, defaults to TMPDIR
86 ; Custom exception store path, defaults to TMPDIR
147 ; This is used to store exception from RhodeCode in shared directory
87 ; This is used to store exception from RhodeCode in shared directory
148 #exception_tracker.store_path =
88 #exception_tracker.store_path =
149
89
150 ; #############
90 ; #############
151 ; DOGPILE CACHE
91 ; DOGPILE CACHE
152 ; #############
92 ; #############
153
93
154 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
94 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
155 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
95 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
156 cache_dir = %(here)s/data
96 #cache_dir = %(here)s/data
157
97
158 ; ***************************************
98 ; ***************************************
159 ; `repo_object` cache, default file based
99 ; `repo_object` cache, default file based
160 ; ***************************************
100 ; ***************************************
161
101
162 ; `repo_object` cache settings for vcs methods for repositories
102 ; `repo_object` cache settings for vcs methods for repositories
163 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
103 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
164
104
165 ; cache auto-expires after N seconds
105 ; cache auto-expires after N seconds
166 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
106 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
167 rc_cache.repo_object.expiration_time = 2592000
107 #rc_cache.repo_object.expiration_time = 2592000
168
108
169 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
109 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
170 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
110 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
171
111
172 ; ***********************************************************
112 ; ***********************************************************
173 ; `repo_object` cache with redis backend
113 ; `repo_object` cache with redis backend
174 ; recommended for larger instance, and for better performance
114 ; recommended for larger instance, and for better performance
175 ; ***********************************************************
115 ; ***********************************************************
176
116
177 ; `repo_object` cache settings for vcs methods for repositories
117 ; `repo_object` cache settings for vcs methods for repositories
178 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
118 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
179
119
180 ; cache auto-expires after N seconds
120 ; cache auto-expires after N seconds
181 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
121 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
182 #rc_cache.repo_object.expiration_time = 2592000
122 #rc_cache.repo_object.expiration_time = 2592000
183
123
184 ; redis_expiration_time needs to be greater then expiration_time
124 ; redis_expiration_time needs to be greater then expiration_time
185 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
125 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
186
126
187 #rc_cache.repo_object.arguments.host = localhost
127 #rc_cache.repo_object.arguments.host = localhost
188 #rc_cache.repo_object.arguments.port = 6379
128 #rc_cache.repo_object.arguments.port = 6379
189 #rc_cache.repo_object.arguments.db = 5
129 #rc_cache.repo_object.arguments.db = 5
190 #rc_cache.repo_object.arguments.socket_timeout = 30
130 #rc_cache.repo_object.arguments.socket_timeout = 30
191 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
131 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
192 #rc_cache.repo_object.arguments.distributed_lock = true
132 #rc_cache.repo_object.arguments.distributed_lock = true
193
133
194 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
134 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
195 #rc_cache.repo_object.arguments.lock_auto_renewal = true
135 #rc_cache.repo_object.arguments.lock_auto_renewal = true
196
136
197 ; Statsd client config
137 ; Statsd client config, this is used to send metrics to statsd
138 ; We recommend setting statsd_exported and scrape them using Promethues
198 #statsd.enabled = false
139 #statsd.enabled = false
199 #statsd.statsd_host = 0.0.0.0
140 #statsd.statsd_host = 0.0.0.0
200 #statsd.statsd_port = 8125
141 #statsd.statsd_port = 8125
201 #statsd.statsd_prefix =
142 #statsd.statsd_prefix =
202 #statsd.statsd_ipv6 = false
143 #statsd.statsd_ipv6 = false
203
144
145 ; configure logging automatically at server startup set to false
146 ; to use the below custom logging config.
147 ; RC_LOGGING_FORMATTER
148 ; RC_LOGGING_LEVEL
149 ; env variables can control the settings for logging in case of autoconfigure
150
151 #logging.autoconfigure = true
152
153 ; specify your own custom logging config file to configure logging
154 #logging.logging_conf_file = /path/to/custom_logging.ini
155
204 ; #####################
156 ; #####################
205 ; LOGGING CONFIGURATION
157 ; LOGGING CONFIGURATION
206 ; #####################
158 ; #####################
159
207 [loggers]
160 [loggers]
208 keys = root, vcsserver
161 keys = root, vcsserver
209
162
210 [handlers]
163 [handlers]
211 keys = console
164 keys = console
212
165
213 [formatters]
166 [formatters]
214 keys = generic
167 keys = generic, json
215
168
216 ; #######
169 ; #######
217 ; LOGGERS
170 ; LOGGERS
218 ; #######
171 ; #######
219 [logger_root]
172 [logger_root]
220 level = NOTSET
173 level = NOTSET
221 handlers = console
174 handlers = console
222
175
223 [logger_vcsserver]
176 [logger_vcsserver]
224 level = DEBUG
177 level = DEBUG
225 handlers =
178 handlers =
226 qualname = vcsserver
179 qualname = vcsserver
227 propagate = 1
180 propagate = 1
228
181
229
230 ; ########
182 ; ########
231 ; HANDLERS
183 ; HANDLERS
232 ; ########
184 ; ########
233
185
234 [handler_console]
186 [handler_console]
235 class = StreamHandler
187 class = StreamHandler
236 args = (sys.stderr, )
188 args = (sys.stderr, )
237 level = DEBUG
189 level = DEBUG
190 ; To enable JSON formatted logs replace 'generic' with 'json'
191 ; This allows sending properly formatted logs to grafana loki or elasticsearch
238 formatter = generic
192 formatter = generic
239
193
240 ; ##########
194 ; ##########
241 ; FORMATTERS
195 ; FORMATTERS
242 ; ##########
196 ; ##########
243
197
244 [formatter_generic]
198 [formatter_generic]
245 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
199 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
246 datefmt = %Y-%m-%d %H:%M:%S
200 datefmt = %Y-%m-%d %H:%M:%S
201
202 [formatter_json]
203 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
204 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -1,265 +1,520 b''
1 """
1 """
2 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
2 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
3 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
3 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
4 """
4 """
5
5
6 import gc
6 import gc
7 import os
7 import os
8 import sys
8 import sys
9 import math
9 import math
10 import time
10 import time
11 import threading
11 import threading
12 import traceback
12 import traceback
13 import random
13 import random
14 import socket
15 import dataclasses
14 from gunicorn.glogging import Logger
16 from gunicorn.glogging import Logger
15
17
16
18
17 def get_workers():
19 def get_workers():
18 import multiprocessing
20 import multiprocessing
19 return multiprocessing.cpu_count() * 2 + 1
21 return multiprocessing.cpu_count() * 2 + 1
20
22
21 # GLOBAL
23
24 bind = "127.0.0.1:10010"
25
26
27 # Error logging output for gunicorn (-) is stdout
22 errorlog = '-'
28 errorlog = '-'
29
30 # Access logging output for gunicorn (-) is stdout
23 accesslog = '-'
31 accesslog = '-'
24
32
25
33
26 # SERVER MECHANICS
34 # SERVER MECHANICS
27 # None == system temp dir
35 # None == system temp dir
28 # worker_tmp_dir is recommended to be set to some tmpfs
36 # worker_tmp_dir is recommended to be set to some tmpfs
29 worker_tmp_dir = None
37 worker_tmp_dir = None
30 tmp_upload_dir = None
38 tmp_upload_dir = None
31
39
40 # use re-use port logic
41 #reuse_port = True
42
32 # Custom log format
43 # Custom log format
44 #access_log_format = (
45 # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
46
47 # loki format for easier parsing in grafana
33 access_log_format = (
48 access_log_format = (
34 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
49 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
50
51 # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
52 # workers = get_workers()
53
54 # Gunicorn access log level
55 loglevel = 'info'
56
57 # Process name visible in a process list
58 proc_name = "rhodecode_vcsserver"
59
60 # Type of worker class, one of `sync`, `gevent` or `gthread`
61 # currently `sync` is the only option allowed for vcsserver and for rhodecode all of 3 are allowed
62 # gevent:
63 # In this case, the maximum number of concurrent requests is (N workers * X worker_connections)
64 # e.g. workers =3 worker_connections=10 = 3*10, 30 concurrent requests can be handled
65 # gthread:
66 # In this case, the maximum number of concurrent requests is (N workers * X threads)
67 # e.g. workers = 3 threads=3 = 3*3, 9 concurrent requests can be handled
68 worker_class = 'sync'
69
70 # Sets the number of process workers. More workers means more concurrent connections
71 # RhodeCode can handle at the same time. Each additional worker also it increases
72 # memory usage as each has its own set of caches.
73 # The Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
74 # than 8-10 unless for huge deployments .e.g 700-1000 users.
75 # `instance_id = *` must be set in the [app:main] section below (which is the default)
76 # when using more than 1 worker.
77 workers = 2
78
79 # Threads numbers for worker class gthread
80 threads = 1
81
82 # The maximum number of simultaneous clients. Valid only for gevent
83 # In this case, the maximum number of concurrent requests is (N workers * X worker_connections)
84 # e.g workers =3 worker_connections=10 = 3*10, 30 concurrent requests can be handled
85 worker_connections = 10
86
87 # Max number of requests that worker will handle before being gracefully restarted.
88 # Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
89 max_requests = 2000
90 max_requests_jitter = int(max_requests * 0.2) # 20% of max_requests
91
92 # The maximum number of pending connections.
93 # Exceeding this number results in the client getting an error when attempting to connect.
94 backlog = 64
35
95
36 # self adjust workers based on CPU count
96 # The Amount of time a worker can spend with handling a request before it
37 # workers = get_workers()
97 # gets killed and restarted. By default, set to 21600 (6hrs)
98 # Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
99 timeout = 21600
100
101 # The maximum size of HTTP request line in bytes.
102 # 0 for unlimited
103 limit_request_line = 0
104
105 # Limit the number of HTTP headers fields in a request.
106 # By default this value is 100 and can't be larger than 32768.
107 limit_request_fields = 32768
108
109 # Limit the allowed size of an HTTP request header field.
110 # Value is a positive number or 0.
111 # Setting it to 0 will allow unlimited header field sizes.
112 limit_request_field_size = 0
113
114 # Timeout for graceful workers restart.
115 # After receiving a restart signal, workers have this much time to finish
116 # serving requests. Workers still alive after the timeout (starting from the
117 # receipt of the restart signal) are force killed.
118 # Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
119 graceful_timeout = 21600
120
121 # The number of seconds to wait for requests on a Keep-Alive connection.
122 # Generally set in the 1-5 seconds range.
123 keepalive = 2
124
125 # Maximum memory usage that each worker can use before it will receive a
126 # graceful restart signal 0 = memory monitoring is disabled
127 # Examples: 268435456 (256MB), 536870912 (512MB)
128 # 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
129 # Dynamic formula 1024 * 1024 * 256 == 256MBs
130 memory_max_usage = 0
131
132 # How often in seconds to check for memory usage for each gunicorn worker
133 memory_usage_check_interval = 60
134
135 # Threshold value for which we don't recycle worker if GarbageCollection
136 # frees up enough resources. Before each restart, we try to run GC on worker
137 # in case we get enough free memory after that; restart will not happen.
138 memory_usage_recovery_threshold = 0.8
139
140
141 @dataclasses.dataclass
142 class MemoryCheckConfig:
143 max_usage: int
144 check_interval: int
145 recovery_threshold: float
38
146
39
147
40 def _get_process_rss(pid=None):
148 def _get_process_rss(pid=None):
41 try:
149 try:
42 import psutil
150 import psutil
43 if pid:
151 if pid:
44 proc = psutil.Process(pid)
152 proc = psutil.Process(pid)
45 else:
153 else:
46 proc = psutil.Process()
154 proc = psutil.Process()
47 return proc.memory_info().rss
155 return proc.memory_info().rss
48 except Exception:
156 except Exception:
49 return None
157 return None
50
158
51
159
52 def _get_config(ini_path):
160 def _get_config(ini_path):
161 import configparser
53
162
54 try:
163 try:
55 import configparser
56 except ImportError:
57 import ConfigParser as configparser
58 try:
59 config = configparser.RawConfigParser()
164 config = configparser.RawConfigParser()
60 config.read(ini_path)
165 config.read(ini_path)
61 return config
166 return config
62 except Exception:
167 except Exception:
63 return None
168 return None
64
169
65
170
66 def _time_with_offset(memory_usage_check_interval):
171 def get_memory_usage_params(config=None):
67 return time.time() - random.randint(0, memory_usage_check_interval/2.0)
172 # memory spec defaults
173 _memory_max_usage = memory_max_usage
174 _memory_usage_check_interval = memory_usage_check_interval
175 _memory_usage_recovery_threshold = memory_usage_recovery_threshold
176
177 if config:
178 ini_path = os.path.abspath(config)
179 conf = _get_config(ini_path)
180
181 section = 'server:main'
182 if conf and conf.has_section(section):
183
184 if conf.has_option(section, 'memory_max_usage'):
185 _memory_max_usage = conf.getint(section, 'memory_max_usage')
186
187 if conf.has_option(section, 'memory_usage_check_interval'):
188 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
189
190 if conf.has_option(section, 'memory_usage_recovery_threshold'):
191 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
192
193 _memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
194 or _memory_max_usage)
195 _memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
196 or _memory_usage_check_interval)
197 _memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
198 or _memory_usage_recovery_threshold)
199
200 return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
201
202
203 def _time_with_offset(check_interval):
204 return time.time() - random.randint(0, check_interval/2.0)
68
205
69
206
70 def pre_fork(server, worker):
207 def pre_fork(server, worker):
71 pass
208 pass
72
209
73
210
74 def post_fork(server, worker):
211 def post_fork(server, worker):
75
212
76 # memory spec defaults
213 memory_conf = get_memory_usage_params()
77 _memory_max_usage = 0
214 _memory_max_usage = memory_conf.max_usage
78 _memory_usage_check_interval = 60
215 _memory_usage_check_interval = memory_conf.check_interval
79 _memory_usage_recovery_threshold = 0.8
216 _memory_usage_recovery_threshold = memory_conf.recovery_threshold
80
81 ini_path = os.path.abspath(server.cfg.paste)
82 conf = _get_config(ini_path)
83
84 section = 'server:main'
85 if conf and conf.has_section(section):
86
217
87 if conf.has_option(section, 'memory_max_usage'):
218 worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
88 _memory_max_usage = conf.getint(section, 'memory_max_usage')
219 or _memory_max_usage)
89
220 worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
90 if conf.has_option(section, 'memory_usage_check_interval'):
221 or _memory_usage_check_interval)
91 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
222 worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
92
223 or _memory_usage_recovery_threshold)
93 if conf.has_option(section, 'memory_usage_recovery_threshold'):
94 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
95
96 worker._memory_max_usage = _memory_max_usage
97 worker._memory_usage_check_interval = _memory_usage_check_interval
98 worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
99
224
100 # register memory last check time, with some random offset so we don't recycle all
225 # register memory last check time, with some random offset so we don't recycle all
101 # at once
226 # at once
102 worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
227 worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
103
228
104 if _memory_max_usage:
229 if _memory_max_usage:
105 server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
230 server.log.info("pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid,
106 _format_data_size(_memory_max_usage))
231 _format_data_size(_memory_max_usage))
107 else:
232 else:
108 server.log.info("[%-10s] WORKER spawned", worker.pid)
233 server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
109
234
110
235
111 def pre_exec(server):
236 def pre_exec(server):
112 server.log.info("Forked child, re-executing.")
237 server.log.info("Forked child, re-executing.")
113
238
114
239
115 def on_starting(server):
240 def on_starting(server):
116 server_lbl = '{} {}'.format(server.proc_name, server.address)
241 server_lbl = '{} {}'.format(server.proc_name, server.address)
117 server.log.info("Server %s is starting.", server_lbl)
242 server.log.info("Server %s is starting.", server_lbl)
243 server.log.info('Config:')
244 server.log.info(f"\n{server.cfg}")
245 server.log.info(get_memory_usage_params())
118
246
119
247
120 def when_ready(server):
248 def when_ready(server):
121 server.log.info("Server %s is ready. Spawning workers", server)
249 server.log.info("Server %s is ready. Spawning workers", server)
122
250
123
251
124 def on_reload(server):
252 def on_reload(server):
125 pass
253 pass
126
254
127
255
128 def _format_data_size(size, unit="B", precision=1, binary=True):
256 def _format_data_size(size, unit="B", precision=1, binary=True):
129 """Format a number using SI units (kilo, mega, etc.).
257 """Format a number using SI units (kilo, mega, etc.).
130
258
131 ``size``: The number as a float or int.
259 ``size``: The number as a float or int.
132
260
133 ``unit``: The unit name in plural form. Examples: "bytes", "B".
261 ``unit``: The unit name in plural form. Examples: "bytes", "B".
134
262
135 ``precision``: How many digits to the right of the decimal point. Default
263 ``precision``: How many digits to the right of the decimal point. Default
136 is 1. 0 suppresses the decimal point.
264 is 1. 0 suppresses the decimal point.
137
265
138 ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
266 ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
139 If true, use base-2 binary prefixes (kibi = Ki = 1024).
267 If true, use base-2 binary prefixes (kibi = Ki = 1024).
140
268
141 ``full_name``: If false (default), use the prefix abbreviation ("k" or
269 ``full_name``: If false (default), use the prefix abbreviation ("k" or
142 "Ki"). If true, use the full prefix ("kilo" or "kibi"). If false,
270 "Ki"). If true, use the full prefix ("kilo" or "kibi"). If false,
143 use abbreviation ("k" or "Ki").
271 use abbreviation ("k" or "Ki").
144
272
145 """
273 """
146
274
147 if not binary:
275 if not binary:
148 base = 1000
276 base = 1000
149 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
277 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
150 else:
278 else:
151 base = 1024
279 base = 1024
152 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
280 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
153
281
154 sign = ""
282 sign = ""
155 if size > 0:
283 if size > 0:
156 m = int(math.log(size, base))
284 m = int(math.log(size, base))
157 elif size < 0:
285 elif size < 0:
158 sign = "-"
286 sign = "-"
159 size = -size
287 size = -size
160 m = int(math.log(size, base))
288 m = int(math.log(size, base))
161 else:
289 else:
162 m = 0
290 m = 0
163 if m > 8:
291 if m > 8:
164 m = 8
292 m = 8
165
293
166 if m == 0:
294 if m == 0:
167 precision = '%.0f'
295 precision = '%.0f'
168 else:
296 else:
169 precision = '%%.%df' % precision
297 precision = '%%.%df' % precision
170
298
171 size = precision % (size / math.pow(base, m))
299 size = precision % (size / math.pow(base, m))
172
300
173 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
301 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
174
302
175
303
176 def _check_memory_usage(worker):
304 def _check_memory_usage(worker):
177 memory_max_usage = worker._memory_max_usage
305 _memory_max_usage = worker._memory_max_usage
178 if not memory_max_usage:
306 if not _memory_max_usage:
179 return
307 return
180
308
181 memory_usage_check_interval = worker._memory_usage_check_interval
309 _memory_usage_check_interval = worker._memory_usage_check_interval
182 memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
310 _memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
183
311
184 elapsed = time.time() - worker._last_memory_check_time
312 elapsed = time.time() - worker._last_memory_check_time
185 if elapsed > memory_usage_check_interval:
313 if elapsed > _memory_usage_check_interval:
186 mem_usage = _get_process_rss()
314 mem_usage = _get_process_rss()
187 if mem_usage and mem_usage > memory_max_usage:
315 if mem_usage and mem_usage > _memory_max_usage:
188 worker.log.info(
316 worker.log.info(
189 "memory usage %s > %s, forcing gc",
317 "memory usage %s > %s, forcing gc",
190 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
318 _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
191 # Try to clean it up by forcing a full collection.
319 # Try to clean it up by forcing a full collection.
192 gc.collect()
320 gc.collect()
193 mem_usage = _get_process_rss()
321 mem_usage = _get_process_rss()
194 if mem_usage > memory_usage_recovery_threshold:
322 if mem_usage > _memory_usage_recovery_threshold:
195 # Didn't clean up enough, we'll have to terminate.
323 # Didn't clean up enough, we'll have to terminate.
196 worker.log.warning(
324 worker.log.warning(
197 "memory usage %s > %s after gc, quitting",
325 "memory usage %s > %s after gc, quitting",
198 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
326 _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
199 # This will cause worker to auto-restart itself
327 # This will cause worker to auto-restart itself
200 worker.alive = False
328 worker.alive = False
201 worker._last_memory_check_time = time.time()
329 worker._last_memory_check_time = time.time()
202
330
203
331
204 def worker_int(worker):
332 def worker_int(worker):
205 worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
333 worker.log.info("pid=[%-10s] worker received INT or QUIT signal", worker.pid)
206
334
207 # get traceback info, on worker crash
335 # get traceback info, when a worker crashes
208 id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
336 def get_thread_id(t_id):
337 id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
338 return id2name.get(t_id, "unknown_thread_id")
339
209 code = []
340 code = []
210 for thread_id, stack in sys._current_frames().items():
341 for thread_id, stack in sys._current_frames().items(): # noqa
211 code.append(
342 code.append(
212 "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
343 "\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
213 for fname, lineno, name, line in traceback.extract_stack(stack):
344 for fname, lineno, name, line in traceback.extract_stack(stack):
214 code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
345 code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
215 if line:
346 if line:
216 code.append(" %s" % (line.strip()))
347 code.append(" %s" % (line.strip()))
217 worker.log.debug("\n".join(code))
348 worker.log.debug("\n".join(code))
218
349
219
350
220 def worker_abort(worker):
351 def worker_abort(worker):
221 worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
352 worker.log.info("pid=[%-10s] worker received SIGABRT signal", worker.pid)
222
353
223
354
224 def worker_exit(server, worker):
355 def worker_exit(server, worker):
225 worker.log.info("[%-10s] worker exit", worker.pid)
356 worker.log.info("pid=[%-10s] worker exit", worker.pid)
226
357
227
358
228 def child_exit(server, worker):
359 def child_exit(server, worker):
229 worker.log.info("[%-10s] worker child exit", worker.pid)
360 worker.log.info("pid=[%-10s] worker child exit", worker.pid)
230
361
231
362
232 def pre_request(worker, req):
363 def pre_request(worker, req):
233 worker.start_time = time.time()
364 worker.start_time = time.time()
234 worker.log.debug(
365 worker.log.debug(
235 "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
366 "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
236
367
237
368
238 def post_request(worker, req, environ, resp):
369 def post_request(worker, req, environ, resp):
239 total_time = time.time() - worker.start_time
370 total_time = time.time() - worker.start_time
240 # Gunicorn sometimes has problems with reading the status_code
371 # Gunicorn sometimes has problems with reading the status_code
241 status_code = getattr(resp, 'status_code', '')
372 status_code = getattr(resp, 'status_code', '')
242 worker.log.debug(
373 worker.log.debug(
243 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
374 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
244 worker.nr, req.method, req.path, status_code, total_time)
375 worker.nr, req.method, req.path, status_code, total_time)
245 _check_memory_usage(worker)
376 _check_memory_usage(worker)
246
377
247
378
379 def _filter_proxy(ip):
380 """
381 Passed in IP addresses in HEADERS can be in a special format of multiple
382 ips. Those comma separated IPs are passed from various proxies in the
383 chain of request processing. The left-most being the original client.
384 We only care about the first IP which came from the org. client.
385
386 :param ip: ip string from headers
387 """
388 if ',' in ip:
389 _ips = ip.split(',')
390 _first_ip = _ips[0].strip()
391 return _first_ip
392 return ip
393
394
395 def _filter_port(ip):
396 """
397 Removes a port from ip, there are 4 main cases to handle here.
398 - ipv4 eg. 127.0.0.1
399 - ipv6 eg. ::1
400 - ipv4+port eg. 127.0.0.1:8080
401 - ipv6+port eg. [::1]:8080
402
403 :param ip:
404 """
405 def is_ipv6(ip_addr):
406 if hasattr(socket, 'inet_pton'):
407 try:
408 socket.inet_pton(socket.AF_INET6, ip_addr)
409 except socket.error:
410 return False
411 else:
412 return False
413 return True
414
415 if ':' not in ip: # must be ipv4 pure ip
416 return ip
417
418 if '[' in ip and ']' in ip: # ipv6 with port
419 return ip.split(']')[0][1:].lower()
420
421 # must be ipv6 or ipv4 with port
422 if is_ipv6(ip):
423 return ip
424 else:
425 ip, _port = ip.split(':')[:2] # means ipv4+port
426 return ip
427
428
429 def get_ip_addr(environ):
430 proxy_key = 'HTTP_X_REAL_IP'
431 proxy_key2 = 'HTTP_X_FORWARDED_FOR'
432 def_key = 'REMOTE_ADDR'
433
434 def _filters(x):
435 return _filter_port(_filter_proxy(x))
436
437 ip = environ.get(proxy_key)
438 if ip:
439 return _filters(ip)
440
441 ip = environ.get(proxy_key2)
442 if ip:
443 return _filters(ip)
444
445 ip = environ.get(def_key, '0.0.0.0')
446 return _filters(ip)
447
448
248 class RhodeCodeLogger(Logger):
449 class RhodeCodeLogger(Logger):
249 """
450 """
250 Custom Logger that allows some customization that gunicorn doesn't allow
451 Custom Logger that allows some customization that gunicorn doesn't allow
251 """
452 """
252
453
253 datefmt = r"%Y-%m-%d %H:%M:%S"
454 datefmt = r"%Y-%m-%d %H:%M:%S"
254
455
255 def __init__(self, cfg):
456 def __init__(self, cfg):
256 Logger.__init__(self, cfg)
457 Logger.__init__(self, cfg)
257
458
258 def now(self):
459 def now(self):
259 """ return date in RhodeCode Log format """
460 """ return date in RhodeCode Log format """
260 now = time.time()
461 now = time.time()
261 msecs = int((now - long(now)) * 1000)
462 msecs = int((now - int(now)) * 1000)
262 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
463 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
263
464
465 def atoms(self, resp, req, environ, request_time):
466 """ Gets atoms for log formatting.
467 """
468 status = resp.status
469 if isinstance(status, str):
470 status = status.split(None, 1)[0]
471 atoms = {
472 'h': get_ip_addr(environ),
473 'l': '-',
474 'u': self._get_user(environ) or '-',
475 't': self.now(),
476 'r': "%s %s %s" % (environ['REQUEST_METHOD'],
477 environ['RAW_URI'],
478 environ["SERVER_PROTOCOL"]),
479 's': status,
480 'm': environ.get('REQUEST_METHOD'),
481 'U': environ.get('PATH_INFO'),
482 'q': environ.get('QUERY_STRING'),
483 'H': environ.get('SERVER_PROTOCOL'),
484 'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
485 'B': getattr(resp, 'sent', None),
486 'f': environ.get('HTTP_REFERER', '-'),
487 'a': environ.get('HTTP_USER_AGENT', '-'),
488 'T': request_time.seconds,
489 'D': (request_time.seconds * 1000000) + request_time.microseconds,
490 'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
491 'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
492 'p': "<%s>" % os.getpid()
493 }
494
495 # add request headers
496 if hasattr(req, 'headers'):
497 req_headers = req.headers
498 else:
499 req_headers = req
500
501 if hasattr(req_headers, "items"):
502 req_headers = req_headers.items()
503
504 atoms.update({"{%s}i" % k.lower(): v for k, v in req_headers})
505
506 resp_headers = resp.headers
507 if hasattr(resp_headers, "items"):
508 resp_headers = resp_headers.items()
509
510 # add response headers
511 atoms.update({"{%s}o" % k.lower(): v for k, v in resp_headers})
512
513 # add environ variables
514 environ_variables = environ.items()
515 atoms.update({"{%s}e" % k.lower(): v for k, v in environ_variables})
516
517 return atoms
518
264
519
265 logger_class = RhodeCodeLogger
520 logger_class = RhodeCodeLogger
@@ -1,209 +1,167 b''
1 ## -*- coding: utf-8 -*-
1 #
2
2
3 ; #################################
3 ; #################################
4 ; RHODECODE VCSSERVER CONFIGURATION
4 ; RHODECODE VCSSERVER CONFIGURATION
5 ; #################################
5 ; #################################
6
6
7 [server:main]
7 [server:main]
8 ; COMMON HOST/IP CONFIG
8 ; COMMON HOST/IP CONFIG
9 host = 127.0.0.1
9 host = 127.0.0.1
10 port = 9900
10 port = 10010
11
11
12
12
13 ; ###########################
13 ; ###########################
14 ; GUNICORN APPLICATION SERVER
14 ; GUNICORN APPLICATION SERVER
15 ; ###########################
15 ; ###########################
16
16
17 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
17 ; run with gunicorn --paste rhodecode.ini
18
18
19 ; Module to use, this setting shouldn't be changed
19 ; Module to use, this setting shouldn't be changed
20 use = egg:gunicorn#main
20 use = egg:gunicorn#main
21
21
22 ; Sets the number of process workers. More workers means more concurrent connections
23 ; RhodeCode can handle at the same time. Each additional worker also it increases
24 ; memory usage as each has it's own set of caches.
25 ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
26 ; than 8-10 unless for really big deployments .e.g 700-1000 users.
27 ; `instance_id = *` must be set in the [app:main] section below (which is the default)
28 ; when using more than 1 worker.
29 workers = 2
30
31 ; Gunicorn access log level
32 loglevel = info
33
34 ; Process name visible in process list
35 proc_name = rhodecode_vcsserver
36
37 ; Type of worker class, one of `sync`, `gevent`
38 ; currently `sync` is the only option allowed.
39 worker_class = sync
40
41 ; The maximum number of simultaneous clients. Valid only for gevent
42 worker_connections = 10
43
44 ; Max number of requests that worker will handle before being gracefully restarted.
45 ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
46 max_requests = 1000
47 max_requests_jitter = 30
48
49 ; Amount of time a worker can spend with handling a request before it
50 ; gets killed and restarted. By default set to 21600 (6hrs)
51 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
52 timeout = 21600
53
54 ; The maximum size of HTTP request line in bytes.
55 ; 0 for unlimited
56 limit_request_line = 0
57
58 ; Limit the number of HTTP headers fields in a request.
59 ; By default this value is 100 and can't be larger than 32768.
60 limit_request_fields = 32768
61
62 ; Limit the allowed size of an HTTP request header field.
63 ; Value is a positive number or 0.
64 ; Setting it to 0 will allow unlimited header field sizes.
65 limit_request_field_size = 0
66
67 ; Timeout for graceful workers restart.
68 ; After receiving a restart signal, workers have this much time to finish
69 ; serving requests. Workers still alive after the timeout (starting from the
70 ; receipt of the restart signal) are force killed.
71 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
72 graceful_timeout = 3600
73
74 # The number of seconds to wait for requests on a Keep-Alive connection.
75 # Generally set in the 1-5 seconds range.
76 keepalive = 2
77
78 ; Maximum memory usage that each worker can use before it will receive a
79 ; graceful restart signal 0 = memory monitoring is disabled
80 ; Examples: 268435456 (256MB), 536870912 (512MB)
81 ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
82 memory_max_usage = 0
83
84 ; How often in seconds to check for memory usage for each gunicorn worker
85 memory_usage_check_interval = 60
86
87 ; Threshold value for which we don't recycle worker if GarbageCollection
88 ; frees up enough resources. Before each restart we try to run GC on worker
89 ; in case we get enough free memory after that, restart will not happen.
90 memory_usage_recovery_threshold = 0.8
91
92
93 [app:main]
22 [app:main]
94 ; The %(here)s variable will be replaced with the absolute path of parent directory
23 ; The %(here)s variable will be replaced with the absolute path of parent directory
95 ; of this file
24 ; of this file
25 ; Each option in the app:main can be override by an environmental variable
26 ;
27 ;To override an option:
28 ;
29 ;RC_<KeyName>
30 ;Everything should be uppercase, . and - should be replaced by _.
31 ;For example, if you have these configuration settings:
32 ;rc_cache.repo_object.backend = foo
33 ;can be overridden by
34 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
35
96 use = egg:rhodecode-vcsserver
36 use = egg:rhodecode-vcsserver
97
37
98 ; Pyramid default locales, we need this to be set
38 ; Pyramid default locales, we need this to be set
99 pyramid.default_locale_name = en
39 #pyramid.default_locale_name = en
100
40
101 ; default locale used by VCS systems
41 ; default locale used by VCS systems
102 locale = en_US.UTF-8
42 #locale = en_US.UTF-8
103
43
104 ; path to binaries for vcsserver, it should be set by the installer
44 ; path to binaries for vcsserver, it should be set by the installer
105 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
45 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
106 ; it can also be a path to nix-build output in case of development
46 ; it can also be a path to nix-build output in case of development
107 core.binary_dir = ""
47 core.binary_dir = ""
108
48
109 ; Custom exception store path, defaults to TMPDIR
49 ; Custom exception store path, defaults to TMPDIR
110 ; This is used to store exception from RhodeCode in shared directory
50 ; This is used to store exception from RhodeCode in shared directory
111 #exception_tracker.store_path =
51 #exception_tracker.store_path =
112
52
113 ; #############
53 ; #############
114 ; DOGPILE CACHE
54 ; DOGPILE CACHE
115 ; #############
55 ; #############
116
56
117 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
57 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
118 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
58 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
119 cache_dir = %(here)s/data
59 #cache_dir = %(here)s/data
120
60
121 ; ***************************************
61 ; ***************************************
122 ; `repo_object` cache, default file based
62 ; `repo_object` cache, default file based
123 ; ***************************************
63 ; ***************************************
124
64
125 ; `repo_object` cache settings for vcs methods for repositories
65 ; `repo_object` cache settings for vcs methods for repositories
126 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
66 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
127
67
128 ; cache auto-expires after N seconds
68 ; cache auto-expires after N seconds
129 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
69 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
130 rc_cache.repo_object.expiration_time = 2592000
70 #rc_cache.repo_object.expiration_time = 2592000
131
71
132 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
72 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
133 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
73 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
134
74
135 ; ***********************************************************
75 ; ***********************************************************
136 ; `repo_object` cache with redis backend
76 ; `repo_object` cache with redis backend
137 ; recommended for larger instance, and for better performance
77 ; recommended for larger instance, and for better performance
138 ; ***********************************************************
78 ; ***********************************************************
139
79
140 ; `repo_object` cache settings for vcs methods for repositories
80 ; `repo_object` cache settings for vcs methods for repositories
141 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
81 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
142
82
143 ; cache auto-expires after N seconds
83 ; cache auto-expires after N seconds
144 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
84 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
145 #rc_cache.repo_object.expiration_time = 2592000
85 #rc_cache.repo_object.expiration_time = 2592000
146
86
147 ; redis_expiration_time needs to be greater then expiration_time
87 ; redis_expiration_time needs to be greater then expiration_time
148 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
88 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
149
89
150 #rc_cache.repo_object.arguments.host = localhost
90 #rc_cache.repo_object.arguments.host = localhost
151 #rc_cache.repo_object.arguments.port = 6379
91 #rc_cache.repo_object.arguments.port = 6379
152 #rc_cache.repo_object.arguments.db = 5
92 #rc_cache.repo_object.arguments.db = 5
153 #rc_cache.repo_object.arguments.socket_timeout = 30
93 #rc_cache.repo_object.arguments.socket_timeout = 30
154 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
94 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
155 #rc_cache.repo_object.arguments.distributed_lock = true
95 #rc_cache.repo_object.arguments.distributed_lock = true
156
96
157 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
97 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
158 #rc_cache.repo_object.arguments.lock_auto_renewal = true
98 #rc_cache.repo_object.arguments.lock_auto_renewal = true
159
99
160 ; Statsd client config
100 ; Statsd client config, this is used to send metrics to statsd
101 ; We recommend setting statsd_exported and scrape them using Promethues
161 #statsd.enabled = false
102 #statsd.enabled = false
162 #statsd.statsd_host = 0.0.0.0
103 #statsd.statsd_host = 0.0.0.0
163 #statsd.statsd_port = 8125
104 #statsd.statsd_port = 8125
164 #statsd.statsd_prefix =
105 #statsd.statsd_prefix =
165 #statsd.statsd_ipv6 = false
106 #statsd.statsd_ipv6 = false
166
107
108 ; configure logging automatically at server startup set to false
109 ; to use the below custom logging config.
110 ; RC_LOGGING_FORMATTER
111 ; RC_LOGGING_LEVEL
112 ; env variables can control the settings for logging in case of autoconfigure
113
114 #logging.autoconfigure = true
115
116 ; specify your own custom logging config file to configure logging
117 #logging.logging_conf_file = /path/to/custom_logging.ini
118
167 ; #####################
119 ; #####################
168 ; LOGGING CONFIGURATION
120 ; LOGGING CONFIGURATION
169 ; #####################
121 ; #####################
122
170 [loggers]
123 [loggers]
171 keys = root, vcsserver
124 keys = root, vcsserver
172
125
173 [handlers]
126 [handlers]
174 keys = console
127 keys = console
175
128
176 [formatters]
129 [formatters]
177 keys = generic
130 keys = generic, json
178
131
179 ; #######
132 ; #######
180 ; LOGGERS
133 ; LOGGERS
181 ; #######
134 ; #######
182 [logger_root]
135 [logger_root]
183 level = NOTSET
136 level = NOTSET
184 handlers = console
137 handlers = console
185
138
186 [logger_vcsserver]
139 [logger_vcsserver]
187 level = DEBUG
140 level = INFO
188 handlers =
141 handlers =
189 qualname = vcsserver
142 qualname = vcsserver
190 propagate = 1
143 propagate = 1
191
144
192
193 ; ########
145 ; ########
194 ; HANDLERS
146 ; HANDLERS
195 ; ########
147 ; ########
196
148
197 [handler_console]
149 [handler_console]
198 class = StreamHandler
150 class = StreamHandler
199 args = (sys.stderr, )
151 args = (sys.stderr, )
200 level = INFO
152 level = INFO
153 ; To enable JSON formatted logs replace 'generic' with 'json'
154 ; This allows sending properly formatted logs to grafana loki or elasticsearch
201 formatter = generic
155 formatter = generic
202
156
203 ; ##########
157 ; ##########
204 ; FORMATTERS
158 ; FORMATTERS
205 ; ##########
159 ; ##########
206
160
207 [formatter_generic]
161 [formatter_generic]
208 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
162 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
209 datefmt = %Y-%m-%d %H:%M:%S
163 datefmt = %Y-%m-%d %H:%M:%S
164
165 [formatter_json]
166 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
167 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -1,57 +1,56 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import socket
18 import socket
19
20 import pytest
19 import pytest
21
20
22
21
23 def pytest_addoption(parser):
22 def pytest_addoption(parser):
24 parser.addoption(
23 parser.addoption(
25 '--repeat', type=int, default=100,
24 '--perf-repeat-vcs', type=int, default=100,
26 help="Number of repetitions in performance tests.")
25 help="Number of repetitions in performance tests.")
27
26
28
27
29 @pytest.fixture(scope='session')
28 @pytest.fixture(scope='session')
30 def repeat(request):
29 def repeat(request):
31 """
30 """
32 The number of repetitions is based on this fixture.
31 The number of repetitions is based on this fixture.
33
32
34 Slower calls may divide it by 10 or 100. It is chosen in a way so that the
33 Slower calls may divide it by 10 or 100. It is chosen in a way so that the
35 tests are not too slow in our default test suite.
34 tests are not too slow in our default test suite.
36 """
35 """
37 return request.config.getoption('--repeat')
36 return request.config.getoption('--perf-repeat-vcs')
38
37
39
38
40 @pytest.fixture(scope='session')
39 @pytest.fixture(scope='session')
41 def vcsserver_port(request):
40 def vcsserver_port(request):
42 port = get_available_port()
41 port = get_available_port()
43 print('Using vcsserver port %s' % (port, ))
42 print(f'Using vcsserver port {port}')
44 return port
43 return port
45
44
46
45
47 def get_available_port():
46 def get_available_port():
48 family = socket.AF_INET
47 family = socket.AF_INET
49 socktype = socket.SOCK_STREAM
48 socktype = socket.SOCK_STREAM
50 host = '127.0.0.1'
49 host = '127.0.0.1'
51
50
52 mysocket = socket.socket(family, socktype)
51 mysocket = socket.socket(family, socktype)
53 mysocket.bind((host, 0))
52 mysocket.bind((host, 0))
54 port = mysocket.getsockname()[1]
53 port = mysocket.getsockname()[1]
55 mysocket.close()
54 mysocket.close()
56 del mysocket
55 del mysocket
57 return port
56 return port
@@ -1,48 +1,77 b''
1 ## dependencies
1 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
2
3 # our custom configobj
4 https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
5
6 dogpile.cache==0.9.0
7 dogpile.core==0.4.1
8 decorator==4.1.2
9 dulwich==0.13.0
10 hgsubversion==1.9.3
11 hg-evolve==9.1.0
12 mako==1.1.0
13 markupsafe==1.1.1
14 mercurial==5.1.1
15 msgpack-python==0.5.6
16
17 pastedeploy==2.1.0
18 pyramid==1.10.4
19 pyramid-mako==1.1.0
20 pygit2==0.28.2
21
2
3 async-timeout==4.0.3
4 atomicwrites==1.4.1
5 celery==5.3.6
6 billiard==4.2.0
7 click==8.1.3
8 click-didyoumean==0.3.0
9 click==8.1.3
10 click-plugins==1.1.1
11 click==8.1.3
12 click-repl==0.2.0
13 click==8.1.3
14 prompt-toolkit==3.0.38
15 wcwidth==0.2.6
16 six==1.16.0
17 kombu==5.3.5
18 amqp==5.2.0
19 vine==5.1.0
20 vine==5.1.0
21 python-dateutil==2.8.2
22 six==1.16.0
23 tzdata==2023.4
24 vine==5.1.0
25 contextlib2==21.6.0
26 cov-core==1.15.0
27 coverage==7.2.3
28 diskcache==5.6.3
29 dogpile.cache==1.3.0
30 decorator==5.1.1
31 stevedore==5.1.0
32 pbr==5.11.1
33 dulwich==0.21.6
34 urllib3==1.26.14
35 gunicorn==21.2.0
36 packaging==23.1
37 hg-evolve==11.0.2
38 importlib-metadata==6.0.0
39 zipp==3.15.0
40 mercurial==6.3.3
41 mock==5.0.2
42 more-itertools==9.1.0
43 msgpack==1.0.7
44 orjson==3.9.13
45 psutil==5.9.8
46 py==1.11.0
47 pygit2==1.13.3
48 cffi==1.16.0
49 pycparser==2.21
50 pygments==2.15.1
51 pyparsing==3.1.1
52 pyramid==2.0.2
53 hupper==1.12
54 plaster==1.1.2
55 plaster-pastedeploy==1.0.1
56 pastedeploy==3.1.0
57 plaster==1.1.2
58 translationstring==1.4
59 venusian==3.0.0
60 webob==1.8.7
61 zope.deprecation==5.0.0
62 zope.interface==6.1.0
63 redis==5.0.1
64 async-timeout==4.0.3
22 repoze.lru==0.7
65 repoze.lru==0.7
23 redis==3.5.3
66 scandir==1.10.0
24 simplejson==3.16.0
67 setproctitle==1.3.3
25 subprocess32==3.5.4
68 subvertpy==0.11.0
26 subvertpy==0.10.1
69 waitress==3.0.0
70 wcwidth==0.2.6
27
71
28 six==1.11.0
29 translationstring==1.3
30 webob==1.8.5
31 zope.deprecation==4.4.0
32 zope.interface==4.6.0
33
34 ## http servers
35 gevent==1.5.0
36 greenlet==0.4.15
37 gunicorn==19.9.0
38 waitress==1.3.1
39
40 ## debug
41 ipdb==0.13.2
42 ipython==5.1.0
43
72
44 ## test related requirements
73 ## test related requirements
45 -r requirements_test.txt
74 #-r requirements_test.txt
46
75
47 ## uncomment to add the debug libraries
76 ## uncomment to add the debug libraries
48 #-r requirements_debug.txt
77 #-r requirements_debug.txt
@@ -1,8 +1,28 b''
1 ## special libraries we could extend the requirements.txt file with to add some
1 ## special libraries we could extend the requirements.txt file with to add some
2 ## custom libraries useful for debug and memory tracing
2 ## custom libraries usefull for debug and memory tracing
3
4 ## uncomment inclusion of this file in requirements.txt run make generate-pkgs and nix-shell
5
3
6 objgraph
4 objgraph
7 memory-profiler
5 memory-profiler
8 pympler
6 pympler
7
8 ## debug
9 ipdb
10 ipython
11 rich
12
13 # format
14 flake8
15 ruff
16
17 pipdeptree==2.7.1
18 invoke==2.0.0
19 bumpversion==0.6.0
20 bump2version==1.0.1
21
22 docutils-stubs
23 types-redis
24 types-requests==2.31.0.6
25 types-sqlalchemy
26 types-psutil
27 types-pycurl
28 types-ujson
@@ -1,16 +1,45 b''
1 # test related requirements
1 # test related requirements
2 pytest==4.6.5
2
3 py==1.8.0
3 cov-core==1.15.0
4 pytest-cov==2.7.1
4 coverage==7.2.3
5 pytest-sugar==0.9.2
5 mock==5.0.2
6 pytest-runner==5.1.0
6 py==1.11.0
7 pytest-cov==4.0.0
8 coverage==7.2.3
9 pytest==7.3.1
10 attrs==22.2.0
11 iniconfig==2.0.0
12 packaging==23.1
13 pluggy==1.0.0
7 pytest-profiling==1.7.0
14 pytest-profiling==1.7.0
8 pytest-timeout==1.3.3
15 gprof2dot==2022.7.29
9 gprof2dot==2017.9.19
16 pytest==7.3.1
17 attrs==22.2.0
18 iniconfig==2.0.0
19 packaging==23.1
20 pluggy==1.0.0
21 six==1.16.0
22 pytest-runner==6.0.0
23 pytest-sugar==0.9.7
24 packaging==23.1
25 pytest==7.3.1
26 attrs==22.2.0
27 iniconfig==2.0.0
28 packaging==23.1
29 pluggy==1.0.0
30 termcolor==2.3.0
31 pytest-timeout==2.1.0
32 pytest==7.3.1
33 attrs==22.2.0
34 iniconfig==2.0.0
35 packaging==23.1
36 pluggy==1.0.0
37 webtest==3.0.0
38 beautifulsoup4==4.11.2
39 soupsieve==2.4
40 waitress==3.0.0
41 webob==1.8.7
10
42
11 mock==3.0.5
43 # RhodeCode test-data
12 cov-core==1.15.0
44 rc_testdata @ https://code.rhodecode.com/upstream/rc-testdata-dist/raw/77378e9097f700b4c1b9391b56199fe63566b5c9/rc_testdata-0.11.0.tar.gz#egg=rc_testdata
13 coverage==4.5.4
45 rc_testdata==0.11.0
14
15 webtest==2.0.34
16 beautifulsoup4==4.6.3
@@ -1,1 +1,1 b''
1 4.27.1 No newline at end of file
1 5.0.0 No newline at end of file
@@ -1,28 +1,41 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import pkgutil
18 import os
19
20 __version__ = ''
19
21
20
22
21 __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
23 def get_version():
24 global __version__
25 if __version__:
26 return __version__
27
28 here = os.path.abspath(os.path.dirname(__file__))
29 ver_file = os.path.join(here, "VERSION")
30 with open(ver_file, "rt") as f:
31 version = f.read().strip()
32
33 __version__ = version
34 return version
22
35
23 # link to config for pyramid
36 # link to config for pyramid
24 CONFIG = {}
37 CONFIG = {}
25
38
26 # Populated with the settings dictionary from application init in
39 # Populated with the settings dictionary from application init in
27 #
40 #
28 PYRAMID_SETTINGS = {}
41 PYRAMID_SETTINGS = {}
@@ -1,130 +1,193 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import os
17 import os
18 import sys
18 import sys
19 import traceback
19 import tempfile
20 import logging
20 import logging
21 import urlparse
21 import urllib.parse
22
23 from vcsserver.lib.rc_cache.archive_cache import get_archival_cache_store
22
24
23 from vcsserver import exceptions
25 from vcsserver import exceptions
24 from vcsserver.exceptions import NoContentException
26 from vcsserver.exceptions import NoContentException
25 from vcsserver.hgcompat import (archival)
27 from vcsserver.hgcompat import archival
26
28 from vcsserver.str_utils import safe_bytes
29 from vcsserver.lib.exc_tracking import format_exc
27 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
28
31
29
32
30 class RepoFactory(object):
33 class RepoFactory:
31 """
34 """
32 Utility to create instances of repository
35 Utility to create instances of repository
33
36
34 It provides internal caching of the `repo` object based on
37 It provides internal caching of the `repo` object based on
35 the :term:`call context`.
38 the :term:`call context`.
36 """
39 """
37 repo_type = None
40 repo_type = None
38
41
39 def __init__(self):
42 def __init__(self):
40 pass
43 pass
41
44
42 def _create_config(self, path, config):
45 def _create_config(self, path, config):
43 config = {}
46 config = {}
44 return config
47 return config
45
48
46 def _create_repo(self, wire, create):
49 def _create_repo(self, wire, create):
47 raise NotImplementedError()
50 raise NotImplementedError()
48
51
49 def repo(self, wire, create=False):
52 def repo(self, wire, create=False):
50 raise NotImplementedError()
53 raise NotImplementedError()
51
54
52
55
53 def obfuscate_qs(query_string):
56 def obfuscate_qs(query_string):
54 if query_string is None:
57 if query_string is None:
55 return None
58 return None
56
59
57 parsed = []
60 parsed = []
58 for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
61 for k, v in urllib.parse.parse_qsl(query_string, keep_blank_values=True):
59 if k in ['auth_token', 'api_key']:
62 if k in ['auth_token', 'api_key']:
60 v = "*****"
63 v = "*****"
61 parsed.append((k, v))
64 parsed.append((k, v))
62
65
63 return '&'.join('{}{}'.format(
66 return '&'.join('{}{}'.format(
64 k, '={}'.format(v) if v else '') for k, v in parsed)
67 k, f'={v}' if v else '') for k, v in parsed)
65
68
66
69
67 def raise_from_original(new_type):
70 def raise_from_original(new_type, org_exc: Exception):
68 """
71 """
69 Raise a new exception type with original args and traceback.
72 Raise a new exception type with original args and traceback.
70 """
73 """
71 exc_type, exc_value, exc_traceback = sys.exc_info()
74 exc_info = sys.exc_info()
75 exc_type, exc_value, exc_traceback = exc_info
72 new_exc = new_type(*exc_value.args)
76 new_exc = new_type(*exc_value.args)
77
73 # store the original traceback into the new exc
78 # store the original traceback into the new exc
74 new_exc._org_exc_tb = traceback.format_exc(exc_traceback)
79 new_exc._org_exc_tb = format_exc(exc_info)
75
80
76 try:
81 try:
77 raise new_exc, None, exc_traceback
82 raise new_exc.with_traceback(exc_traceback)
78 finally:
83 finally:
79 del exc_traceback
84 del exc_traceback
80
85
81
86
82 class ArchiveNode(object):
87 class ArchiveNode:
83 def __init__(self, path, mode, is_link, raw_bytes):
88 def __init__(self, path, mode, is_link, raw_bytes):
84 self.path = path
89 self.path = path
85 self.mode = mode
90 self.mode = mode
86 self.is_link = is_link
91 self.is_link = is_link
87 self.raw_bytes = raw_bytes
92 self.raw_bytes = raw_bytes
88
93
89
94
90 def archive_repo(walker, archive_dest_path, kind, mtime, archive_at_path,
95 def store_archive_in_cache(node_walker, archive_key, kind, mtime, archive_at_path, archive_dir_name,
91 archive_dir_name, commit_id, write_metadata=True, extra_metadata=None):
96 commit_id, write_metadata=True, extra_metadata=None, cache_config=None):
92 """
97 """
93 walker should be a file walker, for example:
98 Function that would store generate archive and send it to a dedicated backend store
94 def walker():
99 In here we use diskcache
100
101 :param node_walker: a generator returning nodes to add to archive
102 :param archive_key: key used to store the path
103 :param kind: archive kind
104 :param mtime: time of creation
105 :param archive_at_path: default '/' the path at archive was started.
106 If this is not '/' it means it's a partial archive
107 :param archive_dir_name: inside dir name when creating an archive
108 :param commit_id: commit sha of revision archive was created at
109 :param write_metadata:
110 :param extra_metadata:
111 :param cache_config:
112
113 walker should be a file walker, for example,
114 def node_walker():
95 for file_info in files:
115 for file_info in files:
96 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
116 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
97 """
117 """
98 extra_metadata = extra_metadata or {}
118 extra_metadata = extra_metadata or {}
99
119
120 d_cache = get_archival_cache_store(config=cache_config)
121
122 if archive_key in d_cache:
123 with d_cache as d_cache_reader:
124 reader, tag = d_cache_reader.get(archive_key, read=True, tag=True, retry=True)
125 return reader.name
126
127 archive_tmp_path = safe_bytes(tempfile.mkstemp()[1])
128 log.debug('Creating new temp archive in %s', archive_tmp_path)
129
100 if kind == "tgz":
130 if kind == "tgz":
101 archiver = archival.tarit(archive_dest_path, mtime, "gz")
131 archiver = archival.tarit(archive_tmp_path, mtime, b"gz")
102 elif kind == "tbz2":
132 elif kind == "tbz2":
103 archiver = archival.tarit(archive_dest_path, mtime, "bz2")
133 archiver = archival.tarit(archive_tmp_path, mtime, b"bz2")
104 elif kind == 'zip':
134 elif kind == 'zip':
105 archiver = archival.zipit(archive_dest_path, mtime)
135 archiver = archival.zipit(archive_tmp_path, mtime)
106 else:
136 else:
107 raise exceptions.ArchiveException()(
137 raise exceptions.ArchiveException()(
108 'Remote does not support: "%s" archive type.' % kind)
138 f'Remote does not support: "{kind}" archive type.')
109
139
110 for f in walker(commit_id, archive_at_path):
140 for f in node_walker(commit_id, archive_at_path):
111 f_path = os.path.join(archive_dir_name, f.path.lstrip('/'))
141 f_path = os.path.join(safe_bytes(archive_dir_name), safe_bytes(f.path).lstrip(b'/'))
112 try:
142 try:
113 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
143 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
114 except NoContentException:
144 except NoContentException:
115 # NOTE(marcink): this is a special case for SVN so we can create "empty"
145 # NOTE(marcink): this is a special case for SVN so we can create "empty"
116 # directories which arent supported by archiver
146 # directories which are not supported by archiver
117 archiver.addfile(os.path.join(f_path, '.dir'), f.mode, f.is_link, '')
147 archiver.addfile(os.path.join(f_path, b'.dir'), f.mode, f.is_link, b'')
118
148
119 if write_metadata:
149 if write_metadata:
120 metadata = dict([
150 metadata = dict([
121 ('commit_id', commit_id),
151 ('commit_id', commit_id),
122 ('mtime', mtime),
152 ('mtime', mtime),
123 ])
153 ])
124 metadata.update(extra_metadata)
154 metadata.update(extra_metadata)
125
155
126 meta = ["%s:%s" % (f_name, value) for f_name, value in metadata.items()]
156 meta = [safe_bytes(f"{f_name}:{value}") for f_name, value in metadata.items()]
127 f_path = os.path.join(archive_dir_name, '.archival.txt')
157 f_path = os.path.join(safe_bytes(archive_dir_name), b'.archival.txt')
128 archiver.addfile(f_path, 0o644, False, '\n'.join(meta))
158 archiver.addfile(f_path, 0o644, False, b'\n'.join(meta))
159
160 archiver.done()
161
162 # ensure set & get are atomic
163 with d_cache.transact():
164
165 with open(archive_tmp_path, 'rb') as archive_file:
166 add_result = d_cache.set(archive_key, archive_file, read=True, tag='db-name', retry=True)
167 if not add_result:
168 log.error('Failed to store cache for key=%s', archive_key)
169
170 os.remove(archive_tmp_path)
129
171
130 return archiver.done()
172 reader, tag = d_cache.get(archive_key, read=True, tag=True, retry=True)
173 if not reader:
174 raise AssertionError(f'empty reader on key={archive_key} added={add_result}')
175
176 return reader.name
177
178
179 class BinaryEnvelope:
180 def __init__(self, val):
181 self.val = val
182
183
184 class BytesEnvelope(bytes):
185 def __new__(cls, content):
186 if isinstance(content, bytes):
187 return super().__new__(cls, content)
188 else:
189 raise TypeError('BytesEnvelope content= param must be bytes. Use BinaryEnvelope to wrap other types')
190
191
192 class BinaryBytesEnvelope(BytesEnvelope):
193 pass
@@ -1,8 +1,10 b''
1 # Copyright (C) 2014-2023 RhodeCode GmbH
2
1 """
3 """
2 Provides a stub implementation for VCS operations.
4 Provides a stub implementation for VCS operations.
3
5
4 Intended usage is to help in performance measurements. The basic idea is to
6 Intended usage is to help in performance measurements. The basic idea is to
5 implement an `EchoApp` which sends back what it gets. Based on a configuration
7 implement an `EchoApp` which sends back what it gets. Based on a configuration
6 parameter this app can be activated, so that it replaced the endpoints for Git
8 parameter this app can be activated, so that it replaced the endpoints for Git
7 and Mercurial.
9 and Mercurial.
8 """
10 """
@@ -1,54 +1,56 b''
1 # Copyright (C) 2014-2023 RhodeCode GmbH
2
1 """
3 """
2 Implementation of :class:`EchoApp`.
4 Implementation of :class:`EchoApp`.
3
5
4 This WSGI application will just echo back the data which it recieves.
6 This WSGI application will just echo back the data which it recieves.
5 """
7 """
6
8
7 import logging
9 import logging
8
10
9
11
10 log = logging.getLogger(__name__)
12 log = logging.getLogger(__name__)
11
13
12
14
13 class EchoApp(object):
15 class EchoApp:
14
16
15 def __init__(self, repo_path, repo_name, config):
17 def __init__(self, repo_path, repo_name, config):
16 self._repo_path = repo_path
18 self._repo_path = repo_path
17 log.info("EchoApp initialized for %s", repo_path)
19 log.info("EchoApp initialized for %s", repo_path)
18
20
19 def __call__(self, environ, start_response):
21 def __call__(self, environ, start_response):
20 log.debug("EchoApp called for %s", self._repo_path)
22 log.debug("EchoApp called for %s", self._repo_path)
21 log.debug("Content-Length: %s", environ.get('CONTENT_LENGTH'))
23 log.debug("Content-Length: %s", environ.get('CONTENT_LENGTH'))
22 environ['wsgi.input'].read()
24 environ['wsgi.input'].read()
23 status = '200 OK'
25 status = '200 OK'
24 headers = [('Content-Type', 'text/plain')]
26 headers = [('Content-Type', 'text/plain')]
25 start_response(status, headers)
27 start_response(status, headers)
26 return ["ECHO"]
28 return [b"ECHO"]
27
29
28
30
29 class EchoAppStream(object):
31 class EchoAppStream:
30
32
31 def __init__(self, repo_path, repo_name, config):
33 def __init__(self, repo_path, repo_name, config):
32 self._repo_path = repo_path
34 self._repo_path = repo_path
33 log.info("EchoApp initialized for %s", repo_path)
35 log.info("EchoApp initialized for %s", repo_path)
34
36
35 def __call__(self, environ, start_response):
37 def __call__(self, environ, start_response):
36 log.debug("EchoApp called for %s", self._repo_path)
38 log.debug("EchoApp called for %s", self._repo_path)
37 log.debug("Content-Length: %s", environ.get('CONTENT_LENGTH'))
39 log.debug("Content-Length: %s", environ.get('CONTENT_LENGTH'))
38 environ['wsgi.input'].read()
40 environ['wsgi.input'].read()
39 status = '200 OK'
41 status = '200 OK'
40 headers = [('Content-Type', 'text/plain')]
42 headers = [('Content-Type', 'text/plain')]
41 start_response(status, headers)
43 start_response(status, headers)
42
44
43 def generator():
45 def generator():
44 for _ in xrange(1000000):
46 for _ in range(1000000):
45 yield "ECHO"
47 yield b"ECHO_STREAM"
46 return generator()
48 return generator()
47
49
48
50
49 def create_app():
51 def create_app():
50 """
52 """
51 Allows to run this app directly in a WSGI server.
53 Allows to run this app directly in a WSGI server.
52 """
54 """
53 stub_config = {}
55 stub_config = {}
54 return EchoApp('stub_path', 'stub_name', stub_config)
56 return EchoApp('stub_path', 'stub_name', stub_config)
@@ -1,45 +1,47 b''
1 # Copyright (C) 2014-2023 RhodeCode GmbH
2
1 """
3 """
2 Provides the same API as :mod:`remote_wsgi`.
4 Provides the same API as :mod:`remote_wsgi`.
3
5
4 Uses the `EchoApp` instead of real implementations.
6 Uses the `EchoApp` instead of real implementations.
5 """
7 """
6
8
7 import logging
9 import logging
8
10
9 from .echo_app import EchoApp
11 from .echo_app import EchoApp
10 from vcsserver import wsgi_app_caller
12 from vcsserver import wsgi_app_caller
11
13
12
14
13 log = logging.getLogger(__name__)
15 log = logging.getLogger(__name__)
14
16
15
17
16 class GitRemoteWsgi(object):
18 class GitRemoteWsgi:
17 def handle(self, environ, input_data, *args, **kwargs):
19 def handle(self, environ, input_data, *args, **kwargs):
18 app = wsgi_app_caller.WSGIAppCaller(
20 app = wsgi_app_caller.WSGIAppCaller(
19 create_echo_wsgi_app(*args, **kwargs))
21 create_echo_wsgi_app(*args, **kwargs))
20
22
21 return app.handle(environ, input_data)
23 return app.handle(environ, input_data)
22
24
23
25
24 class HgRemoteWsgi(object):
26 class HgRemoteWsgi:
25 def handle(self, environ, input_data, *args, **kwargs):
27 def handle(self, environ, input_data, *args, **kwargs):
26 app = wsgi_app_caller.WSGIAppCaller(
28 app = wsgi_app_caller.WSGIAppCaller(
27 create_echo_wsgi_app(*args, **kwargs))
29 create_echo_wsgi_app(*args, **kwargs))
28
30
29 return app.handle(environ, input_data)
31 return app.handle(environ, input_data)
30
32
31
33
32 def create_echo_wsgi_app(repo_path, repo_name, config):
34 def create_echo_wsgi_app(repo_path, repo_name, config):
33 log.debug("Creating EchoApp WSGI application")
35 log.debug("Creating EchoApp WSGI application")
34
36
35 _assert_valid_config(config)
37 _assert_valid_config(config)
36
38
37 # Remaining items are forwarded to have the extras available
39 # Remaining items are forwarded to have the extras available
38 return EchoApp(repo_path, repo_name, config=config)
40 return EchoApp(repo_path, repo_name, config=config)
39
41
40
42
41 def _assert_valid_config(config):
43 def _assert_valid_config(config):
42 config = config.copy()
44 config = config.copy()
43
45
44 # This is what git needs from config at this stage
46 # This is what git needs from config at this stage
45 config.pop('git_update_server_info')
47 config.pop('git_update_server_info')
@@ -1,125 +1,125 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """
18 """
19 Special exception handling over the wire.
19 Special exception handling over the wire.
20
20
21 Since we cannot assume that our client is able to import our exception classes,
21 Since we cannot assume that our client is able to import our exception classes,
22 this module provides a "wrapping" mechanism to raise plain exceptions
22 this module provides a "wrapping" mechanism to raise plain exceptions
23 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
23 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
24 different error conditions.
24 different error conditions.
25 """
25 """
26
26
27 from pyramid.httpexceptions import HTTPLocked, HTTPForbidden
27 from pyramid.httpexceptions import HTTPLocked, HTTPForbidden
28
28
29
29
30 def _make_exception(kind, org_exc, *args):
30 def _make_exception(kind, org_exc, *args):
31 """
31 """
32 Prepares a base `Exception` instance to be sent over the wire.
32 Prepares a base `Exception` instance to be sent over the wire.
33
33
34 To give our caller a hint what this is about, it will attach an attribute
34 To give our caller a hint what this is about, it will attach an attribute
35 `_vcs_kind` to the exception.
35 `_vcs_kind` to the exception.
36 """
36 """
37 exc = Exception(*args)
37 exc = Exception(*args)
38 exc._vcs_kind = kind
38 exc._vcs_kind = kind
39 exc._org_exc = org_exc
39 exc._org_exc = org_exc
40 exc._org_exc_tb = getattr(org_exc, '_org_exc_tb', '')
40 exc._org_exc_tb = getattr(org_exc, '_org_exc_tb', '')
41 return exc
41 return exc
42
42
43
43
44 def AbortException(org_exc=None):
44 def AbortException(org_exc=None):
45 def _make_exception_wrapper(*args):
45 def _make_exception_wrapper(*args):
46 return _make_exception('abort', org_exc, *args)
46 return _make_exception('abort', org_exc, *args)
47 return _make_exception_wrapper
47 return _make_exception_wrapper
48
48
49
49
50 def ArchiveException(org_exc=None):
50 def ArchiveException(org_exc=None):
51 def _make_exception_wrapper(*args):
51 def _make_exception_wrapper(*args):
52 return _make_exception('archive', org_exc, *args)
52 return _make_exception('archive', org_exc, *args)
53 return _make_exception_wrapper
53 return _make_exception_wrapper
54
54
55
55
56 def LookupException(org_exc=None):
56 def LookupException(org_exc=None):
57 def _make_exception_wrapper(*args):
57 def _make_exception_wrapper(*args):
58 return _make_exception('lookup', org_exc, *args)
58 return _make_exception('lookup', org_exc, *args)
59 return _make_exception_wrapper
59 return _make_exception_wrapper
60
60
61
61
62 def VcsException(org_exc=None):
62 def VcsException(org_exc=None):
63 def _make_exception_wrapper(*args):
63 def _make_exception_wrapper(*args):
64 return _make_exception('error', org_exc, *args)
64 return _make_exception('error', org_exc, *args)
65 return _make_exception_wrapper
65 return _make_exception_wrapper
66
66
67
67
68 def RepositoryLockedException(org_exc=None):
68 def RepositoryLockedException(org_exc=None):
69 def _make_exception_wrapper(*args):
69 def _make_exception_wrapper(*args):
70 return _make_exception('repo_locked', org_exc, *args)
70 return _make_exception('repo_locked', org_exc, *args)
71 return _make_exception_wrapper
71 return _make_exception_wrapper
72
72
73
73
74 def RepositoryBranchProtectedException(org_exc=None):
74 def RepositoryBranchProtectedException(org_exc=None):
75 def _make_exception_wrapper(*args):
75 def _make_exception_wrapper(*args):
76 return _make_exception('repo_branch_protected', org_exc, *args)
76 return _make_exception('repo_branch_protected', org_exc, *args)
77 return _make_exception_wrapper
77 return _make_exception_wrapper
78
78
79
79
80 def RequirementException(org_exc=None):
80 def RequirementException(org_exc=None):
81 def _make_exception_wrapper(*args):
81 def _make_exception_wrapper(*args):
82 return _make_exception('requirement', org_exc, *args)
82 return _make_exception('requirement', org_exc, *args)
83 return _make_exception_wrapper
83 return _make_exception_wrapper
84
84
85
85
86 def UnhandledException(org_exc=None):
86 def UnhandledException(org_exc=None):
87 def _make_exception_wrapper(*args):
87 def _make_exception_wrapper(*args):
88 return _make_exception('unhandled', org_exc, *args)
88 return _make_exception('unhandled', org_exc, *args)
89 return _make_exception_wrapper
89 return _make_exception_wrapper
90
90
91
91
92 def URLError(org_exc=None):
92 def URLError(org_exc=None):
93 def _make_exception_wrapper(*args):
93 def _make_exception_wrapper(*args):
94 return _make_exception('url_error', org_exc, *args)
94 return _make_exception('url_error', org_exc, *args)
95 return _make_exception_wrapper
95 return _make_exception_wrapper
96
96
97
97
98 def SubrepoMergeException(org_exc=None):
98 def SubrepoMergeException(org_exc=None):
99 def _make_exception_wrapper(*args):
99 def _make_exception_wrapper(*args):
100 return _make_exception('subrepo_merge_error', org_exc, *args)
100 return _make_exception('subrepo_merge_error', org_exc, *args)
101 return _make_exception_wrapper
101 return _make_exception_wrapper
102
102
103
103
104 class HTTPRepoLocked(HTTPLocked):
104 class HTTPRepoLocked(HTTPLocked):
105 """
105 """
106 Subclass of HTTPLocked response that allows to set the title and status
106 Subclass of HTTPLocked response that allows to set the title and status
107 code via constructor arguments.
107 code via constructor arguments.
108 """
108 """
109 def __init__(self, title, status_code=None, **kwargs):
109 def __init__(self, title, status_code=None, **kwargs):
110 self.code = status_code or HTTPLocked.code
110 self.code = status_code or HTTPLocked.code
111 self.title = title
111 self.title = title
112 super(HTTPRepoLocked, self).__init__(**kwargs)
112 super().__init__(**kwargs)
113
113
114
114
115 class HTTPRepoBranchProtected(HTTPForbidden):
115 class HTTPRepoBranchProtected(HTTPForbidden):
116 def __init__(self, *args, **kwargs):
116 def __init__(self, *args, **kwargs):
117 super(HTTPForbidden, self).__init__(*args, **kwargs)
117 super(HTTPForbidden, self).__init__(*args, **kwargs)
118
118
119
119
120 class RefNotFoundException(KeyError):
120 class RefNotFoundException(KeyError):
121 pass
121 pass
122
122
123
123
124 class NoContentException(ValueError):
124 class NoContentException(ValueError):
125 pass
125 pass
@@ -1,19 +1,19 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18
19 from app import create_app
19 from .app import create_app # noqa
@@ -1,292 +1,296 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import re
18 import re
19 import logging
19 import logging
20 from wsgiref.util import FileWrapper
21
20
22 import simplejson as json
23 from pyramid.config import Configurator
21 from pyramid.config import Configurator
24 from pyramid.response import Response, FileIter
22 from pyramid.response import Response, FileIter
25 from pyramid.httpexceptions import (
23 from pyramid.httpexceptions import (
26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
24 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
27 HTTPUnprocessableEntity)
25 HTTPUnprocessableEntity)
28
26
27 from vcsserver.lib.rc_json import json
29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
28 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
29 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
31 from vcsserver.utils import safe_int
30 from vcsserver.str_utils import safe_int
32
31
33 log = logging.getLogger(__name__)
32 log = logging.getLogger(__name__)
34
33
35
34
36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
35 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' # +json ?
37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
36 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
38
37
39
38
40 def write_response_error(http_exception, text=None):
39 def write_response_error(http_exception, text=None):
41 content_type = GIT_LFS_CONTENT_TYPE + '+json'
40 content_type = GIT_LFS_CONTENT_TYPE + '+json'
42 _exception = http_exception(content_type=content_type)
41 _exception = http_exception(content_type=content_type)
43 _exception.content_type = content_type
42 _exception.content_type = content_type
44 if text:
43 if text:
45 _exception.body = json.dumps({'message': text})
44 _exception.body = json.dumps({'message': text})
46 log.debug('LFS: writing response of type %s to client with text:%s',
45 log.debug('LFS: writing response of type %s to client with text:%s',
47 http_exception, text)
46 http_exception, text)
48 return _exception
47 return _exception
49
48
50
49
51 class AuthHeaderRequired(object):
50 class AuthHeaderRequired:
52 """
51 """
53 Decorator to check if request has proper auth-header
52 Decorator to check if request has proper auth-header
54 """
53 """
55
54
56 def __call__(self, func):
55 def __call__(self, func):
57 return get_cython_compat_decorator(self.__wrapper, func)
56 return get_cython_compat_decorator(self.__wrapper, func)
58
57
59 def __wrapper(self, func, *fargs, **fkwargs):
58 def __wrapper(self, func, *fargs, **fkwargs):
60 request = fargs[1]
59 request = fargs[1]
61 auth = request.authorization
60 auth = request.authorization
62 if not auth:
61 if not auth:
63 return write_response_error(HTTPForbidden)
62 return write_response_error(HTTPForbidden)
64 return func(*fargs[1:], **fkwargs)
63 return func(*fargs[1:], **fkwargs)
65
64
66
65
67 # views
66 # views
68
67
69 def lfs_objects(request):
68 def lfs_objects(request):
70 # indicate not supported, V1 API
69 # indicate not supported, V1 API
71 log.warning('LFS: v1 api not supported, reporting it back to client')
70 log.warning('LFS: v1 api not supported, reporting it back to client')
72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
71 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
73
72
74
73
75 @AuthHeaderRequired()
74 @AuthHeaderRequired()
76 def lfs_objects_batch(request):
75 def lfs_objects_batch(request):
77 """
76 """
78 The client sends the following information to the Batch endpoint to transfer some objects:
77 The client sends the following information to the Batch endpoint to transfer some objects:
79
78
80 operation - Should be download or upload.
79 operation - Should be download or upload.
81 transfers - An optional Array of String identifiers for transfer
80 transfers - An optional Array of String identifiers for transfer
82 adapters that the client has configured. If omitted, the basic
81 adapters that the client has configured. If omitted, the basic
83 transfer adapter MUST be assumed by the server.
82 transfer adapter MUST be assumed by the server.
84 objects - An Array of objects to download.
83 objects - An Array of objects to download.
85 oid - String OID of the LFS object.
84 oid - String OID of the LFS object.
86 size - Integer byte size of the LFS object. Must be at least zero.
85 size - Integer byte size of the LFS object. Must be at least zero.
87 """
86 """
88 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
87 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
89 auth = request.authorization
88 auth = request.authorization
90 repo = request.matchdict.get('repo')
89 repo = request.matchdict.get('repo')
91 data = request.json
90 data = request.json
92 operation = data.get('operation')
91 operation = data.get('operation')
93 http_scheme = request.registry.git_lfs_http_scheme
92 http_scheme = request.registry.git_lfs_http_scheme
94
93
95 if operation not in ('download', 'upload'):
94 if operation not in ('download', 'upload'):
96 log.debug('LFS: unsupported operation:%s', operation)
95 log.debug('LFS: unsupported operation:%s', operation)
97 return write_response_error(
96 return write_response_error(
98 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
97 HTTPBadRequest, f'unsupported operation mode: `{operation}`')
99
98
100 if 'objects' not in data:
99 if 'objects' not in data:
101 log.debug('LFS: missing objects data')
100 log.debug('LFS: missing objects data')
102 return write_response_error(
101 return write_response_error(
103 HTTPBadRequest, 'missing objects data')
102 HTTPBadRequest, 'missing objects data')
104
103
105 log.debug('LFS: handling operation of type: %s', operation)
104 log.debug('LFS: handling operation of type: %s', operation)
106
105
107 objects = []
106 objects = []
108 for o in data['objects']:
107 for o in data['objects']:
109 try:
108 try:
110 oid = o['oid']
109 oid = o['oid']
111 obj_size = o['size']
110 obj_size = o['size']
112 except KeyError:
111 except KeyError:
113 log.exception('LFS, failed to extract data')
112 log.exception('LFS, failed to extract data')
114 return write_response_error(
113 return write_response_error(
115 HTTPBadRequest, 'unsupported data in objects')
114 HTTPBadRequest, 'unsupported data in objects')
116
115
117 obj_data = {'oid': oid}
116 obj_data = {'oid': oid}
117 if http_scheme == 'http':
118 # Note(marcink): when using http, we might have a custom port
119 # so we skip setting it to http, url dispatch then wont generate a port in URL
120 # for development we need this
121 http_scheme = None
118
122
119 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
123 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
120 _scheme=http_scheme)
124 _scheme=http_scheme)
121 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
125 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
122 _scheme=http_scheme)
126 _scheme=http_scheme)
123 store = LFSOidStore(
127 store = LFSOidStore(
124 oid, repo, store_location=request.registry.git_lfs_store_path)
128 oid, repo, store_location=request.registry.git_lfs_store_path)
125 handler = OidHandler(
129 handler = OidHandler(
126 store, repo, auth, oid, obj_size, obj_data,
130 store, repo, auth, oid, obj_size, obj_data,
127 obj_href, obj_verify_href)
131 obj_href, obj_verify_href)
128
132
129 # this verifies also OIDs
133 # this verifies also OIDs
130 actions, errors = handler.exec_operation(operation)
134 actions, errors = handler.exec_operation(operation)
131 if errors:
135 if errors:
132 log.warning('LFS: got following errors: %s', errors)
136 log.warning('LFS: got following errors: %s', errors)
133 obj_data['errors'] = errors
137 obj_data['errors'] = errors
134
138
135 if actions:
139 if actions:
136 obj_data['actions'] = actions
140 obj_data['actions'] = actions
137
141
138 obj_data['size'] = obj_size
142 obj_data['size'] = obj_size
139 obj_data['authenticated'] = True
143 obj_data['authenticated'] = True
140 objects.append(obj_data)
144 objects.append(obj_data)
141
145
142 result = {'objects': objects, 'transfer': 'basic'}
146 result = {'objects': objects, 'transfer': 'basic'}
143 log.debug('LFS Response %s', safe_result(result))
147 log.debug('LFS Response %s', safe_result(result))
144
148
145 return result
149 return result
146
150
147
151
148 def lfs_objects_oid_upload(request):
152 def lfs_objects_oid_upload(request):
149 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
153 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
150 repo = request.matchdict.get('repo')
154 repo = request.matchdict.get('repo')
151 oid = request.matchdict.get('oid')
155 oid = request.matchdict.get('oid')
152 store = LFSOidStore(
156 store = LFSOidStore(
153 oid, repo, store_location=request.registry.git_lfs_store_path)
157 oid, repo, store_location=request.registry.git_lfs_store_path)
154 engine = store.get_engine(mode='wb')
158 engine = store.get_engine(mode='wb')
155 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
159 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
156
160
157 body = request.environ['wsgi.input']
161 body = request.environ['wsgi.input']
158
162
159 with engine as f:
163 with engine as f:
160 blksize = 64 * 1024 # 64kb
164 blksize = 64 * 1024 # 64kb
161 while True:
165 while True:
162 # read in chunks as stream comes in from Gunicorn
166 # read in chunks as stream comes in from Gunicorn
163 # this is a specific Gunicorn support function.
167 # this is a specific Gunicorn support function.
164 # might work differently on waitress
168 # might work differently on waitress
165 chunk = body.read(blksize)
169 chunk = body.read(blksize)
166 if not chunk:
170 if not chunk:
167 break
171 break
168 f.write(chunk)
172 f.write(chunk)
169
173
170 return {'upload': 'ok'}
174 return {'upload': 'ok'}
171
175
172
176
173 def lfs_objects_oid_download(request):
177 def lfs_objects_oid_download(request):
174 repo = request.matchdict.get('repo')
178 repo = request.matchdict.get('repo')
175 oid = request.matchdict.get('oid')
179 oid = request.matchdict.get('oid')
176
180
177 store = LFSOidStore(
181 store = LFSOidStore(
178 oid, repo, store_location=request.registry.git_lfs_store_path)
182 oid, repo, store_location=request.registry.git_lfs_store_path)
179 if not store.has_oid():
183 if not store.has_oid():
180 log.debug('LFS: oid %s does not exists in store', oid)
184 log.debug('LFS: oid %s does not exists in store', oid)
181 return write_response_error(
185 return write_response_error(
182 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
186 HTTPNotFound, f'requested file with oid `{oid}` not found in store')
183
187
184 # TODO(marcink): support range header ?
188 # TODO(marcink): support range header ?
185 # Range: bytes=0-, `bytes=(\d+)\-.*`
189 # Range: bytes=0-, `bytes=(\d+)\-.*`
186
190
187 f = open(store.oid_path, 'rb')
191 f = open(store.oid_path, 'rb')
188 response = Response(
192 response = Response(
189 content_type='application/octet-stream', app_iter=FileIter(f))
193 content_type='application/octet-stream', app_iter=FileIter(f))
190 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
194 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
191 return response
195 return response
192
196
193
197
194 def lfs_objects_verify(request):
198 def lfs_objects_verify(request):
195 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
199 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
196 repo = request.matchdict.get('repo')
200 repo = request.matchdict.get('repo')
197
201
198 data = request.json
202 data = request.json
199 oid = data.get('oid')
203 oid = data.get('oid')
200 size = safe_int(data.get('size'))
204 size = safe_int(data.get('size'))
201
205
202 if not (oid and size):
206 if not (oid and size):
203 return write_response_error(
207 return write_response_error(
204 HTTPBadRequest, 'missing oid and size in request data')
208 HTTPBadRequest, 'missing oid and size in request data')
205
209
206 store = LFSOidStore(
210 store = LFSOidStore(
207 oid, repo, store_location=request.registry.git_lfs_store_path)
211 oid, repo, store_location=request.registry.git_lfs_store_path)
208 if not store.has_oid():
212 if not store.has_oid():
209 log.debug('LFS: oid %s does not exists in store', oid)
213 log.debug('LFS: oid %s does not exists in store', oid)
210 return write_response_error(
214 return write_response_error(
211 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
215 HTTPNotFound, f'oid `{oid}` does not exists in store')
212
216
213 store_size = store.size_oid()
217 store_size = store.size_oid()
214 if store_size != size:
218 if store_size != size:
215 msg = 'requested file size mismatch store size:%s requested:%s' % (
219 msg = 'requested file size mismatch store size:{} requested:{}'.format(
216 store_size, size)
220 store_size, size)
217 return write_response_error(
221 return write_response_error(
218 HTTPUnprocessableEntity, msg)
222 HTTPUnprocessableEntity, msg)
219
223
220 return {'message': {'size': 'ok', 'in_store': 'ok'}}
224 return {'message': {'size': 'ok', 'in_store': 'ok'}}
221
225
222
226
223 def lfs_objects_lock(request):
227 def lfs_objects_lock(request):
224 return write_response_error(
228 return write_response_error(
225 HTTPNotImplemented, 'GIT LFS locking api not supported')
229 HTTPNotImplemented, 'GIT LFS locking api not supported')
226
230
227
231
228 def not_found(request):
232 def not_found(request):
229 return write_response_error(
233 return write_response_error(
230 HTTPNotFound, 'request path not found')
234 HTTPNotFound, 'request path not found')
231
235
232
236
233 def lfs_disabled(request):
237 def lfs_disabled(request):
234 return write_response_error(
238 return write_response_error(
235 HTTPNotImplemented, 'GIT LFS disabled for this repo')
239 HTTPNotImplemented, 'GIT LFS disabled for this repo')
236
240
237
241
238 def git_lfs_app(config):
242 def git_lfs_app(config):
239
243
240 # v1 API deprecation endpoint
244 # v1 API deprecation endpoint
241 config.add_route('lfs_objects',
245 config.add_route('lfs_objects',
242 '/{repo:.*?[^/]}/info/lfs/objects')
246 '/{repo:.*?[^/]}/info/lfs/objects')
243 config.add_view(lfs_objects, route_name='lfs_objects',
247 config.add_view(lfs_objects, route_name='lfs_objects',
244 request_method='POST', renderer='json')
248 request_method='POST', renderer='json')
245
249
246 # locking API
250 # locking API
247 config.add_route('lfs_objects_lock',
251 config.add_route('lfs_objects_lock',
248 '/{repo:.*?[^/]}/info/lfs/locks')
252 '/{repo:.*?[^/]}/info/lfs/locks')
249 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
253 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
250 request_method=('POST', 'GET'), renderer='json')
254 request_method=('POST', 'GET'), renderer='json')
251
255
252 config.add_route('lfs_objects_lock_verify',
256 config.add_route('lfs_objects_lock_verify',
253 '/{repo:.*?[^/]}/info/lfs/locks/verify')
257 '/{repo:.*?[^/]}/info/lfs/locks/verify')
254 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
258 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
255 request_method=('POST', 'GET'), renderer='json')
259 request_method=('POST', 'GET'), renderer='json')
256
260
257 # batch API
261 # batch API
258 config.add_route('lfs_objects_batch',
262 config.add_route('lfs_objects_batch',
259 '/{repo:.*?[^/]}/info/lfs/objects/batch')
263 '/{repo:.*?[^/]}/info/lfs/objects/batch')
260 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
264 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
261 request_method='POST', renderer='json')
265 request_method='POST', renderer='json')
262
266
263 # oid upload/download API
267 # oid upload/download API
264 config.add_route('lfs_objects_oid',
268 config.add_route('lfs_objects_oid',
265 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
269 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
266 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
270 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
267 request_method='PUT', renderer='json')
271 request_method='PUT', renderer='json')
268 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
272 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
269 request_method='GET', renderer='json')
273 request_method='GET', renderer='json')
270
274
271 # verification API
275 # verification API
272 config.add_route('lfs_objects_verify',
276 config.add_route('lfs_objects_verify',
273 '/{repo:.*?[^/]}/info/lfs/verify')
277 '/{repo:.*?[^/]}/info/lfs/verify')
274 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
278 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
275 request_method='POST', renderer='json')
279 request_method='POST', renderer='json')
276
280
277 # not found handler for API
281 # not found handler for API
278 config.add_notfound_view(not_found, renderer='json')
282 config.add_notfound_view(not_found, renderer='json')
279
283
280
284
281 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
285 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
282 config = Configurator()
286 config = Configurator()
283 if git_lfs_enabled:
287 if git_lfs_enabled:
284 config.include(git_lfs_app)
288 config.include(git_lfs_app)
285 config.registry.git_lfs_store_path = git_lfs_store_path
289 config.registry.git_lfs_store_path = git_lfs_store_path
286 config.registry.git_lfs_http_scheme = git_lfs_http_scheme
290 config.registry.git_lfs_http_scheme = git_lfs_http_scheme
287 else:
291 else:
288 # not found handler for API, reporting disabled LFS support
292 # not found handler for API, reporting disabled LFS support
289 config.add_notfound_view(lfs_disabled, renderer='json')
293 config.add_notfound_view(lfs_disabled, renderer='json')
290
294
291 app = config.make_wsgi_app()
295 app = config.make_wsgi_app()
292 return app
296 return app
@@ -1,175 +1,177 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import shutil
19 import shutil
20 import logging
20 import logging
21 from collections import OrderedDict
21 from collections import OrderedDict
22
22
23 log = logging.getLogger(__name__)
23 log = logging.getLogger(__name__)
24
24
25
25
26 class OidHandler(object):
26 class OidHandler:
27
27
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 obj_verify_href=None):
29 obj_verify_href=None):
30 self.current_store = store
30 self.current_store = store
31 self.repo_name = repo_name
31 self.repo_name = repo_name
32 self.auth = auth
32 self.auth = auth
33 self.oid = oid
33 self.oid = oid
34 self.obj_size = obj_size
34 self.obj_size = obj_size
35 self.obj_data = obj_data
35 self.obj_data = obj_data
36 self.obj_href = obj_href
36 self.obj_href = obj_href
37 self.obj_verify_href = obj_verify_href
37 self.obj_verify_href = obj_verify_href
38
38
39 def get_store(self, mode=None):
39 def get_store(self, mode=None):
40 return self.current_store
40 return self.current_store
41
41
42 def get_auth(self):
42 def get_auth(self):
43 """returns auth header for re-use in upload/download"""
43 """returns auth header for re-use in upload/download"""
44 return " ".join(self.auth)
44 return " ".join(self.auth)
45
45
46 def download(self):
46 def download(self):
47
47
48 store = self.get_store()
48 store = self.get_store()
49 response = None
49 response = None
50 has_errors = None
50 has_errors = None
51
51
52 if not store.has_oid():
52 if not store.has_oid():
53 # error reply back to client that something is wrong with dl
53 # error reply back to client that something is wrong with dl
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
54 err_msg = f'object: {store.oid} does not exist in store'
55 has_errors = OrderedDict(
55 has_errors = OrderedDict(
56 error=OrderedDict(
56 error=OrderedDict(
57 code=404,
57 code=404,
58 message=err_msg
58 message=err_msg
59 )
59 )
60 )
60 )
61
61
62 download_action = OrderedDict(
62 download_action = OrderedDict(
63 href=self.obj_href,
63 href=self.obj_href,
64 header=OrderedDict([("Authorization", self.get_auth())])
64 header=OrderedDict([("Authorization", self.get_auth())])
65 )
65 )
66 if not has_errors:
66 if not has_errors:
67 response = OrderedDict(download=download_action)
67 response = OrderedDict(download=download_action)
68 return response, has_errors
68 return response, has_errors
69
69
70 def upload(self, skip_existing=True):
70 def upload(self, skip_existing=True):
71 """
71 """
72 Write upload action for git-lfs server
72 Write upload action for git-lfs server
73 """
73 """
74
74
75 store = self.get_store()
75 store = self.get_store()
76 response = None
76 response = None
77 has_errors = None
77 has_errors = None
78
78
79 # verify if we have the OID before, if we do, reply with empty
79 # verify if we have the OID before, if we do, reply with empty
80 if store.has_oid():
80 if store.has_oid():
81 log.debug('LFS: store already has oid %s', store.oid)
81 log.debug('LFS: store already has oid %s', store.oid)
82
82
83 # validate size
83 # validate size
84 store_size = store.size_oid()
84 store_size = store.size_oid()
85 size_match = store_size == self.obj_size
85 size_match = store_size == self.obj_size
86 if not size_match:
86 if not size_match:
87 log.warning(
87 log.warning(
88 'LFS: size mismatch for oid:%s, in store:%s expected: %s',
88 'LFS: size mismatch for oid:%s, in store:%s expected: %s',
89 self.oid, store_size, self.obj_size)
89 self.oid, store_size, self.obj_size)
90 elif skip_existing:
90 elif skip_existing:
91 log.debug('LFS: skipping further action as oid is existing')
91 log.debug('LFS: skipping further action as oid is existing')
92 return response, has_errors
92 return response, has_errors
93
93
94 chunked = ("Transfer-Encoding", "chunked")
94 chunked = ("Transfer-Encoding", "chunked")
95 upload_action = OrderedDict(
95 upload_action = OrderedDict(
96 href=self.obj_href,
96 href=self.obj_href,
97 header=OrderedDict([("Authorization", self.get_auth()), chunked])
97 header=OrderedDict([("Authorization", self.get_auth()), chunked])
98 )
98 )
99 if not has_errors:
99 if not has_errors:
100 response = OrderedDict(upload=upload_action)
100 response = OrderedDict(upload=upload_action)
101 # if specified in handler, return the verification endpoint
101 # if specified in handler, return the verification endpoint
102 if self.obj_verify_href:
102 if self.obj_verify_href:
103 verify_action = OrderedDict(
103 verify_action = OrderedDict(
104 href=self.obj_verify_href,
104 href=self.obj_verify_href,
105 header=OrderedDict([("Authorization", self.get_auth())])
105 header=OrderedDict([("Authorization", self.get_auth())])
106 )
106 )
107 response['verify'] = verify_action
107 response['verify'] = verify_action
108 return response, has_errors
108 return response, has_errors
109
109
110 def exec_operation(self, operation, *args, **kwargs):
110 def exec_operation(self, operation, *args, **kwargs):
111 handler = getattr(self, operation)
111 handler = getattr(self, operation)
112 log.debug('LFS: handling request using %s handler', handler)
112 log.debug('LFS: handling request using %s handler', handler)
113 return handler(*args, **kwargs)
113 return handler(*args, **kwargs)
114
114
115
115
116 class LFSOidStore(object):
116 class LFSOidStore:
117
117
118 def __init__(self, oid, repo, store_location=None):
118 def __init__(self, oid, repo, store_location=None):
119 self.oid = oid
119 self.oid = oid
120 self.repo = repo
120 self.repo = repo
121 self.store_path = store_location or self.get_default_store()
121 defined_store_path = store_location or self.get_default_store()
122 self.store_suffix = f"/objects/{oid[:2]}/{oid[2:4]}"
123 self.store_path = f"{defined_store_path.rstrip('/')}{self.store_suffix}"
122 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
124 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
123 self.oid_path = os.path.join(self.store_path, oid)
125 self.oid_path = os.path.join(self.store_path, oid)
124 self.fd = None
126 self.fd = None
125
127
126 def get_engine(self, mode):
128 def get_engine(self, mode):
127 """
129 """
128 engine = .get_engine(mode='wb')
130 engine = .get_engine(mode='wb')
129 with engine as f:
131 with engine as f:
130 f.write('...')
132 f.write('...')
131 """
133 """
132
134
133 class StoreEngine(object):
135 class StoreEngine:
134 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
136 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
135 self.mode = mode
137 self.mode = mode
136 self.store_path = store_path
138 self.store_path = store_path
137 self.oid_path = oid_path
139 self.oid_path = oid_path
138 self.tmp_oid_path = tmp_oid_path
140 self.tmp_oid_path = tmp_oid_path
139
141
140 def __enter__(self):
142 def __enter__(self):
141 if not os.path.isdir(self.store_path):
143 if not os.path.isdir(self.store_path):
142 os.makedirs(self.store_path)
144 os.makedirs(self.store_path)
143
145
144 # TODO(marcink): maybe write metadata here with size/oid ?
146 # TODO(marcink): maybe write metadata here with size/oid ?
145 fd = open(self.tmp_oid_path, self.mode)
147 fd = open(self.tmp_oid_path, self.mode)
146 self.fd = fd
148 self.fd = fd
147 return fd
149 return fd
148
150
149 def __exit__(self, exc_type, exc_value, traceback):
151 def __exit__(self, exc_type, exc_value, traceback):
150 # close tmp file, and rename to final destination
152 # close tmp file, and rename to final destination
151 self.fd.close()
153 self.fd.close()
152 shutil.move(self.tmp_oid_path, self.oid_path)
154 shutil.move(self.tmp_oid_path, self.oid_path)
153
155
154 return StoreEngine(
156 return StoreEngine(
155 mode, self.store_path, self.oid_path, self.tmp_oid_path)
157 mode, self.store_path, self.oid_path, self.tmp_oid_path)
156
158
157 def get_default_store(self):
159 def get_default_store(self):
158 """
160 """
159 Default store, consistent with defaults of Mercurial large files store
161 Default store, consistent with defaults of Mercurial large files store
160 which is /home/username/.cache/largefiles
162 which is /home/username/.cache/largefiles
161 """
163 """
162 user_home = os.path.expanduser("~")
164 user_home = os.path.expanduser("~")
163 return os.path.join(user_home, '.cache', 'lfs-store')
165 return os.path.join(user_home, '.cache', 'lfs-store')
164
166
165 def has_oid(self):
167 def has_oid(self):
166 return os.path.exists(os.path.join(self.store_path, self.oid))
168 return os.path.exists(os.path.join(self.store_path, self.oid))
167
169
168 def size_oid(self):
170 def size_oid(self):
169 size = -1
171 size = -1
170
172
171 if self.has_oid():
173 if self.has_oid():
172 oid = os.path.join(self.store_path, self.oid)
174 oid = os.path.join(self.store_path, self.oid)
173 size = os.stat(oid).st_size
175 size = os.stat(oid).st_size
174
176
175 return size
177 return size
@@ -1,16 +1,16 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@@ -1,272 +1,274 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from webtest.app import TestApp as WebObTestApp
20 from webtest.app import TestApp as WebObTestApp
21 import simplejson as json
22
21
22 from vcsserver.lib.rc_json import json
23 from vcsserver.str_utils import safe_bytes
23 from vcsserver.git_lfs.app import create_app
24 from vcsserver.git_lfs.app import create_app
25 from vcsserver.git_lfs.lib import LFSOidStore
24
26
25
27
26 @pytest.fixture(scope='function')
28 @pytest.fixture(scope='function')
27 def git_lfs_app(tmpdir):
29 def git_lfs_app(tmpdir):
28 custom_app = WebObTestApp(create_app(
30 custom_app = WebObTestApp(create_app(
29 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
31 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
30 git_lfs_http_scheme='http'))
32 git_lfs_http_scheme='http'))
31 custom_app._store = str(tmpdir)
33 custom_app._store = str(tmpdir)
32 return custom_app
34 return custom_app
33
35
34
36
35 @pytest.fixture(scope='function')
37 @pytest.fixture(scope='function')
36 def git_lfs_https_app(tmpdir):
38 def git_lfs_https_app(tmpdir):
37 custom_app = WebObTestApp(create_app(
39 custom_app = WebObTestApp(create_app(
38 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
40 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
39 git_lfs_http_scheme='https'))
41 git_lfs_http_scheme='https'))
40 custom_app._store = str(tmpdir)
42 custom_app._store = str(tmpdir)
41 return custom_app
43 return custom_app
42
44
43
45
44 @pytest.fixture()
46 @pytest.fixture()
45 def http_auth():
47 def http_auth():
46 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
48 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
47
49
48
50
49 class TestLFSApplication(object):
51 class TestLFSApplication:
50
52
51 def test_app_wrong_path(self, git_lfs_app):
53 def test_app_wrong_path(self, git_lfs_app):
52 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
54 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
53
55
54 def test_app_deprecated_endpoint(self, git_lfs_app):
56 def test_app_deprecated_endpoint(self, git_lfs_app):
55 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
57 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
56 assert response.status_code == 501
58 assert response.status_code == 501
57 assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
59 assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
58
60
59 def test_app_lock_verify_api_not_available(self, git_lfs_app):
61 def test_app_lock_verify_api_not_available(self, git_lfs_app):
60 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
62 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
61 assert response.status_code == 501
63 assert response.status_code == 501
62 assert json.loads(response.text) == {
64 assert json.loads(response.text) == {
63 u'message': u'GIT LFS locking api not supported'}
65 'message': 'GIT LFS locking api not supported'}
64
66
65 def test_app_lock_api_not_available(self, git_lfs_app):
67 def test_app_lock_api_not_available(self, git_lfs_app):
66 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
68 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
67 assert response.status_code == 501
69 assert response.status_code == 501
68 assert json.loads(response.text) == {
70 assert json.loads(response.text) == {
69 u'message': u'GIT LFS locking api not supported'}
71 'message': 'GIT LFS locking api not supported'}
70
72
71 def test_app_batch_api_missing_auth(self, git_lfs_app):
73 def test_app_batch_api_missing_auth(self, git_lfs_app):
72 git_lfs_app.post_json(
74 git_lfs_app.post_json(
73 '/repo/info/lfs/objects/batch', params={}, status=403)
75 '/repo/info/lfs/objects/batch', params={}, status=403)
74
76
75 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
77 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
76 response = git_lfs_app.post_json(
78 response = git_lfs_app.post_json(
77 '/repo/info/lfs/objects/batch', params={}, status=400,
79 '/repo/info/lfs/objects/batch', params={}, status=400,
78 extra_environ=http_auth)
80 extra_environ=http_auth)
79 assert json.loads(response.text) == {
81 assert json.loads(response.text) == {
80 u'message': u'unsupported operation mode: `None`'}
82 'message': 'unsupported operation mode: `None`'}
81
83
82 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
84 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
83 response = git_lfs_app.post_json(
85 response = git_lfs_app.post_json(
84 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
86 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
85 status=400, extra_environ=http_auth)
87 status=400, extra_environ=http_auth)
86 assert json.loads(response.text) == {
88 assert json.loads(response.text) == {
87 u'message': u'missing objects data'}
89 'message': 'missing objects data'}
88
90
89 def test_app_batch_api_unsupported_data_in_objects(
91 def test_app_batch_api_unsupported_data_in_objects(
90 self, git_lfs_app, http_auth):
92 self, git_lfs_app, http_auth):
91 params = {'operation': 'download',
93 params = {'operation': 'download',
92 'objects': [{}]}
94 'objects': [{}]}
93 response = git_lfs_app.post_json(
95 response = git_lfs_app.post_json(
94 '/repo/info/lfs/objects/batch', params=params, status=400,
96 '/repo/info/lfs/objects/batch', params=params, status=400,
95 extra_environ=http_auth)
97 extra_environ=http_auth)
96 assert json.loads(response.text) == {
98 assert json.loads(response.text) == {
97 u'message': u'unsupported data in objects'}
99 'message': 'unsupported data in objects'}
98
100
99 def test_app_batch_api_download_missing_object(
101 def test_app_batch_api_download_missing_object(
100 self, git_lfs_app, http_auth):
102 self, git_lfs_app, http_auth):
101 params = {'operation': 'download',
103 params = {'operation': 'download',
102 'objects': [{'oid': '123', 'size': '1024'}]}
104 'objects': [{'oid': '123', 'size': '1024'}]}
103 response = git_lfs_app.post_json(
105 response = git_lfs_app.post_json(
104 '/repo/info/lfs/objects/batch', params=params,
106 '/repo/info/lfs/objects/batch', params=params,
105 extra_environ=http_auth)
107 extra_environ=http_auth)
106
108
107 expected_objects = [
109 expected_objects = [
108 {u'authenticated': True,
110 {'authenticated': True,
109 u'errors': {u'error': {
111 'errors': {'error': {
110 u'code': 404,
112 'code': 404,
111 u'message': u'object: 123 does not exist in store'}},
113 'message': 'object: 123 does not exist in store'}},
112 u'oid': u'123',
114 'oid': '123',
113 u'size': u'1024'}
115 'size': '1024'}
114 ]
116 ]
115 assert json.loads(response.text) == {
117 assert json.loads(response.text) == {
116 'objects': expected_objects, 'transfer': 'basic'}
118 'objects': expected_objects, 'transfer': 'basic'}
117
119
118 def test_app_batch_api_download(self, git_lfs_app, http_auth):
120 def test_app_batch_api_download(self, git_lfs_app, http_auth):
119 oid = '456'
121 oid = '456'
120 oid_path = os.path.join(git_lfs_app._store, oid)
122 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
121 if not os.path.isdir(os.path.dirname(oid_path)):
123 if not os.path.isdir(os.path.dirname(oid_path)):
122 os.makedirs(os.path.dirname(oid_path))
124 os.makedirs(os.path.dirname(oid_path))
123 with open(oid_path, 'wb') as f:
125 with open(oid_path, 'wb') as f:
124 f.write('OID_CONTENT')
126 f.write(safe_bytes('OID_CONTENT'))
125
127
126 params = {'operation': 'download',
128 params = {'operation': 'download',
127 'objects': [{'oid': oid, 'size': '1024'}]}
129 'objects': [{'oid': oid, 'size': '1024'}]}
128 response = git_lfs_app.post_json(
130 response = git_lfs_app.post_json(
129 '/repo/info/lfs/objects/batch', params=params,
131 '/repo/info/lfs/objects/batch', params=params,
130 extra_environ=http_auth)
132 extra_environ=http_auth)
131
133
132 expected_objects = [
134 expected_objects = [
133 {u'authenticated': True,
135 {'authenticated': True,
134 u'actions': {
136 'actions': {
135 u'download': {
137 'download': {
136 u'header': {u'Authorization': u'Basic XXXXX'},
138 'header': {'Authorization': 'Basic XXXXX'},
137 u'href': u'http://localhost/repo/info/lfs/objects/456'},
139 'href': 'http://localhost/repo/info/lfs/objects/456'},
138 },
140 },
139 u'oid': u'456',
141 'oid': '456',
140 u'size': u'1024'}
142 'size': '1024'}
141 ]
143 ]
142 assert json.loads(response.text) == {
144 assert json.loads(response.text) == {
143 'objects': expected_objects, 'transfer': 'basic'}
145 'objects': expected_objects, 'transfer': 'basic'}
144
146
145 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
147 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
146 params = {'operation': 'upload',
148 params = {'operation': 'upload',
147 'objects': [{'oid': '123', 'size': '1024'}]}
149 'objects': [{'oid': '123', 'size': '1024'}]}
148 response = git_lfs_app.post_json(
150 response = git_lfs_app.post_json(
149 '/repo/info/lfs/objects/batch', params=params,
151 '/repo/info/lfs/objects/batch', params=params,
150 extra_environ=http_auth)
152 extra_environ=http_auth)
151 expected_objects = [
153 expected_objects = [
152 {u'authenticated': True,
154 {'authenticated': True,
153 u'actions': {
155 'actions': {
154 u'upload': {
156 'upload': {
155 u'header': {u'Authorization': u'Basic XXXXX',
157 'header': {'Authorization': 'Basic XXXXX',
156 u'Transfer-Encoding': u'chunked'},
158 'Transfer-Encoding': 'chunked'},
157 u'href': u'http://localhost/repo/info/lfs/objects/123'},
159 'href': 'http://localhost/repo/info/lfs/objects/123'},
158 u'verify': {
160 'verify': {
159 u'header': {u'Authorization': u'Basic XXXXX'},
161 'header': {'Authorization': 'Basic XXXXX'},
160 u'href': u'http://localhost/repo/info/lfs/verify'}
162 'href': 'http://localhost/repo/info/lfs/verify'}
161 },
163 },
162 u'oid': u'123',
164 'oid': '123',
163 u'size': u'1024'}
165 'size': '1024'}
164 ]
166 ]
165 assert json.loads(response.text) == {
167 assert json.loads(response.text) == {
166 'objects': expected_objects, 'transfer': 'basic'}
168 'objects': expected_objects, 'transfer': 'basic'}
167
169
168 def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
170 def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
169 params = {'operation': 'upload',
171 params = {'operation': 'upload',
170 'objects': [{'oid': '123', 'size': '1024'}]}
172 'objects': [{'oid': '123', 'size': '1024'}]}
171 response = git_lfs_https_app.post_json(
173 response = git_lfs_https_app.post_json(
172 '/repo/info/lfs/objects/batch', params=params,
174 '/repo/info/lfs/objects/batch', params=params,
173 extra_environ=http_auth)
175 extra_environ=http_auth)
174 expected_objects = [
176 expected_objects = [
175 {u'authenticated': True,
177 {'authenticated': True,
176 u'actions': {
178 'actions': {
177 u'upload': {
179 'upload': {
178 u'header': {u'Authorization': u'Basic XXXXX',
180 'header': {'Authorization': 'Basic XXXXX',
179 u'Transfer-Encoding': u'chunked'},
181 'Transfer-Encoding': 'chunked'},
180 u'href': u'https://localhost/repo/info/lfs/objects/123'},
182 'href': 'https://localhost/repo/info/lfs/objects/123'},
181 u'verify': {
183 'verify': {
182 u'header': {u'Authorization': u'Basic XXXXX'},
184 'header': {'Authorization': 'Basic XXXXX'},
183 u'href': u'https://localhost/repo/info/lfs/verify'}
185 'href': 'https://localhost/repo/info/lfs/verify'}
184 },
186 },
185 u'oid': u'123',
187 'oid': '123',
186 u'size': u'1024'}
188 'size': '1024'}
187 ]
189 ]
188 assert json.loads(response.text) == {
190 assert json.loads(response.text) == {
189 'objects': expected_objects, 'transfer': 'basic'}
191 'objects': expected_objects, 'transfer': 'basic'}
190
192
191 def test_app_verify_api_missing_data(self, git_lfs_app):
193 def test_app_verify_api_missing_data(self, git_lfs_app):
192 params = {'oid': 'missing'}
194 params = {'oid': 'missing'}
193 response = git_lfs_app.post_json(
195 response = git_lfs_app.post_json(
194 '/repo/info/lfs/verify', params=params,
196 '/repo/info/lfs/verify', params=params,
195 status=400)
197 status=400)
196
198
197 assert json.loads(response.text) == {
199 assert json.loads(response.text) == {
198 u'message': u'missing oid and size in request data'}
200 'message': 'missing oid and size in request data'}
199
201
200 def test_app_verify_api_missing_obj(self, git_lfs_app):
202 def test_app_verify_api_missing_obj(self, git_lfs_app):
201 params = {'oid': 'missing', 'size': '1024'}
203 params = {'oid': 'missing', 'size': '1024'}
202 response = git_lfs_app.post_json(
204 response = git_lfs_app.post_json(
203 '/repo/info/lfs/verify', params=params,
205 '/repo/info/lfs/verify', params=params,
204 status=404)
206 status=404)
205
207
206 assert json.loads(response.text) == {
208 assert json.loads(response.text) == {
207 u'message': u'oid `missing` does not exists in store'}
209 'message': 'oid `missing` does not exists in store'}
208
210
209 def test_app_verify_api_size_mismatch(self, git_lfs_app):
211 def test_app_verify_api_size_mismatch(self, git_lfs_app):
210 oid = 'existing'
212 oid = 'existing'
211 oid_path = os.path.join(git_lfs_app._store, oid)
213 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
212 if not os.path.isdir(os.path.dirname(oid_path)):
214 if not os.path.isdir(os.path.dirname(oid_path)):
213 os.makedirs(os.path.dirname(oid_path))
215 os.makedirs(os.path.dirname(oid_path))
214 with open(oid_path, 'wb') as f:
216 with open(oid_path, 'wb') as f:
215 f.write('OID_CONTENT')
217 f.write(safe_bytes('OID_CONTENT'))
216
218
217 params = {'oid': oid, 'size': '1024'}
219 params = {'oid': oid, 'size': '1024'}
218 response = git_lfs_app.post_json(
220 response = git_lfs_app.post_json(
219 '/repo/info/lfs/verify', params=params, status=422)
221 '/repo/info/lfs/verify', params=params, status=422)
220
222
221 assert json.loads(response.text) == {
223 assert json.loads(response.text) == {
222 u'message': u'requested file size mismatch '
224 'message': 'requested file size mismatch '
223 u'store size:11 requested:1024'}
225 'store size:11 requested:1024'}
224
226
225 def test_app_verify_api(self, git_lfs_app):
227 def test_app_verify_api(self, git_lfs_app):
226 oid = 'existing'
228 oid = 'existing'
227 oid_path = os.path.join(git_lfs_app._store, oid)
229 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
228 if not os.path.isdir(os.path.dirname(oid_path)):
230 if not os.path.isdir(os.path.dirname(oid_path)):
229 os.makedirs(os.path.dirname(oid_path))
231 os.makedirs(os.path.dirname(oid_path))
230 with open(oid_path, 'wb') as f:
232 with open(oid_path, 'wb') as f:
231 f.write('OID_CONTENT')
233 f.write(safe_bytes('OID_CONTENT'))
232
234
233 params = {'oid': oid, 'size': 11}
235 params = {'oid': oid, 'size': 11}
234 response = git_lfs_app.post_json(
236 response = git_lfs_app.post_json(
235 '/repo/info/lfs/verify', params=params)
237 '/repo/info/lfs/verify', params=params)
236
238
237 assert json.loads(response.text) == {
239 assert json.loads(response.text) == {
238 u'message': {u'size': u'ok', u'in_store': u'ok'}}
240 'message': {'size': 'ok', 'in_store': 'ok'}}
239
241
240 def test_app_download_api_oid_not_existing(self, git_lfs_app):
242 def test_app_download_api_oid_not_existing(self, git_lfs_app):
241 oid = 'missing'
243 oid = 'missing'
242
244
243 response = git_lfs_app.get(
245 response = git_lfs_app.get(
244 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
246 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
245
247
246 assert json.loads(response.text) == {
248 assert json.loads(response.text) == {
247 u'message': u'requested file with oid `missing` not found in store'}
249 'message': 'requested file with oid `missing` not found in store'}
248
250
249 def test_app_download_api(self, git_lfs_app):
251 def test_app_download_api(self, git_lfs_app):
250 oid = 'existing'
252 oid = 'existing'
251 oid_path = os.path.join(git_lfs_app._store, oid)
253 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
252 if not os.path.isdir(os.path.dirname(oid_path)):
254 if not os.path.isdir(os.path.dirname(oid_path)):
253 os.makedirs(os.path.dirname(oid_path))
255 os.makedirs(os.path.dirname(oid_path))
254 with open(oid_path, 'wb') as f:
256 with open(oid_path, 'wb') as f:
255 f.write('OID_CONTENT')
257 f.write(safe_bytes('OID_CONTENT'))
256
258
257 response = git_lfs_app.get(
259 response = git_lfs_app.get(
258 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
260 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
259 assert response
261 assert response
260
262
261 def test_app_upload(self, git_lfs_app):
263 def test_app_upload(self, git_lfs_app):
262 oid = 'uploaded'
264 oid = 'uploaded'
263
265
264 response = git_lfs_app.put(
266 response = git_lfs_app.put(
265 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
267 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
266
268
267 assert json.loads(response.text) == {u'upload': u'ok'}
269 assert json.loads(response.text) == {'upload': 'ok'}
268
270
269 # verify that we actually wrote that OID
271 # verify that we actually wrote that OID
270 oid_path = os.path.join(git_lfs_app._store, oid)
272 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
271 assert os.path.isfile(oid_path)
273 assert os.path.isfile(oid_path)
272 assert 'CONTENT' == open(oid_path).read()
274 assert 'CONTENT' == open(oid_path).read()
@@ -1,141 +1,142 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from vcsserver.str_utils import safe_bytes
20 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
21 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
21
22
22
23
23 @pytest.fixture()
24 @pytest.fixture()
24 def lfs_store(tmpdir):
25 def lfs_store(tmpdir):
25 repo = 'test'
26 repo = 'test'
26 oid = '123456789'
27 oid = '123456789'
27 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
28 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
28 return store
29 return store
29
30
30
31
31 @pytest.fixture()
32 @pytest.fixture()
32 def oid_handler(lfs_store):
33 def oid_handler(lfs_store):
33 store = lfs_store
34 store = lfs_store
34 repo = store.repo
35 repo = store.repo
35 oid = store.oid
36 oid = store.oid
36
37
37 oid_handler = OidHandler(
38 oid_handler = OidHandler(
38 store=store, repo_name=repo, auth=('basic', 'xxxx'),
39 store=store, repo_name=repo, auth=('basic', 'xxxx'),
39 oid=oid,
40 oid=oid,
40 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
41 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
41 obj_verify_href='http://localhost/verify')
42 obj_verify_href='http://localhost/verify')
42 return oid_handler
43 return oid_handler
43
44
44
45
45 class TestOidHandler(object):
46 class TestOidHandler:
46
47
47 @pytest.mark.parametrize('exec_action', [
48 @pytest.mark.parametrize('exec_action', [
48 'download',
49 'download',
49 'upload',
50 'upload',
50 ])
51 ])
51 def test_exec_action(self, exec_action, oid_handler):
52 def test_exec_action(self, exec_action, oid_handler):
52 handler = oid_handler.exec_operation(exec_action)
53 handler = oid_handler.exec_operation(exec_action)
53 assert handler
54 assert handler
54
55
55 def test_exec_action_undefined(self, oid_handler):
56 def test_exec_action_undefined(self, oid_handler):
56 with pytest.raises(AttributeError):
57 with pytest.raises(AttributeError):
57 oid_handler.exec_operation('wrong')
58 oid_handler.exec_operation('wrong')
58
59
59 def test_download_oid_not_existing(self, oid_handler):
60 def test_download_oid_not_existing(self, oid_handler):
60 response, has_errors = oid_handler.exec_operation('download')
61 response, has_errors = oid_handler.exec_operation('download')
61
62
62 assert response is None
63 assert response is None
63 assert has_errors['error'] == {
64 assert has_errors['error'] == {
64 'code': 404,
65 'code': 404,
65 'message': 'object: 123456789 does not exist in store'}
66 'message': 'object: 123456789 does not exist in store'}
66
67
67 def test_download_oid(self, oid_handler):
68 def test_download_oid(self, oid_handler):
68 store = oid_handler.get_store()
69 store = oid_handler.get_store()
69 if not os.path.isdir(os.path.dirname(store.oid_path)):
70 if not os.path.isdir(os.path.dirname(store.oid_path)):
70 os.makedirs(os.path.dirname(store.oid_path))
71 os.makedirs(os.path.dirname(store.oid_path))
71
72
72 with open(store.oid_path, 'wb') as f:
73 with open(store.oid_path, 'wb') as f:
73 f.write('CONTENT')
74 f.write(safe_bytes('CONTENT'))
74
75
75 response, has_errors = oid_handler.exec_operation('download')
76 response, has_errors = oid_handler.exec_operation('download')
76
77
77 assert has_errors is None
78 assert has_errors is None
78 assert response['download'] == {
79 assert response['download'] == {
79 'header': {'Authorization': 'basic xxxx'},
80 'header': {'Authorization': 'basic xxxx'},
80 'href': 'http://localhost/handle_oid'
81 'href': 'http://localhost/handle_oid'
81 }
82 }
82
83
83 def test_upload_oid_that_exists(self, oid_handler):
84 def test_upload_oid_that_exists(self, oid_handler):
84 store = oid_handler.get_store()
85 store = oid_handler.get_store()
85 if not os.path.isdir(os.path.dirname(store.oid_path)):
86 if not os.path.isdir(os.path.dirname(store.oid_path)):
86 os.makedirs(os.path.dirname(store.oid_path))
87 os.makedirs(os.path.dirname(store.oid_path))
87
88
88 with open(store.oid_path, 'wb') as f:
89 with open(store.oid_path, 'wb') as f:
89 f.write('CONTENT')
90 f.write(safe_bytes('CONTENT'))
90 oid_handler.obj_size = 7
91 oid_handler.obj_size = 7
91 response, has_errors = oid_handler.exec_operation('upload')
92 response, has_errors = oid_handler.exec_operation('upload')
92 assert has_errors is None
93 assert has_errors is None
93 assert response is None
94 assert response is None
94
95
95 def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
96 def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
96 store = oid_handler.get_store()
97 store = oid_handler.get_store()
97 if not os.path.isdir(os.path.dirname(store.oid_path)):
98 if not os.path.isdir(os.path.dirname(store.oid_path)):
98 os.makedirs(os.path.dirname(store.oid_path))
99 os.makedirs(os.path.dirname(store.oid_path))
99
100
100 with open(store.oid_path, 'wb') as f:
101 with open(store.oid_path, 'wb') as f:
101 f.write('CONTENT')
102 f.write(safe_bytes('CONTENT'))
102
103
103 oid_handler.obj_size = 10240
104 oid_handler.obj_size = 10240
104 response, has_errors = oid_handler.exec_operation('upload')
105 response, has_errors = oid_handler.exec_operation('upload')
105 assert has_errors is None
106 assert has_errors is None
106 assert response['upload'] == {
107 assert response['upload'] == {
107 'header': {'Authorization': 'basic xxxx',
108 'header': {'Authorization': 'basic xxxx',
108 'Transfer-Encoding': 'chunked'},
109 'Transfer-Encoding': 'chunked'},
109 'href': 'http://localhost/handle_oid',
110 'href': 'http://localhost/handle_oid',
110 }
111 }
111
112
112 def test_upload_oid(self, oid_handler):
113 def test_upload_oid(self, oid_handler):
113 response, has_errors = oid_handler.exec_operation('upload')
114 response, has_errors = oid_handler.exec_operation('upload')
114 assert has_errors is None
115 assert has_errors is None
115 assert response['upload'] == {
116 assert response['upload'] == {
116 'header': {'Authorization': 'basic xxxx',
117 'header': {'Authorization': 'basic xxxx',
117 'Transfer-Encoding': 'chunked'},
118 'Transfer-Encoding': 'chunked'},
118 'href': 'http://localhost/handle_oid'
119 'href': 'http://localhost/handle_oid'
119 }
120 }
120
121
121
122
122 class TestLFSStore(object):
123 class TestLFSStore:
123 def test_write_oid(self, lfs_store):
124 def test_write_oid(self, lfs_store):
124 oid_location = lfs_store.oid_path
125 oid_location = lfs_store.oid_path
125
126
126 assert not os.path.isfile(oid_location)
127 assert not os.path.isfile(oid_location)
127
128
128 engine = lfs_store.get_engine(mode='wb')
129 engine = lfs_store.get_engine(mode='wb')
129 with engine as f:
130 with engine as f:
130 f.write('CONTENT')
131 f.write(safe_bytes('CONTENT'))
131
132
132 assert os.path.isfile(oid_location)
133 assert os.path.isfile(oid_location)
133
134
134 def test_detect_has_oid(self, lfs_store):
135 def test_detect_has_oid(self, lfs_store):
135
136
136 assert lfs_store.has_oid() is False
137 assert lfs_store.has_oid() is False
137 engine = lfs_store.get_engine(mode='wb')
138 engine = lfs_store.get_engine(mode='wb')
138 with engine as f:
139 with engine as f:
139 f.write('CONTENT')
140 f.write(safe_bytes('CONTENT'))
140
141
141 assert lfs_store.has_oid() is True No newline at end of file
142 assert lfs_store.has_oid() is True
@@ -1,50 +1,50 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import copy
17 import copy
18 from functools import wraps
18 from functools import wraps
19
19
20
20
21 def get_cython_compat_decorator(wrapper, func):
21 def get_cython_compat_decorator(wrapper, func):
22 """
22 """
23 Creates a cython compatible decorator. The previously used
23 Creates a cython compatible decorator. The previously used
24 decorator.decorator() function seems to be incompatible with cython.
24 decorator.decorator() function seems to be incompatible with cython.
25
25
26 :param wrapper: __wrapper method of the decorator class
26 :param wrapper: __wrapper method of the decorator class
27 :param func: decorated function
27 :param func: decorated function
28 """
28 """
29 @wraps(func)
29 @wraps(func)
30 def local_wrapper(*args, **kwds):
30 def local_wrapper(*args, **kwds):
31 return wrapper(func, *args, **kwds)
31 return wrapper(func, *args, **kwds)
32 local_wrapper.__wrapped__ = func
32 local_wrapper.__wrapped__ = func
33 return local_wrapper
33 return local_wrapper
34
34
35
35
36 def safe_result(result):
36 def safe_result(result):
37 """clean result for better representation in logs"""
37 """clean result for better representation in logs"""
38 clean_copy = copy.deepcopy(result)
38 clean_copy = copy.deepcopy(result)
39
39
40 try:
40 try:
41 if 'objects' in clean_copy:
41 if 'objects' in clean_copy:
42 for oid_data in clean_copy['objects']:
42 for oid_data in clean_copy['objects']:
43 if 'actions' in oid_data:
43 if 'actions' in oid_data:
44 for action_name, data in oid_data['actions'].items():
44 for action_name, data in oid_data['actions'].items():
45 if 'header' in data:
45 if 'header' in data:
46 data['header'] = {'Authorization': '*****'}
46 data['header'] = {'Authorization': '*****'}
47 except Exception:
47 except Exception:
48 return result
48 return result
49
49
50 return clean_copy
50 return clean_copy
@@ -1,79 +1,92 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """
18 """
19 Mercurial libs compatibility
19 Mercurial libs compatibility
20 """
20 """
21
21
22 import mercurial
22 import mercurial
23 from mercurial import demandimport
23 from mercurial import demandimport
24
24 # patch demandimport, due to bug in mercurial when it always triggers
25 # patch demandimport, due to bug in mercurial when it always triggers
25 # demandimport.enable()
26 # demandimport.enable()
27 from vcsserver.str_utils import safe_bytes
28
26 demandimport.enable = lambda *args, **kwargs: 1
29 demandimport.enable = lambda *args, **kwargs: 1
27
30
28 from mercurial import ui
31 from mercurial import ui
29 from mercurial import patch
32 from mercurial import patch
30 from mercurial import config
33 from mercurial import config
31 from mercurial import extensions
34 from mercurial import extensions
32 from mercurial import scmutil
35 from mercurial import scmutil
33 from mercurial import archival
36 from mercurial import archival
34 from mercurial import discovery
37 from mercurial import discovery
35 from mercurial import unionrepo
38 from mercurial import unionrepo
36 from mercurial import localrepo
39 from mercurial import localrepo
37 from mercurial import merge as hg_merge
40 from mercurial import merge as hg_merge
38 from mercurial import subrepo
41 from mercurial import subrepo
39 from mercurial import subrepoutil
42 from mercurial import subrepoutil
40 from mercurial import tags as hg_tag
43 from mercurial import tags as hg_tag
41 from mercurial import util as hgutil
44 from mercurial import util as hgutil
42 from mercurial.commands import clone, nullid, pull
45 from mercurial.commands import clone, pull
46 from mercurial.node import nullid
43 from mercurial.context import memctx, memfilectx
47 from mercurial.context import memctx, memfilectx
44 from mercurial.error import (
48 from mercurial.error import (
45 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
49 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
46 RequirementError, ProgrammingError)
50 RequirementError, ProgrammingError)
47 from mercurial.hgweb import hgweb_mod
51 from mercurial.hgweb import hgweb_mod
48 from mercurial.localrepo import instance
52 from mercurial.localrepo import instance
49 from mercurial.match import match, alwaysmatcher, patternmatcher
53 from mercurial.match import match, alwaysmatcher, patternmatcher
50 from mercurial.mdiff import diffopts
54 from mercurial.mdiff import diffopts
51 from mercurial.node import bin, hex
55 from mercurial.node import bin, hex
52 from mercurial.encoding import tolocal
56 from mercurial.encoding import tolocal
53 from mercurial.discovery import findcommonoutgoing
57 from mercurial.discovery import findcommonoutgoing
54 from mercurial.hg import peer
58 from mercurial.hg import peer
55 from mercurial.httppeer import makepeer
59 from mercurial.httppeer import makepeer
56 from mercurial.util import url as hg_url
60 from mercurial.utils.urlutil import url as hg_url
57 from mercurial.scmutil import revrange, revsymbol
61 from mercurial.scmutil import revrange, revsymbol
58 from mercurial.node import nullrev
62 from mercurial.node import nullrev
59 from mercurial import exchange
63 from mercurial import exchange
60 from hgext import largefiles
64 from hgext import largefiles
61
65
62 # those authnadlers are patched for python 2.6.5 bug an
66 # those authnadlers are patched for python 2.6.5 bug an
63 # infinit looping when given invalid resources
67 # infinit looping when given invalid resources
64 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
68 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
65
69
70 # hg strip is in core now
71 from mercurial import strip as hgext_strip
72
66
73
67 def get_ctx(repo, ref):
74 def get_ctx(repo, ref):
75 if not isinstance(ref, int):
76 ref = safe_bytes(ref)
77
68 try:
78 try:
69 ctx = repo[ref]
79 ctx = repo[ref]
80 return ctx
70 except (ProgrammingError, TypeError):
81 except (ProgrammingError, TypeError):
71 # we're unable to find the rev using a regular lookup, we fallback
82 # we're unable to find the rev using a regular lookup, we fallback
72 # to slower, but backward compat revsymbol usage
83 # to slower, but backward compat revsymbol usage
73 ctx = revsymbol(repo, ref)
84 pass
74 except (LookupError, RepoLookupError):
85 except (LookupError, RepoLookupError):
75 # Similar case as above but only for refs that are not numeric
86 # Similar case as above but only for refs that are not numeric
76 if isinstance(ref, (int, long)):
87 if isinstance(ref, int):
77 raise
88 raise
78 ctx = revsymbol(repo, ref)
89
90 ctx = revsymbol(repo, ref)
91
79 return ctx
92 return ctx
@@ -1,134 +1,134 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """
18 """
19 Adjustments to Mercurial
19 Adjustments to Mercurial
20
20
21 Intentionally kept separate from `hgcompat` and `hg`, so that these patches can
21 Intentionally kept separate from `hgcompat` and `hg`, so that these patches can
22 be applied without having to import the whole Mercurial machinery.
22 be applied without having to import the whole Mercurial machinery.
23
23
24 Imports are function local, so that just importing this module does not cause
24 Imports are function local, so that just importing this module does not cause
25 side-effects other than these functions being defined.
25 side-effects other than these functions being defined.
26 """
26 """
27
27
28 import logging
28 import logging
29
29
30
30
31 def patch_largefiles_capabilities():
31 def patch_largefiles_capabilities():
32 """
32 """
33 Patches the capabilities function in the largefiles extension.
33 Patches the capabilities function in the largefiles extension.
34 """
34 """
35 from vcsserver import hgcompat
35 from vcsserver import hgcompat
36 lfproto = hgcompat.largefiles.proto
36 lfproto = hgcompat.largefiles.proto
37 wrapper = _dynamic_capabilities_wrapper(
37 wrapper = _dynamic_capabilities_wrapper(
38 lfproto, hgcompat.extensions.extensions)
38 lfproto, hgcompat.extensions.extensions)
39 lfproto._capabilities = wrapper
39 lfproto._capabilities = wrapper
40
40
41
41
42 def _dynamic_capabilities_wrapper(lfproto, extensions):
42 def _dynamic_capabilities_wrapper(lfproto, extensions):
43
43
44 wrapped_capabilities = lfproto._capabilities
44 wrapped_capabilities = lfproto._capabilities
45 logger = logging.getLogger('vcsserver.hg')
45 logger = logging.getLogger('vcsserver.hg')
46
46
47 def _dynamic_capabilities(orig, repo, proto):
47 def _dynamic_capabilities(orig, repo, proto):
48 """
48 """
49 Adds dynamic behavior, so that the capability is only added if the
49 Adds dynamic behavior, so that the capability is only added if the
50 extension is enabled in the current ui object.
50 extension is enabled in the current ui object.
51 """
51 """
52 if 'largefiles' in dict(extensions(repo.ui)):
52 if 'largefiles' in dict(extensions(repo.ui)):
53 logger.debug('Extension largefiles enabled')
53 logger.debug('Extension largefiles enabled')
54 calc_capabilities = wrapped_capabilities
54 calc_capabilities = wrapped_capabilities
55 return calc_capabilities(orig, repo, proto)
55 return calc_capabilities(orig, repo, proto)
56 else:
56 else:
57 logger.debug('Extension largefiles disabled')
57 logger.debug('Extension largefiles disabled')
58 return orig(repo, proto)
58 return orig(repo, proto)
59
59
60 return _dynamic_capabilities
60 return _dynamic_capabilities
61
61
62
62
63 def patch_subrepo_type_mapping():
63 def patch_subrepo_type_mapping():
64 from collections import defaultdict
64 from collections import defaultdict
65 from hgcompat import subrepo, subrepoutil
65 from .hgcompat import subrepo, subrepoutil
66 from vcsserver.exceptions import SubrepoMergeException
66 from vcsserver.exceptions import SubrepoMergeException
67
67
68 class NoOpSubrepo(subrepo.abstractsubrepo):
68 class NoOpSubrepo(subrepo.abstractsubrepo):
69
69
70 def __init__(self, ctx, path, *args, **kwargs):
70 def __init__(self, ctx, path, *args, **kwargs):
71 """Initialize abstractsubrepo part
71 """Initialize abstractsubrepo part
72
72
73 ``ctx`` is the context referring this subrepository in the
73 ``ctx`` is the context referring this subrepository in the
74 parent repository.
74 parent repository.
75
75
76 ``path`` is the path to this subrepository as seen from
76 ``path`` is the path to this subrepository as seen from
77 innermost repository.
77 innermost repository.
78 """
78 """
79 self.ui = ctx.repo().ui
79 self.ui = ctx.repo().ui
80 self._ctx = ctx
80 self._ctx = ctx
81 self._path = path
81 self._path = path
82
82
83 def storeclean(self, path):
83 def storeclean(self, path):
84 """
84 """
85 returns true if the repository has not changed since it was last
85 returns true if the repository has not changed since it was last
86 cloned from or pushed to a given repository.
86 cloned from or pushed to a given repository.
87 """
87 """
88 return True
88 return True
89
89
90 def dirty(self, ignoreupdate=False, missing=False):
90 def dirty(self, ignoreupdate=False, missing=False):
91 """returns true if the dirstate of the subrepo is dirty or does not
91 """returns true if the dirstate of the subrepo is dirty or does not
92 match current stored state. If ignoreupdate is true, only check
92 match current stored state. If ignoreupdate is true, only check
93 whether the subrepo has uncommitted changes in its dirstate.
93 whether the subrepo has uncommitted changes in its dirstate.
94 """
94 """
95 return False
95 return False
96
96
97 def basestate(self):
97 def basestate(self):
98 """current working directory base state, disregarding .hgsubstate
98 """current working directory base state, disregarding .hgsubstate
99 state and working directory modifications"""
99 state and working directory modifications"""
100 substate = subrepoutil.state(self._ctx, self.ui)
100 substate = subrepoutil.state(self._ctx, self.ui)
101 file_system_path, rev, repotype = substate.get(self._path)
101 file_system_path, rev, repotype = substate.get(self._path)
102 return rev
102 return rev
103
103
104 def remove(self):
104 def remove(self):
105 """remove the subrepo
105 """remove the subrepo
106
106
107 (should verify the dirstate is not dirty first)
107 (should verify the dirstate is not dirty first)
108 """
108 """
109 pass
109 pass
110
110
111 def get(self, state, overwrite=False):
111 def get(self, state, overwrite=False):
112 """run whatever commands are needed to put the subrepo into
112 """run whatever commands are needed to put the subrepo into
113 this state
113 this state
114 """
114 """
115 pass
115 pass
116
116
117 def merge(self, state):
117 def merge(self, state):
118 """merge currently-saved state with the new state."""
118 """merge currently-saved state with the new state."""
119 raise SubrepoMergeException()()
119 raise SubrepoMergeException()()
120
120
121 def push(self, opts):
121 def push(self, opts):
122 """perform whatever action is analogous to 'hg push'
122 """perform whatever action is analogous to 'hg push'
123
123
124 This may be a no-op on some systems.
124 This may be a no-op on some systems.
125 """
125 """
126 pass
126 pass
127
127
128 # Patch subrepo type mapping to always return our NoOpSubrepo class
128 # Patch subrepo type mapping to always return our NoOpSubrepo class
129 # whenever a subrepo class is looked up.
129 # whenever a subrepo class is looked up.
130 subrepo.types = {
130 subrepo.types = {
131 'hg': NoOpSubrepo,
131 'hg': NoOpSubrepo,
132 'git': NoOpSubrepo,
132 'git': NoOpSubrepo,
133 'svn': NoOpSubrepo
133 'svn': NoOpSubrepo
134 }
134 }
@@ -1,205 +1,220 b''
1 # -*- coding: utf-8 -*-
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
5 #
3 #
6 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
7 # (at your option) any later version.
10 #
8 #
11 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
12 # GNU General Public License for more details.
15 #
13 #
16 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
17
20 import re
18 import re
21 import os
19 import os
22 import sys
20 import sys
23 import datetime
21 import datetime
24 import logging
22 import logging
25 import pkg_resources
23 import pkg_resources
26
24
27 import vcsserver
25 import vcsserver
26 from vcsserver.str_utils import safe_bytes
28
27
29 log = logging.getLogger(__name__)
28 log = logging.getLogger(__name__)
30
29
30 HOOKS_DIR_MODE = 0o755
31 HOOKS_FILE_MODE = 0o755
32
33
34 def set_permissions_if_needed(path_to_check, perms: oct):
35 # Get current permissions
36 current_permissions = os.stat(path_to_check).st_mode & 0o777 # Extract permission bits
37
38 # Check if current permissions are lower than required
39 if current_permissions < int(perms):
40 # Change the permissions if they are lower than required
41 os.chmod(path_to_check, perms)
42
31
43
32 def get_git_hooks_path(repo_path, bare):
44 def get_git_hooks_path(repo_path, bare):
33 hooks_path = os.path.join(repo_path, 'hooks')
45 hooks_path = os.path.join(repo_path, 'hooks')
34 if not bare:
46 if not bare:
35 hooks_path = os.path.join(repo_path, '.git', 'hooks')
47 hooks_path = os.path.join(repo_path, '.git', 'hooks')
36
48
37 return hooks_path
49 return hooks_path
38
50
39
51
40 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
52 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
41 """
53 """
42 Creates a RhodeCode hook inside a git repository
54 Creates a RhodeCode hook inside a git repository
43
55
44 :param repo_path: path to repository
56 :param repo_path: path to repository
57 :param bare: defines if repository is considered a bare git repo
45 :param executable: binary executable to put in the hooks
58 :param executable: binary executable to put in the hooks
46 :param force_create: Create even if same name hook exists
59 :param force_create: Creates even if the same name hook exists
47 """
60 """
48 executable = executable or sys.executable
61 executable = executable or sys.executable
49 hooks_path = get_git_hooks_path(repo_path, bare)
62 hooks_path = get_git_hooks_path(repo_path, bare)
50
63
51 if not os.path.isdir(hooks_path):
64 # we always call it to ensure dir exists and it has a proper mode
52 os.makedirs(hooks_path, mode=0o777)
65 if not os.path.exists(hooks_path):
66 # If it doesn't exist, create a new directory with the specified mode
67 os.makedirs(hooks_path, mode=HOOKS_DIR_MODE, exist_ok=True)
68 # If it exists, change the directory's mode to the specified mode
69 set_permissions_if_needed(hooks_path, perms=HOOKS_DIR_MODE)
53
70
54 tmpl_post = pkg_resources.resource_string(
71 tmpl_post = pkg_resources.resource_string(
55 'vcsserver', '/'.join(
72 'vcsserver', '/'.join(
56 ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
73 ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
57 tmpl_pre = pkg_resources.resource_string(
74 tmpl_pre = pkg_resources.resource_string(
58 'vcsserver', '/'.join(
75 'vcsserver', '/'.join(
59 ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
76 ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
60
77
61 path = '' # not used for now
78 path = '' # not used for now
62 timestamp = datetime.datetime.utcnow().isoformat()
79 timestamp = datetime.datetime.utcnow().isoformat()
63
80
64 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
81 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
65 log.debug('Installing git hook in repo %s', repo_path)
82 log.debug('Installing git hook in repo %s', repo_path)
66 _hook_file = os.path.join(hooks_path, '%s-receive' % h_type)
83 _hook_file = os.path.join(hooks_path, f'{h_type}-receive')
67 _rhodecode_hook = check_rhodecode_hook(_hook_file)
84 _rhodecode_hook = check_rhodecode_hook(_hook_file)
68
85
69 if _rhodecode_hook or force_create:
86 if _rhodecode_hook or force_create:
70 log.debug('writing git %s hook file at %s !', h_type, _hook_file)
87 log.debug('writing git %s hook file at %s !', h_type, _hook_file)
71 try:
88 try:
72 with open(_hook_file, 'wb') as f:
89 with open(_hook_file, 'wb') as f:
73 template = template.replace(
90 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
74 '_TMPL_', vcsserver.__version__)
91 template = template.replace(b'_DATE_', safe_bytes(timestamp))
75 template = template.replace('_DATE_', timestamp)
92 template = template.replace(b'_ENV_', safe_bytes(executable))
76 template = template.replace('_ENV_', executable)
93 template = template.replace(b'_PATH_', safe_bytes(path))
77 template = template.replace('_PATH_', path)
78 f.write(template)
94 f.write(template)
79 os.chmod(_hook_file, 0o755)
95 set_permissions_if_needed(_hook_file, perms=HOOKS_FILE_MODE)
80 except IOError:
96 except OSError:
81 log.exception('error writing hook file %s', _hook_file)
97 log.exception('error writing hook file %s', _hook_file)
82 else:
98 else:
83 log.debug('skipping writing hook file')
99 log.debug('skipping writing hook file')
84
100
85 return True
101 return True
86
102
87
103
88 def get_svn_hooks_path(repo_path):
104 def get_svn_hooks_path(repo_path):
89 hooks_path = os.path.join(repo_path, 'hooks')
105 hooks_path = os.path.join(repo_path, 'hooks')
90
106
91 return hooks_path
107 return hooks_path
92
108
93
109
94 def install_svn_hooks(repo_path, executable=None, force_create=False):
110 def install_svn_hooks(repo_path, executable=None, force_create=False):
95 """
111 """
96 Creates RhodeCode hooks inside a svn repository
112 Creates RhodeCode hooks inside a svn repository
97
113
98 :param repo_path: path to repository
114 :param repo_path: path to repository
99 :param executable: binary executable to put in the hooks
115 :param executable: binary executable to put in the hooks
100 :param force_create: Create even if same name hook exists
116 :param force_create: Create even if same name hook exists
101 """
117 """
102 executable = executable or sys.executable
118 executable = executable or sys.executable
103 hooks_path = get_svn_hooks_path(repo_path)
119 hooks_path = get_svn_hooks_path(repo_path)
104 if not os.path.isdir(hooks_path):
120 if not os.path.isdir(hooks_path):
105 os.makedirs(hooks_path, mode=0o777)
121 os.makedirs(hooks_path, mode=0o777, exist_ok=True)
106
122
107 tmpl_post = pkg_resources.resource_string(
123 tmpl_post = pkg_resources.resource_string(
108 'vcsserver', '/'.join(
124 'vcsserver', '/'.join(
109 ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
125 ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
110 tmpl_pre = pkg_resources.resource_string(
126 tmpl_pre = pkg_resources.resource_string(
111 'vcsserver', '/'.join(
127 'vcsserver', '/'.join(
112 ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
128 ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
113
129
114 path = '' # not used for now
130 path = '' # not used for now
115 timestamp = datetime.datetime.utcnow().isoformat()
131 timestamp = datetime.datetime.utcnow().isoformat()
116
132
117 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
133 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
118 log.debug('Installing svn hook in repo %s', repo_path)
134 log.debug('Installing svn hook in repo %s', repo_path)
119 _hook_file = os.path.join(hooks_path, '%s-commit' % h_type)
135 _hook_file = os.path.join(hooks_path, f'{h_type}-commit')
120 _rhodecode_hook = check_rhodecode_hook(_hook_file)
136 _rhodecode_hook = check_rhodecode_hook(_hook_file)
121
137
122 if _rhodecode_hook or force_create:
138 if _rhodecode_hook or force_create:
123 log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
139 log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
124
140
125 try:
141 try:
126 with open(_hook_file, 'wb') as f:
142 with open(_hook_file, 'wb') as f:
127 template = template.replace(
143 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
128 '_TMPL_', vcsserver.__version__)
144 template = template.replace(b'_DATE_', safe_bytes(timestamp))
129 template = template.replace('_DATE_', timestamp)
145 template = template.replace(b'_ENV_', safe_bytes(executable))
130 template = template.replace('_ENV_', executable)
146 template = template.replace(b'_PATH_', safe_bytes(path))
131 template = template.replace('_PATH_', path)
132
147
133 f.write(template)
148 f.write(template)
134 os.chmod(_hook_file, 0o755)
149 os.chmod(_hook_file, 0o755)
135 except IOError:
150 except OSError:
136 log.exception('error writing hook file %s', _hook_file)
151 log.exception('error writing hook file %s', _hook_file)
137 else:
152 else:
138 log.debug('skipping writing hook file')
153 log.debug('skipping writing hook file')
139
154
140 return True
155 return True
141
156
142
157
143 def get_version_from_hook(hook_path):
158 def get_version_from_hook(hook_path):
144 version = ''
159 version = b''
145 hook_content = read_hook_content(hook_path)
160 hook_content = read_hook_content(hook_path)
146 matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
161 matches = re.search(rb'RC_HOOK_VER\s*=\s*(.*)', hook_content)
147 if matches:
162 if matches:
148 try:
163 try:
149 version = matches.groups()[0]
164 version = matches.groups()[0]
150 log.debug('got version %s from hooks.', version)
165 log.debug('got version %s from hooks.', version)
151 except Exception:
166 except Exception:
152 log.exception("Exception while reading the hook version.")
167 log.exception("Exception while reading the hook version.")
153 return version.replace("'", "")
168 return version.replace(b"'", b"")
154
169
155
170
156 def check_rhodecode_hook(hook_path):
171 def check_rhodecode_hook(hook_path):
157 """
172 """
158 Check if the hook was created by RhodeCode
173 Check if the hook was created by RhodeCode
159 """
174 """
160 if not os.path.exists(hook_path):
175 if not os.path.exists(hook_path):
161 return True
176 return True
162
177
163 log.debug('hook exists, checking if it is from RhodeCode')
178 log.debug('hook exists, checking if it is from RhodeCode')
164
179
165 version = get_version_from_hook(hook_path)
180 version = get_version_from_hook(hook_path)
166 if version:
181 if version:
167 return True
182 return True
168
183
169 return False
184 return False
170
185
171
186
172 def read_hook_content(hook_path):
187 def read_hook_content(hook_path) -> bytes:
173 content = ''
188 content = b''
174 if os.path.isfile(hook_path):
189 if os.path.isfile(hook_path):
175 with open(hook_path, 'rb') as f:
190 with open(hook_path, 'rb') as f:
176 content = f.read()
191 content = f.read()
177 return content
192 return content
178
193
179
194
180 def get_git_pre_hook_version(repo_path, bare):
195 def get_git_pre_hook_version(repo_path, bare):
181 hooks_path = get_git_hooks_path(repo_path, bare)
196 hooks_path = get_git_hooks_path(repo_path, bare)
182 _hook_file = os.path.join(hooks_path, 'pre-receive')
197 _hook_file = os.path.join(hooks_path, 'pre-receive')
183 version = get_version_from_hook(_hook_file)
198 version = get_version_from_hook(_hook_file)
184 return version
199 return version
185
200
186
201
187 def get_git_post_hook_version(repo_path, bare):
202 def get_git_post_hook_version(repo_path, bare):
188 hooks_path = get_git_hooks_path(repo_path, bare)
203 hooks_path = get_git_hooks_path(repo_path, bare)
189 _hook_file = os.path.join(hooks_path, 'post-receive')
204 _hook_file = os.path.join(hooks_path, 'post-receive')
190 version = get_version_from_hook(_hook_file)
205 version = get_version_from_hook(_hook_file)
191 return version
206 return version
192
207
193
208
194 def get_svn_pre_hook_version(repo_path):
209 def get_svn_pre_hook_version(repo_path):
195 hooks_path = get_svn_hooks_path(repo_path)
210 hooks_path = get_svn_hooks_path(repo_path)
196 _hook_file = os.path.join(hooks_path, 'pre-commit')
211 _hook_file = os.path.join(hooks_path, 'pre-commit')
197 version = get_version_from_hook(_hook_file)
212 version = get_version_from_hook(_hook_file)
198 return version
213 return version
199
214
200
215
201 def get_svn_post_hook_version(repo_path):
216 def get_svn_post_hook_version(repo_path):
202 hooks_path = get_svn_hooks_path(repo_path)
217 hooks_path = get_svn_hooks_path(repo_path)
203 _hook_file = os.path.join(hooks_path, 'post-commit')
218 _hook_file = os.path.join(hooks_path, 'post-commit')
204 version = get_version_from_hook(_hook_file)
219 version = get_version_from_hook(_hook_file)
205 return version
220 return version
@@ -1,51 +1,51 b''
1 #!_ENV_
1 #!_ENV_
2 import os
2 import os
3 import sys
3 import sys
4 path_adjust = [_PATH_]
4 path_adjust = [_PATH_]
5
5
6 if path_adjust:
6 if path_adjust:
7 sys.path = path_adjust
7 sys.path = path_adjust
8
8
9 try:
9 try:
10 from vcsserver import hooks
10 from vcsserver import hooks
11 except ImportError:
11 except ImportError:
12 if os.environ.get('RC_DEBUG_GIT_HOOK'):
12 if os.environ.get('RC_DEBUG_GIT_HOOK'):
13 import traceback
13 import traceback
14 print traceback.format_exc()
14 print(traceback.format_exc())
15 hooks = None
15 hooks = None
16
16
17
17
18 # TIMESTAMP: _DATE_
18 # TIMESTAMP: _DATE_
19 RC_HOOK_VER = '_TMPL_'
19 RC_HOOK_VER = '_TMPL_'
20
20
21
21
22 def main():
22 def main():
23 if hooks is None:
23 if hooks is None:
24 # exit with success if we cannot import vcsserver.hooks !!
24 # exit with success if we cannot import vcsserver.hooks !!
25 # this allows simply push to this repo even without rhodecode
25 # this allows simply push to this repo even without rhodecode
26 sys.exit(0)
26 sys.exit(0)
27
27
28 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
28 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
29 sys.exit(0)
29 sys.exit(0)
30
30
31 repo_path = os.getcwd()
31 repo_path = os.getcwd()
32 push_data = sys.stdin.readlines()
32 push_data = sys.stdin.readlines()
33 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
33 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
34 # os.environ is modified here by a subprocess call that
34 # os.environ is modified here by a subprocess call that
35 # runs git and later git executes this hook.
35 # runs git and later git executes this hook.
36 # Environ gets some additional info from rhodecode system
36 # Environ gets some additional info from rhodecode system
37 # like IP or username from basic-auth
37 # like IP or username from basic-auth
38 try:
38 try:
39 result = hooks.git_post_receive(repo_path, push_data, os.environ)
39 result = hooks.git_post_receive(repo_path, push_data, os.environ)
40 sys.exit(result)
40 sys.exit(result)
41 except Exception as error:
41 except Exception as error:
42 # TODO: johbo: Improve handling of this special case
42 # TODO: johbo: Improve handling of this special case
43 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
43 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
44 raise
44 raise
45 print 'ERROR:', error
45 print(f'ERROR: {error}')
46 sys.exit(1)
46 sys.exit(1)
47 sys.exit(0)
47 sys.exit(0)
48
48
49
49
50 if __name__ == '__main__':
50 if __name__ == '__main__':
51 main()
51 main()
@@ -1,51 +1,51 b''
1 #!_ENV_
1 #!_ENV_
2 import os
2 import os
3 import sys
3 import sys
4 path_adjust = [_PATH_]
4 path_adjust = [_PATH_]
5
5
6 if path_adjust:
6 if path_adjust:
7 sys.path = path_adjust
7 sys.path = path_adjust
8
8
9 try:
9 try:
10 from vcsserver import hooks
10 from vcsserver import hooks
11 except ImportError:
11 except ImportError:
12 if os.environ.get('RC_DEBUG_GIT_HOOK'):
12 if os.environ.get('RC_DEBUG_GIT_HOOK'):
13 import traceback
13 import traceback
14 print traceback.format_exc()
14 print(traceback.format_exc())
15 hooks = None
15 hooks = None
16
16
17
17
18 # TIMESTAMP: _DATE_
18 # TIMESTAMP: _DATE_
19 RC_HOOK_VER = '_TMPL_'
19 RC_HOOK_VER = '_TMPL_'
20
20
21
21
22 def main():
22 def main():
23 if hooks is None:
23 if hooks is None:
24 # exit with success if we cannot import vcsserver.hooks !!
24 # exit with success if we cannot import vcsserver.hooks !!
25 # this allows simply push to this repo even without rhodecode
25 # this allows simply push to this repo even without rhodecode
26 sys.exit(0)
26 sys.exit(0)
27
27
28 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
28 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_GIT_HOOKS'):
29 sys.exit(0)
29 sys.exit(0)
30
30
31 repo_path = os.getcwd()
31 repo_path = os.getcwd()
32 push_data = sys.stdin.readlines()
32 push_data = sys.stdin.readlines()
33 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
33 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
34 # os.environ is modified here by a subprocess call that
34 # os.environ is modified here by a subprocess call that
35 # runs git and later git executes this hook.
35 # runs git and later git executes this hook.
36 # Environ gets some additional info from rhodecode system
36 # Environ gets some additional info from rhodecode system
37 # like IP or username from basic-auth
37 # like IP or username from basic-auth
38 try:
38 try:
39 result = hooks.git_pre_receive(repo_path, push_data, os.environ)
39 result = hooks.git_pre_receive(repo_path, push_data, os.environ)
40 sys.exit(result)
40 sys.exit(result)
41 except Exception as error:
41 except Exception as error:
42 # TODO: johbo: Improve handling of this special case
42 # TODO: johbo: Improve handling of this special case
43 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
43 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
44 raise
44 raise
45 print 'ERROR:', error
45 print(f'ERROR: {error}')
46 sys.exit(1)
46 sys.exit(1)
47 sys.exit(0)
47 sys.exit(0)
48
48
49
49
50 if __name__ == '__main__':
50 if __name__ == '__main__':
51 main()
51 main()
@@ -1,50 +1,50 b''
1 #!_ENV_
1 #!_ENV_
2
2
3 import os
3 import os
4 import sys
4 import sys
5 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
6
6
7 if path_adjust:
7 if path_adjust:
8 sys.path = path_adjust
8 sys.path = path_adjust
9
9
10 try:
10 try:
11 from vcsserver import hooks
11 from vcsserver import hooks
12 except ImportError:
12 except ImportError:
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
14 import traceback
14 import traceback
15 print traceback.format_exc()
15 print(traceback.format_exc())
16 hooks = None
16 hooks = None
17
17
18
18
19 # TIMESTAMP: _DATE_
19 # TIMESTAMP: _DATE_
20 RC_HOOK_VER = '_TMPL_'
20 RC_HOOK_VER = '_TMPL_'
21
21
22
22
23 def main():
23 def main():
24 if hooks is None:
24 if hooks is None:
25 # exit with success if we cannot import vcsserver.hooks !!
25 # exit with success if we cannot import vcsserver.hooks !!
26 # this allows simply push to this repo even without rhodecode
26 # this allows simply push to this repo even without rhodecode
27 sys.exit(0)
27 sys.exit(0)
28
28
29 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
29 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
30 sys.exit(0)
30 sys.exit(0)
31 repo_path = os.getcwd()
31 repo_path = os.getcwd()
32 push_data = sys.argv[1:]
32 push_data = sys.argv[1:]
33
33
34 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
34 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
35
35
36 try:
36 try:
37 result = hooks.svn_post_commit(repo_path, push_data, os.environ)
37 result = hooks.svn_post_commit(repo_path, push_data, os.environ)
38 sys.exit(result)
38 sys.exit(result)
39 except Exception as error:
39 except Exception as error:
40 # TODO: johbo: Improve handling of this special case
40 # TODO: johbo: Improve handling of this special case
41 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
41 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
42 raise
42 raise
43 print 'ERROR:', error
43 print(f'ERROR: {error}')
44 sys.exit(1)
44 sys.exit(1)
45 sys.exit(0)
45 sys.exit(0)
46
46
47
47
48
48
49 if __name__ == '__main__':
49 if __name__ == '__main__':
50 main()
50 main()
@@ -1,52 +1,52 b''
1 #!_ENV_
1 #!_ENV_
2
2
3 import os
3 import os
4 import sys
4 import sys
5 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
6
6
7 if path_adjust:
7 if path_adjust:
8 sys.path = path_adjust
8 sys.path = path_adjust
9
9
10 try:
10 try:
11 from vcsserver import hooks
11 from vcsserver import hooks
12 except ImportError:
12 except ImportError:
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
14 import traceback
14 import traceback
15 print traceback.format_exc()
15 print(traceback.format_exc())
16 hooks = None
16 hooks = None
17
17
18
18
19 # TIMESTAMP: _DATE_
19 # TIMESTAMP: _DATE_
20 RC_HOOK_VER = '_TMPL_'
20 RC_HOOK_VER = '_TMPL_'
21
21
22
22
23 def main():
23 def main():
24 if os.environ.get('SSH_READ_ONLY') == '1':
24 if os.environ.get('SSH_READ_ONLY') == '1':
25 sys.stderr.write('Only read-only access is allowed')
25 sys.stderr.write('Only read-only access is allowed')
26 sys.exit(1)
26 sys.exit(1)
27
27
28 if hooks is None:
28 if hooks is None:
29 # exit with success if we cannot import vcsserver.hooks !!
29 # exit with success if we cannot import vcsserver.hooks !!
30 # this allows simply push to this repo even without rhodecode
30 # this allows simply push to this repo even without rhodecode
31 sys.exit(0)
31 sys.exit(0)
32 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
32 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
33 sys.exit(0)
33 sys.exit(0)
34 repo_path = os.getcwd()
34 repo_path = os.getcwd()
35 push_data = sys.argv[1:]
35 push_data = sys.argv[1:]
36
36
37 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
37 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
38
38
39 try:
39 try:
40 result = hooks.svn_pre_commit(repo_path, push_data, os.environ)
40 result = hooks.svn_pre_commit(repo_path, push_data, os.environ)
41 sys.exit(result)
41 sys.exit(result)
42 except Exception as error:
42 except Exception as error:
43 # TODO: johbo: Improve handling of this special case
43 # TODO: johbo: Improve handling of this special case
44 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
44 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
45 raise
45 raise
46 print 'ERROR:', error
46 print(f'ERROR: {error}')
47 sys.exit(1)
47 sys.exit(1)
48 sys.exit(0)
48 sys.exit(0)
49
49
50
50
51 if __name__ == '__main__':
51 if __name__ == '__main__':
52 main()
52 main()
@@ -1,729 +1,795 b''
1 # -*- coding: utf-8 -*-
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
5 #
3 #
6 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
7 # (at your option) any later version.
10 #
8 #
11 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
12 # GNU General Public License for more details.
15 #
13 #
16 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
17
20 import io
18 import io
21 import os
19 import os
22 import sys
20 import sys
23 import logging
21 import logging
24 import collections
22 import collections
25 import importlib
26 import base64
23 import base64
24 import msgpack
25 import dataclasses
26 import pygit2
27
27
28 from httplib import HTTPConnection
28 import http.client
29
29 from celery import Celery
30
30
31 import mercurial.scmutil
31 import mercurial.scmutil
32 import mercurial.node
32 import mercurial.node
33 import simplejson as json
34
33
34 from vcsserver.lib.rc_json import json
35 from vcsserver import exceptions, subprocessio, settings
35 from vcsserver import exceptions, subprocessio, settings
36 from vcsserver.str_utils import ascii_str, safe_str
37 from vcsserver.remote.git_remote import Repository
36
38
39 celery_app = Celery('__vcsserver__')
37 log = logging.getLogger(__name__)
40 log = logging.getLogger(__name__)
38
41
39
42
40 class HooksHttpClient(object):
43 class HooksHttpClient:
44 proto = 'msgpack.v1'
41 connection = None
45 connection = None
42
46
43 def __init__(self, hooks_uri):
47 def __init__(self, hooks_uri):
44 self.hooks_uri = hooks_uri
48 self.hooks_uri = hooks_uri
45
49
50 def __repr__(self):
51 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
52
46 def __call__(self, method, extras):
53 def __call__(self, method, extras):
47 connection = HTTPConnection(self.hooks_uri)
54 connection = http.client.HTTPConnection(self.hooks_uri)
48 body = self._serialize(method, extras)
55 # binary msgpack body
49 try:
56 headers, body = self._serialize(method, extras)
50 connection.request('POST', '/', body)
57 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
51 except Exception:
52 log.error('Hooks calling Connection failed on %s', connection.__dict__)
53 raise
54 response = connection.getresponse()
55
56 response_data = response.read()
57
58
58 try:
59 try:
59 return json.loads(response_data)
60 try:
60 except Exception:
61 connection.request('POST', '/', body, headers)
61 log.exception('Failed to decode hook response json data. '
62 except Exception as error:
62 'response_code:%s, raw_data:%s',
63 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
63 response.status, response_data)
64 raise
64 raise
65
65
66 def _serialize(self, hook_name, extras):
66 response = connection.getresponse()
67 try:
68 return msgpack.load(response)
69 except Exception:
70 response_data = response.read()
71 log.exception('Failed to decode hook response json data. '
72 'response_code:%s, raw_data:%s',
73 response.status, response_data)
74 raise
75 finally:
76 connection.close()
77
78 @classmethod
79 def _serialize(cls, hook_name, extras):
67 data = {
80 data = {
68 'method': hook_name,
81 'method': hook_name,
69 'extras': extras
82 'extras': extras
70 }
83 }
71 return json.dumps(data)
84 headers = {
85 "rc-hooks-protocol": cls.proto,
86 "Connection": "keep-alive"
87 }
88 return headers, msgpack.packb(data)
72
89
73
90
74 class HooksDummyClient(object):
91 class HooksCeleryClient:
75 def __init__(self, hooks_module):
92 TASK_TIMEOUT = 60 # time in seconds
76 self._hooks_module = importlib.import_module(hooks_module)
77
93
78 def __call__(self, hook_name, extras):
94 def __init__(self, queue, backend):
79 with self._hooks_module.Hooks() as hooks:
95 celery_app.config_from_object({
80 return getattr(hooks, hook_name)(extras)
96 'broker_url': queue, 'result_backend': backend,
97 'broker_connection_retry_on_startup': True,
98 'task_serializer': 'msgpack',
99 'accept_content': ['json', 'msgpack'],
100 'result_serializer': 'msgpack',
101 'result_accept_content': ['json', 'msgpack']
102 })
103 self.celery_app = celery_app
104
105 def __call__(self, method, extras):
106 inquired_task = self.celery_app.signature(
107 f'rhodecode.lib.celerylib.tasks.{method}'
108 )
109 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
81
110
82
111
83 class HooksShadowRepoClient(object):
112 class HooksShadowRepoClient:
84
113
85 def __call__(self, hook_name, extras):
114 def __call__(self, hook_name, extras):
86 return {'output': '', 'status': 0}
115 return {'output': '', 'status': 0}
87
116
88
117
89 class RemoteMessageWriter(object):
118 class RemoteMessageWriter:
90 """Writer base class."""
119 """Writer base class."""
91 def write(self, message):
120 def write(self, message):
92 raise NotImplementedError()
121 raise NotImplementedError()
93
122
94
123
95 class HgMessageWriter(RemoteMessageWriter):
124 class HgMessageWriter(RemoteMessageWriter):
96 """Writer that knows how to send messages to mercurial clients."""
125 """Writer that knows how to send messages to mercurial clients."""
97
126
98 def __init__(self, ui):
127 def __init__(self, ui):
99 self.ui = ui
128 self.ui = ui
100
129
101 def write(self, message):
130 def write(self, message: str):
102 # TODO: Check why the quiet flag is set by default.
131 # TODO: Check why the quiet flag is set by default.
103 old = self.ui.quiet
132 old = self.ui.quiet
104 self.ui.quiet = False
133 self.ui.quiet = False
105 self.ui.status(message.encode('utf-8'))
134 self.ui.status(message.encode('utf-8'))
106 self.ui.quiet = old
135 self.ui.quiet = old
107
136
108
137
109 class GitMessageWriter(RemoteMessageWriter):
138 class GitMessageWriter(RemoteMessageWriter):
110 """Writer that knows how to send messages to git clients."""
139 """Writer that knows how to send messages to git clients."""
111
140
112 def __init__(self, stdout=None):
141 def __init__(self, stdout=None):
113 self.stdout = stdout or sys.stdout
142 self.stdout = stdout or sys.stdout
114
143
115 def write(self, message):
144 def write(self, message: str):
116 self.stdout.write(message.encode('utf-8'))
145 self.stdout.write(message)
117
146
118
147
119 class SvnMessageWriter(RemoteMessageWriter):
148 class SvnMessageWriter(RemoteMessageWriter):
120 """Writer that knows how to send messages to svn clients."""
149 """Writer that knows how to send messages to svn clients."""
121
150
122 def __init__(self, stderr=None):
151 def __init__(self, stderr=None):
123 # SVN needs data sent to stderr for back-to-client messaging
152 # SVN needs data sent to stderr for back-to-client messaging
124 self.stderr = stderr or sys.stderr
153 self.stderr = stderr or sys.stderr
125
154
126 def write(self, message):
155 def write(self, message):
127 self.stderr.write(message.encode('utf-8'))
156 self.stderr.write(message.encode('utf-8'))
128
157
129
158
130 def _handle_exception(result):
159 def _handle_exception(result):
131 exception_class = result.get('exception')
160 exception_class = result.get('exception')
132 exception_traceback = result.get('exception_traceback')
161 exception_traceback = result.get('exception_traceback')
162 log.debug('Handling hook-call exception: %s', exception_class)
133
163
134 if exception_traceback:
164 if exception_traceback:
135 log.error('Got traceback from remote call:%s', exception_traceback)
165 log.error('Got traceback from remote call:%s', exception_traceback)
136
166
137 if exception_class == 'HTTPLockedRC':
167 if exception_class == 'HTTPLockedRC':
138 raise exceptions.RepositoryLockedException()(*result['exception_args'])
168 raise exceptions.RepositoryLockedException()(*result['exception_args'])
139 elif exception_class == 'HTTPBranchProtected':
169 elif exception_class == 'HTTPBranchProtected':
140 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
170 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
141 elif exception_class == 'RepositoryError':
171 elif exception_class == 'RepositoryError':
142 raise exceptions.VcsException()(*result['exception_args'])
172 raise exceptions.VcsException()(*result['exception_args'])
143 elif exception_class:
173 elif exception_class:
144 raise Exception('Got remote exception "%s" with args "%s"' %
174 raise Exception(
145 (exception_class, result['exception_args']))
175 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
176 )
146
177
147
178
148 def _get_hooks_client(extras):
179 def _get_hooks_client(extras):
149 hooks_uri = extras.get('hooks_uri')
180 hooks_uri = extras.get('hooks_uri')
181 task_queue = extras.get('task_queue')
182 task_backend = extras.get('task_backend')
150 is_shadow_repo = extras.get('is_shadow_repo')
183 is_shadow_repo = extras.get('is_shadow_repo')
184
151 if hooks_uri:
185 if hooks_uri:
152 return HooksHttpClient(extras['hooks_uri'])
186 return HooksHttpClient(hooks_uri)
187 elif task_queue and task_backend:
188 return HooksCeleryClient(task_queue, task_backend)
153 elif is_shadow_repo:
189 elif is_shadow_repo:
154 return HooksShadowRepoClient()
190 return HooksShadowRepoClient()
155 else:
191 else:
156 return HooksDummyClient(extras['hooks_module'])
192 raise Exception("Hooks client not found!")
157
193
158
194
159 def _call_hook(hook_name, extras, writer):
195 def _call_hook(hook_name, extras, writer):
160 hooks_client = _get_hooks_client(extras)
196 hooks_client = _get_hooks_client(extras)
161 log.debug('Hooks, using client:%s', hooks_client)
197 log.debug('Hooks, using client:%s', hooks_client)
162 result = hooks_client(hook_name, extras)
198 result = hooks_client(hook_name, extras)
163 log.debug('Hooks got result: %s', result)
199 log.debug('Hooks got result: %s', result)
164
165 _handle_exception(result)
200 _handle_exception(result)
166 writer.write(result['output'])
201 writer.write(result['output'])
167
202
168 return result['status']
203 return result['status']
169
204
170
205
171 def _extras_from_ui(ui):
206 def _extras_from_ui(ui):
172 hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
207 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
173 if not hook_data:
208 if not hook_data:
174 # maybe it's inside environ ?
209 # maybe it's inside environ ?
175 env_hook_data = os.environ.get('RC_SCM_DATA')
210 env_hook_data = os.environ.get('RC_SCM_DATA')
176 if env_hook_data:
211 if env_hook_data:
177 hook_data = env_hook_data
212 hook_data = env_hook_data
178
213
179 extras = {}
214 extras = {}
180 if hook_data:
215 if hook_data:
181 extras = json.loads(hook_data)
216 extras = json.loads(hook_data)
182 return extras
217 return extras
183
218
184
219
185 def _rev_range_hash(repo, node, check_heads=False):
220 def _rev_range_hash(repo, node, check_heads=False):
186 from vcsserver.hgcompat import get_ctx
221 from vcsserver.hgcompat import get_ctx
187
222
188 commits = []
223 commits = []
189 revs = []
224 revs = []
190 start = get_ctx(repo, node).rev()
225 start = get_ctx(repo, node).rev()
191 end = len(repo)
226 end = len(repo)
192 for rev in range(start, end):
227 for rev in range(start, end):
193 revs.append(rev)
228 revs.append(rev)
194 ctx = get_ctx(repo, rev)
229 ctx = get_ctx(repo, rev)
195 commit_id = mercurial.node.hex(ctx.node())
230 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
196 branch = ctx.branch()
231 branch = safe_str(ctx.branch())
197 commits.append((commit_id, branch))
232 commits.append((commit_id, branch))
198
233
199 parent_heads = []
234 parent_heads = []
200 if check_heads:
235 if check_heads:
201 parent_heads = _check_heads(repo, start, end, revs)
236 parent_heads = _check_heads(repo, start, end, revs)
202 return commits, parent_heads
237 return commits, parent_heads
203
238
204
239
205 def _check_heads(repo, start, end, commits):
240 def _check_heads(repo, start, end, commits):
206 from vcsserver.hgcompat import get_ctx
241 from vcsserver.hgcompat import get_ctx
207 changelog = repo.changelog
242 changelog = repo.changelog
208 parents = set()
243 parents = set()
209
244
210 for new_rev in commits:
245 for new_rev in commits:
211 for p in changelog.parentrevs(new_rev):
246 for p in changelog.parentrevs(new_rev):
212 if p == mercurial.node.nullrev:
247 if p == mercurial.node.nullrev:
213 continue
248 continue
214 if p < start:
249 if p < start:
215 parents.add(p)
250 parents.add(p)
216
251
217 for p in parents:
252 for p in parents:
218 branch = get_ctx(repo, p).branch()
253 branch = get_ctx(repo, p).branch()
219 # The heads descending from that parent, on the same branch
254 # The heads descending from that parent, on the same branch
220 parent_heads = set([p])
255 parent_heads = {p}
221 reachable = set([p])
256 reachable = {p}
222 for x in xrange(p + 1, end):
257 for x in range(p + 1, end):
223 if get_ctx(repo, x).branch() != branch:
258 if get_ctx(repo, x).branch() != branch:
224 continue
259 continue
225 for pp in changelog.parentrevs(x):
260 for pp in changelog.parentrevs(x):
226 if pp in reachable:
261 if pp in reachable:
227 reachable.add(x)
262 reachable.add(x)
228 parent_heads.discard(pp)
263 parent_heads.discard(pp)
229 parent_heads.add(x)
264 parent_heads.add(x)
230 # More than one head? Suggest merging
265 # More than one head? Suggest merging
231 if len(parent_heads) > 1:
266 if len(parent_heads) > 1:
232 return list(parent_heads)
267 return list(parent_heads)
233
268
234 return []
269 return []
235
270
236
271
237 def _get_git_env():
272 def _get_git_env():
238 env = {}
273 env = {}
239 for k, v in os.environ.items():
274 for k, v in os.environ.items():
240 if k.startswith('GIT'):
275 if k.startswith('GIT'):
241 env[k] = v
276 env[k] = v
242
277
243 # serialized version
278 # serialized version
244 return [(k, v) for k, v in env.items()]
279 return [(k, v) for k, v in env.items()]
245
280
246
281
247 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
282 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
248 env = {}
283 env = {}
249 for k, v in os.environ.items():
284 for k, v in os.environ.items():
250 if k.startswith('HG'):
285 if k.startswith('HG'):
251 env[k] = v
286 env[k] = v
252
287
253 env['HG_NODE'] = old_rev
288 env['HG_NODE'] = old_rev
254 env['HG_NODE_LAST'] = new_rev
289 env['HG_NODE_LAST'] = new_rev
255 env['HG_TXNID'] = txnid
290 env['HG_TXNID'] = txnid
256 env['HG_PENDING'] = repo_path
291 env['HG_PENDING'] = repo_path
257
292
258 return [(k, v) for k, v in env.items()]
293 return [(k, v) for k, v in env.items()]
259
294
260
295
261 def repo_size(ui, repo, **kwargs):
296 def repo_size(ui, repo, **kwargs):
262 extras = _extras_from_ui(ui)
297 extras = _extras_from_ui(ui)
263 return _call_hook('repo_size', extras, HgMessageWriter(ui))
298 return _call_hook('repo_size', extras, HgMessageWriter(ui))
264
299
265
300
266 def pre_pull(ui, repo, **kwargs):
301 def pre_pull(ui, repo, **kwargs):
267 extras = _extras_from_ui(ui)
302 extras = _extras_from_ui(ui)
268 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
303 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
269
304
270
305
271 def pre_pull_ssh(ui, repo, **kwargs):
306 def pre_pull_ssh(ui, repo, **kwargs):
272 extras = _extras_from_ui(ui)
307 extras = _extras_from_ui(ui)
273 if extras and extras.get('SSH'):
308 if extras and extras.get('SSH'):
274 return pre_pull(ui, repo, **kwargs)
309 return pre_pull(ui, repo, **kwargs)
275 return 0
310 return 0
276
311
277
312
278 def post_pull(ui, repo, **kwargs):
313 def post_pull(ui, repo, **kwargs):
279 extras = _extras_from_ui(ui)
314 extras = _extras_from_ui(ui)
280 return _call_hook('post_pull', extras, HgMessageWriter(ui))
315 return _call_hook('post_pull', extras, HgMessageWriter(ui))
281
316
282
317
283 def post_pull_ssh(ui, repo, **kwargs):
318 def post_pull_ssh(ui, repo, **kwargs):
284 extras = _extras_from_ui(ui)
319 extras = _extras_from_ui(ui)
285 if extras and extras.get('SSH'):
320 if extras and extras.get('SSH'):
286 return post_pull(ui, repo, **kwargs)
321 return post_pull(ui, repo, **kwargs)
287 return 0
322 return 0
288
323
289
324
290 def pre_push(ui, repo, node=None, **kwargs):
325 def pre_push(ui, repo, node=None, **kwargs):
291 """
326 """
292 Mercurial pre_push hook
327 Mercurial pre_push hook
293 """
328 """
294 extras = _extras_from_ui(ui)
329 extras = _extras_from_ui(ui)
295 detect_force_push = extras.get('detect_force_push')
330 detect_force_push = extras.get('detect_force_push')
296
331
297 rev_data = []
332 rev_data = []
298 if node and kwargs.get('hooktype') == 'pretxnchangegroup':
333 hook_type: str = safe_str(kwargs.get('hooktype'))
334
335 if node and hook_type == 'pretxnchangegroup':
299 branches = collections.defaultdict(list)
336 branches = collections.defaultdict(list)
300 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
337 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
301 for commit_id, branch in commits:
338 for commit_id, branch in commits:
302 branches[branch].append(commit_id)
339 branches[branch].append(commit_id)
303
340
304 for branch, commits in branches.items():
341 for branch, commits in branches.items():
305 old_rev = kwargs.get('node_last') or commits[0]
342 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
306 rev_data.append({
343 rev_data.append({
307 'total_commits': len(commits),
344 'total_commits': len(commits),
308 'old_rev': old_rev,
345 'old_rev': old_rev,
309 'new_rev': commits[-1],
346 'new_rev': commits[-1],
310 'ref': '',
347 'ref': '',
311 'type': 'branch',
348 'type': 'branch',
312 'name': branch,
349 'name': branch,
313 })
350 })
314
351
315 for push_ref in rev_data:
352 for push_ref in rev_data:
316 push_ref['multiple_heads'] = _heads
353 push_ref['multiple_heads'] = _heads
317
354
318 repo_path = os.path.join(
355 repo_path = os.path.join(
319 extras.get('repo_store', ''), extras.get('repository', ''))
356 extras.get('repo_store', ''), extras.get('repository', ''))
320 push_ref['hg_env'] = _get_hg_env(
357 push_ref['hg_env'] = _get_hg_env(
321 old_rev=push_ref['old_rev'],
358 old_rev=push_ref['old_rev'],
322 new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'),
359 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
323 repo_path=repo_path)
360 repo_path=repo_path)
324
361
325 extras['hook_type'] = kwargs.get('hooktype', 'pre_push')
362 extras['hook_type'] = hook_type or 'pre_push'
326 extras['commit_ids'] = rev_data
363 extras['commit_ids'] = rev_data
327
364
328 return _call_hook('pre_push', extras, HgMessageWriter(ui))
365 return _call_hook('pre_push', extras, HgMessageWriter(ui))
329
366
330
367
331 def pre_push_ssh(ui, repo, node=None, **kwargs):
368 def pre_push_ssh(ui, repo, node=None, **kwargs):
332 extras = _extras_from_ui(ui)
369 extras = _extras_from_ui(ui)
333 if extras.get('SSH'):
370 if extras.get('SSH'):
334 return pre_push(ui, repo, node, **kwargs)
371 return pre_push(ui, repo, node, **kwargs)
335
372
336 return 0
373 return 0
337
374
338
375
339 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
376 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
340 """
377 """
341 Mercurial pre_push hook for SSH
378 Mercurial pre_push hook for SSH
342 """
379 """
343 extras = _extras_from_ui(ui)
380 extras = _extras_from_ui(ui)
344 if extras.get('SSH'):
381 if extras.get('SSH'):
345 permission = extras['SSH_PERMISSIONS']
382 permission = extras['SSH_PERMISSIONS']
346
383
347 if 'repository.write' == permission or 'repository.admin' == permission:
384 if 'repository.write' == permission or 'repository.admin' == permission:
348 return 0
385 return 0
349
386
350 # non-zero ret code
387 # non-zero ret code
351 return 1
388 return 1
352
389
353 return 0
390 return 0
354
391
355
392
356 def post_push(ui, repo, node, **kwargs):
393 def post_push(ui, repo, node, **kwargs):
357 """
394 """
358 Mercurial post_push hook
395 Mercurial post_push hook
359 """
396 """
360 extras = _extras_from_ui(ui)
397 extras = _extras_from_ui(ui)
361
398
362 commit_ids = []
399 commit_ids = []
363 branches = []
400 branches = []
364 bookmarks = []
401 bookmarks = []
365 tags = []
402 tags = []
403 hook_type: str = safe_str(kwargs.get('hooktype'))
366
404
367 commits, _heads = _rev_range_hash(repo, node)
405 commits, _heads = _rev_range_hash(repo, node)
368 for commit_id, branch in commits:
406 for commit_id, branch in commits:
369 commit_ids.append(commit_id)
407 commit_ids.append(commit_id)
370 if branch not in branches:
408 if branch not in branches:
371 branches.append(branch)
409 branches.append(branch)
372
410
373 if hasattr(ui, '_rc_pushkey_branches'):
411 if hasattr(ui, '_rc_pushkey_bookmarks'):
374 bookmarks = ui._rc_pushkey_branches
412 bookmarks = ui._rc_pushkey_bookmarks
375
413
376 extras['hook_type'] = kwargs.get('hooktype', 'post_push')
414 extras['hook_type'] = hook_type or 'post_push'
377 extras['commit_ids'] = commit_ids
415 extras['commit_ids'] = commit_ids
416
378 extras['new_refs'] = {
417 extras['new_refs'] = {
379 'branches': branches,
418 'branches': branches,
380 'bookmarks': bookmarks,
419 'bookmarks': bookmarks,
381 'tags': tags
420 'tags': tags
382 }
421 }
383
422
384 return _call_hook('post_push', extras, HgMessageWriter(ui))
423 return _call_hook('post_push', extras, HgMessageWriter(ui))
385
424
386
425
387 def post_push_ssh(ui, repo, node, **kwargs):
426 def post_push_ssh(ui, repo, node, **kwargs):
388 """
427 """
389 Mercurial post_push hook for SSH
428 Mercurial post_push hook for SSH
390 """
429 """
391 if _extras_from_ui(ui).get('SSH'):
430 if _extras_from_ui(ui).get('SSH'):
392 return post_push(ui, repo, node, **kwargs)
431 return post_push(ui, repo, node, **kwargs)
393 return 0
432 return 0
394
433
395
434
396 def key_push(ui, repo, **kwargs):
435 def key_push(ui, repo, **kwargs):
397 from vcsserver.hgcompat import get_ctx
436 from vcsserver.hgcompat import get_ctx
398 if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
437
438 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
399 # store new bookmarks in our UI object propagated later to post_push
439 # store new bookmarks in our UI object propagated later to post_push
400 ui._rc_pushkey_branches = get_ctx(repo, kwargs['key']).bookmarks()
440 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
401 return
441 return
402
442
403
443
404 # backward compat
444 # backward compat
405 log_pull_action = post_pull
445 log_pull_action = post_pull
406
446
407 # backward compat
447 # backward compat
408 log_push_action = post_push
448 log_push_action = post_push
409
449
410
450
411 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
451 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
412 """
452 """
413 Old hook name: keep here for backward compatibility.
453 Old hook name: keep here for backward compatibility.
414
454
415 This is only required when the installed git hooks are not upgraded.
455 This is only required when the installed git hooks are not upgraded.
416 """
456 """
417 pass
457 pass
418
458
419
459
420 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
460 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
421 """
461 """
422 Old hook name: keep here for backward compatibility.
462 Old hook name: keep here for backward compatibility.
423
463
424 This is only required when the installed git hooks are not upgraded.
464 This is only required when the installed git hooks are not upgraded.
425 """
465 """
426 pass
466 pass
427
467
428
468
429 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
469 @dataclasses.dataclass
470 class HookResponse:
471 status: int
472 output: str
430
473
431
474
432 def git_pre_pull(extras):
475 def git_pre_pull(extras) -> HookResponse:
433 """
476 """
434 Pre pull hook.
477 Pre pull hook.
435
478
436 :param extras: dictionary containing the keys defined in simplevcs
479 :param extras: dictionary containing the keys defined in simplevcs
437 :type extras: dict
480 :type extras: dict
438
481
439 :return: status code of the hook. 0 for success.
482 :return: status code of the hook. 0 for success.
440 :rtype: int
483 :rtype: int
441 """
484 """
485
442 if 'pull' not in extras['hooks']:
486 if 'pull' not in extras['hooks']:
443 return HookResponse(0, '')
487 return HookResponse(0, '')
444
488
445 stdout = io.BytesIO()
489 stdout = io.StringIO()
446 try:
490 try:
447 status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
491 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
492
448 except Exception as error:
493 except Exception as error:
449 status = 128
494 log.exception('Failed to call pre_pull hook')
450 stdout.write('ERROR: %s\n' % str(error))
495 status_code = 128
496 stdout.write(f'ERROR: {error}\n')
451
497
452 return HookResponse(status, stdout.getvalue())
498 return HookResponse(status_code, stdout.getvalue())
453
499
454
500
455 def git_post_pull(extras):
501 def git_post_pull(extras) -> HookResponse:
456 """
502 """
457 Post pull hook.
503 Post pull hook.
458
504
459 :param extras: dictionary containing the keys defined in simplevcs
505 :param extras: dictionary containing the keys defined in simplevcs
460 :type extras: dict
506 :type extras: dict
461
507
462 :return: status code of the hook. 0 for success.
508 :return: status code of the hook. 0 for success.
463 :rtype: int
509 :rtype: int
464 """
510 """
465 if 'pull' not in extras['hooks']:
511 if 'pull' not in extras['hooks']:
466 return HookResponse(0, '')
512 return HookResponse(0, '')
467
513
468 stdout = io.BytesIO()
514 stdout = io.StringIO()
469 try:
515 try:
470 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
516 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
471 except Exception as error:
517 except Exception as error:
472 status = 128
518 status = 128
473 stdout.write('ERROR: %s\n' % error)
519 stdout.write(f'ERROR: {error}\n')
474
520
475 return HookResponse(status, stdout.getvalue())
521 return HookResponse(status, stdout.getvalue())
476
522
477
523
478 def _parse_git_ref_lines(revision_lines):
524 def _parse_git_ref_lines(revision_lines):
479 rev_data = []
525 rev_data = []
480 for revision_line in revision_lines or []:
526 for revision_line in revision_lines or []:
481 old_rev, new_rev, ref = revision_line.strip().split(' ')
527 old_rev, new_rev, ref = revision_line.strip().split(' ')
482 ref_data = ref.split('/', 2)
528 ref_data = ref.split('/', 2)
483 if ref_data[1] in ('tags', 'heads'):
529 if ref_data[1] in ('tags', 'heads'):
484 rev_data.append({
530 rev_data.append({
485 # NOTE(marcink):
531 # NOTE(marcink):
486 # we're unable to tell total_commits for git at this point
532 # we're unable to tell total_commits for git at this point
487 # but we set the variable for consistency with GIT
533 # but we set the variable for consistency with GIT
488 'total_commits': -1,
534 'total_commits': -1,
489 'old_rev': old_rev,
535 'old_rev': old_rev,
490 'new_rev': new_rev,
536 'new_rev': new_rev,
491 'ref': ref,
537 'ref': ref,
492 'type': ref_data[1],
538 'type': ref_data[1],
493 'name': ref_data[2],
539 'name': ref_data[2],
494 })
540 })
495 return rev_data
541 return rev_data
496
542
497
543
498 def git_pre_receive(unused_repo_path, revision_lines, env):
544 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
499 """
545 """
500 Pre push hook.
546 Pre push hook.
501
547
502 :param extras: dictionary containing the keys defined in simplevcs
503 :type extras: dict
504
505 :return: status code of the hook. 0 for success.
548 :return: status code of the hook. 0 for success.
506 :rtype: int
507 """
549 """
508 extras = json.loads(env['RC_SCM_DATA'])
550 extras = json.loads(env['RC_SCM_DATA'])
509 rev_data = _parse_git_ref_lines(revision_lines)
551 rev_data = _parse_git_ref_lines(revision_lines)
510 if 'push' not in extras['hooks']:
552 if 'push' not in extras['hooks']:
511 return 0
553 return 0
512 empty_commit_id = '0' * 40
554 empty_commit_id = '0' * 40
513
555
514 detect_force_push = extras.get('detect_force_push')
556 detect_force_push = extras.get('detect_force_push')
515
557
516 for push_ref in rev_data:
558 for push_ref in rev_data:
517 # store our git-env which holds the temp store
559 # store our git-env which holds the temp store
518 push_ref['git_env'] = _get_git_env()
560 push_ref['git_env'] = _get_git_env()
519 push_ref['pruned_sha'] = ''
561 push_ref['pruned_sha'] = ''
520 if not detect_force_push:
562 if not detect_force_push:
521 # don't check for forced-push when we don't need to
563 # don't check for forced-push when we don't need to
522 continue
564 continue
523
565
524 type_ = push_ref['type']
566 type_ = push_ref['type']
525 new_branch = push_ref['old_rev'] == empty_commit_id
567 new_branch = push_ref['old_rev'] == empty_commit_id
526 delete_branch = push_ref['new_rev'] == empty_commit_id
568 delete_branch = push_ref['new_rev'] == empty_commit_id
527 if type_ == 'heads' and not (new_branch or delete_branch):
569 if type_ == 'heads' and not (new_branch or delete_branch):
528 old_rev = push_ref['old_rev']
570 old_rev = push_ref['old_rev']
529 new_rev = push_ref['new_rev']
571 new_rev = push_ref['new_rev']
530 cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, '^{}'.format(new_rev)]
572 cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, f'^{new_rev}']
531 stdout, stderr = subprocessio.run_command(
573 stdout, stderr = subprocessio.run_command(
532 cmd, env=os.environ.copy())
574 cmd, env=os.environ.copy())
533 # means we're having some non-reachable objects, this forced push was used
575 # means we're having some non-reachable objects, this forced push was used
534 if stdout:
576 if stdout:
535 push_ref['pruned_sha'] = stdout.splitlines()
577 push_ref['pruned_sha'] = stdout.splitlines()
536
578
537 extras['hook_type'] = 'pre_receive'
579 extras['hook_type'] = 'pre_receive'
538 extras['commit_ids'] = rev_data
580 extras['commit_ids'] = rev_data
539 return _call_hook('pre_push', extras, GitMessageWriter())
581
582 stdout = sys.stdout
583 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
584
585 return status_code
540
586
541
587
542 def git_post_receive(unused_repo_path, revision_lines, env):
588 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
543 """
589 """
544 Post push hook.
590 Post push hook.
545
591
546 :param extras: dictionary containing the keys defined in simplevcs
547 :type extras: dict
548
549 :return: status code of the hook. 0 for success.
592 :return: status code of the hook. 0 for success.
550 :rtype: int
551 """
593 """
552 extras = json.loads(env['RC_SCM_DATA'])
594 extras = json.loads(env['RC_SCM_DATA'])
553 if 'push' not in extras['hooks']:
595 if 'push' not in extras['hooks']:
554 return 0
596 return 0
555
597
556 rev_data = _parse_git_ref_lines(revision_lines)
598 rev_data = _parse_git_ref_lines(revision_lines)
557
599
558 git_revs = []
600 git_revs = []
559
601
560 # N.B.(skreft): it is ok to just call git, as git before calling a
602 # N.B.(skreft): it is ok to just call git, as git before calling a
561 # subcommand sets the PATH environment variable so that it point to the
603 # subcommand sets the PATH environment variable so that it point to the
562 # correct version of the git executable.
604 # correct version of the git executable.
563 empty_commit_id = '0' * 40
605 empty_commit_id = '0' * 40
564 branches = []
606 branches = []
565 tags = []
607 tags = []
566 for push_ref in rev_data:
608 for push_ref in rev_data:
567 type_ = push_ref['type']
609 type_ = push_ref['type']
568
610
569 if type_ == 'heads':
611 if type_ == 'heads':
612 # starting new branch case
570 if push_ref['old_rev'] == empty_commit_id:
613 if push_ref['old_rev'] == empty_commit_id:
571 # starting new branch case
614 push_ref_name = push_ref['name']
572 if push_ref['name'] not in branches:
615
573 branches.append(push_ref['name'])
616 if push_ref_name not in branches:
617 branches.append(push_ref_name)
574
618
575 # Fix up head revision if needed
619 need_head_set = ''
576 cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
620 with Repository(os.getcwd()) as repo:
577 try:
621 try:
578 subprocessio.run_command(cmd, env=os.environ.copy())
622 repo.head
579 except Exception:
623 except pygit2.GitError:
580 cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
624 need_head_set = f'refs/heads/{push_ref_name}'
581 'refs/heads/%s' % push_ref['name']]
582 print("Setting default branch to %s" % push_ref['name'])
583 subprocessio.run_command(cmd, env=os.environ.copy())
584
625
585 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
626 if need_head_set:
586 '--format=%(refname)', 'refs/heads/*']
627 repo.set_head(need_head_set)
628 print(f"Setting default branch to {push_ref_name}")
629
630 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
587 stdout, stderr = subprocessio.run_command(
631 stdout, stderr = subprocessio.run_command(
588 cmd, env=os.environ.copy())
632 cmd, env=os.environ.copy())
589 heads = stdout
633 heads = safe_str(stdout)
590 heads = heads.replace(push_ref['ref'], '')
634 heads = heads.replace(push_ref['ref'], '')
591 heads = ' '.join(head for head
635 heads = ' '.join(head for head
592 in heads.splitlines() if head) or '.'
636 in heads.splitlines() if head) or '.'
593 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
637 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
594 '--pretty=format:%H', '--', push_ref['new_rev'],
638 '--pretty=format:%H', '--', push_ref['new_rev'],
595 '--not', heads]
639 '--not', heads]
596 stdout, stderr = subprocessio.run_command(
640 stdout, stderr = subprocessio.run_command(
597 cmd, env=os.environ.copy())
641 cmd, env=os.environ.copy())
598 git_revs.extend(stdout.splitlines())
642 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
643
644 # delete branch case
599 elif push_ref['new_rev'] == empty_commit_id:
645 elif push_ref['new_rev'] == empty_commit_id:
600 # delete branch case
646 git_revs.append(f'delete_branch=>{push_ref["name"]}')
601 git_revs.append('delete_branch=>%s' % push_ref['name'])
602 else:
647 else:
603 if push_ref['name'] not in branches:
648 if push_ref['name'] not in branches:
604 branches.append(push_ref['name'])
649 branches.append(push_ref['name'])
605
650
606 cmd = [settings.GIT_EXECUTABLE, 'log',
651 cmd = [settings.GIT_EXECUTABLE, 'log',
607 '{old_rev}..{new_rev}'.format(**push_ref),
652 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
608 '--reverse', '--pretty=format:%H']
653 '--reverse', '--pretty=format:%H']
609 stdout, stderr = subprocessio.run_command(
654 stdout, stderr = subprocessio.run_command(
610 cmd, env=os.environ.copy())
655 cmd, env=os.environ.copy())
611 git_revs.extend(stdout.splitlines())
656 # we get bytes from stdout, we need str to be consistent
657 log_revs = list(map(ascii_str, stdout.splitlines()))
658 git_revs.extend(log_revs)
659
660 # Pure pygit2 impl. but still 2-3x slower :/
661 # results = []
662 #
663 # with Repository(os.getcwd()) as repo:
664 # repo_new_rev = repo[push_ref['new_rev']]
665 # repo_old_rev = repo[push_ref['old_rev']]
666 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
667 #
668 # for commit in walker:
669 # if commit.id == repo_old_rev.id:
670 # break
671 # results.append(commit.id.hex)
672 # # reverse the order, can't use GIT_SORT_REVERSE
673 # log_revs = results[::-1]
674
612 elif type_ == 'tags':
675 elif type_ == 'tags':
613 if push_ref['name'] not in tags:
676 if push_ref['name'] not in tags:
614 tags.append(push_ref['name'])
677 tags.append(push_ref['name'])
615 git_revs.append('tag=>%s' % push_ref['name'])
678 git_revs.append(f'tag=>{push_ref["name"]}')
616
679
617 extras['hook_type'] = 'post_receive'
680 extras['hook_type'] = 'post_receive'
618 extras['commit_ids'] = git_revs
681 extras['commit_ids'] = git_revs
619 extras['new_refs'] = {
682 extras['new_refs'] = {
620 'branches': branches,
683 'branches': branches,
621 'bookmarks': [],
684 'bookmarks': [],
622 'tags': tags,
685 'tags': tags,
623 }
686 }
624
687
688 stdout = sys.stdout
689
625 if 'repo_size' in extras['hooks']:
690 if 'repo_size' in extras['hooks']:
626 try:
691 try:
627 _call_hook('repo_size', extras, GitMessageWriter())
692 _call_hook('repo_size', extras, GitMessageWriter(stdout))
628 except:
693 except Exception:
629 pass
694 pass
630
695
631 return _call_hook('post_push', extras, GitMessageWriter())
696 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
697 return status_code
632
698
633
699
634 def _get_extras_from_txn_id(path, txn_id):
700 def _get_extras_from_txn_id(path, txn_id):
635 extras = {}
701 extras = {}
636 try:
702 try:
637 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
703 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
638 '-t', txn_id,
704 '-t', txn_id,
639 '--revprop', path, 'rc-scm-extras']
705 '--revprop', path, 'rc-scm-extras']
640 stdout, stderr = subprocessio.run_command(
706 stdout, stderr = subprocessio.run_command(
641 cmd, env=os.environ.copy())
707 cmd, env=os.environ.copy())
642 extras = json.loads(base64.urlsafe_b64decode(stdout))
708 extras = json.loads(base64.urlsafe_b64decode(stdout))
643 except Exception:
709 except Exception:
644 log.exception('Failed to extract extras info from txn_id')
710 log.exception('Failed to extract extras info from txn_id')
645
711
646 return extras
712 return extras
647
713
648
714
649 def _get_extras_from_commit_id(commit_id, path):
715 def _get_extras_from_commit_id(commit_id, path):
650 extras = {}
716 extras = {}
651 try:
717 try:
652 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
718 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
653 '-r', commit_id,
719 '-r', commit_id,
654 '--revprop', path, 'rc-scm-extras']
720 '--revprop', path, 'rc-scm-extras']
655 stdout, stderr = subprocessio.run_command(
721 stdout, stderr = subprocessio.run_command(
656 cmd, env=os.environ.copy())
722 cmd, env=os.environ.copy())
657 extras = json.loads(base64.urlsafe_b64decode(stdout))
723 extras = json.loads(base64.urlsafe_b64decode(stdout))
658 except Exception:
724 except Exception:
659 log.exception('Failed to extract extras info from commit_id')
725 log.exception('Failed to extract extras info from commit_id')
660
726
661 return extras
727 return extras
662
728
663
729
664 def svn_pre_commit(repo_path, commit_data, env):
730 def svn_pre_commit(repo_path, commit_data, env):
665 path, txn_id = commit_data
731 path, txn_id = commit_data
666 branches = []
732 branches = []
667 tags = []
733 tags = []
668
734
669 if env.get('RC_SCM_DATA'):
735 if env.get('RC_SCM_DATA'):
670 extras = json.loads(env['RC_SCM_DATA'])
736 extras = json.loads(env['RC_SCM_DATA'])
671 else:
737 else:
672 # fallback method to read from TXN-ID stored data
738 # fallback method to read from TXN-ID stored data
673 extras = _get_extras_from_txn_id(path, txn_id)
739 extras = _get_extras_from_txn_id(path, txn_id)
674 if not extras:
740 if not extras:
675 return 0
741 return 0
676
742
677 extras['hook_type'] = 'pre_commit'
743 extras['hook_type'] = 'pre_commit'
678 extras['commit_ids'] = [txn_id]
744 extras['commit_ids'] = [txn_id]
679 extras['txn_id'] = txn_id
745 extras['txn_id'] = txn_id
680 extras['new_refs'] = {
746 extras['new_refs'] = {
681 'total_commits': 1,
747 'total_commits': 1,
682 'branches': branches,
748 'branches': branches,
683 'bookmarks': [],
749 'bookmarks': [],
684 'tags': tags,
750 'tags': tags,
685 }
751 }
686
752
687 return _call_hook('pre_push', extras, SvnMessageWriter())
753 return _call_hook('pre_push', extras, SvnMessageWriter())
688
754
689
755
690 def svn_post_commit(repo_path, commit_data, env):
756 def svn_post_commit(repo_path, commit_data, env):
691 """
757 """
692 commit_data is path, rev, txn_id
758 commit_data is path, rev, txn_id
693 """
759 """
694 if len(commit_data) == 3:
760 if len(commit_data) == 3:
695 path, commit_id, txn_id = commit_data
761 path, commit_id, txn_id = commit_data
696 elif len(commit_data) == 2:
762 elif len(commit_data) == 2:
697 log.error('Failed to extract txn_id from commit_data using legacy method. '
763 log.error('Failed to extract txn_id from commit_data using legacy method. '
698 'Some functionality might be limited')
764 'Some functionality might be limited')
699 path, commit_id = commit_data
765 path, commit_id = commit_data
700 txn_id = None
766 txn_id = None
701
767
702 branches = []
768 branches = []
703 tags = []
769 tags = []
704
770
705 if env.get('RC_SCM_DATA'):
771 if env.get('RC_SCM_DATA'):
706 extras = json.loads(env['RC_SCM_DATA'])
772 extras = json.loads(env['RC_SCM_DATA'])
707 else:
773 else:
708 # fallback method to read from TXN-ID stored data
774 # fallback method to read from TXN-ID stored data
709 extras = _get_extras_from_commit_id(commit_id, path)
775 extras = _get_extras_from_commit_id(commit_id, path)
710 if not extras:
776 if not extras:
711 return 0
777 return 0
712
778
713 extras['hook_type'] = 'post_commit'
779 extras['hook_type'] = 'post_commit'
714 extras['commit_ids'] = [commit_id]
780 extras['commit_ids'] = [commit_id]
715 extras['txn_id'] = txn_id
781 extras['txn_id'] = txn_id
716 extras['new_refs'] = {
782 extras['new_refs'] = {
717 'branches': branches,
783 'branches': branches,
718 'bookmarks': [],
784 'bookmarks': [],
719 'tags': tags,
785 'tags': tags,
720 'total_commits': 1,
786 'total_commits': 1,
721 }
787 }
722
788
723 if 'repo_size' in extras['hooks']:
789 if 'repo_size' in extras['hooks']:
724 try:
790 try:
725 _call_hook('repo_size', extras, SvnMessageWriter())
791 _call_hook('repo_size', extras, SvnMessageWriter())
726 except Exception:
792 except Exception:
727 pass
793 pass
728
794
729 return _call_hook('post_push', extras, SvnMessageWriter())
795 return _call_hook('post_push', extras, SvnMessageWriter())
@@ -1,705 +1,775 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import os
19 import os
20 import platform
19 import sys
21 import sys
20 import base64
21 import locale
22 import locale
22 import logging
23 import logging
23 import uuid
24 import uuid
25 import time
24 import wsgiref.util
26 import wsgiref.util
25 import traceback
26 import tempfile
27 import tempfile
27 import psutil
28 import psutil
29
28 from itertools import chain
30 from itertools import chain
29 from cStringIO import StringIO
30
31
31 import simplejson as json
32 import msgpack
32 import msgpack
33 import configparser
34
33 from pyramid.config import Configurator
35 from pyramid.config import Configurator
34 from pyramid.settings import asbool, aslist
35 from pyramid.wsgi import wsgiapp
36 from pyramid.wsgi import wsgiapp
36 from pyramid.compat import configparser
37 from pyramid.response import Response
37 from pyramid.response import Response
38
38
39 from vcsserver.utils import safe_int
39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
40 from vcsserver.lib.rc_json import json
41 from vcsserver.config.settings_maker import SettingsMaker
42 from vcsserver.str_utils import safe_int
43 from vcsserver.lib.statsd_client import StatsdClient
44 from vcsserver.tweens.request_wrapper import get_headers_call_context
45
46 import vcsserver
47 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
48 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
49 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
50 from vcsserver.echo_stub.echo_app import EchoApp
51 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
52 from vcsserver.lib.exc_tracking import store_exception, format_exc
53 from vcsserver.server import VcsServer
54
55 strict_vcs = True
56
57 git_import_err = None
58 try:
59 from vcsserver.remote.git_remote import GitFactory, GitRemote
60 except ImportError as e:
61 GitFactory = None
62 GitRemote = None
63 git_import_err = e
64 if strict_vcs:
65 raise
66
67
68 hg_import_err = None
69 try:
70 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
71 except ImportError as e:
72 MercurialFactory = None
73 HgRemote = None
74 hg_import_err = e
75 if strict_vcs:
76 raise
77
78
79 svn_import_err = None
80 try:
81 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
82 except ImportError as e:
83 SubversionFactory = None
84 SvnRemote = None
85 svn_import_err = e
86 if strict_vcs:
87 raise
40
88
41 log = logging.getLogger(__name__)
89 log = logging.getLogger(__name__)
42
90
43 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
91 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
44 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
92 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
45
93
46 try:
94 try:
47 locale.setlocale(locale.LC_ALL, '')
95 locale.setlocale(locale.LC_ALL, '')
48 except locale.Error as e:
96 except locale.Error as e:
49 log.error(
97 log.error(
50 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
98 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
51 os.environ['LC_ALL'] = 'C'
99 os.environ['LC_ALL'] = 'C'
52
100
53 import vcsserver
54 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
55 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
56 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
57 from vcsserver.echo_stub.echo_app import EchoApp
58 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
59 from vcsserver.lib.exc_tracking import store_exception
60 from vcsserver.server import VcsServer
61
62 try:
63 from vcsserver.git import GitFactory, GitRemote
64 except ImportError:
65 GitFactory = None
66 GitRemote = None
67
68 try:
69 from vcsserver.hg import MercurialFactory, HgRemote
70 except ImportError:
71 MercurialFactory = None
72 HgRemote = None
73
74 try:
75 from vcsserver.svn import SubversionFactory, SvnRemote
76 except ImportError:
77 SubversionFactory = None
78 SvnRemote = None
79
80
101
81 def _is_request_chunked(environ):
102 def _is_request_chunked(environ):
82 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
103 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
83 return stream
104 return stream
84
105
85
106
86 def _int_setting(settings, name, default):
87 settings[name] = int(settings.get(name, default))
88 return settings[name]
89
90
91 def _bool_setting(settings, name, default):
92 input_val = settings.get(name, default)
93 if isinstance(input_val, unicode):
94 input_val = input_val.encode('utf8')
95 settings[name] = asbool(input_val)
96 return settings[name]
97
98
99 def _list_setting(settings, name, default):
100 raw_value = settings.get(name, default)
101
102 # Otherwise we assume it uses pyramids space/newline separation.
103 settings[name] = aslist(raw_value)
104 return settings[name]
105
106
107 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
108 value = settings.get(name, default)
109
110 if default_when_empty and not value:
111 # use default value when value is empty
112 value = default
113
114 if lower:
115 value = value.lower()
116 settings[name] = value
117 return settings[name]
118
119
120 def log_max_fd():
107 def log_max_fd():
121 try:
108 try:
122 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
109 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
123 log.info('Max file descriptors value: %s', maxfd)
110 log.info('Max file descriptors value: %s', maxfd)
124 except Exception:
111 except Exception:
125 pass
112 pass
126
113
127
114
128 class VCS(object):
115 class VCS:
129 def __init__(self, locale_conf=None, cache_config=None):
116 def __init__(self, locale_conf=None, cache_config=None):
130 self.locale = locale_conf
117 self.locale = locale_conf
131 self.cache_config = cache_config
118 self.cache_config = cache_config
132 self._configure_locale()
119 self._configure_locale()
133
120
134 log_max_fd()
121 log_max_fd()
135
122
136 if GitFactory and GitRemote:
123 if GitFactory and GitRemote:
137 git_factory = GitFactory()
124 git_factory = GitFactory()
138 self._git_remote = GitRemote(git_factory)
125 self._git_remote = GitRemote(git_factory)
139 else:
126 else:
140 log.info("Git client import failed")
127 log.error("Git client import failed: %s", git_import_err)
141
128
142 if MercurialFactory and HgRemote:
129 if MercurialFactory and HgRemote:
143 hg_factory = MercurialFactory()
130 hg_factory = MercurialFactory()
144 self._hg_remote = HgRemote(hg_factory)
131 self._hg_remote = HgRemote(hg_factory)
145 else:
132 else:
146 log.info("Mercurial client import failed")
133 log.error("Mercurial client import failed: %s", hg_import_err)
147
134
148 if SubversionFactory and SvnRemote:
135 if SubversionFactory and SvnRemote:
149 svn_factory = SubversionFactory()
136 svn_factory = SubversionFactory()
150
137
151 # hg factory is used for svn url validation
138 # hg factory is used for svn url validation
152 hg_factory = MercurialFactory()
139 hg_factory = MercurialFactory()
153 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
140 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
154 else:
141 else:
155 log.info("Subversion client import failed")
142 log.error("Subversion client import failed: %s", svn_import_err)
156
143
157 self._vcsserver = VcsServer()
144 self._vcsserver = VcsServer()
158
145
159 def _configure_locale(self):
146 def _configure_locale(self):
160 if self.locale:
147 if self.locale:
161 log.info('Settings locale: `LC_ALL` to %s', self.locale)
148 log.info('Settings locale: `LC_ALL` to %s', self.locale)
162 else:
149 else:
163 log.info(
150 log.info('Configuring locale subsystem based on environment variables')
164 'Configuring locale subsystem based on environment variables')
165 try:
151 try:
166 # If self.locale is the empty string, then the locale
152 # If self.locale is the empty string, then the locale
167 # module will use the environment variables. See the
153 # module will use the environment variables. See the
168 # documentation of the package `locale`.
154 # documentation of the package `locale`.
169 locale.setlocale(locale.LC_ALL, self.locale)
155 locale.setlocale(locale.LC_ALL, self.locale)
170
156
171 language_code, encoding = locale.getlocale()
157 language_code, encoding = locale.getlocale()
172 log.info(
158 log.info(
173 'Locale set to language code "%s" with encoding "%s".',
159 'Locale set to language code "%s" with encoding "%s".',
174 language_code, encoding)
160 language_code, encoding)
175 except locale.Error:
161 except locale.Error:
176 log.exception(
162 log.exception('Cannot set locale, not configuring the locale system')
177 'Cannot set locale, not configuring the locale system')
178
163
179
164
180 class WsgiProxy(object):
165 class WsgiProxy:
181 def __init__(self, wsgi):
166 def __init__(self, wsgi):
182 self.wsgi = wsgi
167 self.wsgi = wsgi
183
168
184 def __call__(self, environ, start_response):
169 def __call__(self, environ, start_response):
185 input_data = environ['wsgi.input'].read()
170 input_data = environ['wsgi.input'].read()
186 input_data = msgpack.unpackb(input_data)
171 input_data = msgpack.unpackb(input_data)
187
172
188 error = None
173 error = None
189 try:
174 try:
190 data, status, headers = self.wsgi.handle(
175 data, status, headers = self.wsgi.handle(
191 input_data['environment'], input_data['input_data'],
176 input_data['environment'], input_data['input_data'],
192 *input_data['args'], **input_data['kwargs'])
177 *input_data['args'], **input_data['kwargs'])
193 except Exception as e:
178 except Exception as e:
194 data, status, headers = [], None, None
179 data, status, headers = [], None, None
195 error = {
180 error = {
196 'message': str(e),
181 'message': str(e),
197 '_vcs_kind': getattr(e, '_vcs_kind', None)
182 '_vcs_kind': getattr(e, '_vcs_kind', None)
198 }
183 }
199
184
200 start_response(200, {})
185 start_response(200, {})
201 return self._iterator(error, status, headers, data)
186 return self._iterator(error, status, headers, data)
202
187
203 def _iterator(self, error, status, headers, data):
188 def _iterator(self, error, status, headers, data):
204 initial_data = [
189 initial_data = [
205 error,
190 error,
206 status,
191 status,
207 headers,
192 headers,
208 ]
193 ]
209
194
210 for d in chain(initial_data, data):
195 for d in chain(initial_data, data):
211 yield msgpack.packb(d)
196 yield msgpack.packb(d)
212
197
213
198
214 def not_found(request):
199 def not_found(request):
215 return {'status': '404 NOT FOUND'}
200 return {'status': '404 NOT FOUND'}
216
201
217
202
218 class VCSViewPredicate(object):
203 class VCSViewPredicate:
219 def __init__(self, val, config):
204 def __init__(self, val, config):
220 self.remotes = val
205 self.remotes = val
221
206
222 def text(self):
207 def text(self):
223 return 'vcs view method = %s' % (self.remotes.keys(),)
208 return f'vcs view method = {list(self.remotes.keys())}'
224
209
225 phash = text
210 phash = text
226
211
227 def __call__(self, context, request):
212 def __call__(self, context, request):
228 """
213 """
229 View predicate that returns true if given backend is supported by
214 View predicate that returns true if given backend is supported by
230 defined remotes.
215 defined remotes.
231 """
216 """
232 backend = request.matchdict.get('backend')
217 backend = request.matchdict.get('backend')
233 return backend in self.remotes
218 return backend in self.remotes
234
219
235
220
236 class HTTPApplication(object):
221 class HTTPApplication:
237 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
222 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
238
223
239 remote_wsgi = remote_wsgi
224 remote_wsgi = remote_wsgi
240 _use_echo_app = False
225 _use_echo_app = False
241
226
242 def __init__(self, settings=None, global_config=None):
227 def __init__(self, settings=None, global_config=None):
243 self._sanitize_settings_and_apply_defaults(settings)
244
228
245 self.config = Configurator(settings=settings)
229 self.config = Configurator(settings=settings)
230 # Init our statsd at very start
231 self.config.registry.statsd = StatsdClient.statsd
232 self.config.registry.vcs_call_context = {}
233
246 self.global_config = global_config
234 self.global_config = global_config
247 self.config.include('vcsserver.lib.rc_cache')
235 self.config.include('vcsserver.lib.rc_cache')
236 self.config.include('vcsserver.lib.rc_cache.archive_cache')
248
237
249 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
250 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
251 self._remotes = {
240 self._remotes = {
252 'hg': vcs._hg_remote,
241 'hg': vcs._hg_remote,
253 'git': vcs._git_remote,
242 'git': vcs._git_remote,
254 'svn': vcs._svn_remote,
243 'svn': vcs._svn_remote,
255 'server': vcs._vcsserver,
244 'server': vcs._vcsserver,
256 }
245 }
257 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
258 self._use_echo_app = True
247 self._use_echo_app = True
259 log.warning("Using EchoApp for VCS operations.")
248 log.warning("Using EchoApp for VCS operations.")
260 self.remote_wsgi = remote_wsgi_stub
249 self.remote_wsgi = remote_wsgi_stub
261
250
262 self._configure_settings(global_config, settings)
251 self._configure_settings(global_config, settings)
263
252
264 self._configure()
253 self._configure()
265
254
266 def _configure_settings(self, global_config, app_settings):
255 def _configure_settings(self, global_config, app_settings):
267 """
256 """
268 Configure the settings module.
257 Configure the settings module.
269 """
258 """
270 settings_merged = global_config.copy()
259 settings_merged = global_config.copy()
271 settings_merged.update(app_settings)
260 settings_merged.update(app_settings)
272
261
273 git_path = app_settings.get('git_path', None)
262 git_path = app_settings.get('git_path', None)
274 if git_path:
263 if git_path:
275 settings.GIT_EXECUTABLE = git_path
264 settings.GIT_EXECUTABLE = git_path
276 binary_dir = app_settings.get('core.binary_dir', None)
265 binary_dir = app_settings.get('core.binary_dir', None)
277 if binary_dir:
266 if binary_dir:
278 settings.BINARY_DIR = binary_dir
267 settings.BINARY_DIR = binary_dir
279
268
280 # Store the settings to make them available to other modules.
269 # Store the settings to make them available to other modules.
281 vcsserver.PYRAMID_SETTINGS = settings_merged
270 vcsserver.PYRAMID_SETTINGS = settings_merged
282 vcsserver.CONFIG = settings_merged
271 vcsserver.CONFIG = settings_merged
283
272
284 def _sanitize_settings_and_apply_defaults(self, settings):
285 temp_store = tempfile.gettempdir()
286 default_cache_dir = os.path.join(temp_store, 'rc_cache')
287
288 # save default, cache dir, and use it for all backends later.
289 default_cache_dir = _string_setting(
290 settings,
291 'cache_dir',
292 default_cache_dir, lower=False, default_when_empty=True)
293
294 # ensure we have our dir created
295 if not os.path.isdir(default_cache_dir):
296 os.makedirs(default_cache_dir, mode=0o755)
297
298 # exception store cache
299 _string_setting(
300 settings,
301 'exception_tracker.store_path',
302 temp_store, lower=False, default_when_empty=True)
303
304 # repo_object cache
305 _string_setting(
306 settings,
307 'rc_cache.repo_object.backend',
308 'dogpile.cache.rc.file_namespace', lower=False)
309 _int_setting(
310 settings,
311 'rc_cache.repo_object.expiration_time',
312 30 * 24 * 60 * 60)
313 _string_setting(
314 settings,
315 'rc_cache.repo_object.arguments.filename',
316 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
317
318 def _configure(self):
273 def _configure(self):
319 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
274 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
320
275
321 self.config.add_route('service', '/_service')
276 self.config.add_route('service', '/_service')
322 self.config.add_route('status', '/status')
277 self.config.add_route('status', '/status')
323 self.config.add_route('hg_proxy', '/proxy/hg')
278 self.config.add_route('hg_proxy', '/proxy/hg')
324 self.config.add_route('git_proxy', '/proxy/git')
279 self.config.add_route('git_proxy', '/proxy/git')
325
280
326 # rpc methods
281 # rpc methods
327 self.config.add_route('vcs', '/{backend}')
282 self.config.add_route('vcs', '/{backend}')
328
283
329 # streaming rpc remote methods
284 # streaming rpc remote methods
330 self.config.add_route('vcs_stream', '/{backend}/stream')
285 self.config.add_route('vcs_stream', '/{backend}/stream')
331
286
332 # vcs operations clone/push as streaming
287 # vcs operations clone/push as streaming
333 self.config.add_route('stream_git', '/stream/git/*repo_name')
288 self.config.add_route('stream_git', '/stream/git/*repo_name')
334 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
289 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
335
290
336 self.config.add_view(self.status_view, route_name='status', renderer='json')
291 self.config.add_view(self.status_view, route_name='status', renderer='json')
337 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
292 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
338
293
339 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
294 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
340 self.config.add_view(self.git_proxy(), route_name='git_proxy')
295 self.config.add_view(self.git_proxy(), route_name='git_proxy')
341 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
296 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
342 vcs_view=self._remotes)
297 vcs_view=self._remotes)
343 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
298 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
344 vcs_view=self._remotes)
299 vcs_view=self._remotes)
345
300
346 self.config.add_view(self.hg_stream(), route_name='stream_hg')
301 self.config.add_view(self.hg_stream(), route_name='stream_hg')
347 self.config.add_view(self.git_stream(), route_name='stream_git')
302 self.config.add_view(self.git_stream(), route_name='stream_git')
348
303
349 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
304 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
350
305
351 self.config.add_notfound_view(not_found, renderer='json')
306 self.config.add_notfound_view(not_found, renderer='json')
352
307
353 self.config.add_view(self.handle_vcs_exception, context=Exception)
308 self.config.add_view(self.handle_vcs_exception, context=Exception)
354
309
355 self.config.add_tween(
310 self.config.add_tween(
356 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
311 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
357 )
312 )
358 self.config.add_request_method(
313 self.config.add_request_method(
359 'vcsserver.lib.request_counter.get_request_counter',
314 'vcsserver.lib.request_counter.get_request_counter',
360 'request_count')
315 'request_count')
361
316
362 self.config.add_request_method(
363 'vcsserver.lib._vendor.statsd.get_statsd_client',
364 'statsd', reify=True)
365
366 def wsgi_app(self):
317 def wsgi_app(self):
367 return self.config.make_wsgi_app()
318 return self.config.make_wsgi_app()
368
319
369 def _vcs_view_params(self, request):
320 def _vcs_view_params(self, request):
370 remote = self._remotes[request.matchdict['backend']]
321 remote = self._remotes[request.matchdict['backend']]
371 payload = msgpack.unpackb(request.body, use_list=True)
322 payload = msgpack.unpackb(request.body, use_list=True)
323
372 method = payload.get('method')
324 method = payload.get('method')
373 params = payload['params']
325 params = payload['params']
374 wire = params.get('wire')
326 wire = params.get('wire')
375 args = params.get('args')
327 args = params.get('args')
376 kwargs = params.get('kwargs')
328 kwargs = params.get('kwargs')
377 context_uid = None
329 context_uid = None
378
330
331 request.registry.vcs_call_context = {
332 'method': method,
333 'repo_name': payload.get('_repo_name'),
334 }
335
379 if wire:
336 if wire:
380 try:
337 try:
381 wire['context'] = context_uid = uuid.UUID(wire['context'])
338 wire['context'] = context_uid = uuid.UUID(wire['context'])
382 except KeyError:
339 except KeyError:
383 pass
340 pass
384 args.insert(0, wire)
341 args.insert(0, wire)
385 repo_state_uid = wire.get('repo_state_uid') if wire else None
342 repo_state_uid = wire.get('repo_state_uid') if wire else None
386
343
387 # NOTE(marcink): trading complexity for slight performance
344 # NOTE(marcink): trading complexity for slight performance
388 if log.isEnabledFor(logging.DEBUG):
345 if log.isEnabledFor(logging.DEBUG):
389 no_args_methods = [
346 # also we SKIP printing out any of those methods args since they maybe excessive
390
347 just_args_methods = {
391 ]
348 'commitctx': ('content', 'removed', 'updated'),
392 if method in no_args_methods:
349 'commit': ('content', 'removed', 'updated')
350 }
351 if method in just_args_methods:
352 skip_args = just_args_methods[method]
393 call_args = ''
353 call_args = ''
354 call_kwargs = {}
355 for k in kwargs:
356 if k in skip_args:
357 # replace our skip key with dummy
358 call_kwargs[k] = f'RemovedParam({k})'
359 else:
360 call_kwargs[k] = kwargs[k]
394 else:
361 else:
395 call_args = args[1:]
362 call_args = args[1:]
363 call_kwargs = kwargs
396
364
397 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
365 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
398 method, call_args, kwargs, context_uid, repo_state_uid)
366 method, call_args, call_kwargs, context_uid, repo_state_uid)
399
367
368 statsd = request.registry.statsd
369 if statsd:
370 statsd.incr(
371 'vcsserver_method_total', tags=[
372 f"method:{method}",
373 ])
400 return payload, remote, method, args, kwargs
374 return payload, remote, method, args, kwargs
401
375
402 def vcs_view(self, request):
376 def vcs_view(self, request):
403
377
404 payload, remote, method, args, kwargs = self._vcs_view_params(request)
378 payload, remote, method, args, kwargs = self._vcs_view_params(request)
405 payload_id = payload.get('id')
379 payload_id = payload.get('id')
406
380
407 try:
381 try:
408 resp = getattr(remote, method)(*args, **kwargs)
382 resp = getattr(remote, method)(*args, **kwargs)
409 except Exception as e:
383 except Exception as e:
410 exc_info = list(sys.exc_info())
384 exc_info = list(sys.exc_info())
411 exc_type, exc_value, exc_traceback = exc_info
385 exc_type, exc_value, exc_traceback = exc_info
412
386
413 org_exc = getattr(e, '_org_exc', None)
387 org_exc = getattr(e, '_org_exc', None)
414 org_exc_name = None
388 org_exc_name = None
415 org_exc_tb = ''
389 org_exc_tb = ''
416 if org_exc:
390 if org_exc:
417 org_exc_name = org_exc.__class__.__name__
391 org_exc_name = org_exc.__class__.__name__
418 org_exc_tb = getattr(e, '_org_exc_tb', '')
392 org_exc_tb = getattr(e, '_org_exc_tb', '')
419 # replace our "faked" exception with our org
393 # replace our "faked" exception with our org
420 exc_info[0] = org_exc.__class__
394 exc_info[0] = org_exc.__class__
421 exc_info[1] = org_exc
395 exc_info[1] = org_exc
422
396
423 should_store_exc = True
397 should_store_exc = True
424 if org_exc:
398 if org_exc:
425 def get_exc_fqn(_exc_obj):
399 def get_exc_fqn(_exc_obj):
426 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
400 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
427 return module_name + '.' + org_exc_name
401 return module_name + '.' + org_exc_name
428
402
429 exc_fqn = get_exc_fqn(org_exc)
403 exc_fqn = get_exc_fqn(org_exc)
430
404
431 if exc_fqn in ['mercurial.error.RepoLookupError',
405 if exc_fqn in ['mercurial.error.RepoLookupError',
432 'vcsserver.exceptions.RefNotFoundException']:
406 'vcsserver.exceptions.RefNotFoundException']:
433 should_store_exc = False
407 should_store_exc = False
434
408
435 if should_store_exc:
409 if should_store_exc:
436 store_exception(id(exc_info), exc_info, request_path=request.path)
410 store_exception(id(exc_info), exc_info, request_path=request.path)
437
411
438 tb_info = ''.join(
412 tb_info = format_exc(exc_info)
439 traceback.format_exception(exc_type, exc_value, exc_traceback))
440
413
441 type_ = e.__class__.__name__
414 type_ = e.__class__.__name__
442 if type_ not in self.ALLOWED_EXCEPTIONS:
415 if type_ not in self.ALLOWED_EXCEPTIONS:
443 type_ = None
416 type_ = None
444
417
445 resp = {
418 resp = {
446 'id': payload_id,
419 'id': payload_id,
447 'error': {
420 'error': {
448 'message': e.message,
421 'message': str(e),
449 'traceback': tb_info,
422 'traceback': tb_info,
450 'org_exc': org_exc_name,
423 'org_exc': org_exc_name,
451 'org_exc_tb': org_exc_tb,
424 'org_exc_tb': org_exc_tb,
452 'type': type_
425 'type': type_
453 }
426 }
454 }
427 }
455
428
456 try:
429 try:
457 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
430 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
458 except AttributeError:
431 except AttributeError:
459 pass
432 pass
460 else:
433 else:
461 resp = {
434 resp = {
462 'id': payload_id,
435 'id': payload_id,
463 'result': resp
436 'result': resp
464 }
437 }
465
438 log.debug('Serving data for method %s', method)
466 return resp
439 return resp
467
440
468 def vcs_stream_view(self, request):
441 def vcs_stream_view(self, request):
469 payload, remote, method, args, kwargs = self._vcs_view_params(request)
442 payload, remote, method, args, kwargs = self._vcs_view_params(request)
470 # this method has a stream: marker we remove it here
443 # this method has a stream: marker we remove it here
471 method = method.split('stream:')[-1]
444 method = method.split('stream:')[-1]
472 chunk_size = safe_int(payload.get('chunk_size')) or 4096
445 chunk_size = safe_int(payload.get('chunk_size')) or 4096
473
446
474 try:
447 resp = getattr(remote, method)(*args, **kwargs)
475 resp = getattr(remote, method)(*args, **kwargs)
476 except Exception as e:
477 raise
478
448
479 def get_chunked_data(method_resp):
449 def get_chunked_data(method_resp):
480 stream = StringIO(method_resp)
450 stream = io.BytesIO(method_resp)
481 while 1:
451 while 1:
482 chunk = stream.read(chunk_size)
452 chunk = stream.read(chunk_size)
483 if not chunk:
453 if not chunk:
484 break
454 break
485 yield chunk
455 yield chunk
486
456
487 response = Response(app_iter=get_chunked_data(resp))
457 response = Response(app_iter=get_chunked_data(resp))
488 response.content_type = 'application/octet-stream'
458 response.content_type = 'application/octet-stream'
489
459
490 return response
460 return response
491
461
492 def status_view(self, request):
462 def status_view(self, request):
493 import vcsserver
463 import vcsserver
494 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
464 _platform_id = platform.uname()[1] or 'instance'
495 'pid': os.getpid()}
465
466 return {
467 "status": "OK",
468 "vcsserver_version": vcsserver.get_version(),
469 "platform": _platform_id,
470 "pid": os.getpid(),
471 }
496
472
497 def service_view(self, request):
473 def service_view(self, request):
498 import vcsserver
474 import vcsserver
499
475
500 payload = msgpack.unpackb(request.body, use_list=True)
476 payload = msgpack.unpackb(request.body, use_list=True)
501 server_config, app_config = {}, {}
477 server_config, app_config = {}, {}
502
478
503 try:
479 try:
504 path = self.global_config['__file__']
480 path = self.global_config['__file__']
505 config = configparser.RawConfigParser()
481 config = configparser.RawConfigParser()
506
482
507 config.read(path)
483 config.read(path)
508
484
509 if config.has_section('server:main'):
485 if config.has_section('server:main'):
510 server_config = dict(config.items('server:main'))
486 server_config = dict(config.items('server:main'))
511 if config.has_section('app:main'):
487 if config.has_section('app:main'):
512 app_config = dict(config.items('app:main'))
488 app_config = dict(config.items('app:main'))
513
489
514 except Exception:
490 except Exception:
515 log.exception('Failed to read .ini file for display')
491 log.exception('Failed to read .ini file for display')
516
492
517 environ = os.environ.items()
493 environ = list(os.environ.items())
518
494
519 resp = {
495 resp = {
520 'id': payload.get('id'),
496 'id': payload.get('id'),
521 'result': dict(
497 'result': dict(
522 version=vcsserver.__version__,
498 version=vcsserver.get_version(),
523 config=server_config,
499 config=server_config,
524 app_config=app_config,
500 app_config=app_config,
525 environ=environ,
501 environ=environ,
526 payload=payload,
502 payload=payload,
527 )
503 )
528 }
504 }
529 return resp
505 return resp
530
506
531 def _msgpack_renderer_factory(self, info):
507 def _msgpack_renderer_factory(self, info):
508
532 def _render(value, system):
509 def _render(value, system):
510 bin_type = False
511 res = value.get('result')
512 if isinstance(res, BytesEnvelope):
513 log.debug('Result is wrapped in BytesEnvelope type')
514 bin_type = True
515 elif isinstance(res, BinaryEnvelope):
516 log.debug('Result is wrapped in BinaryEnvelope type')
517 value['result'] = res.val
518 bin_type = True
519
533 request = system.get('request')
520 request = system.get('request')
534 if request is not None:
521 if request is not None:
535 response = request.response
522 response = request.response
536 ct = response.content_type
523 ct = response.content_type
537 if ct == response.default_content_type:
524 if ct == response.default_content_type:
538 response.content_type = 'application/x-msgpack'
525 response.content_type = 'application/x-msgpack'
539 return msgpack.packb(value)
526 if bin_type:
527 response.content_type = 'application/x-msgpack-bin'
528
529 return msgpack.packb(value, use_bin_type=bin_type)
540 return _render
530 return _render
541
531
542 def set_env_from_config(self, environ, config):
532 def set_env_from_config(self, environ, config):
543 dict_conf = {}
533 dict_conf = {}
544 try:
534 try:
545 for elem in config:
535 for elem in config:
546 if elem[0] == 'rhodecode':
536 if elem[0] == 'rhodecode':
547 dict_conf = json.loads(elem[2])
537 dict_conf = json.loads(elem[2])
548 break
538 break
549 except Exception:
539 except Exception:
550 log.exception('Failed to fetch SCM CONFIG')
540 log.exception('Failed to fetch SCM CONFIG')
551 return
541 return
552
542
553 username = dict_conf.get('username')
543 username = dict_conf.get('username')
554 if username:
544 if username:
555 environ['REMOTE_USER'] = username
545 environ['REMOTE_USER'] = username
556 # mercurial specific, some extension api rely on this
546 # mercurial specific, some extension api rely on this
557 environ['HGUSER'] = username
547 environ['HGUSER'] = username
558
548
559 ip = dict_conf.get('ip')
549 ip = dict_conf.get('ip')
560 if ip:
550 if ip:
561 environ['REMOTE_HOST'] = ip
551 environ['REMOTE_HOST'] = ip
562
552
563 if _is_request_chunked(environ):
553 if _is_request_chunked(environ):
564 # set the compatibility flag for webob
554 # set the compatibility flag for webob
565 environ['wsgi.input_terminated'] = True
555 environ['wsgi.input_terminated'] = True
566
556
567 def hg_proxy(self):
557 def hg_proxy(self):
568 @wsgiapp
558 @wsgiapp
569 def _hg_proxy(environ, start_response):
559 def _hg_proxy(environ, start_response):
570 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
560 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
571 return app(environ, start_response)
561 return app(environ, start_response)
572 return _hg_proxy
562 return _hg_proxy
573
563
574 def git_proxy(self):
564 def git_proxy(self):
575 @wsgiapp
565 @wsgiapp
576 def _git_proxy(environ, start_response):
566 def _git_proxy(environ, start_response):
577 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
567 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
578 return app(environ, start_response)
568 return app(environ, start_response)
579 return _git_proxy
569 return _git_proxy
580
570
581 def hg_stream(self):
571 def hg_stream(self):
582 if self._use_echo_app:
572 if self._use_echo_app:
583 @wsgiapp
573 @wsgiapp
584 def _hg_stream(environ, start_response):
574 def _hg_stream(environ, start_response):
585 app = EchoApp('fake_path', 'fake_name', None)
575 app = EchoApp('fake_path', 'fake_name', None)
586 return app(environ, start_response)
576 return app(environ, start_response)
587 return _hg_stream
577 return _hg_stream
588 else:
578 else:
589 @wsgiapp
579 @wsgiapp
590 def _hg_stream(environ, start_response):
580 def _hg_stream(environ, start_response):
591 log.debug('http-app: handling hg stream')
581 log.debug('http-app: handling hg stream')
592 repo_path = environ['HTTP_X_RC_REPO_PATH']
582 call_context = get_headers_call_context(environ)
593 repo_name = environ['HTTP_X_RC_REPO_NAME']
583
594 packed_config = base64.b64decode(
584 repo_path = call_context['repo_path']
595 environ['HTTP_X_RC_REPO_CONFIG'])
585 repo_name = call_context['repo_name']
596 config = msgpack.unpackb(packed_config)
586 config = call_context['repo_config']
587
597 app = scm_app.create_hg_wsgi_app(
588 app = scm_app.create_hg_wsgi_app(
598 repo_path, repo_name, config)
589 repo_path, repo_name, config)
599
590
600 # Consistent path information for hgweb
591 # Consistent path information for hgweb
601 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
592 environ['PATH_INFO'] = call_context['path_info']
602 environ['REPO_NAME'] = repo_name
593 environ['REPO_NAME'] = repo_name
603 self.set_env_from_config(environ, config)
594 self.set_env_from_config(environ, config)
604
595
605 log.debug('http-app: starting app handler '
596 log.debug('http-app: starting app handler '
606 'with %s and process request', app)
597 'with %s and process request', app)
607 return app(environ, ResponseFilter(start_response))
598 return app(environ, ResponseFilter(start_response))
608 return _hg_stream
599 return _hg_stream
609
600
610 def git_stream(self):
601 def git_stream(self):
611 if self._use_echo_app:
602 if self._use_echo_app:
612 @wsgiapp
603 @wsgiapp
613 def _git_stream(environ, start_response):
604 def _git_stream(environ, start_response):
614 app = EchoApp('fake_path', 'fake_name', None)
605 app = EchoApp('fake_path', 'fake_name', None)
615 return app(environ, start_response)
606 return app(environ, start_response)
616 return _git_stream
607 return _git_stream
617 else:
608 else:
618 @wsgiapp
609 @wsgiapp
619 def _git_stream(environ, start_response):
610 def _git_stream(environ, start_response):
620 log.debug('http-app: handling git stream')
611 log.debug('http-app: handling git stream')
621 repo_path = environ['HTTP_X_RC_REPO_PATH']
612
622 repo_name = environ['HTTP_X_RC_REPO_NAME']
613 call_context = get_headers_call_context(environ)
623 packed_config = base64.b64decode(
624 environ['HTTP_X_RC_REPO_CONFIG'])
625 config = msgpack.unpackb(packed_config)
626
614
627 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
615 repo_path = call_context['repo_path']
616 repo_name = call_context['repo_name']
617 config = call_context['repo_config']
618
619 environ['PATH_INFO'] = call_context['path_info']
628 self.set_env_from_config(environ, config)
620 self.set_env_from_config(environ, config)
629
621
630 content_type = environ.get('CONTENT_TYPE', '')
622 content_type = environ.get('CONTENT_TYPE', '')
631
623
632 path = environ['PATH_INFO']
624 path = environ['PATH_INFO']
633 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
625 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
634 log.debug(
626 log.debug(
635 'LFS: Detecting if request `%s` is LFS server path based '
627 'LFS: Detecting if request `%s` is LFS server path based '
636 'on content type:`%s`, is_lfs:%s',
628 'on content type:`%s`, is_lfs:%s',
637 path, content_type, is_lfs_request)
629 path, content_type, is_lfs_request)
638
630
639 if not is_lfs_request:
631 if not is_lfs_request:
640 # fallback detection by path
632 # fallback detection by path
641 if GIT_LFS_PROTO_PAT.match(path):
633 if GIT_LFS_PROTO_PAT.match(path):
642 is_lfs_request = True
634 is_lfs_request = True
643 log.debug(
635 log.debug(
644 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
636 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
645 path, is_lfs_request)
637 path, is_lfs_request)
646
638
647 if is_lfs_request:
639 if is_lfs_request:
648 app = scm_app.create_git_lfs_wsgi_app(
640 app = scm_app.create_git_lfs_wsgi_app(
649 repo_path, repo_name, config)
641 repo_path, repo_name, config)
650 else:
642 else:
651 app = scm_app.create_git_wsgi_app(
643 app = scm_app.create_git_wsgi_app(
652 repo_path, repo_name, config)
644 repo_path, repo_name, config)
653
645
654 log.debug('http-app: starting app handler '
646 log.debug('http-app: starting app handler '
655 'with %s and process request', app)
647 'with %s and process request', app)
656
648
657 return app(environ, start_response)
649 return app(environ, start_response)
658
650
659 return _git_stream
651 return _git_stream
660
652
661 def handle_vcs_exception(self, exception, request):
653 def handle_vcs_exception(self, exception, request):
662 _vcs_kind = getattr(exception, '_vcs_kind', '')
654 _vcs_kind = getattr(exception, '_vcs_kind', '')
655
663 if _vcs_kind == 'repo_locked':
656 if _vcs_kind == 'repo_locked':
664 # Get custom repo-locked status code if present.
657 headers_call_context = get_headers_call_context(request.environ)
665 status_code = request.headers.get('X-RC-Locked-Status-Code')
658 status_code = safe_int(headers_call_context['locked_status_code'])
659
666 return HTTPRepoLocked(
660 return HTTPRepoLocked(
667 title=exception.message, status_code=status_code)
661 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
668
662
669 elif _vcs_kind == 'repo_branch_protected':
663 elif _vcs_kind == 'repo_branch_protected':
670 # Get custom repo-branch-protected status code if present.
664 # Get custom repo-branch-protected status code if present.
671 return HTTPRepoBranchProtected(title=exception.message)
665 return HTTPRepoBranchProtected(
666 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
672
667
673 exc_info = request.exc_info
668 exc_info = request.exc_info
674 store_exception(id(exc_info), exc_info)
669 store_exception(id(exc_info), exc_info)
675
670
676 traceback_info = 'unavailable'
671 traceback_info = 'unavailable'
677 if request.exc_info:
672 if request.exc_info:
678 exc_type, exc_value, exc_tb = request.exc_info
673 traceback_info = format_exc(request.exc_info)
679 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
680
674
681 log.error(
675 log.error(
682 'error occurred handling this request for path: %s, \n tb: %s',
676 'error occurred handling this request for path: %s, \n%s',
683 request.path, traceback_info)
677 request.path, traceback_info)
678
679 statsd = request.registry.statsd
680 if statsd:
681 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
682 statsd.incr('vcsserver_exception_total',
683 tags=[f"type:{exc_type}"])
684 raise exception
684 raise exception
685
685
686
686
687 class ResponseFilter(object):
687 class ResponseFilter:
688
688
689 def __init__(self, start_response):
689 def __init__(self, start_response):
690 self._start_response = start_response
690 self._start_response = start_response
691
691
692 def __call__(self, status, response_headers, exc_info=None):
692 def __call__(self, status, response_headers, exc_info=None):
693 headers = tuple(
693 headers = tuple(
694 (h, v) for h, v in response_headers
694 (h, v) for h, v in response_headers
695 if not wsgiref.util.is_hop_by_hop(h))
695 if not wsgiref.util.is_hop_by_hop(h))
696 return self._start_response(status, headers, exc_info)
696 return self._start_response(status, headers, exc_info)
697
697
698
698
699 def sanitize_settings_and_apply_defaults(global_config, settings):
700 _global_settings_maker = SettingsMaker(global_config)
701 settings_maker = SettingsMaker(settings)
702
703 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
704
705 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
706 settings_maker.enable_logging(logging_conf)
707
708 # Default includes, possible to change as a user
709 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
710 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
711
712 settings_maker.make_setting('__file__', global_config.get('__file__'))
713
714 settings_maker.make_setting('pyramid.default_locale_name', 'en')
715 settings_maker.make_setting('locale', 'en_US.UTF-8')
716
717 settings_maker.make_setting('core.binary_dir', '')
718
719 temp_store = tempfile.gettempdir()
720 default_cache_dir = os.path.join(temp_store, 'rc_cache')
721 # save default, cache dir, and use it for all backends later.
722 default_cache_dir = settings_maker.make_setting(
723 'cache_dir',
724 default=default_cache_dir, default_when_empty=True,
725 parser='dir:ensured')
726
727 # exception store cache
728 settings_maker.make_setting(
729 'exception_tracker.store_path',
730 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
731 parser='dir:ensured'
732 )
733
734 # repo_object cache defaults
735 settings_maker.make_setting(
736 'rc_cache.repo_object.backend',
737 default='dogpile.cache.rc.file_namespace',
738 parser='string')
739 settings_maker.make_setting(
740 'rc_cache.repo_object.expiration_time',
741 default=30 * 24 * 60 * 60, # 30days
742 parser='int')
743 settings_maker.make_setting(
744 'rc_cache.repo_object.arguments.filename',
745 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
746 parser='string')
747
748 # statsd
749 settings_maker.make_setting('statsd.enabled', False, parser='bool')
750 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
751 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
752 settings_maker.make_setting('statsd.statsd_prefix', '')
753 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
754
755 settings_maker.env_expand()
756
757
699 def main(global_config, **settings):
758 def main(global_config, **settings):
759 start_time = time.time()
760 log.info('Pyramid app config starting')
761
700 if MercurialFactory:
762 if MercurialFactory:
701 hgpatches.patch_largefiles_capabilities()
763 hgpatches.patch_largefiles_capabilities()
702 hgpatches.patch_subrepo_type_mapping()
764 hgpatches.patch_subrepo_type_mapping()
703
765
704 app = HTTPApplication(settings=settings, global_config=global_config)
766 # Fill in and sanitize the defaults & do ENV expansion
705 return app.wsgi_app()
767 sanitize_settings_and_apply_defaults(global_config, settings)
768
769 # init and bootstrap StatsdClient
770 StatsdClient.setup(settings)
771
772 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
773 total_time = time.time() - start_time
774 log.info('Pyramid app created and configured in %.2fs', total_time)
775 return pyramid_app
@@ -1,16 +1,16 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@@ -1,26 +1,26 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 # This package contains non rhodecode licensed packages that are
18 # This package contains non rhodecode licensed packages that are
19 # vendored for various reasons
19 # vendored for various reasons
20
20
21 import os
21 import os
22 import sys
22 import sys
23
23
24 vendor_dir = os.path.abspath(os.path.dirname(__file__))
24 vendor_dir = os.path.abspath(os.path.dirname(__file__))
25
25
26 sys.path.append(vendor_dir)
26 sys.path.append(vendor_dir)
@@ -1,390 +1,394 b''
1 import sys
1
2 import threading
2 import threading
3 import weakref
3 import weakref
4 from base64 import b64encode
4 from base64 import b64encode
5 from logging import getLogger
5 from logging import getLogger
6 from os import urandom
6 from os import urandom
7 from typing import Union
7
8
8 from redis import StrictRedis
9 from redis import StrictRedis
9
10
10 __version__ = '3.7.0'
11 __version__ = '4.0.0'
11
12
12 loggers = {
13 loggers = {
13 k: getLogger("vcsserver." + ".".join((__name__, k)))
14 k: getLogger("vcsserver." + ".".join((__name__, k)))
14 for k in [
15 for k in [
15 "acquire",
16 "acquire",
16 "refresh.thread.start",
17 "refresh.thread.start",
17 "refresh.thread.stop",
18 "refresh.thread.stop",
18 "refresh.thread.exit",
19 "refresh.thread.exit",
19 "refresh.start",
20 "refresh.start",
20 "refresh.shutdown",
21 "refresh.shutdown",
21 "refresh.exit",
22 "refresh.exit",
22 "release",
23 "release",
23 ]
24 ]
24 }
25 }
25
26
26 PY3 = sys.version_info[0] == 3
27 text_type = str
27
28 binary_type = bytes
28 if PY3:
29 text_type = str
30 binary_type = bytes
31 else:
32 text_type = unicode # noqa
33 binary_type = str
34
29
35
30
36 # Check if the id match. If not, return an error code.
31 # Check if the id match. If not, return an error code.
37 UNLOCK_SCRIPT = b"""
32 UNLOCK_SCRIPT = b"""
38 if redis.call("get", KEYS[1]) ~= ARGV[1] then
33 if redis.call("get", KEYS[1]) ~= ARGV[1] then
39 return 1
34 return 1
40 else
35 else
41 redis.call("del", KEYS[2])
36 redis.call("del", KEYS[2])
42 redis.call("lpush", KEYS[2], 1)
37 redis.call("lpush", KEYS[2], 1)
43 redis.call("pexpire", KEYS[2], ARGV[2])
38 redis.call("pexpire", KEYS[2], ARGV[2])
44 redis.call("del", KEYS[1])
39 redis.call("del", KEYS[1])
45 return 0
40 return 0
46 end
41 end
47 """
42 """
48
43
49 # Covers both cases when key doesn't exist and doesn't equal to lock's id
44 # Covers both cases when key doesn't exist and doesn't equal to lock's id
50 EXTEND_SCRIPT = b"""
45 EXTEND_SCRIPT = b"""
51 if redis.call("get", KEYS[1]) ~= ARGV[1] then
46 if redis.call("get", KEYS[1]) ~= ARGV[1] then
52 return 1
47 return 1
53 elseif redis.call("ttl", KEYS[1]) < 0 then
48 elseif redis.call("ttl", KEYS[1]) < 0 then
54 return 2
49 return 2
55 else
50 else
56 redis.call("expire", KEYS[1], ARGV[2])
51 redis.call("expire", KEYS[1], ARGV[2])
57 return 0
52 return 0
58 end
53 end
59 """
54 """
60
55
61 RESET_SCRIPT = b"""
56 RESET_SCRIPT = b"""
62 redis.call('del', KEYS[2])
57 redis.call('del', KEYS[2])
63 redis.call('lpush', KEYS[2], 1)
58 redis.call('lpush', KEYS[2], 1)
64 redis.call('pexpire', KEYS[2], ARGV[2])
59 redis.call('pexpire', KEYS[2], ARGV[2])
65 return redis.call('del', KEYS[1])
60 return redis.call('del', KEYS[1])
66 """
61 """
67
62
68 RESET_ALL_SCRIPT = b"""
63 RESET_ALL_SCRIPT = b"""
69 local locks = redis.call('keys', 'lock:*')
64 local locks = redis.call('keys', 'lock:*')
70 local signal
65 local signal
71 for _, lock in pairs(locks) do
66 for _, lock in pairs(locks) do
72 signal = 'lock-signal:' .. string.sub(lock, 6)
67 signal = 'lock-signal:' .. string.sub(lock, 6)
73 redis.call('del', signal)
68 redis.call('del', signal)
74 redis.call('lpush', signal, 1)
69 redis.call('lpush', signal, 1)
75 redis.call('expire', signal, 1)
70 redis.call('expire', signal, 1)
76 redis.call('del', lock)
71 redis.call('del', lock)
77 end
72 end
78 return #locks
73 return #locks
79 """
74 """
80
75
81
76
82 class AlreadyAcquired(RuntimeError):
77 class AlreadyAcquired(RuntimeError):
83 pass
78 pass
84
79
85
80
86 class NotAcquired(RuntimeError):
81 class NotAcquired(RuntimeError):
87 pass
82 pass
88
83
89
84
90 class AlreadyStarted(RuntimeError):
85 class AlreadyStarted(RuntimeError):
91 pass
86 pass
92
87
93
88
94 class TimeoutNotUsable(RuntimeError):
89 class TimeoutNotUsable(RuntimeError):
95 pass
90 pass
96
91
97
92
98 class InvalidTimeout(RuntimeError):
93 class InvalidTimeout(RuntimeError):
99 pass
94 pass
100
95
101
96
102 class TimeoutTooLarge(RuntimeError):
97 class TimeoutTooLarge(RuntimeError):
103 pass
98 pass
104
99
105
100
106 class NotExpirable(RuntimeError):
101 class NotExpirable(RuntimeError):
107 pass
102 pass
108
103
109
104
110 class Lock(object):
105 class Lock:
111 """
106 """
112 A Lock context manager implemented via redis SETNX/BLPOP.
107 A Lock context manager implemented via redis SETNX/BLPOP.
113 """
108 """
109
114 unlock_script = None
110 unlock_script = None
115 extend_script = None
111 extend_script = None
116 reset_script = None
112 reset_script = None
117 reset_all_script = None
113 reset_all_script = None
118
114
115 _lock_renewal_interval: float
116 _lock_renewal_thread: Union[threading.Thread, None]
117
119 def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
118 def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
120 """
119 """
121 :param redis_client:
120 :param redis_client:
122 An instance of :class:`~StrictRedis`.
121 An instance of :class:`~StrictRedis`.
123 :param name:
122 :param name:
124 The name (redis key) the lock should have.
123 The name (redis key) the lock should have.
125 :param expire:
124 :param expire:
126 The lock expiry time in seconds. If left at the default (None)
125 The lock expiry time in seconds. If left at the default (None)
127 the lock will not expire.
126 the lock will not expire.
128 :param id:
127 :param id:
129 The ID (redis value) the lock should have. A random value is
128 The ID (redis value) the lock should have. A random value is
130 generated when left at the default.
129 generated when left at the default.
131
130
132 Note that if you specify this then the lock is marked as "held". Acquires
131 Note that if you specify this then the lock is marked as "held". Acquires
133 won't be possible.
132 won't be possible.
134 :param auto_renewal:
133 :param auto_renewal:
135 If set to ``True``, Lock will automatically renew the lock so that it
134 If set to ``True``, Lock will automatically renew the lock so that it
136 doesn't expire for as long as the lock is held (acquire() called
135 doesn't expire for as long as the lock is held (acquire() called
137 or running in a context manager).
136 or running in a context manager).
138
137
139 Implementation note: Renewal will happen using a daemon thread with
138 Implementation note: Renewal will happen using a daemon thread with
140 an interval of ``expire*2/3``. If wishing to use a different renewal
139 an interval of ``expire*2/3``. If wishing to use a different renewal
141 time, subclass Lock, call ``super().__init__()`` then set
140 time, subclass Lock, call ``super().__init__()`` then set
142 ``self._lock_renewal_interval`` to your desired interval.
141 ``self._lock_renewal_interval`` to your desired interval.
143 :param strict:
142 :param strict:
144 If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
143 If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
145 :param signal_expire:
144 :param signal_expire:
146 Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
145 Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
147 """
146 """
148 if strict and not isinstance(redis_client, StrictRedis):
147 if strict and not isinstance(redis_client, StrictRedis):
149 raise ValueError("redis_client must be instance of StrictRedis. "
148 raise ValueError("redis_client must be instance of StrictRedis. "
150 "Use strict=False if you know what you're doing.")
149 "Use strict=False if you know what you're doing.")
151 if auto_renewal and expire is None:
150 if auto_renewal and expire is None:
152 raise ValueError("Expire may not be None when auto_renewal is set")
151 raise ValueError("Expire may not be None when auto_renewal is set")
153
152
154 self._client = redis_client
153 self._client = redis_client
155
154
156 if expire:
155 if expire:
157 expire = int(expire)
156 expire = int(expire)
158 if expire < 0:
157 if expire < 0:
159 raise ValueError("A negative expire is not acceptable.")
158 raise ValueError("A negative expire is not acceptable.")
160 else:
159 else:
161 expire = None
160 expire = None
162 self._expire = expire
161 self._expire = expire
163
162
164 self._signal_expire = signal_expire
163 self._signal_expire = signal_expire
165 if id is None:
164 if id is None:
166 self._id = b64encode(urandom(18)).decode('ascii')
165 self._id = b64encode(urandom(18)).decode('ascii')
167 elif isinstance(id, binary_type):
166 elif isinstance(id, binary_type):
168 try:
167 try:
169 self._id = id.decode('ascii')
168 self._id = id.decode('ascii')
170 except UnicodeDecodeError:
169 except UnicodeDecodeError:
171 self._id = b64encode(id).decode('ascii')
170 self._id = b64encode(id).decode('ascii')
172 elif isinstance(id, text_type):
171 elif isinstance(id, text_type):
173 self._id = id
172 self._id = id
174 else:
173 else:
175 raise TypeError("Incorrect type for `id`. Must be bytes/str not %s." % type(id))
174 raise TypeError(f"Incorrect type for `id`. Must be bytes/str not {type(id)}.")
176 self._name = 'lock:' + name
175 self._name = 'lock:' + name
177 self._signal = 'lock-signal:' + name
176 self._signal = 'lock-signal:' + name
178 self._lock_renewal_interval = (float(expire) * 2 / 3
177 self._lock_renewal_interval = (float(expire) * 2 / 3
179 if auto_renewal
178 if auto_renewal
180 else None)
179 else None)
181 self._lock_renewal_thread = None
180 self._lock_renewal_thread = None
182
181
183 self.register_scripts(redis_client)
182 self.register_scripts(redis_client)
184
183
185 @classmethod
184 @classmethod
186 def register_scripts(cls, redis_client):
185 def register_scripts(cls, redis_client):
187 global reset_all_script
186 global reset_all_script
188 if reset_all_script is None:
187 if reset_all_script is None:
189 reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
190 cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
188 cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
191 cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
189 cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
192 cls.reset_script = redis_client.register_script(RESET_SCRIPT)
190 cls.reset_script = redis_client.register_script(RESET_SCRIPT)
193 cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
191 cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
192 reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
194
193
195 @property
194 @property
196 def _held(self):
195 def _held(self):
197 return self.id == self.get_owner_id()
196 return self.id == self.get_owner_id()
198
197
199 def reset(self):
198 def reset(self):
200 """
199 """
201 Forcibly deletes the lock. Use this with care.
200 Forcibly deletes the lock. Use this with care.
202 """
201 """
203 self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
202 self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
204
203
205 @property
204 @property
206 def id(self):
205 def id(self):
207 return self._id
206 return self._id
208
207
209 def get_owner_id(self):
208 def get_owner_id(self):
210 owner_id = self._client.get(self._name)
209 owner_id = self._client.get(self._name)
211 if isinstance(owner_id, binary_type):
210 if isinstance(owner_id, binary_type):
212 owner_id = owner_id.decode('ascii', 'replace')
211 owner_id = owner_id.decode('ascii', 'replace')
213 return owner_id
212 return owner_id
214
213
215 def acquire(self, blocking=True, timeout=None):
214 def acquire(self, blocking=True, timeout=None):
216 """
215 """
217 :param blocking:
216 :param blocking:
218 Boolean value specifying whether lock should be blocking or not.
217 Boolean value specifying whether lock should be blocking or not.
219 :param timeout:
218 :param timeout:
220 An integer value specifying the maximum number of seconds to block.
219 An integer value specifying the maximum number of seconds to block.
221 """
220 """
222 logger = loggers["acquire"]
221 logger = loggers["acquire"]
223
222
224 logger.debug("Getting acquire on %r ...", self._name)
223 logger.debug("Getting blocking: %s acquire on %r ...", blocking, self._name)
225
224
226 if self._held:
225 if self._held:
227 owner_id = self.get_owner_id()
226 owner_id = self.get_owner_id()
228 raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
227 raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
229
228
230 if not blocking and timeout is not None:
229 if not blocking and timeout is not None:
231 raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
230 raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
232
231
233 if timeout:
232 if timeout:
234 timeout = int(timeout)
233 timeout = int(timeout)
235 if timeout < 0:
234 if timeout < 0:
236 raise InvalidTimeout("Timeout (%d) cannot be less than or equal to 0" % timeout)
235 raise InvalidTimeout(f"Timeout ({timeout}) cannot be less than or equal to 0")
237
236
238 if self._expire and not self._lock_renewal_interval and timeout > self._expire:
237 if self._expire and not self._lock_renewal_interval and timeout > self._expire:
239 raise TimeoutTooLarge("Timeout (%d) cannot be greater than expire (%d)" % (timeout, self._expire))
238 raise TimeoutTooLarge(f"Timeout ({timeout}) cannot be greater than expire ({self._expire})")
240
239
241 busy = True
240 busy = True
242 blpop_timeout = timeout or self._expire or 0
241 blpop_timeout = timeout or self._expire or 0
243 timed_out = False
242 timed_out = False
244 while busy:
243 while busy:
245 busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
244 busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
246 if busy:
245 if busy:
247 if timed_out:
246 if timed_out:
248 return False
247 return False
249 elif blocking:
248 elif blocking:
250 timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
249 timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
251 else:
250 else:
252 logger.warning("Failed to get %r.", self._name)
251 logger.warning("Failed to acquire Lock(%r).", self._name)
253 return False
252 return False
254
253
255 logger.info("Got lock for %r.", self._name)
254 logger.debug("Acquired Lock(%r).", self._name)
256 if self._lock_renewal_interval is not None:
255 if self._lock_renewal_interval is not None:
257 self._start_lock_renewer()
256 self._start_lock_renewer()
258 return True
257 return True
259
258
260 def extend(self, expire=None):
259 def extend(self, expire=None):
261 """Extends expiration time of the lock.
260 """
261 Extends expiration time of the lock.
262
262
263 :param expire:
263 :param expire:
264 New expiration time. If ``None`` - `expire` provided during
264 New expiration time. If ``None`` - `expire` provided during
265 lock initialization will be taken.
265 lock initialization will be taken.
266 """
266 """
267 if expire:
267 if expire:
268 expire = int(expire)
268 expire = int(expire)
269 if expire < 0:
269 if expire < 0:
270 raise ValueError("A negative expire is not acceptable.")
270 raise ValueError("A negative expire is not acceptable.")
271 elif self._expire is not None:
271 elif self._expire is not None:
272 expire = self._expire
272 expire = self._expire
273 else:
273 else:
274 raise TypeError(
274 raise TypeError(
275 "To extend a lock 'expire' must be provided as an "
275 "To extend a lock 'expire' must be provided as an "
276 "argument to extend() method or at initialization time."
276 "argument to extend() method or at initialization time."
277 )
277 )
278
278
279 error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
279 error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
280 if error == 1:
280 if error == 1:
281 raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
281 raise NotAcquired(f"Lock {self._name} is not acquired or it already expired.")
282 elif error == 2:
282 elif error == 2:
283 raise NotExpirable("Lock %s has no assigned expiration time" % self._name)
283 raise NotExpirable(f"Lock {self._name} has no assigned expiration time")
284 elif error:
284 elif error:
285 raise RuntimeError("Unsupported error code %s from EXTEND script" % error)
285 raise RuntimeError(f"Unsupported error code {error} from EXTEND script")
286
286
287 @staticmethod
287 @staticmethod
288 def _lock_renewer(lockref, interval, stop):
288 def _lock_renewer(name, lockref, interval, stop):
289 """
289 """
290 Renew the lock key in redis every `interval` seconds for as long
290 Renew the lock key in redis every `interval` seconds for as long
291 as `self._lock_renewal_thread.should_exit` is False.
291 as `self._lock_renewal_thread.should_exit` is False.
292 """
292 """
293 while not stop.wait(timeout=interval):
293 while not stop.wait(timeout=interval):
294 loggers["refresh.thread.start"].debug("Refreshing lock")
294 loggers["refresh.thread.start"].debug("Refreshing Lock(%r).", name)
295 lock = lockref()
295 lock: "Lock" = lockref()
296 if lock is None:
296 if lock is None:
297 loggers["refresh.thread.stop"].debug(
297 loggers["refresh.thread.stop"].debug(
298 "The lock no longer exists, stopping lock refreshing"
298 "Stopping loop because Lock(%r) was garbage collected.", name
299 )
299 )
300 break
300 break
301 lock.extend(expire=lock._expire)
301 lock.extend(expire=lock._expire)
302 del lock
302 del lock
303 loggers["refresh.thread.exit"].debug("Exit requested, stopping lock refreshing")
303 loggers["refresh.thread.exit"].debug("Exiting renewal thread for Lock(%r).", name)
304
304
305 def _start_lock_renewer(self):
305 def _start_lock_renewer(self):
306 """
306 """
307 Starts the lock refresher thread.
307 Starts the lock refresher thread.
308 """
308 """
309 if self._lock_renewal_thread is not None:
309 if self._lock_renewal_thread is not None:
310 raise AlreadyStarted("Lock refresh thread already started")
310 raise AlreadyStarted("Lock refresh thread already started")
311
311
312 loggers["refresh.start"].debug(
312 loggers["refresh.start"].debug(
313 "Starting thread to refresh lock every %s seconds",
313 "Starting renewal thread for Lock(%r). Refresh interval: %s seconds.",
314 self._lock_renewal_interval
314 self._name, self._lock_renewal_interval
315 )
315 )
316 self._lock_renewal_stop = threading.Event()
316 self._lock_renewal_stop = threading.Event()
317 self._lock_renewal_thread = threading.Thread(
317 self._lock_renewal_thread = threading.Thread(
318 group=None,
318 group=None,
319 target=self._lock_renewer,
319 target=self._lock_renewer,
320 kwargs={'lockref': weakref.ref(self),
320 kwargs={
321 'interval': self._lock_renewal_interval,
321 'name': self._name,
322 'stop': self._lock_renewal_stop}
322 'lockref': weakref.ref(self),
323 'interval': self._lock_renewal_interval,
324 'stop': self._lock_renewal_stop,
325 },
323 )
326 )
324 self._lock_renewal_thread.setDaemon(True)
327 self._lock_renewal_thread.daemon = True
325 self._lock_renewal_thread.start()
328 self._lock_renewal_thread.start()
326
329
327 def _stop_lock_renewer(self):
330 def _stop_lock_renewer(self):
328 """
331 """
329 Stop the lock renewer.
332 Stop the lock renewer.
330
333
331 This signals the renewal thread and waits for its exit.
334 This signals the renewal thread and waits for its exit.
332 """
335 """
333 if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
336 if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
334 return
337 return
335 loggers["refresh.shutdown"].debug("Signalling the lock refresher to stop")
338 loggers["refresh.shutdown"].debug("Signaling renewal thread for Lock(%r) to exit.", self._name)
336 self._lock_renewal_stop.set()
339 self._lock_renewal_stop.set()
337 self._lock_renewal_thread.join()
340 self._lock_renewal_thread.join()
338 self._lock_renewal_thread = None
341 self._lock_renewal_thread = None
339 loggers["refresh.exit"].debug("Lock refresher has stopped")
342 loggers["refresh.exit"].debug("Renewal thread for Lock(%r) exited.", self._name)
340
343
341 def __enter__(self):
344 def __enter__(self):
342 acquired = self.acquire(blocking=True)
345 acquired = self.acquire(blocking=True)
343 assert acquired, "Lock wasn't acquired, but blocking=True"
346 if not acquired:
347 raise AssertionError(f"Lock({self._name}) wasn't acquired, but blocking=True was used!")
344 return self
348 return self
345
349
346 def __exit__(self, exc_type=None, exc_value=None, traceback=None):
350 def __exit__(self, exc_type=None, exc_value=None, traceback=None):
347 self.release()
351 self.release()
348
352
349 def release(self):
353 def release(self):
350 """Releases the lock, that was acquired with the same object.
354 """Releases the lock, that was acquired with the same object.
351
355
352 .. note::
356 .. note::
353
357
354 If you want to release a lock that you acquired in a different place you have two choices:
358 If you want to release a lock that you acquired in a different place you have two choices:
355
359
356 * Use ``Lock("name", id=id_from_other_place).release()``
360 * Use ``Lock("name", id=id_from_other_place).release()``
357 * Use ``Lock("name").reset()``
361 * Use ``Lock("name").reset()``
358 """
362 """
359 if self._lock_renewal_thread is not None:
363 if self._lock_renewal_thread is not None:
360 self._stop_lock_renewer()
364 self._stop_lock_renewer()
361 loggers["release"].debug("Releasing %r.", self._name)
365 loggers["release"].debug("Releasing Lock(%r).", self._name)
362 error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
366 error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
363 if error == 1:
367 if error == 1:
364 raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
368 raise NotAcquired(f"Lock({self._name}) is not acquired or it already expired.")
365 elif error:
369 elif error:
366 raise RuntimeError("Unsupported error code %s from EXTEND script." % error)
370 raise RuntimeError(f"Unsupported error code {error} from EXTEND script.")
367
371
368 def locked(self):
372 def locked(self):
369 """
373 """
370 Return true if the lock is acquired.
374 Return true if the lock is acquired.
371
375
372 Checks that lock with same name already exists. This method returns true, even if
376 Checks that lock with same name already exists. This method returns true, even if
373 lock have another id.
377 lock have another id.
374 """
378 """
375 return self._client.exists(self._name) == 1
379 return self._client.exists(self._name) == 1
376
380
377
381
378 reset_all_script = None
382 reset_all_script = None
379
383
380
384
381 def reset_all(redis_client):
385 def reset_all(redis_client):
382 """
386 """
383 Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
387 Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
384
388
385 :param redis_client:
389 :param redis_client:
386 An instance of :class:`~StrictRedis`.
390 An instance of :class:`~StrictRedis`.
387 """
391 """
388 Lock.register_scripts(redis_client)
392 Lock.register_scripts(redis_client)
389
393
390 reset_all_script(client=redis_client) # noqa
394 reset_all_script(client=redis_client) # noqa
@@ -1,46 +1,50 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import logging
1 import logging
4
2
5 from .stream import TCPStatsClient, UnixSocketStatsClient # noqa
3 from .stream import TCPStatsClient, UnixSocketStatsClient # noqa
6 from .udp import StatsClient # noqa
4 from .udp import StatsClient # noqa
7
5
8 HOST = 'localhost'
6 HOST = 'localhost'
9 PORT = 8125
7 PORT = 8125
10 IPV6 = False
8 IPV6 = False
11 PREFIX = None
9 PREFIX = None
12 MAXUDPSIZE = 512
10 MAXUDPSIZE = 512
13
11
14 log = logging.getLogger('rhodecode.statsd')
12 log = logging.getLogger('rhodecode.statsd')
15
13
16
14
17 def statsd_config(config, prefix='statsd.'):
15 def statsd_config(config, prefix='statsd.'):
18 _config = {}
16 _config = {}
19 for key in config.keys():
17 for key in config.keys():
20 if key.startswith(prefix):
18 if key.startswith(prefix):
21 _config[key[len(prefix):]] = config[key]
19 _config[key[len(prefix):]] = config[key]
22 return _config
20 return _config
23
21
24
22
25 def client_from_config(configuration, prefix='statsd.', **kwargs):
23 def client_from_config(configuration, prefix='statsd.', **kwargs):
26 from pyramid.settings import asbool
24 from pyramid.settings import asbool
27
25
28 _config = statsd_config(configuration, prefix)
26 _config = statsd_config(configuration, prefix)
29 statsd_enabled = asbool(_config.pop('enabled', False))
27 statsd_enabled = asbool(_config.pop('enabled', False))
30 if not statsd_enabled:
28 if not statsd_enabled:
31 log.debug('statsd client not enabled by statsd.enabled = flag, skipping...')
29 log.debug('statsd client not enabled by statsd.enabled = flag, skipping...')
32 return
30 return
33
31
34 host = _config.pop('statsd_host', HOST)
32 host = _config.pop('statsd_host', HOST)
35 port = _config.pop('statsd_port', PORT)
33 port = _config.pop('statsd_port', PORT)
36 prefix = _config.pop('statsd_prefix', PREFIX)
34 prefix = _config.pop('statsd_prefix', PREFIX)
37 maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
35 maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
38 ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
36 ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
39 log.debug('configured statsd client %s:%s', host, port)
37 log.debug('configured statsd client %s:%s', host, port)
40
38
41 return StatsClient(
39 try:
42 host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
40 client = StatsClient(
41 host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
42 except Exception:
43 log.exception('StatsD is enabled, but failed to connect to statsd server, fallback: disable statsd')
44 client = None
45
46 return client
43
47
44
48
45 def get_statsd_client(request):
49 def get_statsd_client(request):
46 return client_from_config(request.registry.settings)
50 return client_from_config(request.registry.settings)
@@ -1,107 +1,154 b''
1 from __future__ import absolute_import, division, unicode_literals
1 import re
2
3 import random
2 import random
4 from collections import deque
3 from collections import deque
5 from datetime import timedelta
4 from datetime import timedelta
5 from repoze.lru import lru_cache
6
6
7 from .timer import Timer
7 from .timer import Timer
8
8
9 TAG_INVALID_CHARS_RE = re.compile(
10 r"[^\w\d_\-:/\.]",
11 #re.UNICODE
12 )
13 TAG_INVALID_CHARS_SUBS = "_"
9
14
10 class StatsClientBase(object):
15 # we save and expose methods called by statsd for discovery
16 buckets_dict = {
17
18 }
19
20
21 @lru_cache(maxsize=500)
22 def _normalize_tags_with_cache(tag_list):
23 return [TAG_INVALID_CHARS_RE.sub(TAG_INVALID_CHARS_SUBS, tag) for tag in tag_list]
24
25
26 def normalize_tags(tag_list):
27 # We have to turn our input tag list into a non-mutable tuple for it to
28 # be hashable (and thus usable) by the @lru_cache decorator.
29 return _normalize_tags_with_cache(tuple(tag_list))
30
31
32 class StatsClientBase:
11 """A Base class for various statsd clients."""
33 """A Base class for various statsd clients."""
12
34
13 def close(self):
35 def close(self):
14 """Used to close and clean up any underlying resources."""
36 """Used to close and clean up any underlying resources."""
15 raise NotImplementedError()
37 raise NotImplementedError()
16
38
17 def _send(self):
39 def _send(self):
18 raise NotImplementedError()
40 raise NotImplementedError()
19
41
20 def pipeline(self):
42 def pipeline(self):
21 raise NotImplementedError()
43 raise NotImplementedError()
22
44
23 def timer(self, stat, rate=1):
45 def timer(self, stat, rate=1, tags=None, auto_send=True):
24 return Timer(self, stat, rate)
46 """
47 statsd = StatsdClient.statsd
48 with statsd.timer('bucket_name', auto_send=True) as tmr:
49 # This block will be timed.
50 for i in range(0, 100000):
51 i ** 2
52 # you can access time here...
53 elapsed_ms = tmr.ms
54 """
55 return Timer(self, stat, rate, tags, auto_send=auto_send)
25
56
26 def timing(self, stat, delta, rate=1):
57 def timing(self, stat, delta, rate=1, tags=None, use_decimals=True):
27 """
58 """
28 Send new timing information.
59 Send new timing information.
29
60
30 `delta` can be either a number of milliseconds or a timedelta.
61 `delta` can be either a number of milliseconds or a timedelta.
31 """
62 """
32 if isinstance(delta, timedelta):
63 if isinstance(delta, timedelta):
33 # Convert timedelta to number of milliseconds.
64 # Convert timedelta to number of milliseconds.
34 delta = delta.total_seconds() * 1000.
65 delta = delta.total_seconds() * 1000.
35 self._send_stat(stat, '%0.6f|ms' % delta, rate)
66 if use_decimals:
36
67 fmt = '%0.6f|ms'
37 def incr(self, stat, count=1, rate=1):
68 else:
38 """Increment a stat by `count`."""
69 fmt = '%s|ms'
39 self._send_stat(stat, '%s|c' % count, rate)
70 self._send_stat(stat, fmt % delta, rate, tags)
40
71
41 def decr(self, stat, count=1, rate=1):
72 def incr(self, stat, count=1, rate=1, tags=None):
73 """Increment a stat by `count`."""
74 self._send_stat(stat, f'{count}|c', rate, tags)
75
76 def decr(self, stat, count=1, rate=1, tags=None):
42 """Decrement a stat by `count`."""
77 """Decrement a stat by `count`."""
43 self.incr(stat, -count, rate)
78 self.incr(stat, -count, rate, tags)
44
79
45 def gauge(self, stat, value, rate=1, delta=False):
80 def gauge(self, stat, value, rate=1, delta=False, tags=None):
46 """Set a gauge value."""
81 """Set a gauge value."""
47 if value < 0 and not delta:
82 if value < 0 and not delta:
48 if rate < 1:
83 if rate < 1:
49 if random.random() > rate:
84 if random.random() > rate:
50 return
85 return
51 with self.pipeline() as pipe:
86 with self.pipeline() as pipe:
52 pipe._send_stat(stat, '0|g', 1)
87 pipe._send_stat(stat, '0|g', 1)
53 pipe._send_stat(stat, '%s|g' % value, 1)
88 pipe._send_stat(stat, f'{value}|g', 1)
54 else:
89 else:
55 prefix = '+' if delta and value >= 0 else ''
90 prefix = '+' if delta and value >= 0 else ''
56 self._send_stat(stat, '%s%s|g' % (prefix, value), rate)
91 self._send_stat(stat, f'{prefix}{value}|g', rate, tags)
57
92
58 def set(self, stat, value, rate=1):
93 def set(self, stat, value, rate=1):
59 """Set a set value."""
94 """Set a set value."""
60 self._send_stat(stat, '%s|s' % value, rate)
95 self._send_stat(stat, f'{value}|s', rate)
96
97 def histogram(self, stat, value, rate=1, tags=None):
98 """Set a histogram"""
99 self._send_stat(stat, f'{value}|h', rate, tags)
61
100
62 def _send_stat(self, stat, value, rate):
101 def _send_stat(self, stat, value, rate, tags=None):
63 self._after(self._prepare(stat, value, rate))
102 self._after(self._prepare(stat, value, rate, tags))
64
103
65 def _prepare(self, stat, value, rate):
104 def _prepare(self, stat, value, rate, tags=None):
105 global buckets_dict
106 buckets_dict[stat] = 1
107
66 if rate < 1:
108 if rate < 1:
67 if random.random() > rate:
109 if random.random() > rate:
68 return
110 return
69 value = '%s|@%s' % (value, rate)
111 value = f'{value}|@{rate}'
70
112
71 if self._prefix:
113 if self._prefix:
72 stat = '%s.%s' % (self._prefix, stat)
114 stat = f'{self._prefix}.{stat}'
73
115
74 return '%s:%s' % (stat, value)
116 res = '%s:%s%s' % (
117 stat,
118 value,
119 ("|#" + ",".join(normalize_tags(tags))) if tags else "",
120 )
121 return res
75
122
76 def _after(self, data):
123 def _after(self, data):
77 if data:
124 if data:
78 self._send(data)
125 self._send(data)
79
126
80
127
81 class PipelineBase(StatsClientBase):
128 class PipelineBase(StatsClientBase):
82
129
83 def __init__(self, client):
130 def __init__(self, client):
84 self._client = client
131 self._client = client
85 self._prefix = client._prefix
132 self._prefix = client._prefix
86 self._stats = deque()
133 self._stats = deque()
87
134
88 def _send(self):
135 def _send(self):
89 raise NotImplementedError()
136 raise NotImplementedError()
90
137
91 def _after(self, data):
138 def _after(self, data):
92 if data is not None:
139 if data is not None:
93 self._stats.append(data)
140 self._stats.append(data)
94
141
95 def __enter__(self):
142 def __enter__(self):
96 return self
143 return self
97
144
98 def __exit__(self, typ, value, tb):
145 def __exit__(self, typ, value, tb):
99 self.send()
146 self.send()
100
147
101 def send(self):
148 def send(self):
102 if not self._stats:
149 if not self._stats:
103 return
150 return
104 self._send()
151 self._send()
105
152
106 def pipeline(self):
153 def pipeline(self):
107 return self.__class__(self)
154 return self.__class__(self)
@@ -1,75 +1,73 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import socket
1 import socket
4
2
5 from .base import StatsClientBase, PipelineBase
3 from .base import StatsClientBase, PipelineBase
6
4
7
5
8 class StreamPipeline(PipelineBase):
6 class StreamPipeline(PipelineBase):
9 def _send(self):
7 def _send(self):
10 self._client._after('\n'.join(self._stats))
8 self._client._after('\n'.join(self._stats))
11 self._stats.clear()
9 self._stats.clear()
12
10
13
11
14 class StreamClientBase(StatsClientBase):
12 class StreamClientBase(StatsClientBase):
15 def connect(self):
13 def connect(self):
16 raise NotImplementedError()
14 raise NotImplementedError()
17
15
18 def close(self):
16 def close(self):
19 if self._sock and hasattr(self._sock, 'close'):
17 if self._sock and hasattr(self._sock, 'close'):
20 self._sock.close()
18 self._sock.close()
21 self._sock = None
19 self._sock = None
22
20
23 def reconnect(self):
21 def reconnect(self):
24 self.close()
22 self.close()
25 self.connect()
23 self.connect()
26
24
27 def pipeline(self):
25 def pipeline(self):
28 return StreamPipeline(self)
26 return StreamPipeline(self)
29
27
30 def _send(self, data):
28 def _send(self, data):
31 """Send data to statsd."""
29 """Send data to statsd."""
32 if not self._sock:
30 if not self._sock:
33 self.connect()
31 self.connect()
34 self._do_send(data)
32 self._do_send(data)
35
33
36 def _do_send(self, data):
34 def _do_send(self, data):
37 self._sock.sendall(data.encode('ascii') + b'\n')
35 self._sock.sendall(data.encode('ascii') + b'\n')
38
36
39
37
40 class TCPStatsClient(StreamClientBase):
38 class TCPStatsClient(StreamClientBase):
41 """TCP version of StatsClient."""
39 """TCP version of StatsClient."""
42
40
43 def __init__(self, host='localhost', port=8125, prefix=None,
41 def __init__(self, host='localhost', port=8125, prefix=None,
44 timeout=None, ipv6=False):
42 timeout=None, ipv6=False):
45 """Create a new client."""
43 """Create a new client."""
46 self._host = host
44 self._host = host
47 self._port = port
45 self._port = port
48 self._ipv6 = ipv6
46 self._ipv6 = ipv6
49 self._timeout = timeout
47 self._timeout = timeout
50 self._prefix = prefix
48 self._prefix = prefix
51 self._sock = None
49 self._sock = None
52
50
53 def connect(self):
51 def connect(self):
54 fam = socket.AF_INET6 if self._ipv6 else socket.AF_INET
52 fam = socket.AF_INET6 if self._ipv6 else socket.AF_INET
55 family, _, _, _, addr = socket.getaddrinfo(
53 family, _, _, _, addr = socket.getaddrinfo(
56 self._host, self._port, fam, socket.SOCK_STREAM)[0]
54 self._host, self._port, fam, socket.SOCK_STREAM)[0]
57 self._sock = socket.socket(family, socket.SOCK_STREAM)
55 self._sock = socket.socket(family, socket.SOCK_STREAM)
58 self._sock.settimeout(self._timeout)
56 self._sock.settimeout(self._timeout)
59 self._sock.connect(addr)
57 self._sock.connect(addr)
60
58
61
59
62 class UnixSocketStatsClient(StreamClientBase):
60 class UnixSocketStatsClient(StreamClientBase):
63 """Unix domain socket version of StatsClient."""
61 """Unix domain socket version of StatsClient."""
64
62
65 def __init__(self, socket_path, prefix=None, timeout=None):
63 def __init__(self, socket_path, prefix=None, timeout=None):
66 """Create a new client."""
64 """Create a new client."""
67 self._socket_path = socket_path
65 self._socket_path = socket_path
68 self._timeout = timeout
66 self._timeout = timeout
69 self._prefix = prefix
67 self._prefix = prefix
70 self._sock = None
68 self._sock = None
71
69
72 def connect(self):
70 def connect(self):
73 self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
71 self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
74 self._sock.settimeout(self._timeout)
72 self._sock.settimeout(self._timeout)
75 self._sock.connect(self._socket_path)
73 self._sock.connect(self._socket_path)
@@ -1,71 +1,66 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import functools
1 import functools
4
2 from time import perf_counter as time_now
5 # Use timer that's not susceptible to time of day adjustments.
6 try:
7 # perf_counter is only present on Py3.3+
8 from time import perf_counter as time_now
9 except ImportError:
10 # fall back to using time
11 from time import time as time_now
12
3
13
4
14 def safe_wraps(wrapper, *args, **kwargs):
5 def safe_wraps(wrapper, *args, **kwargs):
15 """Safely wraps partial functions."""
6 """Safely wraps partial functions."""
16 while isinstance(wrapper, functools.partial):
7 while isinstance(wrapper, functools.partial):
17 wrapper = wrapper.func
8 wrapper = wrapper.func
18 return functools.wraps(wrapper, *args, **kwargs)
9 return functools.wraps(wrapper, *args, **kwargs)
19
10
20
11
21 class Timer(object):
12 class Timer:
22 """A context manager/decorator for statsd.timing()."""
13 """A context manager/decorator for statsd.timing()."""
23
14
24 def __init__(self, client, stat, rate=1):
15 def __init__(self, client, stat, rate=1, tags=None, use_decimals=True, auto_send=True):
25 self.client = client
16 self.client = client
26 self.stat = stat
17 self.stat = stat
27 self.rate = rate
18 self.rate = rate
19 self.tags = tags
28 self.ms = None
20 self.ms = None
29 self._sent = False
21 self._sent = False
30 self._start_time = None
22 self._start_time = None
23 self.use_decimals = use_decimals
24 self.auto_send = auto_send
31
25
32 def __call__(self, f):
26 def __call__(self, f):
33 """Thread-safe timing function decorator."""
27 """Thread-safe timing function decorator."""
34 @safe_wraps(f)
28 @safe_wraps(f)
35 def _wrapped(*args, **kwargs):
29 def _wrapped(*args, **kwargs):
36 start_time = time_now()
30 start_time = time_now()
37 try:
31 try:
38 return f(*args, **kwargs)
32 return f(*args, **kwargs)
39 finally:
33 finally:
40 elapsed_time_ms = 1000.0 * (time_now() - start_time)
34 elapsed_time_ms = 1000.0 * (time_now() - start_time)
41 self.client.timing(self.stat, elapsed_time_ms, self.rate)
35 self.client.timing(self.stat, elapsed_time_ms, self.rate, self.tags, self.use_decimals)
36 self._sent = True
42 return _wrapped
37 return _wrapped
43
38
44 def __enter__(self):
39 def __enter__(self):
45 return self.start()
40 return self.start()
46
41
47 def __exit__(self, typ, value, tb):
42 def __exit__(self, typ, value, tb):
48 self.stop()
43 self.stop(send=self.auto_send)
49
44
50 def start(self):
45 def start(self):
51 self.ms = None
46 self.ms = None
52 self._sent = False
47 self._sent = False
53 self._start_time = time_now()
48 self._start_time = time_now()
54 return self
49 return self
55
50
56 def stop(self, send=True):
51 def stop(self, send=True):
57 if self._start_time is None:
52 if self._start_time is None:
58 raise RuntimeError('Timer has not started.')
53 raise RuntimeError('Timer has not started.')
59 dt = time_now() - self._start_time
54 dt = time_now() - self._start_time
60 self.ms = 1000.0 * dt # Convert to milliseconds.
55 self.ms = 1000.0 * dt # Convert to milliseconds.
61 if send:
56 if send:
62 self.send()
57 self.send()
63 return self
58 return self
64
59
65 def send(self):
60 def send(self):
66 if self.ms is None:
61 if self.ms is None:
67 raise RuntimeError('No data recorded.')
62 raise RuntimeError('No data recorded.')
68 if self._sent:
63 if self._sent:
69 raise RuntimeError('Already sent data.')
64 raise RuntimeError('Already sent data.')
70 self._sent = True
65 self._sent = True
71 self.client.timing(self.stat, self.ms, self.rate)
66 self.client.timing(self.stat, self.ms, self.rate, self.tags, self.use_decimals)
@@ -1,55 +1,53 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import socket
1 import socket
4
2
5 from .base import StatsClientBase, PipelineBase
3 from .base import StatsClientBase, PipelineBase
6
4
7
5
8 class Pipeline(PipelineBase):
6 class Pipeline(PipelineBase):
9
7
10 def __init__(self, client):
8 def __init__(self, client):
11 super(Pipeline, self).__init__(client)
9 super().__init__(client)
12 self._maxudpsize = client._maxudpsize
10 self._maxudpsize = client._maxudpsize
13
11
14 def _send(self):
12 def _send(self):
15 data = self._stats.popleft()
13 data = self._stats.popleft()
16 while self._stats:
14 while self._stats:
17 # Use popleft to preserve the order of the stats.
15 # Use popleft to preserve the order of the stats.
18 stat = self._stats.popleft()
16 stat = self._stats.popleft()
19 if len(stat) + len(data) + 1 >= self._maxudpsize:
17 if len(stat) + len(data) + 1 >= self._maxudpsize:
20 self._client._after(data)
18 self._client._after(data)
21 data = stat
19 data = stat
22 else:
20 else:
23 data += '\n' + stat
21 data += '\n' + stat
24 self._client._after(data)
22 self._client._after(data)
25
23
26
24
27 class StatsClient(StatsClientBase):
25 class StatsClient(StatsClientBase):
28 """A client for statsd."""
26 """A client for statsd."""
29
27
30 def __init__(self, host='localhost', port=8125, prefix=None,
28 def __init__(self, host='localhost', port=8125, prefix=None,
31 maxudpsize=512, ipv6=False):
29 maxudpsize=512, ipv6=False):
32 """Create a new client."""
30 """Create a new client."""
33 fam = socket.AF_INET6 if ipv6 else socket.AF_INET
31 fam = socket.AF_INET6 if ipv6 else socket.AF_INET
34 family, _, _, _, addr = socket.getaddrinfo(
32 family, _, _, _, addr = socket.getaddrinfo(
35 host, port, fam, socket.SOCK_DGRAM)[0]
33 host, port, fam, socket.SOCK_DGRAM)[0]
36 self._addr = addr
34 self._addr = addr
37 self._sock = socket.socket(family, socket.SOCK_DGRAM)
35 self._sock = socket.socket(family, socket.SOCK_DGRAM)
38 self._prefix = prefix
36 self._prefix = prefix
39 self._maxudpsize = maxudpsize
37 self._maxudpsize = maxudpsize
40
38
41 def _send(self, data):
39 def _send(self, data):
42 """Send data to statsd."""
40 """Send data to statsd."""
43 try:
41 try:
44 self._sock.sendto(data.encode('ascii'), self._addr)
42 self._sock.sendto(data.encode('ascii'), self._addr)
45 except (socket.error, RuntimeError):
43 except (socket.error, RuntimeError):
46 # No time for love, Dr. Jones!
44 # No time for love, Dr. Jones!
47 pass
45 pass
48
46
49 def close(self):
47 def close(self):
50 if self._sock and hasattr(self._sock, 'close'):
48 if self._sock and hasattr(self._sock, 'close'):
51 self._sock.close()
49 self._sock.close()
52 self._sock = None
50 self._sock = None
53
51
54 def pipeline(self):
52 def pipeline(self):
55 return Pipeline(self)
53 return Pipeline(self)
@@ -1,175 +1,273 b''
1 # -*- coding: utf-8 -*-
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
5 #
3 #
6 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
7 # (at your option) any later version.
10 #
8 #
11 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
12 # GNU General Public License for more details.
15 #
13 #
16 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
17
20
18 import io
21 import os
19 import os
22 import time
20 import time
21 import sys
23 import datetime
22 import datetime
24 import msgpack
23 import msgpack
25 import logging
24 import logging
26 import traceback
25 import traceback
27 import tempfile
26 import tempfile
28
27 import glob
29 from pyramid import compat
30
28
31 log = logging.getLogger(__name__)
29 log = logging.getLogger(__name__)
32
30
33 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
31 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
34 global_prefix = 'vcsserver'
32 global_prefix = 'vcsserver'
35 exc_store_dir_name = 'rc_exception_store_v1'
33 exc_store_dir_name = 'rc_exception_store_v1'
36
34
37
35
38 def exc_serialize(exc_id, tb, exc_type):
36 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
39
40 data = {
37 data = {
41 'version': 'v1',
38 "version": "v1",
42 'exc_id': exc_id,
39 "exc_id": exc_id,
43 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
40 "exc_utc_date": datetime.datetime.utcnow().isoformat(),
44 'exc_timestamp': repr(time.time()),
41 "exc_timestamp": repr(time.time()),
45 'exc_message': tb,
42 "exc_message": tb,
46 'exc_type': exc_type,
43 "exc_type": exc_type,
47 }
44 }
45 if extra_data:
46 data.update(extra_data)
48 return msgpack.packb(data), data
47 return msgpack.packb(data), data
49
48
50
49
51 def exc_unserialize(tb):
50 def exc_unserialize(tb):
52 return msgpack.unpackb(tb)
51 return msgpack.unpackb(tb)
53
52
54
53
54 _exc_store = None
55
56
55 def get_exc_store():
57 def get_exc_store():
56 """
58 """
57 Get and create exception store if it's not existing
59 Get and create exception store if it's not existing
58 """
60 """
61 global _exc_store
62
63 if _exc_store is not None:
64 # quick global cache
65 return _exc_store
66
59 import vcsserver as app
67 import vcsserver as app
60
68
61 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
69 exc_store_dir = (
70 app.CONFIG.get("exception_tracker.store_path", "") or tempfile.gettempdir()
71 )
62 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
72 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
63
73
64 _exc_store_path = os.path.abspath(_exc_store_path)
74 _exc_store_path = os.path.abspath(_exc_store_path)
65 if not os.path.isdir(_exc_store_path):
75 if not os.path.isdir(_exc_store_path):
66 os.makedirs(_exc_store_path)
76 os.makedirs(_exc_store_path)
67 log.debug('Initializing exceptions store at %s', _exc_store_path)
77 log.debug("Initializing exceptions store at %s", _exc_store_path)
78 _exc_store = _exc_store_path
79
68 return _exc_store_path
80 return _exc_store_path
69
81
70
82
71 def _store_exception(exc_id, exc_info, prefix, request_path=''):
83 def get_detailed_tb(exc_info):
72 exc_type, exc_value, exc_traceback = exc_info
84 try:
85 from pip._vendor.rich import (
86 traceback as rich_tb,
87 scope as rich_scope,
88 console as rich_console,
89 )
90 except ImportError:
91 try:
92 from rich import (
93 traceback as rich_tb,
94 scope as rich_scope,
95 console as rich_console,
96 )
97 except ImportError:
98 return None
99
100 console = rich_console.Console(width=160, file=io.StringIO())
101
102 exc = rich_tb.Traceback.extract(*exc_info, show_locals=True)
103
104 tb_rich = rich_tb.Traceback(
105 trace=exc,
106 width=160,
107 extra_lines=3,
108 theme=None,
109 word_wrap=False,
110 show_locals=False,
111 max_frames=100,
112 )
73
113
74 tb = ''.join(traceback.format_exception(
114 # last_stack = exc.stacks[-1]
75 exc_type, exc_value, exc_traceback, None))
115 # last_frame = last_stack.frames[-1]
116 # if last_frame and last_frame.locals:
117 # console.print(
118 # rich_scope.render_scope(
119 # last_frame.locals,
120 # title=f'{last_frame.filename}:{last_frame.lineno}'))
121
122 console.print(tb_rich)
123 formatted_locals = console.file.getvalue()
124
125 return formatted_locals
126
76
127
77 detailed_tb = getattr(exc_value, '_org_exc_tb', None)
128 def get_request_metadata(request=None) -> dict:
129 request_metadata = {}
130 if not request:
131 from pyramid.threadlocal import get_current_request
132
133 request = get_current_request()
134
135 # NOTE(marcink): store request information into exc_data
136 if request:
137 request_metadata["client_address"] = getattr(request, "client_addr", "")
138 request_metadata["user_agent"] = getattr(request, "user_agent", "")
139 request_metadata["method"] = getattr(request, "method", "")
140 request_metadata["url"] = getattr(request, "url", "")
141 return request_metadata
142
143
144 def format_exc(exc_info, use_detailed_tb=True):
145 exc_type, exc_value, exc_traceback = exc_info
146 tb = "++ TRACEBACK ++\n\n"
147 tb += "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, None))
148
149 detailed_tb = getattr(exc_value, "_org_exc_tb", None)
78
150
79 if detailed_tb:
151 if detailed_tb:
80 if isinstance(detailed_tb, compat.string_types):
152 remote_tb = detailed_tb
153 if isinstance(detailed_tb, str):
81 remote_tb = [detailed_tb]
154 remote_tb = [detailed_tb]
82
155
83 tb += (
156 tb += (
84 '\n+++ BEG SOURCE EXCEPTION +++\n\n'
157 "\n+++ BEG SOURCE EXCEPTION +++\n\n"
85 '{}\n'
158 "{}\n"
86 '+++ END SOURCE EXCEPTION +++\n'
159 "+++ END SOURCE EXCEPTION +++\n"
87 ''.format('\n'.join(remote_tb))
160 "".format("\n".join(remote_tb))
88 )
161 )
89
162
90 # Avoid that remote_tb also appears in the frame
163 # Avoid that remote_tb also appears in the frame
91 del remote_tb
164 del remote_tb
92
165
166 if use_detailed_tb:
167 locals_tb = get_detailed_tb(exc_info)
168 if locals_tb:
169 tb += f"\n+++ DETAILS +++\n\n{locals_tb}\n" ""
170 return tb
171
172
173 def _store_exception(exc_id, exc_info, prefix, request_path=''):
174 """
175 Low level function to store exception in the exception tracker
176 """
177
178 extra_data = {}
179 extra_data.update(get_request_metadata())
180
181 exc_type, exc_value, exc_traceback = exc_info
182 tb = format_exc(exc_info)
183
93 exc_type_name = exc_type.__name__
184 exc_type_name = exc_type.__name__
185 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name, extra_data=extra_data)
186
187 exc_pref_id = f"{exc_id}_{prefix}_{org_data['exc_timestamp']}"
94 exc_store_path = get_exc_store()
188 exc_store_path = get_exc_store()
95 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
96 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
97 if not os.path.isdir(exc_store_path):
189 if not os.path.isdir(exc_store_path):
98 os.makedirs(exc_store_path)
190 os.makedirs(exc_store_path)
99 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
191 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
100 with open(stored_exc_path, 'wb') as f:
192 with open(stored_exc_path, "wb") as f:
101 f.write(exc_data)
193 f.write(exc_data)
102 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
194 log.debug("Stored generated exception %s as: %s", exc_id, stored_exc_path)
103
195
104 log.error(
196 if request_path:
105 'error occurred handling this request.\n'
197 log.error(
106 'Path: `%s`, tb: %s',
198 'error occurred handling this request.\n'
107 request_path, tb)
199 'Path: `%s`, %s',
200 request_path, tb)
108
201
109
202
110 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
203 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
111 """
204 """
112 Example usage::
205 Example usage::
113
206
114 exc_info = sys.exc_info()
207 exc_info = sys.exc_info()
115 store_exception(id(exc_info), exc_info)
208 store_exception(id(exc_info), exc_info)
116 """
209 """
117
210
118 try:
211 try:
119 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix,
212 exc_type = exc_info[0]
120 request_path=request_path)
213 exc_type_name = exc_type.__name__
214
215 _store_exception(
216 exc_id=exc_id, exc_info=exc_info, prefix=prefix, request_path=request_path,
217 )
218 return exc_id, exc_type_name
121 except Exception:
219 except Exception:
122 log.exception('Failed to store exception `%s` information', exc_id)
220 log.exception("Failed to store exception `%s` information", exc_id)
123 # there's no way this can fail, it will crash server badly if it does.
221 # there's no way this can fail, it will crash server badly if it does.
124 pass
222 pass
125
223
126
224
127 def _find_exc_file(exc_id, prefix=global_prefix):
225 def _find_exc_file(exc_id, prefix=global_prefix):
128 exc_store_path = get_exc_store()
226 exc_store_path = get_exc_store()
129 if prefix:
227 if prefix:
130 exc_id = '{}_{}'.format(exc_id, prefix)
228 exc_id = f"{exc_id}_{prefix}"
131 else:
229 else:
132 # search without a prefix
230 # search without a prefix
133 exc_id = '{}'.format(exc_id)
231 exc_id = f"{exc_id}"
134
232
135 # we need to search the store for such start pattern as above
233 found_exc_id = None
136 for fname in os.listdir(exc_store_path):
234 matches = glob.glob(os.path.join(exc_store_path, exc_id) + "*")
137 if fname.startswith(exc_id):
235 if matches:
138 exc_id = os.path.join(exc_store_path, fname)
236 found_exc_id = matches[0]
139 break
140 continue
141 else:
142 exc_id = None
143
237
144 return exc_id
238 return found_exc_id
145
239
146
240
147 def _read_exception(exc_id, prefix):
241 def _read_exception(exc_id, prefix):
148 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
242 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
149 if exc_id_file_path:
243 if exc_id_file_path:
150 with open(exc_id_file_path, 'rb') as f:
244 with open(exc_id_file_path, "rb") as f:
151 return exc_unserialize(f.read())
245 return exc_unserialize(f.read())
152 else:
246 else:
153 log.debug('Exception File `%s` not found', exc_id_file_path)
247 log.debug("Exception File `%s` not found", exc_id_file_path)
154 return None
248 return None
155
249
156
250
157 def read_exception(exc_id, prefix=global_prefix):
251 def read_exception(exc_id, prefix=global_prefix):
158 try:
252 try:
159 return _read_exception(exc_id=exc_id, prefix=prefix)
253 return _read_exception(exc_id=exc_id, prefix=prefix)
160 except Exception:
254 except Exception:
161 log.exception('Failed to read exception `%s` information', exc_id)
255 log.exception("Failed to read exception `%s` information", exc_id)
162 # there's no way this can fail, it will crash server badly if it does.
256 # there's no way this can fail, it will crash server badly if it does.
163 return None
257 return None
164
258
165
259
166 def delete_exception(exc_id, prefix=global_prefix):
260 def delete_exception(exc_id, prefix=global_prefix):
167 try:
261 try:
168 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
262 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
169 if exc_id_file_path:
263 if exc_id_file_path:
170 os.remove(exc_id_file_path)
264 os.remove(exc_id_file_path)
171
265
172 except Exception:
266 except Exception:
173 log.exception('Failed to remove exception `%s` information', exc_id)
267 log.exception("Failed to remove exception `%s` information", exc_id)
174 # there's no way this can fail, it will crash server badly if it does.
268 # there's no way this can fail, it will crash server badly if it does.
175 pass
269 pass
270
271
272 def generate_id():
273 return id(object())
@@ -1,65 +1,63 b''
1 # -*- coding: utf-8 -*-
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
5 #
3 #
6 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
7 # (at your option) any later version.
10 #
8 #
11 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
12 # GNU General Public License for more details.
15 #
13 #
16 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
17
20
18
21 import logging
19 import logging
22
20
23 from repoze.lru import LRUCache
21 from repoze.lru import LRUCache
24
22
25 from vcsserver.utils import safe_str
23 from vcsserver.str_utils import safe_str
26
24
27 log = logging.getLogger(__name__)
25 log = logging.getLogger(__name__)
28
26
29
27
30 class LRUDict(LRUCache):
28 class LRUDict(LRUCache):
31 """
29 """
32 Wrapper to provide partial dict access
30 Wrapper to provide partial dict access
33 """
31 """
34
32
35 def __setitem__(self, key, value):
33 def __setitem__(self, key, value):
36 return self.put(key, value)
34 return self.put(key, value)
37
35
38 def __getitem__(self, key):
36 def __getitem__(self, key):
39 return self.get(key)
37 return self.get(key)
40
38
41 def __contains__(self, key):
39 def __contains__(self, key):
42 return bool(self.get(key))
40 return bool(self.get(key))
43
41
44 def __delitem__(self, key):
42 def __delitem__(self, key):
45 del self.data[key]
43 del self.data[key]
46
44
47 def keys(self):
45 def keys(self):
48 return self.data.keys()
46 return list(self.data.keys())
49
47
50
48
51 class LRUDictDebug(LRUDict):
49 class LRUDictDebug(LRUDict):
52 """
50 """
53 Wrapper to provide some debug options
51 Wrapper to provide some debug options
54 """
52 """
55 def _report_keys(self):
53 def _report_keys(self):
56 elems_cnt = '%s/%s' % (len(self.keys()), self.size)
54 elems_cnt = f'{len(list(self.keys()))}/{self.size}'
57 # trick for pformat print it more nicely
55 # trick for pformat print it more nicely
58 fmt = '\n'
56 fmt = '\n'
59 for cnt, elem in enumerate(self.keys()):
57 for cnt, elem in enumerate(self.keys()):
60 fmt += '%s - %s\n' % (cnt+1, safe_str(elem))
58 fmt += f'{cnt+1} - {safe_str(elem)}\n'
61 log.debug('current LRU keys (%s):%s', elems_cnt, fmt)
59 log.debug('current LRU keys (%s):%s', elems_cnt, fmt)
62
60
63 def __getitem__(self, key):
61 def __getitem__(self, key):
64 self._report_keys()
62 self._report_keys()
65 return self.get(key)
63 return self.get(key)
@@ -1,79 +1,114 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import logging
18 import logging
19 import threading
20
19 from dogpile.cache import register_backend
21 from dogpile.cache import register_backend
20
22
23 from . import region_meta
24 from .utils import (
25 backend_key_generator,
26 clear_cache_namespace,
27 get_default_cache_settings,
28 get_or_create_region,
29 make_region,
30 str2bool,
31 )
32
33 module_name = 'vcsserver'
34
21 register_backend(
35 register_backend(
22 "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
36 "dogpile.cache.rc.memory_lru", f"{module_name}.lib.rc_cache.backends",
23 "LRUMemoryBackend")
37 "LRUMemoryBackend")
24
38
25 register_backend(
39 register_backend(
26 "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
40 "dogpile.cache.rc.file_namespace", f"{module_name}.lib.rc_cache.backends",
27 "FileNamespaceBackend")
41 "FileNamespaceBackend")
28
42
29 register_backend(
43 register_backend(
30 "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
44 "dogpile.cache.rc.redis", f"{module_name}.lib.rc_cache.backends",
31 "RedisPickleBackend")
45 "RedisPickleBackend")
32
46
33 register_backend(
47 register_backend(
34 "dogpile.cache.rc.redis_msgpack", "vcsserver.lib.rc_cache.backends",
48 "dogpile.cache.rc.redis_msgpack", f"{module_name}.lib.rc_cache.backends",
35 "RedisMsgPackBackend")
49 "RedisMsgPackBackend")
36
50
37
51
38 log = logging.getLogger(__name__)
52 log = logging.getLogger(__name__)
39
53
40 from . import region_meta
54
41 from .utils import (
55 CACHE_OBJ_CACHE_VER = 'v2'
42 get_default_cache_settings, backend_key_generator, get_or_create_region,
56
43 clear_cache_namespace, make_region)
57 CLEAR_DELETE = 'delete'
58 CLEAR_INVALIDATE = 'invalidate'
59
60
61 def async_creation_runner(cache, cache_key, creator, mutex):
62
63 def runner():
64 try:
65 value = creator()
66 cache.set(cache_key, value)
67 finally:
68 mutex.release()
69
70 thread = threading.Thread(target=runner)
71 thread.start()
44
72
45
73
46 def configure_dogpile_cache(settings):
74 def configure_dogpile_cache(settings):
47 cache_dir = settings.get('cache_dir')
75 cache_dir = settings.get('cache_dir')
48 if cache_dir:
76 if cache_dir:
49 region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
77 region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
50
78
51 rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
79 rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
52
80
53 # inspect available namespaces
81 # inspect available namespaces
54 avail_regions = set()
82 avail_regions = set()
55 for key in rc_cache_data.keys():
83 for key in rc_cache_data.keys():
56 namespace_name = key.split('.', 1)[0]
84 namespace_name = key.split('.', 1)[0]
57 if namespace_name in avail_regions:
85 if namespace_name in avail_regions:
58 continue
86 continue
59
87
60 avail_regions.add(namespace_name)
88 avail_regions.add(namespace_name)
61 log.debug('dogpile: found following cache regions: %s', namespace_name)
89 log.debug('dogpile: found following cache regions: %s', namespace_name)
62
90
63 new_region = make_region(
91 new_region = make_region(
64 name=namespace_name,
92 name=namespace_name,
65 function_key_generator=None
93 function_key_generator=None,
94 async_creation_runner=None
66 )
95 )
67
96
68 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(namespace_name))
97 new_region.configure_from_config(settings, f'rc_cache.{namespace_name}.')
69 new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
98 new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
99
100 async_creator = str2bool(settings.pop(f'rc_cache.{namespace_name}.async_creator', 'false'))
101 if async_creator:
102 log.debug('configuring region %s with async creator', new_region)
103 new_region.async_creation_runner = async_creation_runner
104
70 if log.isEnabledFor(logging.DEBUG):
105 if log.isEnabledFor(logging.DEBUG):
71 region_args = dict(backend=new_region.actual_backend.__class__,
106 region_args = dict(backend=new_region.actual_backend,
72 region_invalidator=new_region.region_invalidator.__class__)
107 region_invalidator=new_region.region_invalidator.__class__)
73 log.debug('dogpile: registering a new region `%s` %s', namespace_name, region_args)
108 log.debug('dogpile: registering a new region key=`%s` args=%s', namespace_name, region_args)
74
109
75 region_meta.dogpile_cache_regions[namespace_name] = new_region
110 region_meta.dogpile_cache_regions[namespace_name] = new_region
76
111
77
112
78 def includeme(config):
113 def includeme(config):
79 configure_dogpile_cache(config.registry.settings)
114 configure_dogpile_cache(config.registry.settings)
@@ -1,329 +1,303 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import time
18 #import errno
19 import errno
19 import fcntl
20 import functools
20 import logging
21 import logging
22 import os
23 import pickle
24 #import time
21
25
26 #import gevent
22 import msgpack
27 import msgpack
23 import redis
28 import redis
24
29
25 from dogpile.cache.api import CachedValue
30 flock_org = fcntl.flock
26 from dogpile.cache.backends import memory as memory_backend
31 from typing import Union
32
33 from dogpile.cache.api import Deserializer, Serializer
27 from dogpile.cache.backends import file as file_backend
34 from dogpile.cache.backends import file as file_backend
35 from dogpile.cache.backends import memory as memory_backend
28 from dogpile.cache.backends import redis as redis_backend
36 from dogpile.cache.backends import redis as redis_backend
29 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
37 from dogpile.cache.backends.file import FileLock
30 from dogpile.cache.util import memoized_property
38 from dogpile.cache.util import memoized_property
31
39
32 from pyramid.settings import asbool
33
34 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
40 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
35 from vcsserver.utils import safe_str
41 from vcsserver.str_utils import safe_bytes, safe_str
36
42 from vcsserver.type_utils import str2bool
37
43
38 _default_max_size = 1024
44 _default_max_size = 1024
39
45
40 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
41
47
42
48
43 class LRUMemoryBackend(memory_backend.MemoryBackend):
49 class LRUMemoryBackend(memory_backend.MemoryBackend):
44 key_prefix = 'lru_mem_backend'
50 key_prefix = 'lru_mem_backend'
45 pickle_values = False
51 pickle_values = False
46
52
47 def __init__(self, arguments):
53 def __init__(self, arguments):
48 max_size = arguments.pop('max_size', _default_max_size)
54 self.max_size = arguments.pop('max_size', _default_max_size)
49
55
50 LRUDictClass = LRUDict
56 LRUDictClass = LRUDict
51 if arguments.pop('log_key_count', None):
57 if arguments.pop('log_key_count', None):
52 LRUDictClass = LRUDictDebug
58 LRUDictClass = LRUDictDebug
53
59
54 arguments['cache_dict'] = LRUDictClass(max_size)
60 arguments['cache_dict'] = LRUDictClass(self.max_size)
55 super(LRUMemoryBackend, self).__init__(arguments)
61 super().__init__(arguments)
62
63 def __repr__(self):
64 return f'{self.__class__}(maxsize=`{self.max_size}`)'
65
66 def __str__(self):
67 return self.__repr__()
56
68
57 def delete(self, key):
69 def delete(self, key):
58 try:
70 try:
59 del self._cache[key]
71 del self._cache[key]
60 except KeyError:
72 except KeyError:
61 # we don't care if key isn't there at deletion
73 # we don't care if key isn't there at deletion
62 pass
74 pass
63
75
76 def list_keys(self, prefix):
77 return list(self._cache.keys())
78
64 def delete_multi(self, keys):
79 def delete_multi(self, keys):
65 for key in keys:
80 for key in keys:
66 self.delete(key)
81 self.delete(key)
67
82
68
83 def delete_multi_by_prefix(self, prefix):
69 class PickleSerializer(object):
84 cache_keys = self.list_keys(prefix=prefix)
70
85 num_affected_keys = len(cache_keys)
71 def _dumps(self, value, safe=False):
86 if num_affected_keys:
72 try:
87 self.delete_multi(cache_keys)
73 return compat.pickle.dumps(value)
88 return num_affected_keys
74 except Exception:
75 if safe:
76 return NO_VALUE
77 else:
78 raise
79
80 def _loads(self, value, safe=True):
81 try:
82 return compat.pickle.loads(value)
83 except Exception:
84 if safe:
85 return NO_VALUE
86 else:
87 raise
88
89
89
90
90 class MsgPackSerializer(object):
91 class PickleSerializer:
91
92 serializer: None | Serializer = staticmethod( # type: ignore
92 def _dumps(self, value, safe=False):
93 functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL)
93 try:
94 )
94 return msgpack.packb(value)
95 deserializer: None | Deserializer = staticmethod( # type: ignore
95 except Exception:
96 functools.partial(pickle.loads)
96 if safe:
97 )
97 return NO_VALUE
98 else:
99 raise
100
101 def _loads(self, value, safe=True):
102 """
103 pickle maintained the `CachedValue` wrapper of the tuple
104 msgpack does not, so it must be added back in.
105 """
106 try:
107 value = msgpack.unpackb(value, use_list=False)
108 return CachedValue(*value)
109 except Exception:
110 if safe:
111 return NO_VALUE
112 else:
113 raise
114
98
115
99
116 import fcntl
100 class MsgPackSerializer:
117 flock_org = fcntl.flock
101 serializer: None | Serializer = staticmethod( # type: ignore
102 msgpack.packb
103 )
104 deserializer: None | Deserializer = staticmethod( # type: ignore
105 functools.partial(msgpack.unpackb, use_list=False)
106 )
118
107
119
108
120 class CustomLockFactory(FileLock):
109 class CustomLockFactory(FileLock):
121
110
122 pass
111 pass
123
112
124
113
125 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
114 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
126 key_prefix = 'file_backend'
115 key_prefix = 'file_backend'
127
116
128 def __init__(self, arguments):
117 def __init__(self, arguments):
129 arguments['lock_factory'] = CustomLockFactory
118 arguments['lock_factory'] = CustomLockFactory
130 db_file = arguments.get('filename')
119 db_file = arguments.get('filename')
131
120
132 log.debug('initialing %s DB in %s', self.__class__.__name__, db_file)
121 log.debug('initialing cache-backend=%s db in %s', self.__class__.__name__, db_file)
122 db_file_dir = os.path.dirname(db_file)
123 if not os.path.isdir(db_file_dir):
124 os.makedirs(db_file_dir)
125
133 try:
126 try:
134 super(FileNamespaceBackend, self).__init__(arguments)
127 super().__init__(arguments)
135 except Exception:
128 except Exception:
136 log.error('Failed to initialize db at: %s', db_file)
129 log.exception('Failed to initialize db at: %s', db_file)
137 raise
130 raise
138
131
139 def __repr__(self):
132 def __repr__(self):
140 return '{} `{}`'.format(self.__class__, self.filename)
133 return f'{self.__class__}(file=`{self.filename}`)'
134
135 def __str__(self):
136 return self.__repr__()
141
137
142 def list_keys(self, prefix=''):
138 def _get_keys_pattern(self, prefix: bytes = b''):
143 prefix = '{}:{}'.format(self.key_prefix, prefix)
139 return b'%b:%b' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
144
140
145 def cond(v):
141 def list_keys(self, prefix: bytes = b''):
142 prefix = self._get_keys_pattern(prefix)
143
144 def cond(dbm_key: bytes):
146 if not prefix:
145 if not prefix:
147 return True
146 return True
148
147
149 if v.startswith(prefix):
148 if dbm_key.startswith(prefix):
150 return True
149 return True
151 return False
150 return False
152
151
153 with self._dbm_file(True) as dbm:
152 with self._dbm_file(True) as dbm:
154 try:
153 try:
155 return filter(cond, dbm.keys())
154 return list(filter(cond, dbm.keys()))
156 except Exception:
155 except Exception:
157 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
156 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
158 raise
157 raise
159
158
159 def delete_multi_by_prefix(self, prefix):
160 cache_keys = self.list_keys(prefix=prefix)
161 num_affected_keys = len(cache_keys)
162 if num_affected_keys:
163 self.delete_multi(cache_keys)
164 return num_affected_keys
165
160 def get_store(self):
166 def get_store(self):
161 return self.filename
167 return self.filename
162
168
163 def _dbm_get(self, key):
164 with self._dbm_file(False) as dbm:
165 if hasattr(dbm, 'get'):
166 value = dbm.get(key, NO_VALUE)
167 else:
168 # gdbm objects lack a .get method
169 try:
170 value = dbm[key]
171 except KeyError:
172 value = NO_VALUE
173 if value is not NO_VALUE:
174 value = self._loads(value)
175 return value
176
177 def get(self, key):
178 try:
179 return self._dbm_get(key)
180 except Exception:
181 log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
182 raise
183
184 def set(self, key, value):
185 with self._dbm_file(True) as dbm:
186 dbm[key] = self._dumps(value)
187
188 def set_multi(self, mapping):
189 with self._dbm_file(True) as dbm:
190 for key, value in mapping.items():
191 dbm[key] = self._dumps(value)
192
193
169
194 class BaseRedisBackend(redis_backend.RedisBackend):
170 class BaseRedisBackend(redis_backend.RedisBackend):
195 key_prefix = ''
171 key_prefix = ''
196
172
197 def __init__(self, arguments):
173 def __init__(self, arguments):
198 super(BaseRedisBackend, self).__init__(arguments)
174 self.db_conn = arguments.get('host', '') or arguments.get('url', '') or 'redis-host'
175 super().__init__(arguments)
176
199 self._lock_timeout = self.lock_timeout
177 self._lock_timeout = self.lock_timeout
200 self._lock_auto_renewal = asbool(arguments.pop("lock_auto_renewal", True))
178 self._lock_auto_renewal = str2bool(arguments.pop("lock_auto_renewal", True))
201
179
202 if self._lock_auto_renewal and not self._lock_timeout:
180 if self._lock_auto_renewal and not self._lock_timeout:
203 # set default timeout for auto_renewal
181 # set default timeout for auto_renewal
204 self._lock_timeout = 30
182 self._lock_timeout = 30
205
183
184 def __repr__(self):
185 return f'{self.__class__}(conn=`{self.db_conn}`)'
186
187 def __str__(self):
188 return self.__repr__()
189
206 def _create_client(self):
190 def _create_client(self):
207 args = {}
191 args = {}
208
192
209 if self.url is not None:
193 if self.url is not None:
210 args.update(url=self.url)
194 args.update(url=self.url)
211
195
212 else:
196 else:
213 args.update(
197 args.update(
214 host=self.host, password=self.password,
198 host=self.host, password=self.password,
215 port=self.port, db=self.db
199 port=self.port, db=self.db
216 )
200 )
217
201
218 connection_pool = redis.ConnectionPool(**args)
202 connection_pool = redis.ConnectionPool(**args)
203 self.writer_client = redis.StrictRedis(
204 connection_pool=connection_pool
205 )
206 self.reader_client = self.writer_client
219
207
220 return redis.StrictRedis(connection_pool=connection_pool)
208 def _get_keys_pattern(self, prefix: bytes = b''):
209 return b'%b:%b*' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
210
211 def list_keys(self, prefix: bytes = b''):
212 prefix = self._get_keys_pattern(prefix)
213 return self.reader_client.keys(prefix)
221
214
222 def list_keys(self, prefix=''):
215 def delete_multi_by_prefix(self, prefix, use_lua=False):
223 prefix = '{}:{}*'.format(self.key_prefix, prefix)
216 if use_lua:
224 return self.client.keys(prefix)
217 # high efficient LUA script to delete ALL keys by prefix...
218 lua = """local keys = redis.call('keys', ARGV[1])
219 for i=1,#keys,5000 do
220 redis.call('del', unpack(keys, i, math.min(i+(5000-1), #keys)))
221 end
222 return #keys"""
223 num_affected_keys = self.writer_client.eval(
224 lua,
225 0,
226 f"{prefix}*")
227 else:
228 cache_keys = self.list_keys(prefix=prefix)
229 num_affected_keys = len(cache_keys)
230 if num_affected_keys:
231 self.delete_multi(cache_keys)
232 return num_affected_keys
225
233
226 def get_store(self):
234 def get_store(self):
227 return self.client.connection_pool
235 return self.reader_client.connection_pool
228
229 def get(self, key):
230 value = self.client.get(key)
231 if value is None:
232 return NO_VALUE
233 return self._loads(value)
234
235 def get_multi(self, keys):
236 if not keys:
237 return []
238 values = self.client.mget(keys)
239 loads = self._loads
240 return [
241 loads(v) if v is not None else NO_VALUE
242 for v in values]
243
244 def set(self, key, value):
245 if self.redis_expiration_time:
246 self.client.setex(key, self.redis_expiration_time,
247 self._dumps(value))
248 else:
249 self.client.set(key, self._dumps(value))
250
251 def set_multi(self, mapping):
252 dumps = self._dumps
253 mapping = dict(
254 (k, dumps(v))
255 for k, v in mapping.items()
256 )
257
258 if not self.redis_expiration_time:
259 self.client.mset(mapping)
260 else:
261 pipe = self.client.pipeline()
262 for key, value in mapping.items():
263 pipe.setex(key, self.redis_expiration_time, value)
264 pipe.execute()
265
236
266 def get_mutex(self, key):
237 def get_mutex(self, key):
267 if self.distributed_lock:
238 if self.distributed_lock:
268 lock_key = redis_backend.u('_lock_{0}').format(safe_str(key))
239 lock_key = f'_lock_{safe_str(key)}'
269 return get_mutex_lock(self.client, lock_key, self._lock_timeout,
240 return get_mutex_lock(
270 auto_renewal=self._lock_auto_renewal)
241 self.writer_client, lock_key,
242 self._lock_timeout,
243 auto_renewal=self._lock_auto_renewal
244 )
271 else:
245 else:
272 return None
246 return None
273
247
274
248
275 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
249 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
276 key_prefix = 'redis_pickle_backend'
250 key_prefix = 'redis_pickle_backend'
277 pass
251 pass
278
252
279
253
280 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
254 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
281 key_prefix = 'redis_msgpack_backend'
255 key_prefix = 'redis_msgpack_backend'
282 pass
256 pass
283
257
284
258
285 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
259 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
286 import redis_lock
260 from vcsserver.lib._vendor import redis_lock
287
261
288 class _RedisLockWrapper(object):
262 class _RedisLockWrapper:
289 """LockWrapper for redis_lock"""
263 """LockWrapper for redis_lock"""
290
264
291 @classmethod
265 @classmethod
292 def get_lock(cls):
266 def get_lock(cls):
293 return redis_lock.Lock(
267 return redis_lock.Lock(
294 redis_client=client,
268 redis_client=client,
295 name=lock_key,
269 name=lock_key,
296 expire=lock_timeout,
270 expire=lock_timeout,
297 auto_renewal=auto_renewal,
271 auto_renewal=auto_renewal,
298 strict=True,
272 strict=True,
299 )
273 )
300
274
301 def __repr__(self):
275 def __repr__(self):
302 return "{}:{}".format(self.__class__.__name__, lock_key)
276 return f"{self.__class__.__name__}:{lock_key}"
303
277
304 def __str__(self):
278 def __str__(self):
305 return "{}:{}".format(self.__class__.__name__, lock_key)
279 return f"{self.__class__.__name__}:{lock_key}"
306
280
307 def __init__(self):
281 def __init__(self):
308 self.lock = self.get_lock()
282 self.lock = self.get_lock()
309 self.lock_key = lock_key
283 self.lock_key = lock_key
310
284
311 def acquire(self, wait=True):
285 def acquire(self, wait=True):
312 log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
286 log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
313 try:
287 try:
314 acquired = self.lock.acquire(wait)
288 acquired = self.lock.acquire(wait)
315 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
289 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
316 return acquired
290 return acquired
317 except redis_lock.AlreadyAcquired:
291 except redis_lock.AlreadyAcquired:
318 return False
292 return False
319 except redis_lock.AlreadyStarted:
293 except redis_lock.AlreadyStarted:
320 # refresh thread exists, but it also means we acquired the lock
294 # refresh thread exists, but it also means we acquired the lock
321 return True
295 return True
322
296
323 def release(self):
297 def release(self):
324 try:
298 try:
325 self.lock.release()
299 self.lock.release()
326 except redis_lock.NotAcquired:
300 except redis_lock.NotAcquired:
327 pass
301 pass
328
302
329 return _RedisLockWrapper()
303 return _RedisLockWrapper()
@@ -1,26 +1,26 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import tempfile
19 import tempfile
20
20
21 dogpile_config_defaults = {
21 dogpile_config_defaults = {
22 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
22 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
23 }
23 }
24
24
25 # GLOBAL TO STORE ALL REGISTERED REGIONS
25 # GLOBAL TO STORE ALL REGISTERED REGIONS
26 dogpile_cache_regions = {}
26 dogpile_cache_regions = {}
@@ -1,263 +1,245 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import functools
19 import time
20 import logging
19 import logging
21 import functools
20 import os
21 import threading
22 import time
22
23
24 import decorator
23 from dogpile.cache import CacheRegion
25 from dogpile.cache import CacheRegion
24 from dogpile.cache.util import compat
26
25
27
26 from vcsserver.utils import safe_str, sha1
28 from vcsserver.utils import sha1
29 from vcsserver.str_utils import safe_bytes
30 from vcsserver.type_utils import str2bool # noqa :required by imports from .utils
27
31
28 from vcsserver.lib.rc_cache import region_meta
32 from . import region_meta
29
33
30 log = logging.getLogger(__name__)
34 log = logging.getLogger(__name__)
31
35
32
36
33 class RhodeCodeCacheRegion(CacheRegion):
37 class RhodeCodeCacheRegion(CacheRegion):
34
38
39 def __repr__(self):
40 return f'`{self.__class__.__name__}(name={self.name}, backend={self.backend.__class__})`'
41
35 def conditional_cache_on_arguments(
42 def conditional_cache_on_arguments(
36 self, namespace=None,
43 self, namespace=None,
37 expiration_time=None,
44 expiration_time=None,
38 should_cache_fn=None,
45 should_cache_fn=None,
39 to_str=compat.string_type,
46 to_str=str,
40 function_key_generator=None,
47 function_key_generator=None,
41 condition=True):
48 condition=True):
42 """
49 """
43 Custom conditional decorator, that will not touch any dogpile internals if
50 Custom conditional decorator, that will not touch any dogpile internals if
44 condition isn't meet. This works a bit different than should_cache_fn
51 condition isn't meet. This works a bit different from should_cache_fn
45 And it's faster in cases we don't ever want to compute cached values
52 And it's faster in cases we don't ever want to compute cached values
46 """
53 """
47 expiration_time_is_callable = compat.callable(expiration_time)
54 expiration_time_is_callable = callable(expiration_time)
55 if not namespace:
56 namespace = getattr(self, '_default_namespace', None)
48
57
49 if function_key_generator is None:
58 if function_key_generator is None:
50 function_key_generator = self.function_key_generator
59 function_key_generator = self.function_key_generator
51
60
52 # workaround for py2 and cython problems, this block should be removed
61 def get_or_create_for_user_func(func_key_generator, user_func, *arg, **kw):
53 # once we've migrated to py3
54 if 'cython' == 'cython':
55 def decorator(fn):
56 if to_str is compat.string_type:
57 # backwards compatible
58 key_generator = function_key_generator(namespace, fn)
59 else:
60 key_generator = function_key_generator(namespace, fn, to_str=to_str)
61
62 @functools.wraps(fn)
63 def decorate(*arg, **kw):
64 key = key_generator(*arg, **kw)
65
66 @functools.wraps(fn)
67 def creator():
68 return fn(*arg, **kw)
69
70 if not condition:
71 return creator()
72
73 timeout = expiration_time() if expiration_time_is_callable \
74 else expiration_time
75
76 return self.get_or_create(key, creator, timeout, should_cache_fn)
77
78 def invalidate(*arg, **kw):
79 key = key_generator(*arg, **kw)
80 self.delete(key)
81
82 def set_(value, *arg, **kw):
83 key = key_generator(*arg, **kw)
84 self.set(key, value)
85
86 def get(*arg, **kw):
87 key = key_generator(*arg, **kw)
88 return self.get(key)
89
90 def refresh(*arg, **kw):
91 key = key_generator(*arg, **kw)
92 value = fn(*arg, **kw)
93 self.set(key, value)
94 return value
95
96 decorate.set = set_
97 decorate.invalidate = invalidate
98 decorate.refresh = refresh
99 decorate.get = get
100 decorate.original = fn
101 decorate.key_generator = key_generator
102 decorate.__wrapped__ = fn
103
104 return decorate
105 return decorator
106
107 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
108
62
109 if not condition:
63 if not condition:
110 log.debug('Calling un-cached method:%s', user_func.func_name)
64 log.debug('Calling un-cached method:%s', user_func.__name__)
111 start = time.time()
65 start = time.time()
112 result = user_func(*arg, **kw)
66 result = user_func(*arg, **kw)
113 total = time.time() - start
67 total = time.time() - start
114 log.debug('un-cached method:%s took %.4fs', user_func.func_name, total)
68 log.debug('un-cached method:%s took %.4fs', user_func.__name__, total)
115 return result
69 return result
116
70
117 key = key_generator(*arg, **kw)
71 key = func_key_generator(*arg, **kw)
118
72
119 timeout = expiration_time() if expiration_time_is_callable \
73 timeout = expiration_time() if expiration_time_is_callable \
120 else expiration_time
74 else expiration_time
121
75
122 log.debug('Calling cached method:`%s`', user_func.func_name)
76 log.debug('Calling cached method:`%s`', user_func.__name__)
123 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
77 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
124
78
125 def cache_decorator(user_func):
79 def cache_decorator(user_func):
126 if to_str is compat.string_type:
80 if to_str is str:
127 # backwards compatible
81 # backwards compatible
128 key_generator = function_key_generator(namespace, user_func)
82 key_generator = function_key_generator(namespace, user_func)
129 else:
83 else:
130 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
84 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
131
85
132 def refresh(*arg, **kw):
86 def refresh(*arg, **kw):
133 """
87 """
134 Like invalidate, but regenerates the value instead
88 Like invalidate, but regenerates the value instead
135 """
89 """
136 key = key_generator(*arg, **kw)
90 key = key_generator(*arg, **kw)
137 value = user_func(*arg, **kw)
91 value = user_func(*arg, **kw)
138 self.set(key, value)
92 self.set(key, value)
139 return value
93 return value
140
94
141 def invalidate(*arg, **kw):
95 def invalidate(*arg, **kw):
142 key = key_generator(*arg, **kw)
96 key = key_generator(*arg, **kw)
143 self.delete(key)
97 self.delete(key)
144
98
145 def set_(value, *arg, **kw):
99 def set_(value, *arg, **kw):
146 key = key_generator(*arg, **kw)
100 key = key_generator(*arg, **kw)
147 self.set(key, value)
101 self.set(key, value)
148
102
149 def get(*arg, **kw):
103 def get(*arg, **kw):
150 key = key_generator(*arg, **kw)
104 key = key_generator(*arg, **kw)
151 return self.get(key)
105 return self.get(key)
152
106
153 user_func.set = set_
107 user_func.set = set_
154 user_func.invalidate = invalidate
108 user_func.invalidate = invalidate
155 user_func.get = get
109 user_func.get = get
156 user_func.refresh = refresh
110 user_func.refresh = refresh
157 user_func.key_generator = key_generator
111 user_func.key_generator = key_generator
158 user_func.original = user_func
112 user_func.original = user_func
159
113
160 # Use `decorate` to preserve the signature of :param:`user_func`.
114 # Use `decorate` to preserve the signature of :param:`user_func`.
161 return decorator.decorate(user_func, functools.partial(
115 return decorator.decorate(user_func, functools.partial(
162 get_or_create_for_user_func, key_generator))
116 get_or_create_for_user_func, key_generator))
163
117
164 return cache_decorator
118 return cache_decorator
165
119
166
120
167 def make_region(*arg, **kw):
121 def make_region(*arg, **kw):
168 return RhodeCodeCacheRegion(*arg, **kw)
122 return RhodeCodeCacheRegion(*arg, **kw)
169
123
170
124
171 def get_default_cache_settings(settings, prefixes=None):
125 def get_default_cache_settings(settings, prefixes=None):
172 prefixes = prefixes or []
126 prefixes = prefixes or []
173 cache_settings = {}
127 cache_settings = {}
174 for key in settings.keys():
128 for key in settings.keys():
175 for prefix in prefixes:
129 for prefix in prefixes:
176 if key.startswith(prefix):
130 if key.startswith(prefix):
177 name = key.split(prefix)[1].strip()
131 name = key.split(prefix)[1].strip()
178 val = settings[key]
132 val = settings[key]
179 if isinstance(val, compat.string_types):
133 if isinstance(val, str):
180 val = val.strip()
134 val = val.strip()
181 cache_settings[name] = val
135 cache_settings[name] = val
182 return cache_settings
136 return cache_settings
183
137
184
138
185 def compute_key_from_params(*args):
139 def compute_key_from_params(*args):
186 """
140 """
187 Helper to compute key from given params to be used in cache manager
141 Helper to compute key from given params to be used in cache manager
188 """
142 """
189 return sha1("_".join(map(safe_str, args)))
143 return sha1(safe_bytes("_".join(map(str, args))))
144
145
146 def custom_key_generator(backend, namespace, fn):
147 func_name = fn.__name__
148
149 def generate_key(*args):
150 backend_pref = getattr(backend, 'key_prefix', None) or 'backend_prefix'
151 namespace_pref = namespace or 'default_namespace'
152 arg_key = compute_key_from_params(*args)
153 final_key = f"{backend_pref}:{namespace_pref}:{func_name}_{arg_key}"
154
155 return final_key
156
157 return generate_key
190
158
191
159
192 def backend_key_generator(backend):
160 def backend_key_generator(backend):
193 """
161 """
194 Special wrapper that also sends over the backend to the key generator
162 Special wrapper that also sends over the backend to the key generator
195 """
163 """
196 def wrapper(namespace, fn):
164 def wrapper(namespace, fn):
197 return key_generator(backend, namespace, fn)
165 return custom_key_generator(backend, namespace, fn)
198 return wrapper
166 return wrapper
199
167
200
168
201 def key_generator(backend, namespace, fn):
169 def get_or_create_region(region_name, region_namespace: str = None, use_async_runner=False):
202 fname = fn.__name__
170 from .backends import FileNamespaceBackend
171 from . import async_creation_runner
203
172
204 def generate_key(*args):
205 backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
206 namespace_pref = namespace or 'default_namespace'
207 arg_key = compute_key_from_params(*args)
208 final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
209
210 return final_key
211
212 return generate_key
213
214
215 def get_or_create_region(region_name, region_namespace=None):
216 from vcsserver.lib.rc_cache.backends import FileNamespaceBackend
217 region_obj = region_meta.dogpile_cache_regions.get(region_name)
173 region_obj = region_meta.dogpile_cache_regions.get(region_name)
218 if not region_obj:
174 if not region_obj:
219 raise EnvironmentError(
175 reg_keys = list(region_meta.dogpile_cache_regions.keys())
220 'Region `{}` not in configured: {}.'.format(
176 raise OSError(f'Region `{region_name}` not in configured: {reg_keys}.')
221 region_name, region_meta.dogpile_cache_regions.keys()))
177
178 region_uid_name = f'{region_name}:{region_namespace}'
222
179
223 region_uid_name = '{}:{}'.format(region_name, region_namespace)
180 # Special case for ONLY the FileNamespaceBackend backend. We register one-file-per-region
224 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
181 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
182 if not region_namespace:
183 raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
184
225 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
185 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
226 if region_exist:
186 if region_exist:
227 log.debug('Using already configured region: %s', region_namespace)
187 log.debug('Using already configured region: %s', region_namespace)
228 return region_exist
188 return region_exist
229 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
189
230 expiration_time = region_obj.expiration_time
190 expiration_time = region_obj.expiration_time
231
191
232 if not os.path.isdir(cache_dir):
192 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
233 os.makedirs(cache_dir)
193 namespace_cache_dir = cache_dir
194
195 # we default the namespace_cache_dir to our default cache dir.
196 # however, if this backend is configured with filename= param, we prioritize that
197 # so all caches within that particular region, even those namespaced end up in the same path
198 if region_obj.actual_backend.filename:
199 namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
200
201 if not os.path.isdir(namespace_cache_dir):
202 os.makedirs(namespace_cache_dir)
234 new_region = make_region(
203 new_region = make_region(
235 name=region_uid_name,
204 name=region_uid_name,
236 function_key_generator=backend_key_generator(region_obj.actual_backend)
205 function_key_generator=backend_key_generator(region_obj.actual_backend)
237 )
206 )
207
238 namespace_filename = os.path.join(
208 namespace_filename = os.path.join(
239 cache_dir, "{}.cache.dbm".format(region_namespace))
209 namespace_cache_dir, f"{region_name}_{region_namespace}.cache_db")
240 # special type that allows 1db per namespace
210 # special type that allows 1db per namespace
241 new_region.configure(
211 new_region.configure(
242 backend='dogpile.cache.rc.file_namespace',
212 backend='dogpile.cache.rc.file_namespace',
243 expiration_time=expiration_time,
213 expiration_time=expiration_time,
244 arguments={"filename": namespace_filename}
214 arguments={"filename": namespace_filename}
245 )
215 )
246
216
247 # create and save in region caches
217 # create and save in region caches
248 log.debug('configuring new region: %s', region_uid_name)
218 log.debug('configuring new region: %s', region_uid_name)
249 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
219 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
250
220
221 region_obj._default_namespace = region_namespace
222 if use_async_runner:
223 region_obj.async_creation_runner = async_creation_runner
251 return region_obj
224 return region_obj
252
225
253
226
254 def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
227 def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str) -> int:
255 region = get_or_create_region(cache_region, cache_namespace_uid)
228 from . import CLEAR_DELETE, CLEAR_INVALIDATE
256 cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
229
257 num_delete_keys = len(cache_keys)
230 if not isinstance(cache_region, RhodeCodeCacheRegion):
258 if invalidate:
231 cache_region = get_or_create_region(cache_region, cache_namespace_uid)
259 region.invalidate(hard=False)
232 log.debug('clearing cache region: %s [prefix:%s] with method=%s',
260 else:
233 cache_region, cache_namespace_uid, method)
261 if num_delete_keys:
234
262 region.delete_multi(cache_keys)
235 num_affected_keys = 0
263 return num_delete_keys
236
237 if method == CLEAR_INVALIDATE:
238 # NOTE: The CacheRegion.invalidate() method’s default mode of
239 # operation is to set a timestamp local to this CacheRegion in this Python process only.
240 # It does not impact other Python processes or regions as the timestamp is only stored locally in memory.
241 cache_region.invalidate(hard=True)
242
243 if method == CLEAR_DELETE:
244 num_affected_keys = cache_region.backend.delete_multi_by_prefix(prefix=cache_namespace_uid)
245 return num_affected_keys
@@ -1,27 +1,25 b''
1 # -*- coding: utf-8 -*-
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
5 #
3 #
6 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
7 # (at your option) any later version.
10 #
8 #
11 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
12 # GNU General Public License for more details.
15 #
13 #
16 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
17
20
18
21 counter = 0
19 counter = 0
22
20
23
21
24 def get_request_counter(request):
22 def get_request_counter(request):
25 global counter
23 global counter
26 counter += 1
24 counter += 1
27 return counter
25 return counter
@@ -1,386 +1,417 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """Handles the Git smart protocol."""
18 """Handles the Git smart protocol."""
19
19
20 import os
20 import os
21 import socket
21 import socket
22 import logging
22 import logging
23
23
24 import simplejson as json
25 import dulwich.protocol
24 import dulwich.protocol
25 from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
26 from webob import Request, Response, exc
26 from webob import Request, Response, exc
27
27
28 from vcsserver.lib.rc_json import json
28 from vcsserver import hooks, subprocessio
29 from vcsserver import hooks, subprocessio
30 from vcsserver.str_utils import ascii_bytes
29
31
30
32
31 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
32
34
33
35
34 class FileWrapper(object):
36 class FileWrapper:
35 """File wrapper that ensures how much data is read from it."""
37 """File wrapper that ensures how much data is read from it."""
36
38
37 def __init__(self, fd, content_length):
39 def __init__(self, fd, content_length):
38 self.fd = fd
40 self.fd = fd
39 self.content_length = content_length
41 self.content_length = content_length
40 self.remain = content_length
42 self.remain = content_length
41
43
42 def read(self, size):
44 def read(self, size):
43 if size <= self.remain:
45 if size <= self.remain:
44 try:
46 try:
45 data = self.fd.read(size)
47 data = self.fd.read(size)
46 except socket.error:
48 except socket.error:
47 raise IOError(self)
49 raise IOError(self)
48 self.remain -= size
50 self.remain -= size
49 elif self.remain:
51 elif self.remain:
50 data = self.fd.read(self.remain)
52 data = self.fd.read(self.remain)
51 self.remain = 0
53 self.remain = 0
52 else:
54 else:
53 data = None
55 data = None
54 return data
56 return data
55
57
56 def __repr__(self):
58 def __repr__(self):
57 return '<FileWrapper %s len: %s, read: %s>' % (
59 return '<FileWrapper {} len: {}, read: {}>'.format(
58 self.fd, self.content_length, self.content_length - self.remain
60 self.fd, self.content_length, self.content_length - self.remain
59 )
61 )
60
62
61
63
62 class GitRepository(object):
64 class GitRepository:
63 """WSGI app for handling Git smart protocol endpoints."""
65 """WSGI app for handling Git smart protocol endpoints."""
64
66
65 git_folder_signature = frozenset(
67 git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
66 ('config', 'head', 'info', 'objects', 'refs'))
67 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
68 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
68 valid_accepts = frozenset(('application/x-%s-result' %
69 valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
69 c for c in commands))
70
70
71 # The last bytes are the SHA1 of the first 12 bytes.
71 # The last bytes are the SHA1 of the first 12 bytes.
72 EMPTY_PACK = (
72 EMPTY_PACK = (
73 'PACK\x00\x00\x00\x02\x00\x00\x00\x00' +
73 b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
74 '\x02\x9d\x08\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
74 b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
75 )
75 )
76 SIDE_BAND_CAPS = frozenset(('side-band', 'side-band-64k'))
76 FLUSH_PACKET = b"0000"
77
77
78 def __init__(self, repo_name, content_path, git_path, update_server_info,
78 SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
79 extras):
79
80 def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
80 files = frozenset(f.lower() for f in os.listdir(content_path))
81 files = frozenset(f.lower() for f in os.listdir(content_path))
81 valid_dir_signature = self.git_folder_signature.issubset(files)
82 valid_dir_signature = self.git_folder_signature.issubset(files)
82
83
83 if not valid_dir_signature:
84 if not valid_dir_signature:
84 raise OSError('%s missing git signature' % content_path)
85 raise OSError(f'{content_path} missing git signature')
85
86
86 self.content_path = content_path
87 self.content_path = content_path
87 self.repo_name = repo_name
88 self.repo_name = repo_name
88 self.extras = extras
89 self.extras = extras
89 self.git_path = git_path
90 self.git_path = git_path
90 self.update_server_info = update_server_info
91 self.update_server_info = update_server_info
91
92
92 def _get_fixedpath(self, path):
93 def _get_fixedpath(self, path):
93 """
94 """
94 Small fix for repo_path
95 Small fix for repo_path
95
96
96 :param path:
97 :param path:
97 """
98 """
98 path = path.split(self.repo_name, 1)[-1]
99 path = path.split(self.repo_name, 1)[-1]
99 if path.startswith('.git'):
100 if path.startswith('.git'):
100 # for bare repos we still get the .git prefix inside, we skip it
101 # for bare repos we still get the .git prefix inside, we skip it
101 # here, and remove from the service command
102 # here, and remove from the service command
102 path = path[4:]
103 path = path[4:]
103
104
104 return path.strip('/')
105 return path.strip('/')
105
106
106 def inforefs(self, request, unused_environ):
107 def inforefs(self, request, unused_environ):
107 """
108 """
108 WSGI Response producer for HTTP GET Git Smart
109 WSGI Response producer for HTTP GET Git Smart
109 HTTP /info/refs request.
110 HTTP /info/refs request.
110 """
111 """
111
112
112 git_command = request.GET.get('service')
113 git_command = request.GET.get('service')
113 if git_command not in self.commands:
114 if git_command not in self.commands:
114 log.debug('command %s not allowed', git_command)
115 log.debug('command %s not allowed', git_command)
115 return exc.HTTPForbidden()
116 return exc.HTTPForbidden()
116
117
117 # please, resist the urge to add '\n' to git capture and increment
118 # please, resist the urge to add '\n' to git capture and increment
118 # line count by 1.
119 # line count by 1.
119 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
120 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
120 # a part of protocol.
121 # a part of protocol.
121 # The code in Git client not only does NOT need '\n', but actually
122 # The code in Git client not only does NOT need '\n', but actually
122 # blows up if you sprinkle "flush" (0000) as "0001\n".
123 # blows up if you sprinkle "flush" (0000) as "0001\n".
123 # It reads binary, per number of bytes specified.
124 # It reads binary, per number of bytes specified.
124 # if you do add '\n' as part of data, count it.
125 # if you do add '\n' as part of data, count it.
125 server_advert = '# service=%s\n' % git_command
126 server_advert = f'# service={git_command}\n'
126 packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
127 packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
127 try:
128 try:
128 gitenv = dict(os.environ)
129 gitenv = dict(os.environ)
129 # forget all configs
130 # forget all configs
130 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
131 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
131 command = [self.git_path, git_command[4:], '--stateless-rpc',
132 command = [self.git_path, git_command[4:], '--stateless-rpc',
132 '--advertise-refs', self.content_path]
133 '--advertise-refs', self.content_path]
133 out = subprocessio.SubprocessIOChunker(
134 out = subprocessio.SubprocessIOChunker(
134 command,
135 command,
135 env=gitenv,
136 env=gitenv,
136 starting_values=[packet_len + server_advert + '0000'],
137 starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
137 shell=False
138 shell=False
138 )
139 )
139 except EnvironmentError:
140 except OSError:
140 log.exception('Error processing command')
141 log.exception('Error processing command')
141 raise exc.HTTPExpectationFailed()
142 raise exc.HTTPExpectationFailed()
142
143
143 resp = Response()
144 resp = Response()
144 resp.content_type = 'application/x-%s-advertisement' % str(git_command)
145 resp.content_type = f'application/x-{git_command}-advertisement'
145 resp.charset = None
146 resp.charset = None
146 resp.app_iter = out
147 resp.app_iter = out
147
148
148 return resp
149 return resp
149
150
150 def _get_want_capabilities(self, request):
151 def _get_want_capabilities(self, request):
151 """Read the capabilities found in the first want line of the request."""
152 """Read the capabilities found in the first want line of the request."""
152 pos = request.body_file_seekable.tell()
153 pos = request.body_file_seekable.tell()
153 first_line = request.body_file_seekable.readline()
154 first_line = request.body_file_seekable.readline()
154 request.body_file_seekable.seek(pos)
155 request.body_file_seekable.seek(pos)
155
156
156 return frozenset(
157 return frozenset(
157 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
158 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
158
159
159 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
160 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
160 """
161 """
161 Construct a response with an empty PACK file.
162 Construct a response with an empty PACK file.
162
163
163 We use an empty PACK file, as that would trigger the failure of the pull
164 We use an empty PACK file, as that would trigger the failure of the pull
164 or clone command.
165 or clone command.
165
166
166 We also print in the error output a message explaining why the command
167 We also print in the error output a message explaining why the command
167 was aborted.
168 was aborted.
168
169
169 If aditionally, the user is accepting messages we send them the output
170 If additionally, the user is accepting messages we send them the output
170 of the pre-pull hook.
171 of the pre-pull hook.
171
172
172 Note that for clients not supporting side-band we just send them the
173 Note that for clients not supporting side-band we just send them the
173 emtpy PACK file.
174 emtpy PACK file.
174 """
175 """
176
175 if self.SIDE_BAND_CAPS.intersection(capabilities):
177 if self.SIDE_BAND_CAPS.intersection(capabilities):
176 response = []
178 response = []
177 proto = dulwich.protocol.Protocol(None, response.append)
179 proto = dulwich.protocol.Protocol(None, response.append)
178 proto.write_pkt_line('NAK\n')
180 proto.write_pkt_line(dulwich.protocol.NAK_LINE)
179 self._write_sideband_to_proto(pre_pull_messages, proto,
181
180 capabilities)
182 self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
181 # N.B.(skreft): Do not change the sideband channel to 3, as that
183 # N.B.(skreft): Do not change the sideband channel to 3, as that
182 # produces a fatal error in the client:
184 # produces a fatal error in the client:
183 # fatal: error in sideband demultiplexer
185 # fatal: error in sideband demultiplexer
184 proto.write_sideband(2, 'Pre pull hook failed: aborting\n')
186 proto.write_sideband(
185 proto.write_sideband(1, self.EMPTY_PACK)
187 dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
188 ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
189 proto.write_sideband(
190 dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
191 ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
186
192
187 # writes 0000
193 # writes b"0000" as default
188 proto.write_pkt_line(None)
194 proto.write_pkt_line(None)
189
195
190 return response
196 return response
191 else:
197 else:
192 return [self.EMPTY_PACK]
198 return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
199
200 def _build_post_pull_response(self, response, capabilities, start_message, end_message):
201 """
202 Given a list response we inject the post-pull messages.
203
204 We only inject the messages if the client supports sideband, and the
205 response has the format:
206 0008NAK\n...0000
207
208 Note that we do not check the no-progress capability as by default, git
209 sends it, which effectively would block all messages.
210 """
211
212 if not self.SIDE_BAND_CAPS.intersection(capabilities):
213 return response
214
215 if not start_message and not end_message:
216 return response
217
218 try:
219 iter(response)
220 # iterator probably will work, we continue
221 except TypeError:
222 raise TypeError(f'response must be an iterator: got {type(response)}')
223 if isinstance(response, (list, tuple)):
224 raise TypeError(f'response must be an iterator: got {type(response)}')
225
226 def injected_response():
193
227
194 def _write_sideband_to_proto(self, data, proto, capabilities):
228 do_loop = 1
229 header_injected = 0
230 next_item = None
231 has_item = False
232 item = b''
233
234 while do_loop:
235
236 try:
237 next_item = next(response)
238 except StopIteration:
239 do_loop = 0
240
241 if has_item:
242 # last item ! alter it now
243 if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
244 new_response = [item[:-4]]
245 new_response.extend(self._get_messages(end_message, capabilities))
246 new_response.append(self.FLUSH_PACKET)
247 item = b''.join(new_response)
248
249 yield item
250
251 has_item = True
252 item = next_item
253
254 # alter item if it's the initial chunk
255 if not header_injected and item.startswith(b'0008NAK\n'):
256 new_response = [b'0008NAK\n']
257 new_response.extend(self._get_messages(start_message, capabilities))
258 new_response.append(item[8:])
259 item = b''.join(new_response)
260 header_injected = 1
261
262 return injected_response()
263
264 def _write_sideband_to_proto(self, proto, data, capabilities):
195 """
265 """
196 Write the data to the proto's sideband number 2.
266 Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
197
267
198 We do not use dulwich's write_sideband directly as it only supports
268 We do not use dulwich's write_sideband directly as it only supports
199 side-band-64k.
269 side-band-64k.
200 """
270 """
201 if not data:
271 if not data:
202 return
272 return
203
273
204 # N.B.(skreft): The values below are explained in the pack protocol
274 # N.B.(skreft): The values below are explained in the pack protocol
205 # documentation, section Packfile Data.
275 # documentation, section Packfile Data.
206 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
276 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
207 if 'side-band-64k' in capabilities:
277 if CAPABILITY_SIDE_BAND_64K in capabilities:
208 chunk_size = 65515
278 chunk_size = 65515
209 elif 'side-band' in capabilities:
279 elif CAPABILITY_SIDE_BAND in capabilities:
210 chunk_size = 995
280 chunk_size = 995
211 else:
281 else:
212 return
282 return
213
283
214 chunker = (
284 chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
215 data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size))
216
285
217 for chunk in chunker:
286 for chunk in chunker:
218 proto.write_sideband(2, chunk)
287 proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
219
288
220 def _get_messages(self, data, capabilities):
289 def _get_messages(self, data, capabilities):
221 """Return a list with packets for sending data in sideband number 2."""
290 """Return a list with packets for sending data in sideband number 2."""
222 response = []
291 response = []
223 proto = dulwich.protocol.Protocol(None, response.append)
292 proto = dulwich.protocol.Protocol(None, response.append)
224
293
225 self._write_sideband_to_proto(data, proto, capabilities)
294 self._write_sideband_to_proto(proto, data, capabilities)
226
295
227 return response
296 return response
228
297
229 def _inject_messages_to_response(self, response, capabilities,
230 start_messages, end_messages):
231 """
232 Given a list response we inject the pre/post-pull messages.
233
234 We only inject the messages if the client supports sideband, and the
235 response has the format:
236 0008NAK\n...0000
237
238 Note that we do not check the no-progress capability as by default, git
239 sends it, which effectively would block all messages.
240 """
241 if not self.SIDE_BAND_CAPS.intersection(capabilities):
242 return response
243
244 if not start_messages and not end_messages:
245 return response
246
247 # make a list out of response if it's an iterator
248 # so we can investigate it for message injection.
249 if hasattr(response, '__iter__'):
250 response = list(response)
251
252 if (not response[0].startswith('0008NAK\n') or
253 not response[-1].endswith('0000')):
254 return response
255
256 new_response = ['0008NAK\n']
257 new_response.extend(self._get_messages(start_messages, capabilities))
258 if len(response) == 1:
259 new_response.append(response[0][8:-4])
260 else:
261 new_response.append(response[0][8:])
262 new_response.extend(response[1:-1])
263 new_response.append(response[-1][:-4])
264 new_response.extend(self._get_messages(end_messages, capabilities))
265 new_response.append('0000')
266
267 return new_response
268
269 def backend(self, request, environ):
298 def backend(self, request, environ):
270 """
299 """
271 WSGI Response producer for HTTP POST Git Smart HTTP requests.
300 WSGI Response producer for HTTP POST Git Smart HTTP requests.
272 Reads commands and data from HTTP POST's body.
301 Reads commands and data from HTTP POST's body.
273 returns an iterator obj with contents of git command's
302 returns an iterator obj with contents of git command's
274 response to stdout
303 response to stdout
275 """
304 """
276 # TODO(skreft): think how we could detect an HTTPLockedException, as
305 # TODO(skreft): think how we could detect an HTTPLockedException, as
277 # we probably want to have the same mechanism used by mercurial and
306 # we probably want to have the same mechanism used by mercurial and
278 # simplevcs.
307 # simplevcs.
279 # For that we would need to parse the output of the command looking for
308 # For that we would need to parse the output of the command looking for
280 # some signs of the HTTPLockedError, parse the data and reraise it in
309 # some signs of the HTTPLockedError, parse the data and reraise it in
281 # pygrack. However, that would interfere with the streaming.
310 # pygrack. However, that would interfere with the streaming.
282 #
311 #
283 # Now the output of a blocked push is:
312 # Now the output of a blocked push is:
284 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
313 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
285 # POST git-receive-pack (1047 bytes)
314 # POST git-receive-pack (1047 bytes)
286 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
315 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
287 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
316 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
288 # ! [remote rejected] master -> master (pre-receive hook declined)
317 # ! [remote rejected] master -> master (pre-receive hook declined)
289 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
318 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
290
319
291 git_command = self._get_fixedpath(request.path_info)
320 git_command = self._get_fixedpath(request.path_info)
292 if git_command not in self.commands:
321 if git_command not in self.commands:
293 log.debug('command %s not allowed', git_command)
322 log.debug('command %s not allowed', git_command)
294 return exc.HTTPForbidden()
323 return exc.HTTPForbidden()
295
324
296 capabilities = None
325 capabilities = None
297 if git_command == 'git-upload-pack':
326 if git_command == 'git-upload-pack':
298 capabilities = self._get_want_capabilities(request)
327 capabilities = self._get_want_capabilities(request)
299
328
300 if 'CONTENT_LENGTH' in environ:
329 if 'CONTENT_LENGTH' in environ:
301 inputstream = FileWrapper(request.body_file_seekable,
330 inputstream = FileWrapper(request.body_file_seekable,
302 request.content_length)
331 request.content_length)
303 else:
332 else:
304 inputstream = request.body_file_seekable
333 inputstream = request.body_file_seekable
305
334
306 resp = Response()
335 resp = Response()
307 resp.content_type = ('application/x-%s-result' %
336 resp.content_type = f'application/x-{git_command}-result'
308 git_command.encode('utf8'))
309 resp.charset = None
337 resp.charset = None
310
338
311 pre_pull_messages = ''
339 pre_pull_messages = ''
340 # Upload-pack == clone
312 if git_command == 'git-upload-pack':
341 if git_command == 'git-upload-pack':
313 status, pre_pull_messages = hooks.git_pre_pull(self.extras)
342 hook_response = hooks.git_pre_pull(self.extras)
314 if status != 0:
343 if hook_response.status != 0:
344 pre_pull_messages = hook_response.output
315 resp.app_iter = self._build_failed_pre_pull_response(
345 resp.app_iter = self._build_failed_pre_pull_response(
316 capabilities, pre_pull_messages)
346 capabilities, pre_pull_messages)
317 return resp
347 return resp
318
348
319 gitenv = dict(os.environ)
349 gitenv = dict(os.environ)
320 # forget all configs
350 # forget all configs
321 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
351 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
322 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
352 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
323 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
353 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
324 self.content_path]
354 self.content_path]
325 log.debug('handling cmd %s', cmd)
355 log.debug('handling cmd %s', cmd)
326
356
327 out = subprocessio.SubprocessIOChunker(
357 out = subprocessio.SubprocessIOChunker(
328 cmd,
358 cmd,
329 inputstream=inputstream,
359 input_stream=inputstream,
330 env=gitenv,
360 env=gitenv,
331 cwd=self.content_path,
361 cwd=self.content_path,
332 shell=False,
362 shell=False,
333 fail_on_stderr=False,
363 fail_on_stderr=False,
334 fail_on_return_code=False
364 fail_on_return_code=False
335 )
365 )
336
366
337 if self.update_server_info and git_command == 'git-receive-pack':
367 if self.update_server_info and git_command == 'git-receive-pack':
338 # We need to fully consume the iterator here, as the
368 # We need to fully consume the iterator here, as the
339 # update-server-info command needs to be run after the push.
369 # update-server-info command needs to be run after the push.
340 out = list(out)
370 out = list(out)
341
371
342 # Updating refs manually after each push.
372 # Updating refs manually after each push.
343 # This is required as some clients are exposing Git repos internally
373 # This is required as some clients are exposing Git repos internally
344 # with the dumb protocol.
374 # with the dumb protocol.
345 cmd = [self.git_path, 'update-server-info']
375 cmd = [self.git_path, 'update-server-info']
346 log.debug('handling cmd %s', cmd)
376 log.debug('handling cmd %s', cmd)
347 output = subprocessio.SubprocessIOChunker(
377 output = subprocessio.SubprocessIOChunker(
348 cmd,
378 cmd,
349 inputstream=inputstream,
379 input_stream=inputstream,
350 env=gitenv,
380 env=gitenv,
351 cwd=self.content_path,
381 cwd=self.content_path,
352 shell=False,
382 shell=False,
353 fail_on_stderr=False,
383 fail_on_stderr=False,
354 fail_on_return_code=False
384 fail_on_return_code=False
355 )
385 )
356 # Consume all the output so the subprocess finishes
386 # Consume all the output so the subprocess finishes
357 for _ in output:
387 for _ in output:
358 pass
388 pass
359
389
390 # Upload-pack == clone
360 if git_command == 'git-upload-pack':
391 if git_command == 'git-upload-pack':
361 unused_status, post_pull_messages = hooks.git_post_pull(self.extras)
392 hook_response = hooks.git_post_pull(self.extras)
362 resp.app_iter = self._inject_messages_to_response(
393 post_pull_messages = hook_response.output
363 out, capabilities, pre_pull_messages, post_pull_messages)
394 resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
364 else:
395 else:
365 resp.app_iter = out
396 resp.app_iter = out
366
397
367 return resp
398 return resp
368
399
369 def __call__(self, environ, start_response):
400 def __call__(self, environ, start_response):
370 request = Request(environ)
401 request = Request(environ)
371 _path = self._get_fixedpath(request.path_info)
402 _path = self._get_fixedpath(request.path_info)
372 if _path.startswith('info/refs'):
403 if _path.startswith('info/refs'):
373 app = self.inforefs
404 app = self.inforefs
374 else:
405 else:
375 app = self.backend
406 app = self.backend
376
407
377 try:
408 try:
378 resp = app(request, environ)
409 resp = app(request, environ)
379 except exc.HTTPException as error:
410 except exc.HTTPException as error:
380 log.exception('HTTP Error')
411 log.exception('HTTP Error')
381 resp = error
412 resp = error
382 except Exception:
413 except Exception:
383 log.exception('Unknown error')
414 log.exception('Unknown error')
384 resp = exc.HTTPInternalServerError()
415 resp = exc.HTTPInternalServerError()
385
416
386 return resp(environ, start_response)
417 return resp(environ, start_response)
This diff has been collapsed as it changes many lines, (787 lines changed) Show them Hide them
@@ -1,1281 +1,1518 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import collections
18 import collections
19 import logging
19 import logging
20 import os
20 import os
21 import posixpath as vcspath
22 import re
21 import re
23 import stat
22 import stat
24 import traceback
23 import traceback
25 import urllib
24 import urllib.request
26 import urllib2
25 import urllib.parse
26 import urllib.error
27 from functools import wraps
27 from functools import wraps
28
28
29 import more_itertools
29 import more_itertools
30 import pygit2
30 import pygit2
31 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
32 from pygit2 import index as LibGit2Index
32 from pygit2 import index as LibGit2Index
33 from dulwich import index, objects
33 from dulwich import index, objects
34 from dulwich.client import HttpGitClient, LocalGitClient
34 from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
35 from dulwich.errors import (
35 from dulwich.errors import (
36 NotGitRepository, ChecksumMismatch, WrongObjectException,
36 NotGitRepository, ChecksumMismatch, WrongObjectException,
37 MissingCommitError, ObjectMissing, HangupException,
37 MissingCommitError, ObjectMissing, HangupException,
38 UnexpectedCommandError)
38 UnexpectedCommandError)
39 from dulwich.repo import Repo as DulwichRepo
39 from dulwich.repo import Repo as DulwichRepo
40 from dulwich.server import update_server_info
41
40
41 import rhodecode
42 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver import exceptions, settings, subprocessio
43 from vcsserver.utils import safe_str, safe_int, safe_unicode
43 from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes, convert_to_str
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
45 from vcsserver.hgcompat import (
45 from vcsserver.hgcompat import (
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
47 from vcsserver.git_lfs.lib import LFSOidStore
47 from vcsserver.git_lfs.lib import LFSOidStore
48 from vcsserver.vcs_base import RemoteBase
48 from vcsserver.vcs_base import RemoteBase
49
49
50 DIR_STAT = stat.S_IFDIR
50 DIR_STAT = stat.S_IFDIR
51 FILE_MODE = stat.S_IFMT
51 FILE_MODE = stat.S_IFMT
52 GIT_LINK = objects.S_IFGITLINK
52 GIT_LINK = objects.S_IFGITLINK
53 PEELED_REF_MARKER = '^{}'
53 PEELED_REF_MARKER = b'^{}'
54
54 HEAD_MARKER = b'HEAD'
55
55
56 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
57
57
58
58
59 def str_to_dulwich(value):
60 """
61 Dulwich 0.10.1a requires `unicode` objects to be passed in.
62 """
63 return value.decode(settings.WIRE_ENCODING)
64
65
66 def reraise_safe_exceptions(func):
59 def reraise_safe_exceptions(func):
67 """Converts Dulwich exceptions to something neutral."""
60 """Converts Dulwich exceptions to something neutral."""
68
61
69 @wraps(func)
62 @wraps(func)
70 def wrapper(*args, **kwargs):
63 def wrapper(*args, **kwargs):
71 try:
64 try:
72 return func(*args, **kwargs)
65 return func(*args, **kwargs)
73 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
66 except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
74 exc = exceptions.LookupException(org_exc=e)
67 exc = exceptions.LookupException(org_exc=e)
75 raise exc(safe_str(e))
68 raise exc(safe_str(e))
76 except (HangupException, UnexpectedCommandError) as e:
69 except (HangupException, UnexpectedCommandError) as e:
77 exc = exceptions.VcsException(org_exc=e)
70 exc = exceptions.VcsException(org_exc=e)
78 raise exc(safe_str(e))
71 raise exc(safe_str(e))
79 except Exception as e:
72 except Exception:
80 # NOTE(marcink): becuase of how dulwich handles some exceptions
73 # NOTE(marcink): because of how dulwich handles some exceptions
81 # (KeyError on empty repos), we cannot track this and catch all
74 # (KeyError on empty repos), we cannot track this and catch all
82 # exceptions, it's an exceptions from other handlers
75 # exceptions, it's an exceptions from other handlers
83 #if not hasattr(e, '_vcs_kind'):
76 #if not hasattr(e, '_vcs_kind'):
84 #log.exception("Unhandled exception in git remote call")
77 #log.exception("Unhandled exception in git remote call")
85 #raise_from_original(exceptions.UnhandledException)
78 #raise_from_original(exceptions.UnhandledException)
86 raise
79 raise
87 return wrapper
80 return wrapper
88
81
89
82
90 class Repo(DulwichRepo):
83 class Repo(DulwichRepo):
91 """
84 """
92 A wrapper for dulwich Repo class.
85 A wrapper for dulwich Repo class.
93
86
94 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
87 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
95 "Too many open files" error. We need to close all opened file descriptors
88 "Too many open files" error. We need to close all opened file descriptors
96 once the repo object is destroyed.
89 once the repo object is destroyed.
97 """
90 """
98 def __del__(self):
91 def __del__(self):
99 if hasattr(self, 'object_store'):
92 if hasattr(self, 'object_store'):
100 self.close()
93 self.close()
101
94
102
95
103 class Repository(LibGit2Repo):
96 class Repository(LibGit2Repo):
104
97
105 def __enter__(self):
98 def __enter__(self):
106 return self
99 return self
107
100
108 def __exit__(self, exc_type, exc_val, exc_tb):
101 def __exit__(self, exc_type, exc_val, exc_tb):
109 self.free()
102 self.free()
110
103
111
104
112 class GitFactory(RepoFactory):
105 class GitFactory(RepoFactory):
113 repo_type = 'git'
106 repo_type = 'git'
114
107
115 def _create_repo(self, wire, create, use_libgit2=False):
108 def _create_repo(self, wire, create, use_libgit2=False):
116 if use_libgit2:
109 if use_libgit2:
117 return Repository(wire['path'])
110 repo = Repository(safe_bytes(wire['path']))
118 else:
111 else:
119 repo_path = str_to_dulwich(wire['path'])
112 # dulwich mode
120 return Repo(repo_path)
113 repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
114 repo = Repo(repo_path)
115
116 log.debug('repository created: got GIT object: %s', repo)
117 return repo
121
118
122 def repo(self, wire, create=False, use_libgit2=False):
119 def repo(self, wire, create=False, use_libgit2=False):
123 """
120 """
124 Get a repository instance for the given path.
121 Get a repository instance for the given path.
125 """
122 """
126 return self._create_repo(wire, create, use_libgit2)
123 return self._create_repo(wire, create, use_libgit2)
127
124
128 def repo_libgit2(self, wire):
125 def repo_libgit2(self, wire):
129 return self.repo(wire, use_libgit2=True)
126 return self.repo(wire, use_libgit2=True)
130
127
131
128
129 def create_signature_from_string(author_str, **kwargs):
130 """
131 Creates a pygit2.Signature object from a string of the format 'Name <email>'.
132
133 :param author_str: String of the format 'Name <email>'
134 :return: pygit2.Signature object
135 """
136 match = re.match(r'^(.+) <(.+)>$', author_str)
137 if match is None:
138 raise ValueError(f"Invalid format: {author_str}")
139
140 name, email = match.groups()
141 return pygit2.Signature(name, email, **kwargs)
142
143
144 def get_obfuscated_url(url_obj):
145 url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
146 url_obj.query = obfuscate_qs(url_obj.query)
147 obfuscated_uri = str(url_obj)
148 return obfuscated_uri
149
150
132 class GitRemote(RemoteBase):
151 class GitRemote(RemoteBase):
133
152
134 def __init__(self, factory):
153 def __init__(self, factory):
135 self._factory = factory
154 self._factory = factory
136 self._bulk_methods = {
155 self._bulk_methods = {
137 "date": self.date,
156 "date": self.date,
138 "author": self.author,
157 "author": self.author,
139 "branch": self.branch,
158 "branch": self.branch,
140 "message": self.message,
159 "message": self.message,
141 "parents": self.parents,
160 "parents": self.parents,
142 "_commit": self.revision,
161 "_commit": self.revision,
143 }
162 }
163 self._bulk_file_methods = {
164 "size": self.get_node_size,
165 "data": self.get_node_data,
166 "flags": self.get_node_flags,
167 "is_binary": self.get_node_is_binary,
168 "md5": self.md5_hash
169 }
144
170
145 def _wire_to_config(self, wire):
171 def _wire_to_config(self, wire):
146 if 'config' in wire:
172 if 'config' in wire:
147 return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
173 return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
148 return {}
174 return {}
149
175
150 def _remote_conf(self, config):
176 def _remote_conf(self, config):
151 params = [
177 params = [
152 '-c', 'core.askpass=""',
178 '-c', 'core.askpass=""',
153 ]
179 ]
154 ssl_cert_dir = config.get('vcs_ssl_dir')
180 config_attrs = {
155 if ssl_cert_dir:
181 'vcs_ssl_dir': 'http.sslCAinfo={}',
156 params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
182 'vcs_git_lfs_store_location': 'lfs.storage={}'
183 }
184 for key, param in config_attrs.items():
185 if value := config.get(key):
186 params.extend(['-c', param.format(value)])
157 return params
187 return params
158
188
159 @reraise_safe_exceptions
189 @reraise_safe_exceptions
160 def discover_git_version(self):
190 def discover_git_version(self):
161 stdout, _ = self.run_git_command(
191 stdout, _ = self.run_git_command(
162 {}, ['--version'], _bare=True, _safe=True)
192 {}, ['--version'], _bare=True, _safe=True)
163 prefix = 'git version'
193 prefix = b'git version'
164 if stdout.startswith(prefix):
194 if stdout.startswith(prefix):
165 stdout = stdout[len(prefix):]
195 stdout = stdout[len(prefix):]
166 return stdout.strip()
196 return safe_str(stdout.strip())
167
197
168 @reraise_safe_exceptions
198 @reraise_safe_exceptions
169 def is_empty(self, wire):
199 def is_empty(self, wire):
170 repo_init = self._factory.repo_libgit2(wire)
200 repo_init = self._factory.repo_libgit2(wire)
171 with repo_init as repo:
201 with repo_init as repo:
172
173 try:
202 try:
174 has_head = repo.head.name
203 has_head = repo.head.name
175 if has_head:
204 if has_head:
176 return False
205 return False
177
206
178 # NOTE(marcink): check again using more expensive method
207 # NOTE(marcink): check again using more expensive method
179 return repo.is_empty
208 return repo.is_empty
180 except Exception:
209 except Exception:
181 pass
210 pass
182
211
183 return True
212 return True
184
213
185 @reraise_safe_exceptions
214 @reraise_safe_exceptions
186 def assert_correct_path(self, wire):
215 def assert_correct_path(self, wire):
187 cache_on, context_uid, repo_id = self._cache_on(wire)
216 cache_on, context_uid, repo_id = self._cache_on(wire)
188 region = self._region(wire)
217 region = self._region(wire)
218
189 @region.conditional_cache_on_arguments(condition=cache_on)
219 @region.conditional_cache_on_arguments(condition=cache_on)
190 def _assert_correct_path(_context_uid, _repo_id):
220 def _assert_correct_path(_context_uid, _repo_id, fast_check):
191 try:
221 if fast_check:
192 repo_init = self._factory.repo_libgit2(wire)
222 path = safe_str(wire['path'])
193 with repo_init as repo:
223 if pygit2.discover_repository(path):
194 pass
224 return True
195 except pygit2.GitError:
196 path = wire.get('path')
197 tb = traceback.format_exc()
198 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
199 return False
225 return False
226 else:
227 try:
228 repo_init = self._factory.repo_libgit2(wire)
229 with repo_init:
230 pass
231 except pygit2.GitError:
232 path = wire.get('path')
233 tb = traceback.format_exc()
234 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
235 return False
236 return True
200
237
201 return True
238 return _assert_correct_path(context_uid, repo_id, True)
202 return _assert_correct_path(context_uid, repo_id)
203
239
204 @reraise_safe_exceptions
240 @reraise_safe_exceptions
205 def bare(self, wire):
241 def bare(self, wire):
206 repo_init = self._factory.repo_libgit2(wire)
242 repo_init = self._factory.repo_libgit2(wire)
207 with repo_init as repo:
243 with repo_init as repo:
208 return repo.is_bare
244 return repo.is_bare
209
245
210 @reraise_safe_exceptions
246 @reraise_safe_exceptions
247 def get_node_data(self, wire, commit_id, path):
248 repo_init = self._factory.repo_libgit2(wire)
249 with repo_init as repo:
250 commit = repo[commit_id]
251 blob_obj = commit.tree[path]
252
253 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
254 raise exceptions.LookupException()(
255 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
256
257 return BytesEnvelope(blob_obj.data)
258
259 @reraise_safe_exceptions
260 def get_node_size(self, wire, commit_id, path):
261 repo_init = self._factory.repo_libgit2(wire)
262 with repo_init as repo:
263 commit = repo[commit_id]
264 blob_obj = commit.tree[path]
265
266 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
267 raise exceptions.LookupException()(
268 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
269
270 return blob_obj.size
271
272 @reraise_safe_exceptions
273 def get_node_flags(self, wire, commit_id, path):
274 repo_init = self._factory.repo_libgit2(wire)
275 with repo_init as repo:
276 commit = repo[commit_id]
277 blob_obj = commit.tree[path]
278
279 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
280 raise exceptions.LookupException()(
281 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
282
283 return blob_obj.filemode
284
285 @reraise_safe_exceptions
286 def get_node_is_binary(self, wire, commit_id, path):
287 repo_init = self._factory.repo_libgit2(wire)
288 with repo_init as repo:
289 commit = repo[commit_id]
290 blob_obj = commit.tree[path]
291
292 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
293 raise exceptions.LookupException()(
294 f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
295
296 return blob_obj.is_binary
297
298 @reraise_safe_exceptions
211 def blob_as_pretty_string(self, wire, sha):
299 def blob_as_pretty_string(self, wire, sha):
212 repo_init = self._factory.repo_libgit2(wire)
300 repo_init = self._factory.repo_libgit2(wire)
213 with repo_init as repo:
301 with repo_init as repo:
214 blob_obj = repo[sha]
302 blob_obj = repo[sha]
215 blob = blob_obj.data
303 return BytesEnvelope(blob_obj.data)
216 return blob
217
304
218 @reraise_safe_exceptions
305 @reraise_safe_exceptions
219 def blob_raw_length(self, wire, sha):
306 def blob_raw_length(self, wire, sha):
220 cache_on, context_uid, repo_id = self._cache_on(wire)
307 cache_on, context_uid, repo_id = self._cache_on(wire)
221 region = self._region(wire)
308 region = self._region(wire)
309
222 @region.conditional_cache_on_arguments(condition=cache_on)
310 @region.conditional_cache_on_arguments(condition=cache_on)
223 def _blob_raw_length(_repo_id, _sha):
311 def _blob_raw_length(_repo_id, _sha):
224
312
225 repo_init = self._factory.repo_libgit2(wire)
313 repo_init = self._factory.repo_libgit2(wire)
226 with repo_init as repo:
314 with repo_init as repo:
227 blob = repo[sha]
315 blob = repo[sha]
228 return blob.size
316 return blob.size
229
317
230 return _blob_raw_length(repo_id, sha)
318 return _blob_raw_length(repo_id, sha)
231
319
232 def _parse_lfs_pointer(self, raw_content):
320 def _parse_lfs_pointer(self, raw_content):
321 spec_string = b'version https://git-lfs.github.com/spec'
322 if raw_content and raw_content.startswith(spec_string):
233
323
234 spec_string = 'version https://git-lfs.github.com/spec'
324 pattern = re.compile(rb"""
235 if raw_content and raw_content.startswith(spec_string):
236 pattern = re.compile(r"""
237 (?:\n)?
325 (?:\n)?
238 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
326 ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
239 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
327 ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
240 ^size[ ](?P<oid_size>[0-9]+)\n
328 ^size[ ](?P<oid_size>[0-9]+)\n
241 (?:\n)?
329 (?:\n)?
242 """, re.VERBOSE | re.MULTILINE)
330 """, re.VERBOSE | re.MULTILINE)
243 match = pattern.match(raw_content)
331 match = pattern.match(raw_content)
244 if match:
332 if match:
245 return match.groupdict()
333 return match.groupdict()
246
334
247 return {}
335 return {}
248
336
249 @reraise_safe_exceptions
337 @reraise_safe_exceptions
250 def is_large_file(self, wire, commit_id):
338 def is_large_file(self, wire, commit_id):
251 cache_on, context_uid, repo_id = self._cache_on(wire)
339 cache_on, context_uid, repo_id = self._cache_on(wire)
340 region = self._region(wire)
252
341
253 region = self._region(wire)
254 @region.conditional_cache_on_arguments(condition=cache_on)
342 @region.conditional_cache_on_arguments(condition=cache_on)
255 def _is_large_file(_repo_id, _sha):
343 def _is_large_file(_repo_id, _sha):
256 repo_init = self._factory.repo_libgit2(wire)
344 repo_init = self._factory.repo_libgit2(wire)
257 with repo_init as repo:
345 with repo_init as repo:
258 blob = repo[commit_id]
346 blob = repo[commit_id]
259 if blob.is_binary:
347 if blob.is_binary:
260 return {}
348 return {}
261
349
262 return self._parse_lfs_pointer(blob.data)
350 return self._parse_lfs_pointer(blob.data)
263
351
264 return _is_large_file(repo_id, commit_id)
352 return _is_large_file(repo_id, commit_id)
265
353
266 @reraise_safe_exceptions
354 @reraise_safe_exceptions
267 def is_binary(self, wire, tree_id):
355 def is_binary(self, wire, tree_id):
268 cache_on, context_uid, repo_id = self._cache_on(wire)
356 cache_on, context_uid, repo_id = self._cache_on(wire)
357 region = self._region(wire)
269
358
270 region = self._region(wire)
271 @region.conditional_cache_on_arguments(condition=cache_on)
359 @region.conditional_cache_on_arguments(condition=cache_on)
272 def _is_binary(_repo_id, _tree_id):
360 def _is_binary(_repo_id, _tree_id):
273 repo_init = self._factory.repo_libgit2(wire)
361 repo_init = self._factory.repo_libgit2(wire)
274 with repo_init as repo:
362 with repo_init as repo:
275 blob_obj = repo[tree_id]
363 blob_obj = repo[tree_id]
276 return blob_obj.is_binary
364 return blob_obj.is_binary
277
365
278 return _is_binary(repo_id, tree_id)
366 return _is_binary(repo_id, tree_id)
279
367
280 @reraise_safe_exceptions
368 @reraise_safe_exceptions
369 def md5_hash(self, wire, commit_id, path):
370 cache_on, context_uid, repo_id = self._cache_on(wire)
371 region = self._region(wire)
372
373 @region.conditional_cache_on_arguments(condition=cache_on)
374 def _md5_hash(_repo_id, _commit_id, _path):
375 repo_init = self._factory.repo_libgit2(wire)
376 with repo_init as repo:
377 commit = repo[_commit_id]
378 blob_obj = commit.tree[_path]
379
380 if blob_obj.type != pygit2.GIT_OBJ_BLOB:
381 raise exceptions.LookupException()(
382 f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
383
384 return ''
385
386 return _md5_hash(repo_id, commit_id, path)
387
388 @reraise_safe_exceptions
281 def in_largefiles_store(self, wire, oid):
389 def in_largefiles_store(self, wire, oid):
282 conf = self._wire_to_config(wire)
390 conf = self._wire_to_config(wire)
283 repo_init = self._factory.repo_libgit2(wire)
391 repo_init = self._factory.repo_libgit2(wire)
284 with repo_init as repo:
392 with repo_init as repo:
285 repo_name = repo.path
393 repo_name = repo.path
286
394
287 store_location = conf.get('vcs_git_lfs_store_location')
395 store_location = conf.get('vcs_git_lfs_store_location')
288 if store_location:
396 if store_location:
289
397
290 store = LFSOidStore(
398 store = LFSOidStore(
291 oid=oid, repo=repo_name, store_location=store_location)
399 oid=oid, repo=repo_name, store_location=store_location)
292 return store.has_oid()
400 return store.has_oid()
293
401
294 return False
402 return False
295
403
296 @reraise_safe_exceptions
404 @reraise_safe_exceptions
297 def store_path(self, wire, oid):
405 def store_path(self, wire, oid):
298 conf = self._wire_to_config(wire)
406 conf = self._wire_to_config(wire)
299 repo_init = self._factory.repo_libgit2(wire)
407 repo_init = self._factory.repo_libgit2(wire)
300 with repo_init as repo:
408 with repo_init as repo:
301 repo_name = repo.path
409 repo_name = repo.path
302
410
303 store_location = conf.get('vcs_git_lfs_store_location')
411 store_location = conf.get('vcs_git_lfs_store_location')
304 if store_location:
412 if store_location:
305 store = LFSOidStore(
413 store = LFSOidStore(
306 oid=oid, repo=repo_name, store_location=store_location)
414 oid=oid, repo=repo_name, store_location=store_location)
307 return store.oid_path
415 return store.oid_path
308 raise ValueError('Unable to fetch oid with path {}'.format(oid))
416 raise ValueError(f'Unable to fetch oid with path {oid}')
309
417
310 @reraise_safe_exceptions
418 @reraise_safe_exceptions
311 def bulk_request(self, wire, rev, pre_load):
419 def bulk_request(self, wire, rev, pre_load):
312 cache_on, context_uid, repo_id = self._cache_on(wire)
420 cache_on, context_uid, repo_id = self._cache_on(wire)
313 region = self._region(wire)
421 region = self._region(wire)
422
314 @region.conditional_cache_on_arguments(condition=cache_on)
423 @region.conditional_cache_on_arguments(condition=cache_on)
315 def _bulk_request(_repo_id, _rev, _pre_load):
424 def _bulk_request(_repo_id, _rev, _pre_load):
316 result = {}
425 result = {}
317 for attr in pre_load:
426 for attr in pre_load:
318 try:
427 try:
319 method = self._bulk_methods[attr]
428 method = self._bulk_methods[attr]
429 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
320 args = [wire, rev]
430 args = [wire, rev]
321 result[attr] = method(*args)
431 result[attr] = method(*args)
322 except KeyError as e:
432 except KeyError as e:
323 raise exceptions.VcsException(e)(
433 raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
324 "Unknown bulk attribute: %s" % attr)
325 return result
434 return result
326
435
327 return _bulk_request(repo_id, rev, sorted(pre_load))
436 return _bulk_request(repo_id, rev, sorted(pre_load))
328
437
329 def _build_opener(self, url):
438 @reraise_safe_exceptions
439 def bulk_file_request(self, wire, commit_id, path, pre_load):
440 cache_on, context_uid, repo_id = self._cache_on(wire)
441 region = self._region(wire)
442
443 @region.conditional_cache_on_arguments(condition=cache_on)
444 def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
445 result = {}
446 for attr in pre_load:
447 try:
448 method = self._bulk_file_methods[attr]
449 wire.update({'cache': False}) # disable cache for bulk calls so we don't double cache
450 result[attr] = method(wire, _commit_id, _path)
451 except KeyError as e:
452 raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
453 return result
454
455 return BinaryEnvelope(_bulk_file_request(repo_id, commit_id, path, sorted(pre_load)))
456
457 def _build_opener(self, url: str):
330 handlers = []
458 handlers = []
331 url_obj = url_parser(url)
459 url_obj = url_parser(safe_bytes(url))
332 _, authinfo = url_obj.authinfo()
460 authinfo = url_obj.authinfo()[1]
333
461
334 if authinfo:
462 if authinfo:
335 # create a password manager
463 # create a password manager
336 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
464 passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
337 passmgr.add_password(*authinfo)
465 passmgr.add_password(*convert_to_str(authinfo))
338
466
339 handlers.extend((httpbasicauthhandler(passmgr),
467 handlers.extend((httpbasicauthhandler(passmgr),
340 httpdigestauthhandler(passmgr)))
468 httpdigestauthhandler(passmgr)))
341
469
342 return urllib2.build_opener(*handlers)
470 return urllib.request.build_opener(*handlers)
343
344 def _type_id_to_name(self, type_id):
345 return {
346 1: b'commit',
347 2: b'tree',
348 3: b'blob',
349 4: b'tag'
350 }[type_id]
351
471
352 @reraise_safe_exceptions
472 @reraise_safe_exceptions
353 def check_url(self, url, config):
473 def check_url(self, url, config):
354 url_obj = url_parser(url)
474 url_obj = url_parser(safe_bytes(url))
355 test_uri, _ = url_obj.authinfo()
475
356 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
476 test_uri = safe_str(url_obj.authinfo()[0])
357 url_obj.query = obfuscate_qs(url_obj.query)
477 obfuscated_uri = get_obfuscated_url(url_obj)
358 cleaned_uri = str(url_obj)
478
359 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
479 log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
360
480
361 if not test_uri.endswith('info/refs'):
481 if not test_uri.endswith('info/refs'):
362 test_uri = test_uri.rstrip('/') + '/info/refs'
482 test_uri = test_uri.rstrip('/') + '/info/refs'
363
483
364 o = self._build_opener(url)
484 o = self._build_opener(url=url)
365 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
485 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
366
486
367 q = {"service": 'git-upload-pack'}
487 q = {"service": 'git-upload-pack'}
368 qs = '?%s' % urllib.urlencode(q)
488 qs = f'?{urllib.parse.urlencode(q)}'
369 cu = "%s%s" % (test_uri, qs)
489 cu = f"{test_uri}{qs}"
370 req = urllib2.Request(cu, None, {})
371
490
372 try:
491 try:
373 log.debug("Trying to open URL %s", cleaned_uri)
492 req = urllib.request.Request(cu, None, {})
493 log.debug("Trying to open URL %s", obfuscated_uri)
374 resp = o.open(req)
494 resp = o.open(req)
375 if resp.code != 200:
495 if resp.code != 200:
376 raise exceptions.URLError()('Return Code is not 200')
496 raise exceptions.URLError()('Return Code is not 200')
377 except Exception as e:
497 except Exception as e:
378 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
498 log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
379 # means it cannot be cloned
499 # means it cannot be cloned
380 raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
500 raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
381
501
382 # now detect if it's proper git repo
502 # now detect if it's proper git repo
383 gitdata = resp.read()
503 gitdata: bytes = resp.read()
384 if 'service=git-upload-pack' in gitdata:
504
505 if b'service=git-upload-pack' in gitdata:
385 pass
506 pass
386 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
507 elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
387 # old style git can return some other format !
508 # old style git can return some other format!
388 pass
509 pass
389 else:
510 else:
390 raise exceptions.URLError()(
511 e = None
391 "url [%s] does not look like an git" % (cleaned_uri,))
512 raise exceptions.URLError(e)(
513 f"url [{obfuscated_uri}] does not look like an hg repo org_exc: {e}")
392
514
393 return True
515 return True
394
516
395 @reraise_safe_exceptions
517 @reraise_safe_exceptions
396 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
518 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
397 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
519 # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
398 remote_refs = self.pull(wire, url, apply_refs=False)
520 remote_refs = self.pull(wire, url, apply_refs=False)
399 repo = self._factory.repo(wire)
521 repo = self._factory.repo(wire)
400 if isinstance(valid_refs, list):
522 if isinstance(valid_refs, list):
401 valid_refs = tuple(valid_refs)
523 valid_refs = tuple(valid_refs)
402
524
403 for k in remote_refs:
525 for k in remote_refs:
404 # only parse heads/tags and skip so called deferred tags
526 # only parse heads/tags and skip so called deferred tags
405 if k.startswith(valid_refs) and not k.endswith(deferred):
527 if k.startswith(valid_refs) and not k.endswith(deferred):
406 repo[k] = remote_refs[k]
528 repo[k] = remote_refs[k]
407
529
408 if update_after_clone:
530 if update_after_clone:
409 # we want to checkout HEAD
531 # we want to checkout HEAD
410 repo["HEAD"] = remote_refs["HEAD"]
532 repo["HEAD"] = remote_refs["HEAD"]
411 index.build_index_from_tree(repo.path, repo.index_path(),
533 index.build_index_from_tree(repo.path, repo.index_path(),
412 repo.object_store, repo["HEAD"].tree)
534 repo.object_store, repo["HEAD"].tree)
413
535
414 @reraise_safe_exceptions
536 @reraise_safe_exceptions
415 def branch(self, wire, commit_id):
537 def branch(self, wire, commit_id):
416 cache_on, context_uid, repo_id = self._cache_on(wire)
538 cache_on, context_uid, repo_id = self._cache_on(wire)
417 region = self._region(wire)
539 region = self._region(wire)
540
418 @region.conditional_cache_on_arguments(condition=cache_on)
541 @region.conditional_cache_on_arguments(condition=cache_on)
419 def _branch(_context_uid, _repo_id, _commit_id):
542 def _branch(_context_uid, _repo_id, _commit_id):
420 regex = re.compile('^refs/heads')
543 regex = re.compile('^refs/heads')
421
544
422 def filter_with(ref):
545 def filter_with(ref):
423 return regex.match(ref[0]) and ref[1] == _commit_id
546 return regex.match(ref[0]) and ref[1] == _commit_id
424
547
425 branches = filter(filter_with, self.get_refs(wire).items())
548 branches = list(filter(filter_with, list(self.get_refs(wire).items())))
426 return [x[0].split('refs/heads/')[-1] for x in branches]
549 return [x[0].split('refs/heads/')[-1] for x in branches]
427
550
428 return _branch(context_uid, repo_id, commit_id)
551 return _branch(context_uid, repo_id, commit_id)
429
552
430 @reraise_safe_exceptions
553 @reraise_safe_exceptions
431 def commit_branches(self, wire, commit_id):
554 def commit_branches(self, wire, commit_id):
432 cache_on, context_uid, repo_id = self._cache_on(wire)
555 cache_on, context_uid, repo_id = self._cache_on(wire)
433 region = self._region(wire)
556 region = self._region(wire)
557
434 @region.conditional_cache_on_arguments(condition=cache_on)
558 @region.conditional_cache_on_arguments(condition=cache_on)
435 def _commit_branches(_context_uid, _repo_id, _commit_id):
559 def _commit_branches(_context_uid, _repo_id, _commit_id):
436 repo_init = self._factory.repo_libgit2(wire)
560 repo_init = self._factory.repo_libgit2(wire)
437 with repo_init as repo:
561 with repo_init as repo:
438 branches = [x for x in repo.branches.with_commit(_commit_id)]
562 branches = [x for x in repo.branches.with_commit(_commit_id)]
439 return branches
563 return branches
440
564
441 return _commit_branches(context_uid, repo_id, commit_id)
565 return _commit_branches(context_uid, repo_id, commit_id)
442
566
443 @reraise_safe_exceptions
567 @reraise_safe_exceptions
444 def add_object(self, wire, content):
568 def add_object(self, wire, content):
445 repo_init = self._factory.repo_libgit2(wire)
569 repo_init = self._factory.repo_libgit2(wire)
446 with repo_init as repo:
570 with repo_init as repo:
447 blob = objects.Blob()
571 blob = objects.Blob()
448 blob.set_raw_string(content)
572 blob.set_raw_string(content)
449 repo.object_store.add_object(blob)
573 repo.object_store.add_object(blob)
450 return blob.id
574 return blob.id
451
575
452 # TODO: this is quite complex, check if that can be simplified
576 @reraise_safe_exceptions
577 def create_commit(self, wire, author, committer, message, branch, new_tree_id,
578 date_args: list[int, int] = None,
579 parents: list | None = None):
580
581 repo_init = self._factory.repo_libgit2(wire)
582 with repo_init as repo:
583
584 if date_args:
585 current_time, offset = date_args
586
587 kw = {
588 'time': current_time,
589 'offset': offset
590 }
591 author = create_signature_from_string(author, **kw)
592 committer = create_signature_from_string(committer, **kw)
593
594 tree = new_tree_id
595 if isinstance(tree, (bytes, str)):
596 # validate this tree is in the repo...
597 tree = repo[safe_str(tree)].id
598
599 if parents:
600 # run via sha's and validate them in repo
601 parents = [repo[c].id for c in parents]
602 else:
603 parents = []
604 # ensure we COMMIT on top of given branch head
605 # check if this repo has ANY branches, otherwise it's a new branch case we need to make
606 if branch in repo.branches.local:
607 parents += [repo.branches[branch].target]
608 elif [x for x in repo.branches.local]:
609 parents += [repo.head.target]
610 #else:
611 # in case we want to commit on new branch we create it on top of HEAD
612 #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
613
614 # # Create a new commit
615 commit_oid = repo.create_commit(
616 f'refs/heads/{branch}', # the name of the reference to update
617 author, # the author of the commit
618 committer, # the committer of the commit
619 message, # the commit message
620 tree, # the tree produced by the index
621 parents # list of parents for the new commit, usually just one,
622 )
623
624 new_commit_id = safe_str(commit_oid)
625
626 return new_commit_id
627
453 @reraise_safe_exceptions
628 @reraise_safe_exceptions
454 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
629 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
455 # Defines the root tree
456 class _Root(object):
457 def __repr__(self):
458 return 'ROOT TREE'
459 ROOT = _Root()
460
630
461 repo = self._factory.repo(wire)
631 def mode2pygit(mode):
462 object_store = repo.object_store
632 """
463
633 git only supports two filemode 644 and 755
464 # Create tree and populates it with blobs
465
466 if commit_tree and repo[commit_tree]:
467 git_commit = repo[commit_data['parents'][0]]
468 commit_tree = repo[git_commit.tree] # root tree
469 else:
470 commit_tree = objects.Tree()
471
472 for node in updated:
473 # Compute subdirs if needed
474 dirpath, nodename = vcspath.split(node['path'])
475 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
476 parent = commit_tree
477 ancestors = [('', parent)]
478
634
479 # Tries to dig for the deepest existing tree
635 0o100755 -> 33261
480 while dirnames:
636 0o100644 -> 33188
481 curdir = dirnames.pop(0)
637 """
482 try:
638 return {
483 dir_id = parent[curdir][1]
639 0o100644: pygit2.GIT_FILEMODE_BLOB,
484 except KeyError:
640 0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
485 # put curdir back into dirnames and stops
641 0o120000: pygit2.GIT_FILEMODE_LINK
486 dirnames.insert(0, curdir)
642 }.get(mode) or pygit2.GIT_FILEMODE_BLOB
487 break
488 else:
489 # If found, updates parent
490 parent = repo[dir_id]
491 ancestors.append((curdir, parent))
492 # Now parent is deepest existing tree and we need to create
493 # subtrees for dirnames (in reverse order)
494 # [this only applies for nodes from added]
495 new_trees = []
496
643
497 blob = objects.Blob.from_string(node['content'])
644 repo_init = self._factory.repo_libgit2(wire)
645 with repo_init as repo:
646 repo_index = repo.index
498
647
499 if dirnames:
648 commit_parents = None
500 # If there are trees which should be created we need to build
649 if commit_tree and commit_data['parents']:
501 # them now (in reverse order)
650 commit_parents = commit_data['parents']
502 reversed_dirnames = list(reversed(dirnames))
651 parent_commit = repo[commit_parents[0]]
503 curtree = objects.Tree()
652 repo_index.read_tree(parent_commit.tree)
504 curtree[node['node_path']] = node['mode'], blob.id
505 new_trees.append(curtree)
506 for dirname in reversed_dirnames[:-1]:
507 newtree = objects.Tree()
508 newtree[dirname] = (DIR_STAT, curtree.id)
509 new_trees.append(newtree)
510 curtree = newtree
511 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
512 else:
513 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
514
653
515 new_trees.append(parent)
654 for pathspec in updated:
516 # Update ancestors
655 blob_id = repo.create_blob(pathspec['content'])
517 reversed_ancestors = reversed(
656 ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
518 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
657 repo_index.add(ie)
519 for parent, tree, path in reversed_ancestors:
658
520 parent[path] = (DIR_STAT, tree.id)
659 for pathspec in removed:
521 object_store.add_object(tree)
660 repo_index.remove(pathspec)
522
661
523 object_store.add_object(blob)
662 # Write changes to the index
524 for tree in new_trees:
663 repo_index.write()
525 object_store.add_object(tree)
664
665 # Create a tree from the updated index
666 written_commit_tree = repo_index.write_tree()
667
668 new_tree_id = written_commit_tree
526
669
527 for node_path in removed:
670 author = commit_data['author']
528 paths = node_path.split('/')
671 committer = commit_data['committer']
529 tree = commit_tree # start with top-level
672 message = commit_data['message']
530 trees = [{'tree': tree, 'path': ROOT}]
673
531 # Traverse deep into the forest...
674 date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
532 # resolve final tree by iterating the path.
533 # e.g a/b/c.txt will get
534 # - root as tree then
535 # - 'a' as tree,
536 # - 'b' as tree,
537 # - stop at c as blob.
538 for path in paths:
539 try:
540 obj = repo[tree[path][1]]
541 if isinstance(obj, objects.Tree):
542 trees.append({'tree': obj, 'path': path})
543 tree = obj
544 except KeyError:
545 break
546 #PROBLEM:
547 """
548 We're not editing same reference tree object
549 """
550 # Cut down the blob and all rotten trees on the way back...
551 for path, tree_data in reversed(zip(paths, trees)):
552 tree = tree_data['tree']
553 tree.__delitem__(path)
554 # This operation edits the tree, we need to mark new commit back
555
675
556 if len(tree) > 0:
676 new_commit_id = self.create_commit(wire, author, committer, message, branch,
557 # This tree still has elements - don't remove it or any
677 new_tree_id, date_args=date_args, parents=commit_parents)
558 # of it's parents
559 break
560
561 object_store.add_object(commit_tree)
562
678
563 # Create commit
679 # libgit2, ensure the branch is there and exists
564 commit = objects.Commit()
680 self.create_branch(wire, branch, new_commit_id)
565 commit.tree = commit_tree.id
566 for k, v in commit_data.items():
567 setattr(commit, k, v)
568 object_store.add_object(commit)
569
681
570 self.create_branch(wire, branch, commit.id)
682 # libgit2, set new ref to this created commit
683 self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
571
684
572 # dulwich set-ref
685 return new_commit_id
573 ref = 'refs/heads/%s' % branch
574 repo.refs[ref] = commit.id
575
576 return commit.id
577
686
578 @reraise_safe_exceptions
687 @reraise_safe_exceptions
579 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
688 def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
580 if url != 'default' and '://' not in url:
689 if url != 'default' and '://' not in url:
581 client = LocalGitClient(url)
690 client = LocalGitClient(url)
582 else:
691 else:
583 url_obj = url_parser(url)
692 url_obj = url_parser(safe_bytes(url))
584 o = self._build_opener(url)
693 o = self._build_opener(url)
585 url, _ = url_obj.authinfo()
694 url = url_obj.authinfo()[0]
586 client = HttpGitClient(base_url=url, opener=o)
695 client = HttpGitClient(base_url=url, opener=o)
587 repo = self._factory.repo(wire)
696 repo = self._factory.repo(wire)
588
697
589 determine_wants = repo.object_store.determine_wants_all
698 determine_wants = repo.object_store.determine_wants_all
699
590 if refs:
700 if refs:
591 def determine_wants_requested(references):
701 refs: list[bytes] = [ascii_bytes(x) for x in refs]
592 return [references[r] for r in references if r in refs]
702
703 def determine_wants_requested(_remote_refs):
704 determined = []
705 for ref_name, ref_hash in _remote_refs.items():
706 bytes_ref_name = safe_bytes(ref_name)
707
708 if bytes_ref_name in refs:
709 bytes_ref_hash = safe_bytes(ref_hash)
710 determined.append(bytes_ref_hash)
711 return determined
712
713 # swap with our custom requested wants
593 determine_wants = determine_wants_requested
714 determine_wants = determine_wants_requested
594
715
595 try:
716 try:
596 remote_refs = client.fetch(
717 remote_refs = client.fetch(
597 path=url, target=repo, determine_wants=determine_wants)
718 path=url, target=repo, determine_wants=determine_wants)
719
598 except NotGitRepository as e:
720 except NotGitRepository as e:
599 log.warning(
721 log.warning(
600 'Trying to fetch from "%s" failed, not a Git repository.', url)
722 'Trying to fetch from "%s" failed, not a Git repository.', url)
601 # Exception can contain unicode which we convert
723 # Exception can contain unicode which we convert
602 raise exceptions.AbortException(e)(repr(e))
724 raise exceptions.AbortException(e)(repr(e))
603
725
604 # mikhail: client.fetch() returns all the remote refs, but fetches only
726 # mikhail: client.fetch() returns all the remote refs, but fetches only
605 # refs filtered by `determine_wants` function. We need to filter result
727 # refs filtered by `determine_wants` function. We need to filter result
606 # as well
728 # as well
607 if refs:
729 if refs:
608 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
730 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
609
731
610 if apply_refs:
732 if apply_refs:
611 # TODO: johbo: Needs proper test coverage with a git repository
733 # TODO: johbo: Needs proper test coverage with a git repository
612 # that contains a tag object, so that we would end up with
734 # that contains a tag object, so that we would end up with
613 # a peeled ref at this point.
735 # a peeled ref at this point.
614 for k in remote_refs:
736 for k in remote_refs:
615 if k.endswith(PEELED_REF_MARKER):
737 if k.endswith(PEELED_REF_MARKER):
616 log.debug("Skipping peeled reference %s", k)
738 log.debug("Skipping peeled reference %s", k)
617 continue
739 continue
618 repo[k] = remote_refs[k]
740 repo[k] = remote_refs[k]
619
741
620 if refs and not update_after:
742 if refs and not update_after:
743 # update to ref
621 # mikhail: explicitly set the head to the last ref.
744 # mikhail: explicitly set the head to the last ref.
622 repo["HEAD"] = remote_refs[refs[-1]]
745 update_to_ref = refs[-1]
746 if isinstance(update_after, str):
747 update_to_ref = update_after
748
749 repo[HEAD_MARKER] = remote_refs[update_to_ref]
623
750
624 if update_after:
751 if update_after:
625 # we want to checkout HEAD
752 # we want to check out HEAD
626 repo["HEAD"] = remote_refs["HEAD"]
753 repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
627 index.build_index_from_tree(repo.path, repo.index_path(),
754 index.build_index_from_tree(repo.path, repo.index_path(),
628 repo.object_store, repo["HEAD"].tree)
755 repo.object_store, repo[HEAD_MARKER].tree)
756
757 if isinstance(remote_refs, FetchPackResult):
758 return remote_refs.refs
629 return remote_refs
759 return remote_refs
630
760
631 @reraise_safe_exceptions
761 @reraise_safe_exceptions
632 def sync_fetch(self, wire, url, refs=None, all_refs=False):
762 def sync_fetch(self, wire, url, refs=None, all_refs=False, **kwargs):
633 repo = self._factory.repo(wire)
763 self._factory.repo(wire)
634 if refs and not isinstance(refs, (list, tuple)):
764 if refs and not isinstance(refs, (list, tuple)):
635 refs = [refs]
765 refs = [refs]
636
766
637 config = self._wire_to_config(wire)
767 config = self._wire_to_config(wire)
638 # get all remote refs we'll use to fetch later
768 # get all remote refs we'll use to fetch later
639 cmd = ['ls-remote']
769 cmd = ['ls-remote']
640 if not all_refs:
770 if not all_refs:
641 cmd += ['--heads', '--tags']
771 cmd += ['--heads', '--tags']
642 cmd += [url]
772 cmd += [url]
643 output, __ = self.run_git_command(
773 output, __ = self.run_git_command(
644 wire, cmd, fail_on_stderr=False,
774 wire, cmd, fail_on_stderr=False,
645 _copts=self._remote_conf(config),
775 _copts=self._remote_conf(config),
646 extra_env={'GIT_TERMINAL_PROMPT': '0'})
776 extra_env={'GIT_TERMINAL_PROMPT': '0'})
647
777
648 remote_refs = collections.OrderedDict()
778 remote_refs = collections.OrderedDict()
649 fetch_refs = []
779 fetch_refs = []
650
780
651 for ref_line in output.splitlines():
781 for ref_line in output.splitlines():
652 sha, ref = ref_line.split('\t')
782 sha, ref = ref_line.split(b'\t')
653 sha = sha.strip()
783 sha = sha.strip()
654 if ref in remote_refs:
784 if ref in remote_refs:
655 # duplicate, skip
785 # duplicate, skip
656 continue
786 continue
657 if ref.endswith(PEELED_REF_MARKER):
787 if ref.endswith(PEELED_REF_MARKER):
658 log.debug("Skipping peeled reference %s", ref)
788 log.debug("Skipping peeled reference %s", ref)
659 continue
789 continue
660 # don't sync HEAD
790 # don't sync HEAD
661 if ref in ['HEAD']:
791 if ref in [HEAD_MARKER]:
662 continue
792 continue
663
793
664 remote_refs[ref] = sha
794 remote_refs[ref] = sha
665
795
666 if refs and sha in refs:
796 if refs and sha in refs:
667 # we filter fetch using our specified refs
797 # we filter fetch using our specified refs
668 fetch_refs.append('{}:{}'.format(ref, ref))
798 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
669 elif not refs:
799 elif not refs:
670 fetch_refs.append('{}:{}'.format(ref, ref))
800 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
671 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
801 log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
672
802
673 if fetch_refs:
803 if fetch_refs:
674 for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
804 for chunk in more_itertools.chunked(fetch_refs, 128):
675 fetch_refs_chunks = list(chunk)
805 fetch_refs_chunks = list(chunk)
676 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
806 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
677 _out, _err = self.run_git_command(
807 self.run_git_command(
678 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
808 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
679 fail_on_stderr=False,
809 fail_on_stderr=False,
680 _copts=self._remote_conf(config),
810 _copts=self._remote_conf(config),
681 extra_env={'GIT_TERMINAL_PROMPT': '0'})
811 extra_env={'GIT_TERMINAL_PROMPT': '0'})
812 if kwargs.get('sync_large_objects'):
813 self.run_git_command(
814 wire, ['lfs', 'fetch', url, '--all'],
815 fail_on_stderr=False,
816 _copts=self._remote_conf(config),
817 )
682
818
683 return remote_refs
819 return remote_refs
684
820
685 @reraise_safe_exceptions
821 @reraise_safe_exceptions
686 def sync_push(self, wire, url, refs=None):
822 def sync_push(self, wire, url, refs=None, **kwargs):
687 if not self.check_url(url, wire):
823 if not self.check_url(url, wire):
688 return
824 return
689 config = self._wire_to_config(wire)
825 config = self._wire_to_config(wire)
690 self._factory.repo(wire)
826 self._factory.repo(wire)
691 self.run_git_command(
827 self.run_git_command(
692 wire, ['push', url, '--mirror'], fail_on_stderr=False,
828 wire, ['push', url, '--mirror'], fail_on_stderr=False,
693 _copts=self._remote_conf(config),
829 _copts=self._remote_conf(config),
694 extra_env={'GIT_TERMINAL_PROMPT': '0'})
830 extra_env={'GIT_TERMINAL_PROMPT': '0'})
831 if kwargs.get('sync_large_objects'):
832 self.run_git_command(
833 wire, ['lfs', 'push', url, '--all'],
834 fail_on_stderr=False,
835 _copts=self._remote_conf(config),
836 )
695
837
696 @reraise_safe_exceptions
838 @reraise_safe_exceptions
697 def get_remote_refs(self, wire, url):
839 def get_remote_refs(self, wire, url):
698 repo = Repo(url)
840 repo = Repo(url)
699 return repo.get_refs()
841 return repo.get_refs()
700
842
701 @reraise_safe_exceptions
843 @reraise_safe_exceptions
702 def get_description(self, wire):
844 def get_description(self, wire):
703 repo = self._factory.repo(wire)
845 repo = self._factory.repo(wire)
704 return repo.get_description()
846 return repo.get_description()
705
847
706 @reraise_safe_exceptions
848 @reraise_safe_exceptions
707 def get_missing_revs(self, wire, rev1, rev2, path2):
849 def get_missing_revs(self, wire, rev1, rev2, other_repo_path):
850 origin_repo_path = wire['path']
708 repo = self._factory.repo(wire)
851 repo = self._factory.repo(wire)
709 LocalGitClient(thin_packs=False).fetch(path2, repo)
852 # fetch from other_repo_path to our origin repo
853 LocalGitClient(thin_packs=False).fetch(other_repo_path, repo)
710
854
711 wire_remote = wire.copy()
855 wire_remote = wire.copy()
712 wire_remote['path'] = path2
856 wire_remote['path'] = other_repo_path
713 repo_remote = self._factory.repo(wire_remote)
857 repo_remote = self._factory.repo(wire_remote)
714 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
858
859 # fetch from origin_repo_path to our remote repo
860 LocalGitClient(thin_packs=False).fetch(origin_repo_path, repo_remote)
715
861
716 revs = [
862 revs = [
717 x.commit.id
863 x.commit.id
718 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
864 for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
719 return revs
865 return revs
720
866
721 @reraise_safe_exceptions
867 @reraise_safe_exceptions
722 def get_object(self, wire, sha, maybe_unreachable=False):
868 def get_object(self, wire, sha, maybe_unreachable=False):
723 cache_on, context_uid, repo_id = self._cache_on(wire)
869 cache_on, context_uid, repo_id = self._cache_on(wire)
724 region = self._region(wire)
870 region = self._region(wire)
871
725 @region.conditional_cache_on_arguments(condition=cache_on)
872 @region.conditional_cache_on_arguments(condition=cache_on)
726 def _get_object(_context_uid, _repo_id, _sha):
873 def _get_object(_context_uid, _repo_id, _sha):
727 repo_init = self._factory.repo_libgit2(wire)
874 repo_init = self._factory.repo_libgit2(wire)
728 with repo_init as repo:
875 with repo_init as repo:
729
876
730 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
877 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
731 try:
878 try:
732 commit = repo.revparse_single(sha)
879 commit = repo.revparse_single(sha)
733 except KeyError:
880 except KeyError:
734 # NOTE(marcink): KeyError doesn't give us any meaningful information
881 # NOTE(marcink): KeyError doesn't give us any meaningful information
735 # here, we instead give something more explicit
882 # here, we instead give something more explicit
736 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
883 e = exceptions.RefNotFoundException('SHA: %s not found', sha)
737 raise exceptions.LookupException(e)(missing_commit_err)
884 raise exceptions.LookupException(e)(missing_commit_err)
738 except ValueError as e:
885 except ValueError as e:
739 raise exceptions.LookupException(e)(missing_commit_err)
886 raise exceptions.LookupException(e)(missing_commit_err)
740
887
741 is_tag = False
888 is_tag = False
742 if isinstance(commit, pygit2.Tag):
889 if isinstance(commit, pygit2.Tag):
743 commit = repo.get(commit.target)
890 commit = repo.get(commit.target)
744 is_tag = True
891 is_tag = True
745
892
746 check_dangling = True
893 check_dangling = True
747 if is_tag:
894 if is_tag:
748 check_dangling = False
895 check_dangling = False
749
896
750 if check_dangling and maybe_unreachable:
897 if check_dangling and maybe_unreachable:
751 check_dangling = False
898 check_dangling = False
752
899
753 # we used a reference and it parsed means we're not having a dangling commit
900 # we used a reference and it parsed means we're not having a dangling commit
754 if sha != commit.hex:
901 if sha != commit.hex:
755 check_dangling = False
902 check_dangling = False
756
903
757 if check_dangling:
904 if check_dangling:
758 # check for dangling commit
905 # check for dangling commit
759 for branch in repo.branches.with_commit(commit.hex):
906 for branch in repo.branches.with_commit(commit.hex):
760 if branch:
907 if branch:
761 break
908 break
762 else:
909 else:
763 # NOTE(marcink): Empty error doesn't give us any meaningful information
910 # NOTE(marcink): Empty error doesn't give us any meaningful information
764 # here, we instead give something more explicit
911 # here, we instead give something more explicit
765 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
912 e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
766 raise exceptions.LookupException(e)(missing_commit_err)
913 raise exceptions.LookupException(e)(missing_commit_err)
767
914
768 commit_id = commit.hex
915 commit_id = commit.hex
769 type_id = commit.type
916 type_str = commit.type_str
770
917
771 return {
918 return {
772 'id': commit_id,
919 'id': commit_id,
773 'type': self._type_id_to_name(type_id),
920 'type': type_str,
774 'commit_id': commit_id,
921 'commit_id': commit_id,
775 'idx': 0
922 'idx': 0
776 }
923 }
777
924
778 return _get_object(context_uid, repo_id, sha)
925 return _get_object(context_uid, repo_id, sha)
779
926
780 @reraise_safe_exceptions
927 @reraise_safe_exceptions
781 def get_refs(self, wire):
928 def get_refs(self, wire):
782 cache_on, context_uid, repo_id = self._cache_on(wire)
929 cache_on, context_uid, repo_id = self._cache_on(wire)
783 region = self._region(wire)
930 region = self._region(wire)
931
784 @region.conditional_cache_on_arguments(condition=cache_on)
932 @region.conditional_cache_on_arguments(condition=cache_on)
785 def _get_refs(_context_uid, _repo_id):
933 def _get_refs(_context_uid, _repo_id):
786
934
787 repo_init = self._factory.repo_libgit2(wire)
935 repo_init = self._factory.repo_libgit2(wire)
788 with repo_init as repo:
936 with repo_init as repo:
789 regex = re.compile('^refs/(heads|tags)/')
937 regex = re.compile('^refs/(heads|tags)/')
790 return {x.name: x.target.hex for x in
938 return {x.name: x.target.hex for x in
791 filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
939 [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
792
940
793 return _get_refs(context_uid, repo_id)
941 return _get_refs(context_uid, repo_id)
794
942
795 @reraise_safe_exceptions
943 @reraise_safe_exceptions
796 def get_branch_pointers(self, wire):
944 def get_branch_pointers(self, wire):
797 cache_on, context_uid, repo_id = self._cache_on(wire)
945 cache_on, context_uid, repo_id = self._cache_on(wire)
798 region = self._region(wire)
946 region = self._region(wire)
947
799 @region.conditional_cache_on_arguments(condition=cache_on)
948 @region.conditional_cache_on_arguments(condition=cache_on)
800 def _get_branch_pointers(_context_uid, _repo_id):
949 def _get_branch_pointers(_context_uid, _repo_id):
801
950
802 repo_init = self._factory.repo_libgit2(wire)
951 repo_init = self._factory.repo_libgit2(wire)
803 regex = re.compile('^refs/heads')
952 regex = re.compile('^refs/heads')
804 with repo_init as repo:
953 with repo_init as repo:
805 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
954 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
806 return {x.target.hex: x.shorthand for x in branches}
955 return {x.target.hex: x.shorthand for x in branches}
807
956
808 return _get_branch_pointers(context_uid, repo_id)
957 return _get_branch_pointers(context_uid, repo_id)
809
958
810 @reraise_safe_exceptions
959 @reraise_safe_exceptions
811 def head(self, wire, show_exc=True):
960 def head(self, wire, show_exc=True):
812 cache_on, context_uid, repo_id = self._cache_on(wire)
961 cache_on, context_uid, repo_id = self._cache_on(wire)
813 region = self._region(wire)
962 region = self._region(wire)
963
814 @region.conditional_cache_on_arguments(condition=cache_on)
964 @region.conditional_cache_on_arguments(condition=cache_on)
815 def _head(_context_uid, _repo_id, _show_exc):
965 def _head(_context_uid, _repo_id, _show_exc):
816 repo_init = self._factory.repo_libgit2(wire)
966 repo_init = self._factory.repo_libgit2(wire)
817 with repo_init as repo:
967 with repo_init as repo:
818 try:
968 try:
819 return repo.head.peel().hex
969 return repo.head.peel().hex
820 except Exception:
970 except Exception:
821 if show_exc:
971 if show_exc:
822 raise
972 raise
823 return _head(context_uid, repo_id, show_exc)
973 return _head(context_uid, repo_id, show_exc)
824
974
825 @reraise_safe_exceptions
975 @reraise_safe_exceptions
826 def init(self, wire):
976 def init(self, wire):
827 repo_path = str_to_dulwich(wire['path'])
977 repo_path = safe_str(wire['path'])
828 self.repo = Repo.init(repo_path)
978 os.makedirs(repo_path, mode=0o755)
979 pygit2.init_repository(repo_path, bare=False)
829
980
830 @reraise_safe_exceptions
981 @reraise_safe_exceptions
831 def init_bare(self, wire):
982 def init_bare(self, wire):
832 repo_path = str_to_dulwich(wire['path'])
983 repo_path = safe_str(wire['path'])
833 self.repo = Repo.init_bare(repo_path)
984 os.makedirs(repo_path, mode=0o755)
985 pygit2.init_repository(repo_path, bare=True)
834
986
835 @reraise_safe_exceptions
987 @reraise_safe_exceptions
836 def revision(self, wire, rev):
988 def revision(self, wire, rev):
837
989
838 cache_on, context_uid, repo_id = self._cache_on(wire)
990 cache_on, context_uid, repo_id = self._cache_on(wire)
839 region = self._region(wire)
991 region = self._region(wire)
992
840 @region.conditional_cache_on_arguments(condition=cache_on)
993 @region.conditional_cache_on_arguments(condition=cache_on)
841 def _revision(_context_uid, _repo_id, _rev):
994 def _revision(_context_uid, _repo_id, _rev):
842 repo_init = self._factory.repo_libgit2(wire)
995 repo_init = self._factory.repo_libgit2(wire)
843 with repo_init as repo:
996 with repo_init as repo:
844 commit = repo[rev]
997 commit = repo[rev]
845 obj_data = {
998 obj_data = {
846 'id': commit.id.hex,
999 'id': commit.id.hex,
847 }
1000 }
848 # tree objects itself don't have tree_id attribute
1001 # tree objects itself don't have tree_id attribute
849 if hasattr(commit, 'tree_id'):
1002 if hasattr(commit, 'tree_id'):
850 obj_data['tree'] = commit.tree_id.hex
1003 obj_data['tree'] = commit.tree_id.hex
851
1004
852 return obj_data
1005 return obj_data
853 return _revision(context_uid, repo_id, rev)
1006 return _revision(context_uid, repo_id, rev)
854
1007
855 @reraise_safe_exceptions
1008 @reraise_safe_exceptions
856 def date(self, wire, commit_id):
1009 def date(self, wire, commit_id):
857 cache_on, context_uid, repo_id = self._cache_on(wire)
1010 cache_on, context_uid, repo_id = self._cache_on(wire)
858 region = self._region(wire)
1011 region = self._region(wire)
1012
859 @region.conditional_cache_on_arguments(condition=cache_on)
1013 @region.conditional_cache_on_arguments(condition=cache_on)
860 def _date(_repo_id, _commit_id):
1014 def _date(_repo_id, _commit_id):
861 repo_init = self._factory.repo_libgit2(wire)
1015 repo_init = self._factory.repo_libgit2(wire)
862 with repo_init as repo:
1016 with repo_init as repo:
863 commit = repo[commit_id]
1017 commit = repo[commit_id]
864
1018
865 if hasattr(commit, 'commit_time'):
1019 if hasattr(commit, 'commit_time'):
866 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
1020 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
867 else:
1021 else:
868 commit = commit.get_object()
1022 commit = commit.get_object()
869 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
1023 commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
870
1024
871 # TODO(marcink): check dulwich difference of offset vs timezone
1025 # TODO(marcink): check dulwich difference of offset vs timezone
872 return [commit_time, commit_time_offset]
1026 return [commit_time, commit_time_offset]
873 return _date(repo_id, commit_id)
1027 return _date(repo_id, commit_id)
874
1028
875 @reraise_safe_exceptions
1029 @reraise_safe_exceptions
876 def author(self, wire, commit_id):
1030 def author(self, wire, commit_id):
877 cache_on, context_uid, repo_id = self._cache_on(wire)
1031 cache_on, context_uid, repo_id = self._cache_on(wire)
878 region = self._region(wire)
1032 region = self._region(wire)
1033
879 @region.conditional_cache_on_arguments(condition=cache_on)
1034 @region.conditional_cache_on_arguments(condition=cache_on)
880 def _author(_repo_id, _commit_id):
1035 def _author(_repo_id, _commit_id):
881 repo_init = self._factory.repo_libgit2(wire)
1036 repo_init = self._factory.repo_libgit2(wire)
882 with repo_init as repo:
1037 with repo_init as repo:
883 commit = repo[commit_id]
1038 commit = repo[commit_id]
884
1039
885 if hasattr(commit, 'author'):
1040 if hasattr(commit, 'author'):
886 author = commit.author
1041 author = commit.author
887 else:
1042 else:
888 author = commit.get_object().author
1043 author = commit.get_object().author
889
1044
890 if author.email:
1045 if author.email:
891 return u"{} <{}>".format(author.name, author.email)
1046 return f"{author.name} <{author.email}>"
892
1047
893 try:
1048 try:
894 return u"{}".format(author.name)
1049 return f"{author.name}"
895 except Exception:
1050 except Exception:
896 return u"{}".format(safe_unicode(author.raw_name))
1051 return f"{safe_str(author.raw_name)}"
897
1052
898 return _author(repo_id, commit_id)
1053 return _author(repo_id, commit_id)
899
1054
900 @reraise_safe_exceptions
1055 @reraise_safe_exceptions
901 def message(self, wire, commit_id):
1056 def message(self, wire, commit_id):
902 cache_on, context_uid, repo_id = self._cache_on(wire)
1057 cache_on, context_uid, repo_id = self._cache_on(wire)
903 region = self._region(wire)
1058 region = self._region(wire)
1059
904 @region.conditional_cache_on_arguments(condition=cache_on)
1060 @region.conditional_cache_on_arguments(condition=cache_on)
905 def _message(_repo_id, _commit_id):
1061 def _message(_repo_id, _commit_id):
906 repo_init = self._factory.repo_libgit2(wire)
1062 repo_init = self._factory.repo_libgit2(wire)
907 with repo_init as repo:
1063 with repo_init as repo:
908 commit = repo[commit_id]
1064 commit = repo[commit_id]
909 return commit.message
1065 return commit.message
910 return _message(repo_id, commit_id)
1066 return _message(repo_id, commit_id)
911
1067
912 @reraise_safe_exceptions
1068 @reraise_safe_exceptions
913 def parents(self, wire, commit_id):
1069 def parents(self, wire, commit_id):
914 cache_on, context_uid, repo_id = self._cache_on(wire)
1070 cache_on, context_uid, repo_id = self._cache_on(wire)
915 region = self._region(wire)
1071 region = self._region(wire)
1072
916 @region.conditional_cache_on_arguments(condition=cache_on)
1073 @region.conditional_cache_on_arguments(condition=cache_on)
917 def _parents(_repo_id, _commit_id):
1074 def _parents(_repo_id, _commit_id):
918 repo_init = self._factory.repo_libgit2(wire)
1075 repo_init = self._factory.repo_libgit2(wire)
919 with repo_init as repo:
1076 with repo_init as repo:
920 commit = repo[commit_id]
1077 commit = repo[commit_id]
921 if hasattr(commit, 'parent_ids'):
1078 if hasattr(commit, 'parent_ids'):
922 parent_ids = commit.parent_ids
1079 parent_ids = commit.parent_ids
923 else:
1080 else:
924 parent_ids = commit.get_object().parent_ids
1081 parent_ids = commit.get_object().parent_ids
925
1082
926 return [x.hex for x in parent_ids]
1083 return [x.hex for x in parent_ids]
927 return _parents(repo_id, commit_id)
1084 return _parents(repo_id, commit_id)
928
1085
929 @reraise_safe_exceptions
1086 @reraise_safe_exceptions
930 def children(self, wire, commit_id):
1087 def children(self, wire, commit_id):
931 cache_on, context_uid, repo_id = self._cache_on(wire)
1088 cache_on, context_uid, repo_id = self._cache_on(wire)
932 region = self._region(wire)
1089 region = self._region(wire)
1090
1091 head = self.head(wire)
1092
933 @region.conditional_cache_on_arguments(condition=cache_on)
1093 @region.conditional_cache_on_arguments(condition=cache_on)
934 def _children(_repo_id, _commit_id):
1094 def _children(_repo_id, _commit_id):
1095
935 output, __ = self.run_git_command(
1096 output, __ = self.run_git_command(
936 wire, ['rev-list', '--all', '--children'])
1097 wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
937
1098
938 child_ids = []
1099 child_ids = []
939 pat = re.compile(r'^%s' % commit_id)
1100 pat = re.compile(fr'^{commit_id}')
940 for l in output.splitlines():
1101 for line in output.splitlines():
941 if pat.match(l):
1102 line = safe_str(line)
942 found_ids = l.split(' ')[1:]
1103 if pat.match(line):
1104 found_ids = line.split(' ')[1:]
943 child_ids.extend(found_ids)
1105 child_ids.extend(found_ids)
1106 break
944
1107
945 return child_ids
1108 return child_ids
946 return _children(repo_id, commit_id)
1109 return _children(repo_id, commit_id)
947
1110
948 @reraise_safe_exceptions
1111 @reraise_safe_exceptions
949 def set_refs(self, wire, key, value):
1112 def set_refs(self, wire, key, value):
950 repo_init = self._factory.repo_libgit2(wire)
1113 repo_init = self._factory.repo_libgit2(wire)
951 with repo_init as repo:
1114 with repo_init as repo:
952 repo.references.create(key, value, force=True)
1115 repo.references.create(key, value, force=True)
953
1116
954 @reraise_safe_exceptions
1117 @reraise_safe_exceptions
1118 def update_refs(self, wire, key, value):
1119 repo_init = self._factory.repo_libgit2(wire)
1120 with repo_init as repo:
1121 if key not in repo.references:
1122 raise ValueError(f'Reference {key} not found in the repository')
1123 repo.references.create(key, value, force=True)
1124
1125 @reraise_safe_exceptions
955 def create_branch(self, wire, branch_name, commit_id, force=False):
1126 def create_branch(self, wire, branch_name, commit_id, force=False):
956 repo_init = self._factory.repo_libgit2(wire)
1127 repo_init = self._factory.repo_libgit2(wire)
957 with repo_init as repo:
1128 with repo_init as repo:
958 commit = repo[commit_id]
1129 if commit_id:
1130 commit = repo[commit_id]
1131 else:
1132 # if commit is not given just use the HEAD
1133 commit = repo.head()
959
1134
960 if force:
1135 if force:
961 repo.branches.local.create(branch_name, commit, force=force)
1136 repo.branches.local.create(branch_name, commit, force=force)
962 elif not repo.branches.get(branch_name):
1137 elif not repo.branches.get(branch_name):
963 # create only if that branch isn't existing
1138 # create only if that branch isn't existing
964 repo.branches.local.create(branch_name, commit, force=force)
1139 repo.branches.local.create(branch_name, commit, force=force)
965
1140
966 @reraise_safe_exceptions
1141 @reraise_safe_exceptions
967 def remove_ref(self, wire, key):
1142 def remove_ref(self, wire, key):
968 repo_init = self._factory.repo_libgit2(wire)
1143 repo_init = self._factory.repo_libgit2(wire)
969 with repo_init as repo:
1144 with repo_init as repo:
970 repo.references.delete(key)
1145 repo.references.delete(key)
971
1146
972 @reraise_safe_exceptions
1147 @reraise_safe_exceptions
973 def tag_remove(self, wire, tag_name):
1148 def tag_remove(self, wire, tag_name):
974 repo_init = self._factory.repo_libgit2(wire)
1149 repo_init = self._factory.repo_libgit2(wire)
975 with repo_init as repo:
1150 with repo_init as repo:
976 key = 'refs/tags/{}'.format(tag_name)
1151 key = f'refs/tags/{tag_name}'
977 repo.references.delete(key)
1152 repo.references.delete(key)
978
1153
979 @reraise_safe_exceptions
1154 @reraise_safe_exceptions
980 def tree_changes(self, wire, source_id, target_id):
1155 def tree_changes(self, wire, source_id, target_id):
981 # TODO(marcink): remove this seems it's only used by tests
982 repo = self._factory.repo(wire)
1156 repo = self._factory.repo(wire)
1157 # source can be empty
1158 source_id = safe_bytes(source_id if source_id else b'')
1159 target_id = safe_bytes(target_id)
1160
983 source = repo[source_id].tree if source_id else None
1161 source = repo[source_id].tree if source_id else None
984 target = repo[target_id].tree
1162 target = repo[target_id].tree
985 result = repo.object_store.tree_changes(source, target)
1163 result = repo.object_store.tree_changes(source, target)
986 return list(result)
1164
1165 added = set()
1166 modified = set()
1167 deleted = set()
1168 for (old_path, new_path), (_, _), (_, _) in list(result):
1169 if new_path and old_path:
1170 modified.add(new_path)
1171 elif new_path and not old_path:
1172 added.add(new_path)
1173 elif not new_path and old_path:
1174 deleted.add(old_path)
1175
1176 return list(added), list(modified), list(deleted)
987
1177
988 @reraise_safe_exceptions
1178 @reraise_safe_exceptions
989 def tree_and_type_for_path(self, wire, commit_id, path):
1179 def tree_and_type_for_path(self, wire, commit_id, path):
990
1180
991 cache_on, context_uid, repo_id = self._cache_on(wire)
1181 cache_on, context_uid, repo_id = self._cache_on(wire)
992 region = self._region(wire)
1182 region = self._region(wire)
1183
993 @region.conditional_cache_on_arguments(condition=cache_on)
1184 @region.conditional_cache_on_arguments(condition=cache_on)
994 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
1185 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
995 repo_init = self._factory.repo_libgit2(wire)
1186 repo_init = self._factory.repo_libgit2(wire)
996
1187
997 with repo_init as repo:
1188 with repo_init as repo:
998 commit = repo[commit_id]
1189 commit = repo[commit_id]
999 try:
1190 try:
1000 tree = commit.tree[path]
1191 tree = commit.tree[path]
1001 except KeyError:
1192 except KeyError:
1002 return None, None, None
1193 return None, None, None
1003
1194
1004 return tree.id.hex, tree.type, tree.filemode
1195 return tree.id.hex, tree.type_str, tree.filemode
1005 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1196 return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
1006
1197
1007 @reraise_safe_exceptions
1198 @reraise_safe_exceptions
1008 def tree_items(self, wire, tree_id):
1199 def tree_items(self, wire, tree_id):
1009 cache_on, context_uid, repo_id = self._cache_on(wire)
1200 cache_on, context_uid, repo_id = self._cache_on(wire)
1010 region = self._region(wire)
1201 region = self._region(wire)
1202
1011 @region.conditional_cache_on_arguments(condition=cache_on)
1203 @region.conditional_cache_on_arguments(condition=cache_on)
1012 def _tree_items(_repo_id, _tree_id):
1204 def _tree_items(_repo_id, _tree_id):
1013
1205
1014 repo_init = self._factory.repo_libgit2(wire)
1206 repo_init = self._factory.repo_libgit2(wire)
1015 with repo_init as repo:
1207 with repo_init as repo:
1016 try:
1208 try:
1017 tree = repo[tree_id]
1209 tree = repo[tree_id]
1018 except KeyError:
1210 except KeyError:
1019 raise ObjectMissing('No tree with id: {}'.format(tree_id))
1211 raise ObjectMissing(f'No tree with id: {tree_id}')
1020
1212
1021 result = []
1213 result = []
1022 for item in tree:
1214 for item in tree:
1023 item_sha = item.hex
1215 item_sha = item.hex
1024 item_mode = item.filemode
1216 item_mode = item.filemode
1025 item_type = item.type
1217 item_type = item.type_str
1026
1218
1027 if item_type == 'commit':
1219 if item_type == 'commit':
1028 # NOTE(marcink): submodules we translate to 'link' for backward compat
1220 # NOTE(marcink): submodules we translate to 'link' for backward compat
1029 item_type = 'link'
1221 item_type = 'link'
1030
1222
1031 result.append((item.name, item_mode, item_sha, item_type))
1223 result.append((item.name, item_mode, item_sha, item_type))
1032 return result
1224 return result
1033 return _tree_items(repo_id, tree_id)
1225 return _tree_items(repo_id, tree_id)
1034
1226
1035 @reraise_safe_exceptions
1227 @reraise_safe_exceptions
1036 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1228 def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1037 """
1229 """
1038 Old version that uses subprocess to call diff
1230 Old version that uses subprocess to call diff
1039 """
1231 """
1040
1232
1041 flags = [
1233 flags = [
1042 '-U%s' % context, '--patch',
1234 f'-U{context}', '--patch',
1043 '--binary',
1235 '--binary',
1044 '--find-renames',
1236 '--find-renames',
1045 '--no-indent-heuristic',
1237 '--no-indent-heuristic',
1046 # '--indent-heuristic',
1238 # '--indent-heuristic',
1047 #'--full-index',
1239 #'--full-index',
1048 #'--abbrev=40'
1240 #'--abbrev=40'
1049 ]
1241 ]
1050
1242
1051 if opt_ignorews:
1243 if opt_ignorews:
1052 flags.append('--ignore-all-space')
1244 flags.append('--ignore-all-space')
1053
1245
1054 if commit_id_1 == self.EMPTY_COMMIT:
1246 if commit_id_1 == self.EMPTY_COMMIT:
1055 cmd = ['show'] + flags + [commit_id_2]
1247 cmd = ['show'] + flags + [commit_id_2]
1056 else:
1248 else:
1057 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1249 cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
1058
1250
1059 if file_filter:
1251 if file_filter:
1060 cmd.extend(['--', file_filter])
1252 cmd.extend(['--', file_filter])
1061
1253
1062 diff, __ = self.run_git_command(wire, cmd)
1254 diff, __ = self.run_git_command(wire, cmd)
1063 # If we used 'show' command, strip first few lines (until actual diff
1255 # If we used 'show' command, strip first few lines (until actual diff
1064 # starts)
1256 # starts)
1065 if commit_id_1 == self.EMPTY_COMMIT:
1257 if commit_id_1 == self.EMPTY_COMMIT:
1066 lines = diff.splitlines()
1258 lines = diff.splitlines()
1067 x = 0
1259 x = 0
1068 for line in lines:
1260 for line in lines:
1069 if line.startswith('diff'):
1261 if line.startswith(b'diff'):
1070 break
1262 break
1071 x += 1
1263 x += 1
1072 # Append new line just like 'diff' command do
1264 # Append new line just like 'diff' command do
1073 diff = '\n'.join(lines[x:]) + '\n'
1265 diff = '\n'.join(lines[x:]) + '\n'
1074 return diff
1266 return diff
1075
1267
1076 @reraise_safe_exceptions
1268 @reraise_safe_exceptions
1077 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1269 def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
1078 repo_init = self._factory.repo_libgit2(wire)
1270 repo_init = self._factory.repo_libgit2(wire)
1271
1079 with repo_init as repo:
1272 with repo_init as repo:
1080 swap = True
1273 swap = True
1081 flags = 0
1274 flags = 0
1082 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1275 flags |= pygit2.GIT_DIFF_SHOW_BINARY
1083
1276
1084 if opt_ignorews:
1277 if opt_ignorews:
1085 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1278 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
1086
1279
1087 if commit_id_1 == self.EMPTY_COMMIT:
1280 if commit_id_1 == self.EMPTY_COMMIT:
1088 comm1 = repo[commit_id_2]
1281 comm1 = repo[commit_id_2]
1089 diff_obj = comm1.tree.diff_to_tree(
1282 diff_obj = comm1.tree.diff_to_tree(
1090 flags=flags, context_lines=context, swap=swap)
1283 flags=flags, context_lines=context, swap=swap)
1091
1284
1092 else:
1285 else:
1093 comm1 = repo[commit_id_2]
1286 comm1 = repo[commit_id_2]
1094 comm2 = repo[commit_id_1]
1287 comm2 = repo[commit_id_1]
1095 diff_obj = comm1.tree.diff_to_tree(
1288 diff_obj = comm1.tree.diff_to_tree(
1096 comm2.tree, flags=flags, context_lines=context, swap=swap)
1289 comm2.tree, flags=flags, context_lines=context, swap=swap)
1097 similar_flags = 0
1290 similar_flags = 0
1098 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1291 similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
1099 diff_obj.find_similar(flags=similar_flags)
1292 diff_obj.find_similar(flags=similar_flags)
1100
1293
1101 if file_filter:
1294 if file_filter:
1102 for p in diff_obj:
1295 for p in diff_obj:
1103 if p.delta.old_file.path == file_filter:
1296 if p.delta.old_file.path == file_filter:
1104 return p.patch or ''
1297 return BytesEnvelope(p.data) or BytesEnvelope(b'')
1105 # fo matching path == no diff
1298 # fo matching path == no diff
1106 return ''
1299 return BytesEnvelope(b'')
1107 return diff_obj.patch or ''
1300
1301 return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
1108
1302
1109 @reraise_safe_exceptions
1303 @reraise_safe_exceptions
1110 def node_history(self, wire, commit_id, path, limit):
1304 def node_history(self, wire, commit_id, path, limit):
1111 cache_on, context_uid, repo_id = self._cache_on(wire)
1305 cache_on, context_uid, repo_id = self._cache_on(wire)
1112 region = self._region(wire)
1306 region = self._region(wire)
1307
1113 @region.conditional_cache_on_arguments(condition=cache_on)
1308 @region.conditional_cache_on_arguments(condition=cache_on)
1114 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1309 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1115 # optimize for n==1, rev-list is much faster for that use-case
1310 # optimize for n==1, rev-list is much faster for that use-case
1116 if limit == 1:
1311 if limit == 1:
1117 cmd = ['rev-list', '-1', commit_id, '--', path]
1312 cmd = ['rev-list', '-1', commit_id, '--', path]
1118 else:
1313 else:
1119 cmd = ['log']
1314 cmd = ['log']
1120 if limit:
1315 if limit:
1121 cmd.extend(['-n', str(safe_int(limit, 0))])
1316 cmd.extend(['-n', str(safe_int(limit, 0))])
1122 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1317 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
1123
1318
1124 output, __ = self.run_git_command(wire, cmd)
1319 output, __ = self.run_git_command(wire, cmd)
1125 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
1320 commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
1126
1321
1127 return [x for x in commit_ids]
1322 return [x for x in commit_ids]
1128 return _node_history(context_uid, repo_id, commit_id, path, limit)
1323 return _node_history(context_uid, repo_id, commit_id, path, limit)
1129
1324
1130 @reraise_safe_exceptions
1325 @reraise_safe_exceptions
1131 def node_annotate(self, wire, commit_id, path):
1326 def node_annotate_legacy(self, wire, commit_id, path):
1132
1327 # note: replaced by pygit2 implementation
1133 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1328 cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
1134 # -l ==> outputs long shas (and we need all 40 characters)
1329 # -l ==> outputs long shas (and we need all 40 characters)
1135 # --root ==> doesn't put '^' character for boundaries
1330 # --root ==> doesn't put '^' character for boundaries
1136 # -r commit_id ==> blames for the given commit
1331 # -r commit_id ==> blames for the given commit
1137 output, __ = self.run_git_command(wire, cmd)
1332 output, __ = self.run_git_command(wire, cmd)
1138
1333
1139 result = []
1334 result = []
1140 for i, blame_line in enumerate(output.split('\n')[:-1]):
1335 for i, blame_line in enumerate(output.splitlines()[:-1]):
1141 line_no = i + 1
1336 line_no = i + 1
1142 commit_id, line = re.split(r' ', blame_line, 1)
1337 blame_commit_id, line = re.split(rb' ', blame_line, 1)
1143 result.append((line_no, commit_id, line))
1338 result.append((line_no, blame_commit_id, line))
1339
1144 return result
1340 return result
1145
1341
1146 @reraise_safe_exceptions
1342 @reraise_safe_exceptions
1147 def update_server_info(self, wire):
1343 def node_annotate(self, wire, commit_id, path):
1148 repo = self._factory.repo(wire)
1344
1149 update_server_info(repo)
1345 result_libgit = []
1346 repo_init = self._factory.repo_libgit2(wire)
1347 with repo_init as repo:
1348 commit = repo[commit_id]
1349 blame_obj = repo.blame(path, newest_commit=commit_id)
1350 for i, line in enumerate(commit.tree[path].data.splitlines()):
1351 line_no = i + 1
1352 hunk = blame_obj.for_line(line_no)
1353 blame_commit_id = hunk.final_commit_id.hex
1354
1355 result_libgit.append((line_no, blame_commit_id, line))
1356
1357 return BinaryEnvelope(result_libgit)
1358
1359 @reraise_safe_exceptions
1360 def update_server_info(self, wire, force=False):
1361 cmd = ['update-server-info']
1362 if force:
1363 cmd += ['--force']
1364 output, __ = self.run_git_command(wire, cmd)
1365 return output.splitlines()
1150
1366
1151 @reraise_safe_exceptions
1367 @reraise_safe_exceptions
1152 def get_all_commit_ids(self, wire):
1368 def get_all_commit_ids(self, wire):
1153
1369
1154 cache_on, context_uid, repo_id = self._cache_on(wire)
1370 cache_on, context_uid, repo_id = self._cache_on(wire)
1155 region = self._region(wire)
1371 region = self._region(wire)
1372
1156 @region.conditional_cache_on_arguments(condition=cache_on)
1373 @region.conditional_cache_on_arguments(condition=cache_on)
1157 def _get_all_commit_ids(_context_uid, _repo_id):
1374 def _get_all_commit_ids(_context_uid, _repo_id):
1158
1375
1159 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1376 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1160 try:
1377 try:
1161 output, __ = self.run_git_command(wire, cmd)
1378 output, __ = self.run_git_command(wire, cmd)
1162 return output.splitlines()
1379 return output.splitlines()
1163 except Exception:
1380 except Exception:
1164 # Can be raised for empty repositories
1381 # Can be raised for empty repositories
1165 return []
1382 return []
1383
1384 @region.conditional_cache_on_arguments(condition=cache_on)
1385 def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
1386 repo_init = self._factory.repo_libgit2(wire)
1387 from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
1388 results = []
1389 with repo_init as repo:
1390 for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
1391 results.append(commit.id.hex)
1392
1166 return _get_all_commit_ids(context_uid, repo_id)
1393 return _get_all_commit_ids(context_uid, repo_id)
1167
1394
1168 @reraise_safe_exceptions
1395 @reraise_safe_exceptions
1169 def run_git_command(self, wire, cmd, **opts):
1396 def run_git_command(self, wire, cmd, **opts):
1170 path = wire.get('path', None)
1397 path = wire.get('path', None)
1398 debug_mode = rhodecode.ConfigGet().get_bool('debug')
1171
1399
1172 if path and os.path.isdir(path):
1400 if path and os.path.isdir(path):
1173 opts['cwd'] = path
1401 opts['cwd'] = path
1174
1402
1175 if '_bare' in opts:
1403 if '_bare' in opts:
1176 _copts = []
1404 _copts = []
1177 del opts['_bare']
1405 del opts['_bare']
1178 else:
1406 else:
1179 _copts = ['-c', 'core.quotepath=false', ]
1407 _copts = ['-c', 'core.quotepath=false', '-c', 'advice.diverging=false']
1180 safe_call = False
1408 safe_call = False
1181 if '_safe' in opts:
1409 if '_safe' in opts:
1182 # no exc on failure
1410 # no exc on failure
1183 del opts['_safe']
1411 del opts['_safe']
1184 safe_call = True
1412 safe_call = True
1185
1413
1186 if '_copts' in opts:
1414 if '_copts' in opts:
1187 _copts.extend(opts['_copts'] or [])
1415 _copts.extend(opts['_copts'] or [])
1188 del opts['_copts']
1416 del opts['_copts']
1189
1417
1190 gitenv = os.environ.copy()
1418 gitenv = os.environ.copy()
1191 gitenv.update(opts.pop('extra_env', {}))
1419 gitenv.update(opts.pop('extra_env', {}))
1192 # need to clean fix GIT_DIR !
1420 # need to clean fix GIT_DIR !
1193 if 'GIT_DIR' in gitenv:
1421 if 'GIT_DIR' in gitenv:
1194 del gitenv['GIT_DIR']
1422 del gitenv['GIT_DIR']
1195 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1423 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
1196 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1424 gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
1197
1425
1198 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1426 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
1199 _opts = {'env': gitenv, 'shell': False}
1427 _opts = {'env': gitenv, 'shell': False}
1200
1428
1201 proc = None
1429 proc = None
1202 try:
1430 try:
1203 _opts.update(opts)
1431 _opts.update(opts)
1204 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1432 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
1205
1433
1206 return ''.join(proc), ''.join(proc.error)
1434 return b''.join(proc), b''.join(proc.stderr)
1207 except (EnvironmentError, OSError) as err:
1435 except OSError as err:
1208 cmd = ' '.join(cmd) # human friendly CMD
1436 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
1209 tb_err = ("Couldn't run git command (%s).\n"
1437 call_opts = {}
1210 "Original error was:%s\n"
1438 if debug_mode:
1211 "Call options:%s\n"
1439 call_opts = _opts
1212 % (cmd, err, _opts))
1440
1441 tb_err = ("Couldn't run git command ({}).\n"
1442 "Original error was:{}\n"
1443 "Call options:{}\n"
1444 .format(cmd, err, call_opts))
1213 log.exception(tb_err)
1445 log.exception(tb_err)
1214 if safe_call:
1446 if safe_call:
1215 return '', err
1447 return '', err
1216 else:
1448 else:
1217 raise exceptions.VcsException()(tb_err)
1449 raise exceptions.VcsException()(tb_err)
1218 finally:
1450 finally:
1219 if proc:
1451 if proc:
1220 proc.close()
1452 proc.close()
1221
1453
1222 @reraise_safe_exceptions
1454 @reraise_safe_exceptions
1223 def install_hooks(self, wire, force=False):
1455 def install_hooks(self, wire, force=False):
1224 from vcsserver.hook_utils import install_git_hooks
1456 from vcsserver.hook_utils import install_git_hooks
1225 bare = self.bare(wire)
1457 bare = self.bare(wire)
1226 path = wire['path']
1458 path = wire['path']
1459 binary_dir = settings.BINARY_DIR
1460 if binary_dir:
1461 os.path.join(binary_dir, 'python3')
1227 return install_git_hooks(path, bare, force_create=force)
1462 return install_git_hooks(path, bare, force_create=force)
1228
1463
1229 @reraise_safe_exceptions
1464 @reraise_safe_exceptions
1230 def get_hooks_info(self, wire):
1465 def get_hooks_info(self, wire):
1231 from vcsserver.hook_utils import (
1466 from vcsserver.hook_utils import (
1232 get_git_pre_hook_version, get_git_post_hook_version)
1467 get_git_pre_hook_version, get_git_post_hook_version)
1233 bare = self.bare(wire)
1468 bare = self.bare(wire)
1234 path = wire['path']
1469 path = wire['path']
1235 return {
1470 return {
1236 'pre_version': get_git_pre_hook_version(path, bare),
1471 'pre_version': get_git_pre_hook_version(path, bare),
1237 'post_version': get_git_post_hook_version(path, bare),
1472 'post_version': get_git_post_hook_version(path, bare),
1238 }
1473 }
1239
1474
1240 @reraise_safe_exceptions
1475 @reraise_safe_exceptions
1241 def set_head_ref(self, wire, head_name):
1476 def set_head_ref(self, wire, head_name):
1242 log.debug('Setting refs/head to `%s`', head_name)
1477 log.debug('Setting refs/head to `%s`', head_name)
1243 cmd = ['symbolic-ref', 'HEAD', 'refs/heads/%s' % head_name]
1478 repo_init = self._factory.repo_libgit2(wire)
1244 output, __ = self.run_git_command(wire, cmd)
1479 with repo_init as repo:
1245 return [head_name] + output.splitlines()
1480 repo.set_head(f'refs/heads/{head_name}')
1481
1482 return [head_name] + [f'set HEAD to refs/heads/{head_name}']
1246
1483
1247 @reraise_safe_exceptions
1484 @reraise_safe_exceptions
1248 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1485 def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
1249 archive_dir_name, commit_id):
1486 archive_dir_name, commit_id, cache_config):
1250
1487
1251 def file_walker(_commit_id, path):
1488 def file_walker(_commit_id, path):
1252 repo_init = self._factory.repo_libgit2(wire)
1489 repo_init = self._factory.repo_libgit2(wire)
1253
1490
1254 with repo_init as repo:
1491 with repo_init as repo:
1255 commit = repo[commit_id]
1492 commit = repo[commit_id]
1256
1493
1257 if path in ['', '/']:
1494 if path in ['', '/']:
1258 tree = commit.tree
1495 tree = commit.tree
1259 else:
1496 else:
1260 tree = commit.tree[path.rstrip('/')]
1497 tree = commit.tree[path.rstrip('/')]
1261 tree_id = tree.id.hex
1498 tree_id = tree.id.hex
1262 try:
1499 try:
1263 tree = repo[tree_id]
1500 tree = repo[tree_id]
1264 except KeyError:
1501 except KeyError:
1265 raise ObjectMissing('No tree with id: {}'.format(tree_id))
1502 raise ObjectMissing(f'No tree with id: {tree_id}')
1266
1503
1267 index = LibGit2Index.Index()
1504 index = LibGit2Index.Index()
1268 index.read_tree(tree)
1505 index.read_tree(tree)
1269 file_iter = index
1506 file_iter = index
1270
1507
1271 for fn in file_iter:
1508 for file_node in file_iter:
1272 file_path = fn.path
1509 file_path = file_node.path
1273 mode = fn.mode
1510 mode = file_node.mode
1274 is_link = stat.S_ISLNK(mode)
1511 is_link = stat.S_ISLNK(mode)
1275 if mode == pygit2.GIT_FILEMODE_COMMIT:
1512 if mode == pygit2.GIT_FILEMODE_COMMIT:
1276 log.debug('Skipping path %s as a commit node', file_path)
1513 log.debug('Skipping path %s as a commit node', file_path)
1277 continue
1514 continue
1278 yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw)
1515 yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
1279
1516
1280 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1517 return store_archive_in_cache(
1281 archive_dir_name, commit_id)
1518 file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
1 NO CONTENT: file renamed from vcsserver/hg.py to vcsserver/remote/hg_remote.py
NO CONTENT: file renamed from vcsserver/hg.py to vcsserver/remote/hg_remote.py
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: file renamed from vcsserver/svn.py to vcsserver/remote/svn_remote.py
NO CONTENT: file renamed from vcsserver/svn.py to vcsserver/remote/svn_remote.py
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
This diff has been collapsed as it changes many lines, (1103 lines changed) Show them Hide them
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now