##// END OF EJS Templates
merge: with lataest default branch
super-admin -
r1041:e1e68d19 merge python3
parent child Browse files
Show More
@@ -0,0 +1,16 b''
1 [DEFAULT]
2 done = false
3
4 [task:bump_version]
5 done = true
6
7 [task:fixes_on_stable]
8 done = true
9
10 [task:pip2nix_generated]
11 done = true
12
13 [release]
14 state = prepared
15 version = 4.27.1
16
@@ -0,0 +1,53 b''
1 ; #####################
2 ; LOGGING CONFIGURATION
3 ; #####################
4 ; Logging template, used for configure the logging
5 ; some variables here are replaced by RhodeCode to default values
6
7 [loggers]
8 keys = root, vcsserver
9
10 [handlers]
11 keys = console
12
13 [formatters]
14 keys = generic, json
15
16 ; #######
17 ; LOGGERS
18 ; #######
19 [logger_root]
20 level = NOTSET
21 handlers = console
22
23 [logger_vcsserver]
24 level = $RC_LOGGING_LEVEL
25 handlers =
26 qualname = vcsserver
27 propagate = 1
28
29 ; ########
30 ; HANDLERS
31 ; ########
32
33 [handler_console]
34 class = StreamHandler
35 args = (sys.stderr, )
36 level = $RC_LOGGING_LEVEL
37 ; To enable JSON formatted logs replace generic with json
38 ; This allows sending properly formatted logs to grafana loki or elasticsearch
39 #formatter = json
40 #formatter = generic
41 formatter = $RC_LOGGING_FORMATTER
42
43 ; ##########
44 ; FORMATTERS
45 ; ##########
46
47 [formatter_generic]
48 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
49 datefmt = %Y-%m-%d %H:%M:%S
50
51 [formatter_json]
52 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
53 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -0,0 +1,10 b''
1 diff -rup configparser-4.0.2-orig/pyproject.toml configparser-4.0.2/pyproject.toml
2 --- configparser-4.0.2-orig/pyproject.toml 2021-03-22 21:28:11.000000000 +0100
3 +++ configparser-4.0.2/pyproject.toml 2021-03-22 21:28:11.000000000 +0100
4 @@ -1,5 +1,5 @@
5 [build-system]
6 -requires = ["setuptools>=40.7", "wheel", "setuptools_scm>=1.15"]
7 +requires = ["setuptools<=42.0", "wheel", "setuptools_scm<6.0.0"]
8 build-backend = "setuptools.build_meta"
9
10 [tool.black]
@@ -0,0 +1,7 b''
1 diff -rup importlib-metadata-1.6.0-orig/yproject.toml importlib-metadata-1.6.0/pyproject.toml
2 --- importlib-metadata-1.6.0-orig/yproject.toml 2021-03-22 22:10:33.000000000 +0100
3 +++ importlib-metadata-1.6.0/pyproject.toml 2021-03-22 22:11:09.000000000 +0100
4 @@ -1,3 +1,3 @@
5 [build-system]
6 -requires = ["setuptools>=30.3", "wheel", "setuptools_scm"]
7 +requires = ["setuptools<42.0", "wheel", "setuptools_scm<6.0.0"]
@@ -0,0 +1,12 b''
1 diff -rup pytest-4.6.9-orig/setup.py pytest-4.6.9/setup.py
2 --- pytest-4.6.9-orig/setup.py 2018-04-10 10:23:04.000000000 +0200
3 +++ pytest-4.6.9/setup.py 2018-04-10 10:23:34.000000000 +0200
4 @@ -24,7 +24,7 @@ def main():
5 def main():
6 setup(
7 use_scm_version={"write_to": "src/_pytest/_version.py"},
8 - setup_requires=["setuptools-scm", "setuptools>=40.0"],
9 + setup_requires=["setuptools-scm<6.0.0", "setuptools<=42.0"],
10 package_dir={"": "src"},
11 # fmt: off
12 extras_require={ No newline at end of file
@@ -0,0 +1,10 b''
1 diff -rup zip-1.2.0-orig/pyproject.toml zip-1.2.0/pyproject.toml
2 --- zip-1.2.0-orig/pyproject.toml 2021-03-23 10:55:37.000000000 +0100
3 +++ zip-1.2.0/pyproject.toml 2021-03-23 10:56:05.000000000 +0100
4 @@ -1,5 +1,5 @@
5 [build-system]
6 -requires = ["setuptools>=34.4", "wheel", "setuptools_scm>=1.15"]
7 +requires = ["setuptools<42.0", "wheel", "setuptools_scm<6.0.0"]
8 build-backend = "setuptools.build_meta"
9
10 [tool.black]
1 NO CONTENT: new file 100644
NO CONTENT: new file 100644
@@ -0,0 +1,207 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2020 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import os
22 import textwrap
23 import string
24 import functools
25 import logging
26 import tempfile
27 import logging.config
28 log = logging.getLogger(__name__)
29
30 # skip keys, that are set here, so we don't double process those
31 set_keys = {
32 '__file__': ''
33 }
34
35
36 def str2bool(_str):
37 """
38 returns True/False value from given string, it tries to translate the
39 string into boolean
40
41 :param _str: string value to translate into boolean
42 :rtype: boolean
43 :returns: boolean from given string
44 """
45 if _str is None:
46 return False
47 if _str in (True, False):
48 return _str
49 _str = str(_str).strip().lower()
50 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
51
52
53 def aslist(obj, sep=None, strip=True):
54 """
55 Returns given string separated by sep as list
56
57 :param obj:
58 :param sep:
59 :param strip:
60 """
61 if isinstance(obj, (basestring,)):
62 if obj in ['', ""]:
63 return []
64
65 lst = obj.split(sep)
66 if strip:
67 lst = [v.strip() for v in lst]
68 return lst
69 elif isinstance(obj, (list, tuple)):
70 return obj
71 elif obj is None:
72 return []
73 else:
74 return [obj]
75
76
77 class SettingsMaker(object):
78
79 def __init__(self, app_settings):
80 self.settings = app_settings
81
82 @classmethod
83 def _bool_func(cls, input_val):
84 if isinstance(input_val, unicode):
85 input_val = input_val.encode('utf8')
86 return str2bool(input_val)
87
88 @classmethod
89 def _int_func(cls, input_val):
90 return int(input_val)
91
92 @classmethod
93 def _list_func(cls, input_val, sep=','):
94 return aslist(input_val, sep=sep)
95
96 @classmethod
97 def _string_func(cls, input_val, lower=True):
98 if lower:
99 input_val = input_val.lower()
100 return input_val
101
102 @classmethod
103 def _float_func(cls, input_val):
104 return float(input_val)
105
106 @classmethod
107 def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
108
109 # ensure we have our dir created
110 if not os.path.isdir(input_val) and ensure_dir:
111 os.makedirs(input_val, mode=mode)
112
113 if not os.path.isdir(input_val):
114 raise Exception('Dir at {} does not exist'.format(input_val))
115 return input_val
116
117 @classmethod
118 def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
119 dirname = os.path.dirname(input_val)
120 cls._dir_func(dirname, ensure_dir=ensure_dir)
121 return input_val
122
123 @classmethod
124 def _key_transformator(cls, key):
125 return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
126
127 def maybe_env_key(self, key):
128 # now maybe we have this KEY in env, search and use the value with higher priority.
129 transformed_key = self._key_transformator(key)
130 envvar_value = os.environ.get(transformed_key)
131 if envvar_value:
132 log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
133
134 return envvar_value
135
136 def env_expand(self):
137 replaced = {}
138 for k, v in self.settings.items():
139 if k not in set_keys:
140 envvar_value = self.maybe_env_key(k)
141 if envvar_value:
142 replaced[k] = envvar_value
143 set_keys[k] = envvar_value
144
145 # replace ALL keys updated
146 self.settings.update(replaced)
147
148 def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
149 """
150 Helper to enable debug on running instance
151 :return:
152 """
153
154 if not str2bool(self.settings.get('logging.autoconfigure')):
155 log.info('logging configuration based on main .ini file')
156 return
157
158 if logging_conf is None:
159 logging_conf = self.settings.get('logging.logging_conf_file') or ''
160
161 if not os.path.isfile(logging_conf):
162 log.error('Unable to setup logging based on %s, '
163 'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
164 return
165
166 with open(logging_conf, 'rb') as f:
167 ini_template = textwrap.dedent(f.read())
168 ini_template = string.Template(ini_template).safe_substitute(
169 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
170 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
171 )
172
173 with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
174 log.info('Saved Temporary LOGGING config at %s', f.name)
175 f.write(ini_template)
176
177 logging.config.fileConfig(f.name)
178 os.remove(f.name)
179
180 def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
181 input_val = self.settings.get(key, default)
182
183 if default_when_empty and not input_val:
184 # use default value when value is set in the config but it is empty
185 input_val = default
186
187 parser_func = {
188 'bool': self._bool_func,
189 'int': self._int_func,
190 'list': self._list_func,
191 'list:newline': functools.partial(self._list_func, sep='/n'),
192 'list:spacesep': functools.partial(self._list_func, sep=' '),
193 'string': functools.partial(self._string_func, lower=lower),
194 'dir': self._dir_func,
195 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
196 'file': self._file_path_func,
197 'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
198 None: lambda i: i
199 }[parser]
200
201 envvar_value = self.maybe_env_key(key)
202 if envvar_value:
203 input_val = envvar_value
204 set_keys[key] = input_val
205
206 self.settings[key] = parser_func(input_val)
207 return self.settings[key]
@@ -0,0 +1,26 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 # This package contains non rhodecode licensed packages that are
19 # vendored for various reasons
20
21 import os
22 import sys
23
24 vendor_dir = os.path.abspath(os.path.dirname(__file__))
25
26 sys.path.append(vendor_dir)
@@ -0,0 +1,243 b''
1 '''
2 This library is provided to allow standard python logging
3 to output log data as JSON formatted strings
4 '''
5 import logging
6 import json
7 import re
8 from datetime import date, datetime, time, tzinfo, timedelta
9 import traceback
10 import importlib
11
12 from inspect import istraceback
13
14 from collections import OrderedDict
15
16
17 def _inject_req_id(record, *args, **kwargs):
18 return record
19
20
21 ExceptionAwareFormatter = logging.Formatter
22
23
24 ZERO = timedelta(0)
25 HOUR = timedelta(hours=1)
26
27
28 class UTC(tzinfo):
29 """UTC"""
30
31 def utcoffset(self, dt):
32 return ZERO
33
34 def tzname(self, dt):
35 return "UTC"
36
37 def dst(self, dt):
38 return ZERO
39
40 utc = UTC()
41
42
43 # skip natural LogRecord attributes
44 # http://docs.python.org/library/logging.html#logrecord-attributes
45 RESERVED_ATTRS = (
46 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
47 'funcName', 'levelname', 'levelno', 'lineno', 'module',
48 'msecs', 'message', 'msg', 'name', 'pathname', 'process',
49 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName')
50
51
52 def merge_record_extra(record, target, reserved):
53 """
54 Merges extra attributes from LogRecord object into target dictionary
55
56 :param record: logging.LogRecord
57 :param target: dict to update
58 :param reserved: dict or list with reserved keys to skip
59 """
60 for key, value in record.__dict__.items():
61 # this allows to have numeric keys
62 if (key not in reserved
63 and not (hasattr(key, "startswith")
64 and key.startswith('_'))):
65 target[key] = value
66 return target
67
68
69 class JsonEncoder(json.JSONEncoder):
70 """
71 A custom encoder extending the default JSONEncoder
72 """
73
74 def default(self, obj):
75 if isinstance(obj, (date, datetime, time)):
76 return self.format_datetime_obj(obj)
77
78 elif istraceback(obj):
79 return ''.join(traceback.format_tb(obj)).strip()
80
81 elif type(obj) == Exception \
82 or isinstance(obj, Exception) \
83 or type(obj) == type:
84 return str(obj)
85
86 try:
87 return super(JsonEncoder, self).default(obj)
88
89 except TypeError:
90 try:
91 return str(obj)
92
93 except Exception:
94 return None
95
96 def format_datetime_obj(self, obj):
97 return obj.isoformat()
98
99
100 class JsonFormatter(ExceptionAwareFormatter):
101 """
102 A custom formatter to format logging records as json strings.
103 Extra values will be formatted as str() if not supported by
104 json default encoder
105 """
106
107 def __init__(self, *args, **kwargs):
108 """
109 :param json_default: a function for encoding non-standard objects
110 as outlined in http://docs.python.org/2/library/json.html
111 :param json_encoder: optional custom encoder
112 :param json_serializer: a :meth:`json.dumps`-compatible callable
113 that will be used to serialize the log record.
114 :param json_indent: an optional :meth:`json.dumps`-compatible numeric value
115 that will be used to customize the indent of the output json.
116 :param prefix: an optional string prefix added at the beginning of
117 the formatted string
118 :param json_indent: indent parameter for json.dumps
119 :param json_ensure_ascii: ensure_ascii parameter for json.dumps
120 :param reserved_attrs: an optional list of fields that will be skipped when
121 outputting json log record. Defaults to all log record attributes:
122 http://docs.python.org/library/logging.html#logrecord-attributes
123 :param timestamp: an optional string/boolean field to add a timestamp when
124 outputting the json log record. If string is passed, timestamp will be added
125 to log record using string as key. If True boolean is passed, timestamp key
126 will be "timestamp". Defaults to False/off.
127 """
128 self.json_default = self._str_to_fn(kwargs.pop("json_default", None))
129 self.json_encoder = self._str_to_fn(kwargs.pop("json_encoder", None))
130 self.json_serializer = self._str_to_fn(kwargs.pop("json_serializer", json.dumps))
131 self.json_indent = kwargs.pop("json_indent", None)
132 self.json_ensure_ascii = kwargs.pop("json_ensure_ascii", True)
133 self.prefix = kwargs.pop("prefix", "")
134 reserved_attrs = kwargs.pop("reserved_attrs", RESERVED_ATTRS)
135 self.reserved_attrs = dict(zip(reserved_attrs, reserved_attrs))
136 self.timestamp = kwargs.pop("timestamp", True)
137
138 # super(JsonFormatter, self).__init__(*args, **kwargs)
139 logging.Formatter.__init__(self, *args, **kwargs)
140 if not self.json_encoder and not self.json_default:
141 self.json_encoder = JsonEncoder
142
143 self._required_fields = self.parse()
144 self._skip_fields = dict(zip(self._required_fields,
145 self._required_fields))
146 self._skip_fields.update(self.reserved_attrs)
147
148 def _str_to_fn(self, fn_as_str):
149 """
150 If the argument is not a string, return whatever was passed in.
151 Parses a string such as package.module.function, imports the module
152 and returns the function.
153
154 :param fn_as_str: The string to parse. If not a string, return it.
155 """
156 if not isinstance(fn_as_str, str):
157 return fn_as_str
158
159 path, _, function = fn_as_str.rpartition('.')
160 module = importlib.import_module(path)
161 return getattr(module, function)
162
163 def parse(self):
164 """
165 Parses format string looking for substitutions
166
167 This method is responsible for returning a list of fields (as strings)
168 to include in all log messages.
169 """
170 standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE)
171 return standard_formatters.findall(self._fmt)
172
173 def add_fields(self, log_record, record, message_dict):
174 """
175 Override this method to implement custom logic for adding fields.
176 """
177 for field in self._required_fields:
178 log_record[field] = record.__dict__.get(field)
179 log_record.update(message_dict)
180 merge_record_extra(record, log_record, reserved=self._skip_fields)
181
182 if self.timestamp:
183 key = self.timestamp if type(self.timestamp) == str else 'timestamp'
184 log_record[key] = datetime.fromtimestamp(record.created, tz=utc)
185
186 def process_log_record(self, log_record):
187 """
188 Override this method to implement custom logic
189 on the possibly ordered dictionary.
190 """
191 return log_record
192
193 def jsonify_log_record(self, log_record):
194 """Returns a json string of the log record."""
195 return self.json_serializer(log_record,
196 default=self.json_default,
197 cls=self.json_encoder,
198 indent=self.json_indent,
199 ensure_ascii=self.json_ensure_ascii)
200
201 def serialize_log_record(self, log_record):
202 """Returns the final representation of the log record."""
203 return "%s%s" % (self.prefix, self.jsonify_log_record(log_record))
204
205 def format(self, record):
206 """Formats a log record and serializes to json"""
207 message_dict = {}
208 # FIXME: logging.LogRecord.msg and logging.LogRecord.message in typeshed
209 # are always type of str. We shouldn't need to override that.
210 if isinstance(record.msg, dict):
211 message_dict = record.msg
212 record.message = None
213 else:
214 record.message = record.getMessage()
215 # only format time if needed
216 if "asctime" in self._required_fields:
217 record.asctime = self.formatTime(record, self.datefmt)
218
219 # Display formatted exception, but allow overriding it in the
220 # user-supplied dict.
221 if record.exc_info and not message_dict.get('exc_info'):
222 message_dict['exc_info'] = self.formatException(record.exc_info)
223 if not message_dict.get('exc_info') and record.exc_text:
224 message_dict['exc_info'] = record.exc_text
225 # Display formatted record of stack frames
226 # default format is a string returned from :func:`traceback.print_stack`
227 try:
228 if record.stack_info and not message_dict.get('stack_info'):
229 message_dict['stack_info'] = self.formatStack(record.stack_info)
230 except AttributeError:
231 # Python2.7 doesn't have stack_info.
232 pass
233
234 try:
235 log_record = OrderedDict()
236 except NameError:
237 log_record = {}
238
239 _inject_req_id(record, with_prefix=False)
240 self.add_fields(log_record, record, message_dict)
241 log_record = self.process_log_record(log_record)
242
243 return self.serialize_log_record(log_record)
@@ -0,0 +1,390 b''
1 import sys
2 import threading
3 import weakref
4 from base64 import b64encode
5 from logging import getLogger
6 from os import urandom
7
8 from redis import StrictRedis
9
10 __version__ = '3.7.0'
11
12 loggers = {
13 k: getLogger("vcsserver." + ".".join((__name__, k)))
14 for k in [
15 "acquire",
16 "refresh.thread.start",
17 "refresh.thread.stop",
18 "refresh.thread.exit",
19 "refresh.start",
20 "refresh.shutdown",
21 "refresh.exit",
22 "release",
23 ]
24 }
25
26 PY3 = sys.version_info[0] == 3
27
28 if PY3:
29 text_type = str
30 binary_type = bytes
31 else:
32 text_type = unicode # noqa
33 binary_type = str
34
35
36 # Check if the id match. If not, return an error code.
37 UNLOCK_SCRIPT = b"""
38 if redis.call("get", KEYS[1]) ~= ARGV[1] then
39 return 1
40 else
41 redis.call("del", KEYS[2])
42 redis.call("lpush", KEYS[2], 1)
43 redis.call("pexpire", KEYS[2], ARGV[2])
44 redis.call("del", KEYS[1])
45 return 0
46 end
47 """
48
49 # Covers both cases when key doesn't exist and doesn't equal to lock's id
50 EXTEND_SCRIPT = b"""
51 if redis.call("get", KEYS[1]) ~= ARGV[1] then
52 return 1
53 elseif redis.call("ttl", KEYS[1]) < 0 then
54 return 2
55 else
56 redis.call("expire", KEYS[1], ARGV[2])
57 return 0
58 end
59 """
60
61 RESET_SCRIPT = b"""
62 redis.call('del', KEYS[2])
63 redis.call('lpush', KEYS[2], 1)
64 redis.call('pexpire', KEYS[2], ARGV[2])
65 return redis.call('del', KEYS[1])
66 """
67
68 RESET_ALL_SCRIPT = b"""
69 local locks = redis.call('keys', 'lock:*')
70 local signal
71 for _, lock in pairs(locks) do
72 signal = 'lock-signal:' .. string.sub(lock, 6)
73 redis.call('del', signal)
74 redis.call('lpush', signal, 1)
75 redis.call('expire', signal, 1)
76 redis.call('del', lock)
77 end
78 return #locks
79 """
80
81
82 class AlreadyAcquired(RuntimeError):
83 pass
84
85
86 class NotAcquired(RuntimeError):
87 pass
88
89
90 class AlreadyStarted(RuntimeError):
91 pass
92
93
94 class TimeoutNotUsable(RuntimeError):
95 pass
96
97
98 class InvalidTimeout(RuntimeError):
99 pass
100
101
102 class TimeoutTooLarge(RuntimeError):
103 pass
104
105
106 class NotExpirable(RuntimeError):
107 pass
108
109
110 class Lock(object):
111 """
112 A Lock context manager implemented via redis SETNX/BLPOP.
113 """
114 unlock_script = None
115 extend_script = None
116 reset_script = None
117 reset_all_script = None
118
119 def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
120 """
121 :param redis_client:
122 An instance of :class:`~StrictRedis`.
123 :param name:
124 The name (redis key) the lock should have.
125 :param expire:
126 The lock expiry time in seconds. If left at the default (None)
127 the lock will not expire.
128 :param id:
129 The ID (redis value) the lock should have. A random value is
130 generated when left at the default.
131
132 Note that if you specify this then the lock is marked as "held". Acquires
133 won't be possible.
134 :param auto_renewal:
135 If set to ``True``, Lock will automatically renew the lock so that it
136 doesn't expire for as long as the lock is held (acquire() called
137 or running in a context manager).
138
139 Implementation note: Renewal will happen using a daemon thread with
140 an interval of ``expire*2/3``. If wishing to use a different renewal
141 time, subclass Lock, call ``super().__init__()`` then set
142 ``self._lock_renewal_interval`` to your desired interval.
143 :param strict:
144 If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
145 :param signal_expire:
146 Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
147 """
148 if strict and not isinstance(redis_client, StrictRedis):
149 raise ValueError("redis_client must be instance of StrictRedis. "
150 "Use strict=False if you know what you're doing.")
151 if auto_renewal and expire is None:
152 raise ValueError("Expire may not be None when auto_renewal is set")
153
154 self._client = redis_client
155
156 if expire:
157 expire = int(expire)
158 if expire < 0:
159 raise ValueError("A negative expire is not acceptable.")
160 else:
161 expire = None
162 self._expire = expire
163
164 self._signal_expire = signal_expire
165 if id is None:
166 self._id = b64encode(urandom(18)).decode('ascii')
167 elif isinstance(id, binary_type):
168 try:
169 self._id = id.decode('ascii')
170 except UnicodeDecodeError:
171 self._id = b64encode(id).decode('ascii')
172 elif isinstance(id, text_type):
173 self._id = id
174 else:
175 raise TypeError("Incorrect type for `id`. Must be bytes/str not %s." % type(id))
176 self._name = 'lock:' + name
177 self._signal = 'lock-signal:' + name
178 self._lock_renewal_interval = (float(expire) * 2 / 3
179 if auto_renewal
180 else None)
181 self._lock_renewal_thread = None
182
183 self.register_scripts(redis_client)
184
185 @classmethod
186 def register_scripts(cls, redis_client):
187 global reset_all_script
188 if reset_all_script is None:
189 reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
190 cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
191 cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
192 cls.reset_script = redis_client.register_script(RESET_SCRIPT)
193 cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
194
195 @property
196 def _held(self):
197 return self.id == self.get_owner_id()
198
199 def reset(self):
200 """
201 Forcibly deletes the lock. Use this with care.
202 """
203 self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
204
205 @property
206 def id(self):
207 return self._id
208
209 def get_owner_id(self):
210 owner_id = self._client.get(self._name)
211 if isinstance(owner_id, binary_type):
212 owner_id = owner_id.decode('ascii', 'replace')
213 return owner_id
214
215 def acquire(self, blocking=True, timeout=None):
216 """
217 :param blocking:
218 Boolean value specifying whether lock should be blocking or not.
219 :param timeout:
220 An integer value specifying the maximum number of seconds to block.
221 """
222 logger = loggers["acquire"]
223
224 logger.debug("Getting blocking: %s acquire on %r ...", blocking, self._name)
225
226 if self._held:
227 owner_id = self.get_owner_id()
228 raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
229
230 if not blocking and timeout is not None:
231 raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
232
233 if timeout:
234 timeout = int(timeout)
235 if timeout < 0:
236 raise InvalidTimeout("Timeout (%d) cannot be less than or equal to 0" % timeout)
237
238 if self._expire and not self._lock_renewal_interval and timeout > self._expire:
239 raise TimeoutTooLarge("Timeout (%d) cannot be greater than expire (%d)" % (timeout, self._expire))
240
241 busy = True
242 blpop_timeout = timeout or self._expire or 0
243 timed_out = False
244 while busy:
245 busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
246 if busy:
247 if timed_out:
248 return False
249 elif blocking:
250 timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
251 else:
252 logger.warning("Failed to get %r.", self._name)
253 return False
254
255 logger.debug("Got lock for %r.", self._name)
256 if self._lock_renewal_interval is not None:
257 self._start_lock_renewer()
258 return True
259
260 def extend(self, expire=None):
261 """Extends expiration time of the lock.
262
263 :param expire:
264 New expiration time. If ``None`` - `expire` provided during
265 lock initialization will be taken.
266 """
267 if expire:
268 expire = int(expire)
269 if expire < 0:
270 raise ValueError("A negative expire is not acceptable.")
271 elif self._expire is not None:
272 expire = self._expire
273 else:
274 raise TypeError(
275 "To extend a lock 'expire' must be provided as an "
276 "argument to extend() method or at initialization time."
277 )
278
279 error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
280 if error == 1:
281 raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
282 elif error == 2:
283 raise NotExpirable("Lock %s has no assigned expiration time" % self._name)
284 elif error:
285 raise RuntimeError("Unsupported error code %s from EXTEND script" % error)
286
287 @staticmethod
288 def _lock_renewer(lockref, interval, stop):
289 """
290 Renew the lock key in redis every `interval` seconds for as long
291 as `self._lock_renewal_thread.should_exit` is False.
292 """
293 while not stop.wait(timeout=interval):
294 loggers["refresh.thread.start"].debug("Refreshing lock")
295 lock = lockref()
296 if lock is None:
297 loggers["refresh.thread.stop"].debug(
298 "The lock no longer exists, stopping lock refreshing"
299 )
300 break
301 lock.extend(expire=lock._expire)
302 del lock
303 loggers["refresh.thread.exit"].debug("Exit requested, stopping lock refreshing")
304
305 def _start_lock_renewer(self):
306 """
307 Starts the lock refresher thread.
308 """
309 if self._lock_renewal_thread is not None:
310 raise AlreadyStarted("Lock refresh thread already started")
311
312 loggers["refresh.start"].debug(
313 "Starting thread to refresh lock every %s seconds",
314 self._lock_renewal_interval
315 )
316 self._lock_renewal_stop = threading.Event()
317 self._lock_renewal_thread = threading.Thread(
318 group=None,
319 target=self._lock_renewer,
320 kwargs={'lockref': weakref.ref(self),
321 'interval': self._lock_renewal_interval,
322 'stop': self._lock_renewal_stop}
323 )
324 self._lock_renewal_thread.setDaemon(True)
325 self._lock_renewal_thread.start()
326
327 def _stop_lock_renewer(self):
328 """
329 Stop the lock renewer.
330
331 This signals the renewal thread and waits for its exit.
332 """
333 if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
334 return
335 loggers["refresh.shutdown"].debug("Signalling the lock refresher to stop")
336 self._lock_renewal_stop.set()
337 self._lock_renewal_thread.join()
338 self._lock_renewal_thread = None
339 loggers["refresh.exit"].debug("Lock refresher has stopped")
340
341 def __enter__(self):
342 acquired = self.acquire(blocking=True)
343 assert acquired, "Lock wasn't acquired, but blocking=True"
344 return self
345
346 def __exit__(self, exc_type=None, exc_value=None, traceback=None):
347 self.release()
348
349 def release(self):
350 """Releases the lock, that was acquired with the same object.
351
352 .. note::
353
354 If you want to release a lock that you acquired in a different place you have two choices:
355
356 * Use ``Lock("name", id=id_from_other_place).release()``
357 * Use ``Lock("name").reset()``
358 """
359 if self._lock_renewal_thread is not None:
360 self._stop_lock_renewer()
361 loggers["release"].debug("Releasing %r.", self._name)
362 error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
363 if error == 1:
364 raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
365 elif error:
366 raise RuntimeError("Unsupported error code %s from EXTEND script." % error)
367
368 def locked(self):
369 """
370 Return true if the lock is acquired.
371
372 Checks that lock with same name already exists. This method returns true, even if
373 lock have another id.
374 """
375 return self._client.exists(self._name) == 1
376
377
378 reset_all_script = None
379
380
381 def reset_all(redis_client):
382 """
383 Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
384
385 :param redis_client:
386 An instance of :class:`~StrictRedis`.
387 """
388 Lock.register_scripts(redis_client)
389
390 reset_all_script(client=redis_client) # noqa
@@ -0,0 +1,52 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import logging
4
5 from .stream import TCPStatsClient, UnixSocketStatsClient # noqa
6 from .udp import StatsClient # noqa
7
8 HOST = 'localhost'
9 PORT = 8125
10 IPV6 = False
11 PREFIX = None
12 MAXUDPSIZE = 512
13
14 log = logging.getLogger('rhodecode.statsd')
15
16
17 def statsd_config(config, prefix='statsd.'):
18 _config = {}
19 for key in config.keys():
20 if key.startswith(prefix):
21 _config[key[len(prefix):]] = config[key]
22 return _config
23
24
25 def client_from_config(configuration, prefix='statsd.', **kwargs):
26 from pyramid.settings import asbool
27
28 _config = statsd_config(configuration, prefix)
29 statsd_enabled = asbool(_config.pop('enabled', False))
30 if not statsd_enabled:
31 log.debug('statsd client not enabled by statsd.enabled = flag, skipping...')
32 return
33
34 host = _config.pop('statsd_host', HOST)
35 port = _config.pop('statsd_port', PORT)
36 prefix = _config.pop('statsd_prefix', PREFIX)
37 maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
38 ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
39 log.debug('configured statsd client %s:%s', host, port)
40
41 try:
42 client = StatsClient(
43 host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
44 except Exception:
45 log.exception('StatsD is enabled, but failed to connect to statsd server, fallback: disable statsd')
46 client = None
47
48 return client
49
50
51 def get_statsd_client(request):
52 return client_from_config(request.registry.settings)
@@ -0,0 +1,156 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import re
4 import random
5 from collections import deque
6 from datetime import timedelta
7 from repoze.lru import lru_cache
8
9 from .timer import Timer
10
11 TAG_INVALID_CHARS_RE = re.compile(
12 r"[^\w\d_\-:/\.]",
13 #re.UNICODE
14 )
15 TAG_INVALID_CHARS_SUBS = "_"
16
17 # we save and expose methods called by statsd for discovery
18 buckets_dict = {
19
20 }
21
22
23 @lru_cache(maxsize=500)
24 def _normalize_tags_with_cache(tag_list):
25 return [TAG_INVALID_CHARS_RE.sub(TAG_INVALID_CHARS_SUBS, tag) for tag in tag_list]
26
27
28 def normalize_tags(tag_list):
29 # We have to turn our input tag list into a non-mutable tuple for it to
30 # be hashable (and thus usable) by the @lru_cache decorator.
31 return _normalize_tags_with_cache(tuple(tag_list))
32
33
34 class StatsClientBase(object):
35 """A Base class for various statsd clients."""
36
37 def close(self):
38 """Used to close and clean up any underlying resources."""
39 raise NotImplementedError()
40
41 def _send(self):
42 raise NotImplementedError()
43
44 def pipeline(self):
45 raise NotImplementedError()
46
47 def timer(self, stat, rate=1, tags=None, auto_send=True):
48 """
49 statsd = StatsdClient.statsd
50 with statsd.timer('bucket_name', auto_send=True) as tmr:
51 # This block will be timed.
52 for i in xrange(0, 100000):
53 i ** 2
54 # you can access time here...
55 elapsed_ms = tmr.ms
56 """
57 return Timer(self, stat, rate, tags, auto_send=auto_send)
58
59 def timing(self, stat, delta, rate=1, tags=None, use_decimals=True):
60 """
61 Send new timing information.
62
63 `delta` can be either a number of milliseconds or a timedelta.
64 """
65 if isinstance(delta, timedelta):
66 # Convert timedelta to number of milliseconds.
67 delta = delta.total_seconds() * 1000.
68 if use_decimals:
69 fmt = '%0.6f|ms'
70 else:
71 fmt = '%s|ms'
72 self._send_stat(stat, fmt % delta, rate, tags)
73
74 def incr(self, stat, count=1, rate=1, tags=None):
75 """Increment a stat by `count`."""
76 self._send_stat(stat, '%s|c' % count, rate, tags)
77
78 def decr(self, stat, count=1, rate=1, tags=None):
79 """Decrement a stat by `count`."""
80 self.incr(stat, -count, rate, tags)
81
82 def gauge(self, stat, value, rate=1, delta=False, tags=None):
83 """Set a gauge value."""
84 if value < 0 and not delta:
85 if rate < 1:
86 if random.random() > rate:
87 return
88 with self.pipeline() as pipe:
89 pipe._send_stat(stat, '0|g', 1)
90 pipe._send_stat(stat, '%s|g' % value, 1)
91 else:
92 prefix = '+' if delta and value >= 0 else ''
93 self._send_stat(stat, '%s%s|g' % (prefix, value), rate, tags)
94
95 def set(self, stat, value, rate=1):
96 """Set a set value."""
97 self._send_stat(stat, '%s|s' % value, rate)
98
99 def histogram(self, stat, value, rate=1, tags=None):
100 """Set a histogram"""
101 self._send_stat(stat, '%s|h' % value, rate, tags)
102
103 def _send_stat(self, stat, value, rate, tags=None):
104 self._after(self._prepare(stat, value, rate, tags))
105
106 def _prepare(self, stat, value, rate, tags=None):
107 global buckets_dict
108 buckets_dict[stat] = 1
109
110 if rate < 1:
111 if random.random() > rate:
112 return
113 value = '%s|@%s' % (value, rate)
114
115 if self._prefix:
116 stat = '%s.%s' % (self._prefix, stat)
117
118 res = '%s:%s%s' % (
119 stat,
120 value,
121 ("|#" + ",".join(normalize_tags(tags))) if tags else "",
122 )
123 return res
124
125 def _after(self, data):
126 if data:
127 self._send(data)
128
129
130 class PipelineBase(StatsClientBase):
131
132 def __init__(self, client):
133 self._client = client
134 self._prefix = client._prefix
135 self._stats = deque()
136
137 def _send(self):
138 raise NotImplementedError()
139
140 def _after(self, data):
141 if data is not None:
142 self._stats.append(data)
143
144 def __enter__(self):
145 return self
146
147 def __exit__(self, typ, value, tb):
148 self.send()
149
150 def send(self):
151 if not self._stats:
152 return
153 self._send()
154
155 def pipeline(self):
156 return self.__class__(self)
@@ -0,0 +1,75 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import socket
4
5 from .base import StatsClientBase, PipelineBase
6
7
8 class StreamPipeline(PipelineBase):
9 def _send(self):
10 self._client._after('\n'.join(self._stats))
11 self._stats.clear()
12
13
14 class StreamClientBase(StatsClientBase):
15 def connect(self):
16 raise NotImplementedError()
17
18 def close(self):
19 if self._sock and hasattr(self._sock, 'close'):
20 self._sock.close()
21 self._sock = None
22
23 def reconnect(self):
24 self.close()
25 self.connect()
26
27 def pipeline(self):
28 return StreamPipeline(self)
29
30 def _send(self, data):
31 """Send data to statsd."""
32 if not self._sock:
33 self.connect()
34 self._do_send(data)
35
36 def _do_send(self, data):
37 self._sock.sendall(data.encode('ascii') + b'\n')
38
39
40 class TCPStatsClient(StreamClientBase):
41 """TCP version of StatsClient."""
42
43 def __init__(self, host='localhost', port=8125, prefix=None,
44 timeout=None, ipv6=False):
45 """Create a new client."""
46 self._host = host
47 self._port = port
48 self._ipv6 = ipv6
49 self._timeout = timeout
50 self._prefix = prefix
51 self._sock = None
52
53 def connect(self):
54 fam = socket.AF_INET6 if self._ipv6 else socket.AF_INET
55 family, _, _, _, addr = socket.getaddrinfo(
56 self._host, self._port, fam, socket.SOCK_STREAM)[0]
57 self._sock = socket.socket(family, socket.SOCK_STREAM)
58 self._sock.settimeout(self._timeout)
59 self._sock.connect(addr)
60
61
62 class UnixSocketStatsClient(StreamClientBase):
63 """Unix domain socket version of StatsClient."""
64
65 def __init__(self, socket_path, prefix=None, timeout=None):
66 """Create a new client."""
67 self._socket_path = socket_path
68 self._timeout = timeout
69 self._prefix = prefix
70 self._sock = None
71
72 def connect(self):
73 self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
74 self._sock.settimeout(self._timeout)
75 self._sock.connect(self._socket_path)
@@ -0,0 +1,75 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import functools
4
5 # Use timer that's not susceptible to time of day adjustments.
6 try:
7 # perf_counter is only present on Py3.3+
8 from time import perf_counter as time_now
9 except ImportError:
10 # fall back to using time
11 from time import time as time_now
12
13
14 def safe_wraps(wrapper, *args, **kwargs):
15 """Safely wraps partial functions."""
16 while isinstance(wrapper, functools.partial):
17 wrapper = wrapper.func
18 return functools.wraps(wrapper, *args, **kwargs)
19
20
21 class Timer(object):
22 """A context manager/decorator for statsd.timing()."""
23
24 def __init__(self, client, stat, rate=1, tags=None, use_decimals=True, auto_send=True):
25 self.client = client
26 self.stat = stat
27 self.rate = rate
28 self.tags = tags
29 self.ms = None
30 self._sent = False
31 self._start_time = None
32 self.use_decimals = use_decimals
33 self.auto_send = auto_send
34
35 def __call__(self, f):
36 """Thread-safe timing function decorator."""
37 @safe_wraps(f)
38 def _wrapped(*args, **kwargs):
39 start_time = time_now()
40 try:
41 return f(*args, **kwargs)
42 finally:
43 elapsed_time_ms = 1000.0 * (time_now() - start_time)
44 self.client.timing(self.stat, elapsed_time_ms, self.rate, self.tags, self.use_decimals)
45 self._sent = True
46 return _wrapped
47
48 def __enter__(self):
49 return self.start()
50
51 def __exit__(self, typ, value, tb):
52 self.stop(send=self.auto_send)
53
54 def start(self):
55 self.ms = None
56 self._sent = False
57 self._start_time = time_now()
58 return self
59
60 def stop(self, send=True):
61 if self._start_time is None:
62 raise RuntimeError('Timer has not started.')
63 dt = time_now() - self._start_time
64 self.ms = 1000.0 * dt # Convert to milliseconds.
65 if send:
66 self.send()
67 return self
68
69 def send(self):
70 if self.ms is None:
71 raise RuntimeError('No data recorded.')
72 if self._sent:
73 raise RuntimeError('Already sent data.')
74 self._sent = True
75 self.client.timing(self.stat, self.ms, self.rate, self.tags, self.use_decimals)
@@ -0,0 +1,55 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import socket
4
5 from .base import StatsClientBase, PipelineBase
6
7
8 class Pipeline(PipelineBase):
9
10 def __init__(self, client):
11 super(Pipeline, self).__init__(client)
12 self._maxudpsize = client._maxudpsize
13
14 def _send(self):
15 data = self._stats.popleft()
16 while self._stats:
17 # Use popleft to preserve the order of the stats.
18 stat = self._stats.popleft()
19 if len(stat) + len(data) + 1 >= self._maxudpsize:
20 self._client._after(data)
21 data = stat
22 else:
23 data += '\n' + stat
24 self._client._after(data)
25
26
27 class StatsClient(StatsClientBase):
28 """A client for statsd."""
29
30 def __init__(self, host='localhost', port=8125, prefix=None,
31 maxudpsize=512, ipv6=False):
32 """Create a new client."""
33 fam = socket.AF_INET6 if ipv6 else socket.AF_INET
34 family, _, _, _, addr = socket.getaddrinfo(
35 host, port, fam, socket.SOCK_DGRAM)[0]
36 self._addr = addr
37 self._sock = socket.socket(family, socket.SOCK_DGRAM)
38 self._prefix = prefix
39 self._maxudpsize = maxudpsize
40
41 def _send(self, data):
42 """Send data to statsd."""
43 try:
44 self._sock.sendto(data.encode('ascii'), self._addr)
45 except (socket.error, RuntimeError):
46 # No time for love, Dr. Jones!
47 pass
48
49 def close(self):
50 if self._sock and hasattr(self._sock, 'close'):
51 self._sock.close()
52 self._sock = None
53
54 def pipeline(self):
55 return Pipeline(self)
@@ -0,0 +1,49 b''
1 from vcsserver.lib._vendor.statsd import client_from_config
2
3
4 class StatsdClientNotInitialised(Exception):
5 pass
6
7
8 class _Singleton(type):
9 """A metaclass that creates a Singleton base class when called."""
10
11 _instances = {}
12
13 def __call__(cls, *args, **kwargs):
14 if cls not in cls._instances:
15 cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs)
16 return cls._instances[cls]
17
18
19 class Singleton(_Singleton("SingletonMeta", (object,), {})):
20 pass
21
22
23 class StatsdClientClass(Singleton):
24 setup_run = False
25 statsd_client = None
26 statsd = None
27
28 def __getattribute__(self, name):
29
30 if name.startswith("statsd"):
31 if self.setup_run:
32 return super(StatsdClientClass, self).__getattribute__(name)
33 else:
34 return None
35 #raise StatsdClientNotInitialised("requested key was %s" % name)
36
37 return super(StatsdClientClass, self).__getattribute__(name)
38
39 def setup(self, settings):
40 """
41 Initialize the client
42 """
43 statsd = client_from_config(settings)
44 self.statsd = statsd
45 self.statsd_client = statsd
46 self.setup_run = True
47
48
49 StatsdClient = StatsdClientClass()
@@ -3,4 +3,3 b' current_version = 5.0.0'
3 message = release: Bump version {current_version} to {new_version}
3 message = release: Bump version {current_version} to {new_version}
4
4
5 [bumpversion:file:vcsserver/VERSION]
5 [bumpversion:file:vcsserver/VERSION]
6
@@ -61,3 +61,22 b' ba6a6dc9ecd7fd8b1dcd6eb0c4ee0210e897c426'
61 17bc818b41bcf6883b9ff0da31f01d8c2a5d0781 v4.18.1
61 17bc818b41bcf6883b9ff0da31f01d8c2a5d0781 v4.18.1
62 1e9f12aa01f82c335abc9017efe94ce1c30b52ba v4.18.2
62 1e9f12aa01f82c335abc9017efe94ce1c30b52ba v4.18.2
63 f4cc6b3c5680bdf4541d7d442fbb7086640fb547 v4.18.3
63 f4cc6b3c5680bdf4541d7d442fbb7086640fb547 v4.18.3
64 5dc0277e4f77bd4cc3042d99625bb5d3ba480c8c v4.19.0
65 3a815eeb1b1efa340dda9b81a8da3cf24a7d605b v4.19.1
66 8841da3680fba841e5a54ebccd8ca56c078f7553 v4.19.2
67 4b0dec7fd80b1ca38e5073e5e562a5a450f73669 v4.19.3
68 1485aa75ffe1b1ec48352dce7b7492d92f85e95f v4.20.0
69 5b740274011766ef2f73803cc196d081e1e7f1d4 v4.20.1
70 5a7835234e2c45e8fb8184c60f548a64b5842af8 v4.21.0
71 26af88343015f8b89d5a66f92bc7547c51fcf0df v4.22.0
72 cf54e5f700fe5dc50af1a1bdf5197c18cf52105f v4.23.0
73 179d989bcfe02c6227f9f6aa9236cbbe1c14c400 v4.23.1
74 383aee8b1652affaa26aefe336a89ee366b2b26d v4.23.2
75 bc1a8141cc51fc23c455ebc50c6609c810b46f8d v4.24.0
76 530a1c03caabc806ea1ef34605f8f67f18c70e55 v4.24.1
77 5908ae65cee1043982e1b26d7b618af5fcfebbb3 v4.25.0
78 cce8bcdf75090d5943a1e9706fe5212d7b5d1fa1 v4.25.1
79 8610c4bf846c63bbc95d3ddfb53fadaaa9c7aa42 v4.25.2
80 d46b7d1be72c76c9f9aaeab6a342951d54459f49 v4.26.0
81 6fba0daab1e20a9e18fb70fa59bd21753e0a5b90 v4.27.0
82 6195da4fc454087173918ae59cae946289458676 v4.27.1
@@ -31,7 +31,7 b' asyncore_use_poll = true'
31 ; GUNICORN APPLICATION SERVER
31 ; GUNICORN APPLICATION SERVER
32 ; ###########################
32 ; ###########################
33
33
34 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
34 ; run with gunicorn --paste rhodecode.ini
35
35
36 ; Module to use, this setting shouldn't be changed
36 ; Module to use, this setting shouldn't be changed
37 #use = egg:gunicorn#main
37 #use = egg:gunicorn#main
@@ -86,7 +86,7 b' asyncore_use_poll = true'
86 ; serving requests. Workers still alive after the timeout (starting from the
86 ; serving requests. Workers still alive after the timeout (starting from the
87 ; receipt of the restart signal) are force killed.
87 ; receipt of the restart signal) are force killed.
88 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
88 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
89 #graceful_timeout = 3600
89 #graceful_timeout = 21600
90
90
91 # The number of seconds to wait for requests on a Keep-Alive connection.
91 # The number of seconds to wait for requests on a Keep-Alive connection.
92 # Generally set in the 1-5 seconds range.
92 # Generally set in the 1-5 seconds range.
@@ -110,6 +110,17 b' asyncore_use_poll = true'
110 [app:main]
110 [app:main]
111 ; The %(here)s variable will be replaced with the absolute path of parent directory
111 ; The %(here)s variable will be replaced with the absolute path of parent directory
112 ; of this file
112 ; of this file
113 ; Each option in the app:main can be override by an environmental variable
114 ;
115 ;To override an option:
116 ;
117 ;RC_<KeyName>
118 ;Everything should be uppercase, . and - should be replaced by _.
119 ;For example, if you have these configuration settings:
120 ;rc_cache.repo_object.backend = foo
121 ;can be overridden by
122 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
123
113 use = egg:rhodecode-vcsserver
124 use = egg:rhodecode-vcsserver
114
125
115
126
@@ -133,13 +144,13 b' debugtoolbar.exclude_prefixes ='
133 ; #################
144 ; #################
134
145
135 ; Pyramid default locales, we need this to be set
146 ; Pyramid default locales, we need this to be set
136 pyramid.default_locale_name = en
147 #pyramid.default_locale_name = en
137
148
138 ; default locale used by VCS systems
149 ; default locale used by VCS systems
139 locale = en_US.UTF-8
150 #locale = en_US.UTF-8
140
151
141 ; path to binaries for vcsserver, it should be set by the installer
152 ; path to binaries for vcsserver, it should be set by the installer
142 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
153 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
143 ; it can also be a path to nix-build output in case of development
154 ; it can also be a path to nix-build output in case of development
144 core.binary_dir = ""
155 core.binary_dir = ""
145
156
@@ -153,21 +164,21 b' core.binary_dir = ""'
153
164
154 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
165 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
155 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
166 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
156 cache_dir = %(here)s/data
167 #cache_dir = %(here)s/data
157
168
158 ; ***************************************
169 ; ***************************************
159 ; `repo_object` cache, default file based
170 ; `repo_object` cache, default file based
160 ; ***************************************
171 ; ***************************************
161
172
162 ; `repo_object` cache settings for vcs methods for repositories
173 ; `repo_object` cache settings for vcs methods for repositories
163 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
174 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
164
175
165 ; cache auto-expires after N seconds
176 ; cache auto-expires after N seconds
166 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
177 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
167 rc_cache.repo_object.expiration_time = 2592000
178 #rc_cache.repo_object.expiration_time = 2592000
168
179
169 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
180 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
170 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
181 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
171
182
172 ; ***********************************************************
183 ; ***********************************************************
173 ; `repo_object` cache with redis backend
184 ; `repo_object` cache with redis backend
@@ -191,10 +202,32 b' rc_cache.repo_object.expiration_time = 2'
191 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
202 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
192 #rc_cache.repo_object.arguments.distributed_lock = true
203 #rc_cache.repo_object.arguments.distributed_lock = true
193
204
205 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
206 #rc_cache.repo_object.arguments.lock_auto_renewal = true
207
208 ; Statsd client config, this is used to send metrics to statsd
209 ; We recommend setting statsd_exported and scrape them using Promethues
210 #statsd.enabled = false
211 #statsd.statsd_host = 0.0.0.0
212 #statsd.statsd_port = 8125
213 #statsd.statsd_prefix =
214 #statsd.statsd_ipv6 = false
215
216 ; configure logging automatically at server startup set to false
217 ; to use the below custom logging config.
218 ; RC_LOGGING_FORMATTER
219 ; RC_LOGGING_LEVEL
220 ; env variables can control the settings for logging in case of autoconfigure
221
222 #logging.autoconfigure = true
223
224 ; specify your own custom logging config file to configure logging
225 #logging.logging_conf_file = /path/to/custom_logging.ini
194
226
195 ; #####################
227 ; #####################
196 ; LOGGING CONFIGURATION
228 ; LOGGING CONFIGURATION
197 ; #####################
229 ; #####################
230
198 [loggers]
231 [loggers]
199 keys = root, vcsserver
232 keys = root, vcsserver
200
233
@@ -202,7 +235,7 b' keys = root, vcsserver'
202 keys = console
235 keys = console
203
236
204 [formatters]
237 [formatters]
205 keys = generic
238 keys = generic, json
206
239
207 ; #######
240 ; #######
208 ; LOGGERS
241 ; LOGGERS
@@ -217,7 +250,6 b' handlers ='
217 qualname = vcsserver
250 qualname = vcsserver
218 propagate = 1
251 propagate = 1
219
252
220
221 ; ########
253 ; ########
222 ; HANDLERS
254 ; HANDLERS
223 ; ########
255 ; ########
@@ -226,6 +258,8 b' propagate = 1'
226 class = StreamHandler
258 class = StreamHandler
227 args = (sys.stderr, )
259 args = (sys.stderr, )
228 level = DEBUG
260 level = DEBUG
261 ; To enable JSON formatted logs replace 'generic' with 'json'
262 ; This allows sending properly formatted logs to grafana loki or elasticsearch
229 formatter = generic
263 formatter = generic
230
264
231 ; ##########
265 ; ##########
@@ -235,3 +269,7 b' formatter = generic'
235 [formatter_generic]
269 [formatter_generic]
236 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
270 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
237 datefmt = %Y-%m-%d %H:%M:%S
271 datefmt = %Y-%m-%d %H:%M:%S
272
273 [formatter_json]
274 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
275 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -11,6 +11,7 b' import time'
11 import threading
11 import threading
12 import traceback
12 import traceback
13 import random
13 import random
14 import socket
14 from gunicorn.glogging import Logger
15 from gunicorn.glogging import Logger
15
16
16
17
@@ -29,9 +30,15 b" accesslog = '-'"
29 worker_tmp_dir = None
30 worker_tmp_dir = None
30 tmp_upload_dir = None
31 tmp_upload_dir = None
31
32
33 #reuse_port = True
34
32 # Custom log format
35 # Custom log format
36 #access_log_format = (
37 # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
38
39 # loki format for easier parsing in grafana
33 access_log_format = (
40 access_log_format = (
34 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
41 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
35
42
36 # self adjust workers based on CPU count
43 # self adjust workers based on CPU count
37 # workers = get_workers()
44 # workers = get_workers()
@@ -50,7 +57,7 b' def _get_process_rss(pid=None):'
50
57
51
58
52 def _get_config(ini_path):
59 def _get_config(ini_path):
53 import configparser
60 import configparser
54
61
55 try:
62 try:
56 config = configparser.RawConfigParser()
63 config = configparser.RawConfigParser()
@@ -90,9 +97,12 b' def post_fork(server, worker):'
90 if conf.has_option(section, 'memory_usage_recovery_threshold'):
97 if conf.has_option(section, 'memory_usage_recovery_threshold'):
91 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
98 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
92
99
93 worker._memory_max_usage = _memory_max_usage
100 worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
94 worker._memory_usage_check_interval = _memory_usage_check_interval
101 or _memory_max_usage)
95 worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
102 worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
103 or _memory_usage_check_interval)
104 worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
105 or _memory_usage_recovery_threshold)
96
106
97 # register memory last check time, with some random offset so we don't recycle all
107 # register memory last check time, with some random offset so we don't recycle all
98 # at once
108 # at once
@@ -242,6 +252,74 b' def post_request(worker, req, environ, r'
242 _check_memory_usage(worker)
252 _check_memory_usage(worker)
243
253
244
254
255 def _filter_proxy(ip):
256 """
257 Passed in IP addresses in HEADERS can be in a special format of multiple
258 ips. Those comma separated IPs are passed from various proxies in the
259 chain of request processing. The left-most being the original client.
260 We only care about the first IP which came from the org. client.
261
262 :param ip: ip string from headers
263 """
264 if ',' in ip:
265 _ips = ip.split(',')
266 _first_ip = _ips[0].strip()
267 return _first_ip
268 return ip
269
270
271 def _filter_port(ip):
272 """
273 Removes a port from ip, there are 4 main cases to handle here.
274 - ipv4 eg. 127.0.0.1
275 - ipv6 eg. ::1
276 - ipv4+port eg. 127.0.0.1:8080
277 - ipv6+port eg. [::1]:8080
278
279 :param ip:
280 """
281 def is_ipv6(ip_addr):
282 if hasattr(socket, 'inet_pton'):
283 try:
284 socket.inet_pton(socket.AF_INET6, ip_addr)
285 except socket.error:
286 return False
287 else:
288 return False
289 return True
290
291 if ':' not in ip: # must be ipv4 pure ip
292 return ip
293
294 if '[' in ip and ']' in ip: # ipv6 with port
295 return ip.split(']')[0][1:].lower()
296
297 # must be ipv6 or ipv4 with port
298 if is_ipv6(ip):
299 return ip
300 else:
301 ip, _port = ip.split(':')[:2] # means ipv4+port
302 return ip
303
304
305 def get_ip_addr(environ):
306 proxy_key = 'HTTP_X_REAL_IP'
307 proxy_key2 = 'HTTP_X_FORWARDED_FOR'
308 def_key = 'REMOTE_ADDR'
309 _filters = lambda x: _filter_port(_filter_proxy(x))
310
311 ip = environ.get(proxy_key)
312 if ip:
313 return _filters(ip)
314
315 ip = environ.get(proxy_key2)
316 if ip:
317 return _filters(ip)
318
319 ip = environ.get(def_key, '0.0.0.0')
320 return _filters(ip)
321
322
245 class RhodeCodeLogger(Logger):
323 class RhodeCodeLogger(Logger):
246 """
324 """
247 Custom Logger that allows some customization that gunicorn doesn't allow
325 Custom Logger that allows some customization that gunicorn doesn't allow
@@ -258,5 +336,58 b' class RhodeCodeLogger(Logger):'
258 msecs = int((now - long(now)) * 1000)
336 msecs = int((now - long(now)) * 1000)
259 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
337 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
260
338
339 def atoms(self, resp, req, environ, request_time):
340 """ Gets atoms for log formatting.
341 """
342 status = resp.status
343 if isinstance(status, str):
344 status = status.split(None, 1)[0]
345 atoms = {
346 'h': get_ip_addr(environ),
347 'l': '-',
348 'u': self._get_user(environ) or '-',
349 't': self.now(),
350 'r': "%s %s %s" % (environ['REQUEST_METHOD'],
351 environ['RAW_URI'],
352 environ["SERVER_PROTOCOL"]),
353 's': status,
354 'm': environ.get('REQUEST_METHOD'),
355 'U': environ.get('PATH_INFO'),
356 'q': environ.get('QUERY_STRING'),
357 'H': environ.get('SERVER_PROTOCOL'),
358 'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
359 'B': getattr(resp, 'sent', None),
360 'f': environ.get('HTTP_REFERER', '-'),
361 'a': environ.get('HTTP_USER_AGENT', '-'),
362 'T': request_time.seconds,
363 'D': (request_time.seconds * 1000000) + request_time.microseconds,
364 'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
365 'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
366 'p': "<%s>" % os.getpid()
367 }
368
369 # add request headers
370 if hasattr(req, 'headers'):
371 req_headers = req.headers
372 else:
373 req_headers = req
374
375 if hasattr(req_headers, "items"):
376 req_headers = req_headers.items()
377
378 atoms.update({"{%s}i" % k.lower(): v for k, v in req_headers})
379
380 resp_headers = resp.headers
381 if hasattr(resp_headers, "items"):
382 resp_headers = resp_headers.items()
383
384 # add response headers
385 atoms.update({"{%s}o" % k.lower(): v for k, v in resp_headers})
386
387 # add environ variables
388 environ_variables = environ.items()
389 atoms.update({"{%s}e" % k.lower(): v for k, v in environ_variables})
390
391 return atoms
261
392
262 logger_class = RhodeCodeLogger
393 logger_class = RhodeCodeLogger
@@ -14,7 +14,7 b' port = 9900'
14 ; GUNICORN APPLICATION SERVER
14 ; GUNICORN APPLICATION SERVER
15 ; ###########################
15 ; ###########################
16
16
17 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
17 ; run with gunicorn --paste rhodecode.ini
18
18
19 ; Module to use, this setting shouldn't be changed
19 ; Module to use, this setting shouldn't be changed
20 use = egg:gunicorn#main
20 use = egg:gunicorn#main
@@ -69,7 +69,7 b' limit_request_field_size = 0'
69 ; serving requests. Workers still alive after the timeout (starting from the
69 ; serving requests. Workers still alive after the timeout (starting from the
70 ; receipt of the restart signal) are force killed.
70 ; receipt of the restart signal) are force killed.
71 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
71 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
72 graceful_timeout = 3600
72 graceful_timeout = 21600
73
73
74 # The number of seconds to wait for requests on a Keep-Alive connection.
74 # The number of seconds to wait for requests on a Keep-Alive connection.
75 # Generally set in the 1-5 seconds range.
75 # Generally set in the 1-5 seconds range.
@@ -93,16 +93,27 b' memory_usage_recovery_threshold = 0.8'
93 [app:main]
93 [app:main]
94 ; The %(here)s variable will be replaced with the absolute path of parent directory
94 ; The %(here)s variable will be replaced with the absolute path of parent directory
95 ; of this file
95 ; of this file
96 ; Each option in the app:main can be override by an environmental variable
97 ;
98 ;To override an option:
99 ;
100 ;RC_<KeyName>
101 ;Everything should be uppercase, . and - should be replaced by _.
102 ;For example, if you have these configuration settings:
103 ;rc_cache.repo_object.backend = foo
104 ;can be overridden by
105 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
106
96 use = egg:rhodecode-vcsserver
107 use = egg:rhodecode-vcsserver
97
108
98 ; Pyramid default locales, we need this to be set
109 ; Pyramid default locales, we need this to be set
99 pyramid.default_locale_name = en
110 #pyramid.default_locale_name = en
100
111
101 ; default locale used by VCS systems
112 ; default locale used by VCS systems
102 locale = en_US.UTF-8
113 #locale = en_US.UTF-8
103
114
104 ; path to binaries for vcsserver, it should be set by the installer
115 ; path to binaries for vcsserver, it should be set by the installer
105 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
116 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
106 ; it can also be a path to nix-build output in case of development
117 ; it can also be a path to nix-build output in case of development
107 core.binary_dir = ""
118 core.binary_dir = ""
108
119
@@ -116,21 +127,21 b' core.binary_dir = ""'
116
127
117 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
128 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
118 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
129 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
119 cache_dir = %(here)s/data
130 #cache_dir = %(here)s/data
120
131
121 ; ***************************************
132 ; ***************************************
122 ; `repo_object` cache, default file based
133 ; `repo_object` cache, default file based
123 ; ***************************************
134 ; ***************************************
124
135
125 ; `repo_object` cache settings for vcs methods for repositories
136 ; `repo_object` cache settings for vcs methods for repositories
126 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
137 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
127
138
128 ; cache auto-expires after N seconds
139 ; cache auto-expires after N seconds
129 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
140 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
130 rc_cache.repo_object.expiration_time = 2592000
141 #rc_cache.repo_object.expiration_time = 2592000
131
142
132 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
143 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
133 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
144 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
134
145
135 ; ***********************************************************
146 ; ***********************************************************
136 ; `repo_object` cache with redis backend
147 ; `repo_object` cache with redis backend
@@ -154,10 +165,32 b' rc_cache.repo_object.expiration_time = 2'
154 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
165 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
155 #rc_cache.repo_object.arguments.distributed_lock = true
166 #rc_cache.repo_object.arguments.distributed_lock = true
156
167
168 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
169 #rc_cache.repo_object.arguments.lock_auto_renewal = true
170
171 ; Statsd client config, this is used to send metrics to statsd
172 ; We recommend setting statsd_exported and scrape them using Promethues
173 #statsd.enabled = false
174 #statsd.statsd_host = 0.0.0.0
175 #statsd.statsd_port = 8125
176 #statsd.statsd_prefix =
177 #statsd.statsd_ipv6 = false
178
179 ; configure logging automatically at server startup set to false
180 ; to use the below custom logging config.
181 ; RC_LOGGING_FORMATTER
182 ; RC_LOGGING_LEVEL
183 ; env variables can control the settings for logging in case of autoconfigure
184
185 #logging.autoconfigure = true
186
187 ; specify your own custom logging config file to configure logging
188 #logging.logging_conf_file = /path/to/custom_logging.ini
157
189
158 ; #####################
190 ; #####################
159 ; LOGGING CONFIGURATION
191 ; LOGGING CONFIGURATION
160 ; #####################
192 ; #####################
193
161 [loggers]
194 [loggers]
162 keys = root, vcsserver
195 keys = root, vcsserver
163
196
@@ -165,7 +198,7 b' keys = root, vcsserver'
165 keys = console
198 keys = console
166
199
167 [formatters]
200 [formatters]
168 keys = generic
201 keys = generic, json
169
202
170 ; #######
203 ; #######
171 ; LOGGERS
204 ; LOGGERS
@@ -175,12 +208,11 b' level = NOTSET'
175 handlers = console
208 handlers = console
176
209
177 [logger_vcsserver]
210 [logger_vcsserver]
178 level = DEBUG
211 level = INFO
179 handlers =
212 handlers =
180 qualname = vcsserver
213 qualname = vcsserver
181 propagate = 1
214 propagate = 1
182
215
183
184 ; ########
216 ; ########
185 ; HANDLERS
217 ; HANDLERS
186 ; ########
218 ; ########
@@ -189,6 +221,8 b' propagate = 1'
189 class = StreamHandler
221 class = StreamHandler
190 args = (sys.stderr, )
222 args = (sys.stderr, )
191 level = INFO
223 level = INFO
224 ; To enable JSON formatted logs replace 'generic' with 'json'
225 ; This allows sending properly formatted logs to grafana loki or elasticsearch
192 formatter = generic
226 formatter = generic
193
227
194 ; ##########
228 ; ##########
@@ -198,3 +232,7 b' formatter = generic'
198 [formatter_generic]
232 [formatter_generic]
199 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
233 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
200 datefmt = %Y-%m-%d %H:%M:%S
234 datefmt = %Y-%m-%d %H:%M:%S
235
236 [formatter_json]
237 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
238 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -1,4 +1,5 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2 # RhodeCode VCSServer provides access to different vcs backends via network.
3 # RhodeCode VCSServer provides access to different vcs backends via network.
3 # Copyright (C) 2014-2019 RodeCode GmbH
4 # Copyright (C) 2014-2019 RodeCode GmbH
4 #
5 #
@@ -20,20 +21,47 b''
20 from setuptools import setup, find_packages
21 from setuptools import setup, find_packages
21
22
22 import os
23 import os
24 import re
23 import sys
25 import sys
24 import pkgutil
26 import pkgutil
25 import platform
27 import platform
26 import codecs
28 import codecs
27
29
28 try: # for pip >= 10
30 import pip
31
32 pip_major_version = int(pip.__version__.split(".")[0])
33 if pip_major_version >= 20:
29 from pip._internal.req import parse_requirements
34 from pip._internal.req import parse_requirements
30 except ImportError: # for pip <= 9.0.3
35 from pip._internal.network.session import PipSession
36 elif pip_major_version >= 10:
37 from pip._internal.req import parse_requirements
38 from pip._internal.download import PipSession
39 else:
31 from pip.req import parse_requirements
40 from pip.req import parse_requirements
41 from pip.download import PipSession
42
32
43
33 try: # for pip >= 10
44 def get_package_name(req_object):
34 from pip._internal.download import PipSession
45 package_name = None
35 except ImportError: # for pip <= 9.0.3
46 try:
36 from pip.download import PipSession
47 from pip._internal.req.constructors import install_req_from_parsed_requirement
48 except ImportError:
49 install_req_from_parsed_requirement = None
50
51 # In 20.1 of pip, the requirements object changed
52 if hasattr(req_object, 'req'):
53 package_name = req_object.req.name
54
55 if package_name is None:
56 if install_req_from_parsed_requirement:
57 package = install_req_from_parsed_requirement(req_object)
58 package_name = package.req.name
59
60 if package_name is None:
61 # fallback for older pip
62 package_name = re.split('===|<=|!=|==|>=|~=|<|>', req_object.requirement)[0]
63
64 return package_name
37
65
38
66
39 if sys.version_info < (2, 7):
67 if sys.version_info < (2, 7):
@@ -61,14 +89,15 b' def _get_requirements(req_filename, excl'
61 parsed = parse_requirements(os.path.join(here, req_filename))
89 parsed = parse_requirements(os.path.join(here, req_filename))
62
90
63 requirements = []
91 requirements = []
64 for ir in parsed:
92 for int_req in parsed:
65 if ir.req and ir.name not in exclude:
93 req_name = get_package_name(int_req)
66 requirements.append(str(ir.req))
94 if req_name not in exclude:
95 requirements.append(req_name)
67 return requirements + extras
96 return requirements + extras
68
97
69
98
70 # requirements extract
99 # requirements extract
71 setup_requirements = ['pytest-runner']
100 setup_requirements = []
72 install_requirements = _get_requirements(
101 install_requirements = _get_requirements(
73 'requirements.txt', exclude=['setuptools'])
102 'requirements.txt', exclude=['setuptools'])
74 test_requirements = _get_requirements(
103 test_requirements = _get_requirements(
@@ -14,13 +14,18 b''
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17 import os
18 import sys
18 import sys
19 import traceback
19 import traceback
20 import logging
20 import logging
21 import urllib.parse
21 import urllib.parse
22
22
23 from vcsserver.lib.rc_cache import region_meta
23 from vcsserver.lib.rc_cache import region_meta
24
25 from vcsserver import exceptions
26 from vcsserver.exceptions import NoContentException
27 from vcsserver.hgcompat import (archival)
28
24 log = logging.getLogger(__name__)
29 log = logging.getLogger(__name__)
25
30
26
31
@@ -74,3 +79,54 b' def raise_from_original(new_type):'
74 raise new_exc.with_traceback(exc_traceback)
79 raise new_exc.with_traceback(exc_traceback)
75 finally:
80 finally:
76 del exc_traceback
81 del exc_traceback
82
83
84 class ArchiveNode(object):
85 def __init__(self, path, mode, is_link, raw_bytes):
86 self.path = path
87 self.mode = mode
88 self.is_link = is_link
89 self.raw_bytes = raw_bytes
90
91
92 def archive_repo(walker, archive_dest_path, kind, mtime, archive_at_path,
93 archive_dir_name, commit_id, write_metadata=True, extra_metadata=None):
94 """
95 walker should be a file walker, for example:
96 def walker():
97 for file_info in files:
98 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
99 """
100 extra_metadata = extra_metadata or {}
101
102 if kind == "tgz":
103 archiver = archival.tarit(archive_dest_path, mtime, "gz")
104 elif kind == "tbz2":
105 archiver = archival.tarit(archive_dest_path, mtime, "bz2")
106 elif kind == 'zip':
107 archiver = archival.zipit(archive_dest_path, mtime)
108 else:
109 raise exceptions.ArchiveException()(
110 'Remote does not support: "%s" archive type.' % kind)
111
112 for f in walker(commit_id, archive_at_path):
113 f_path = os.path.join(archive_dir_name, f.path.lstrip('/'))
114 try:
115 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
116 except NoContentException:
117 # NOTE(marcink): this is a special case for SVN so we can create "empty"
118 # directories which arent supported by archiver
119 archiver.addfile(os.path.join(f_path, '.dir'), f.mode, f.is_link, '')
120
121 if write_metadata:
122 metadata = dict([
123 ('commit_id', commit_id),
124 ('mtime', mtime),
125 ])
126 metadata.update(extra_metadata)
127
128 meta = ["%s:%s" % (f_name, value) for f_name, value in metadata.items()]
129 f_path = os.path.join(archive_dir_name, '.archival.txt')
130 archiver.addfile(f_path, 0o644, False, '\n'.join(meta))
131
132 return archiver.done()
@@ -119,3 +119,7 b' class HTTPRepoBranchProtected(HTTPForbid'
119
119
120 class RefNotFoundException(KeyError):
120 class RefNotFoundException(KeyError):
121 pass
121 pass
122
123
124 class NoContentException(ValueError):
125 pass
@@ -1,4 +1,4 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
@@ -29,6 +29,7 b' from functools import wraps'
29 import more_itertools
29 import more_itertools
30 import pygit2
30 import pygit2
31 from pygit2 import Repository as LibGit2Repo
31 from pygit2 import Repository as LibGit2Repo
32 from pygit2 import index as LibGit2Index
32 from dulwich import index, objects
33 from dulwich import index, objects
33 from dulwich.client import HttpGitClient, LocalGitClient
34 from dulwich.client import HttpGitClient, LocalGitClient
34 from dulwich.errors import (
35 from dulwich.errors import (
@@ -40,7 +41,7 b' from dulwich.server import update_server'
40
41
41 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver import exceptions, settings, subprocessio
42 from vcsserver.utils import safe_str, safe_int, safe_unicode
43 from vcsserver.utils import safe_str, safe_int, safe_unicode
43 from vcsserver.base import RepoFactory, obfuscate_qs
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
44 from vcsserver.hgcompat import (
45 from vcsserver.hgcompat import (
45 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 from vcsserver.git_lfs.lib import LFSOidStore
47 from vcsserver.git_lfs.lib import LFSOidStore
@@ -184,7 +185,8 b' class GitRemote(RemoteBase):'
184 @reraise_safe_exceptions
185 @reraise_safe_exceptions
185 def assert_correct_path(self, wire):
186 def assert_correct_path(self, wire):
186 cache_on, context_uid, repo_id = self._cache_on(wire)
187 cache_on, context_uid, repo_id = self._cache_on(wire)
187 @self.region.conditional_cache_on_arguments(condition=cache_on)
188 region = self._region(wire)
189 @region.conditional_cache_on_arguments(condition=cache_on)
188 def _assert_correct_path(_context_uid, _repo_id):
190 def _assert_correct_path(_context_uid, _repo_id):
189 try:
191 try:
190 repo_init = self._factory.repo_libgit2(wire)
192 repo_init = self._factory.repo_libgit2(wire)
@@ -216,7 +218,8 b' class GitRemote(RemoteBase):'
216 @reraise_safe_exceptions
218 @reraise_safe_exceptions
217 def blob_raw_length(self, wire, sha):
219 def blob_raw_length(self, wire, sha):
218 cache_on, context_uid, repo_id = self._cache_on(wire)
220 cache_on, context_uid, repo_id = self._cache_on(wire)
219 @self.region.conditional_cache_on_arguments(condition=cache_on)
221 region = self._region(wire)
222 @region.conditional_cache_on_arguments(condition=cache_on)
220 def _blob_raw_length(_repo_id, _sha):
223 def _blob_raw_length(_repo_id, _sha):
221
224
222 repo_init = self._factory.repo_libgit2(wire)
225 repo_init = self._factory.repo_libgit2(wire)
@@ -247,7 +250,8 b' class GitRemote(RemoteBase):'
247 def is_large_file(self, wire, commit_id):
250 def is_large_file(self, wire, commit_id):
248 cache_on, context_uid, repo_id = self._cache_on(wire)
251 cache_on, context_uid, repo_id = self._cache_on(wire)
249
252
250 @self.region.conditional_cache_on_arguments(condition=cache_on)
253 region = self._region(wire)
254 @region.conditional_cache_on_arguments(condition=cache_on)
251 def _is_large_file(_repo_id, _sha):
255 def _is_large_file(_repo_id, _sha):
252 repo_init = self._factory.repo_libgit2(wire)
256 repo_init = self._factory.repo_libgit2(wire)
253 with repo_init as repo:
257 with repo_init as repo:
@@ -263,7 +267,8 b' class GitRemote(RemoteBase):'
263 def is_binary(self, wire, tree_id):
267 def is_binary(self, wire, tree_id):
264 cache_on, context_uid, repo_id = self._cache_on(wire)
268 cache_on, context_uid, repo_id = self._cache_on(wire)
265
269
266 @self.region.conditional_cache_on_arguments(condition=cache_on)
270 region = self._region(wire)
271 @region.conditional_cache_on_arguments(condition=cache_on)
267 def _is_binary(_repo_id, _tree_id):
272 def _is_binary(_repo_id, _tree_id):
268 repo_init = self._factory.repo_libgit2(wire)
273 repo_init = self._factory.repo_libgit2(wire)
269 with repo_init as repo:
274 with repo_init as repo:
@@ -305,7 +310,8 b' class GitRemote(RemoteBase):'
305 @reraise_safe_exceptions
310 @reraise_safe_exceptions
306 def bulk_request(self, wire, rev, pre_load):
311 def bulk_request(self, wire, rev, pre_load):
307 cache_on, context_uid, repo_id = self._cache_on(wire)
312 cache_on, context_uid, repo_id = self._cache_on(wire)
308 @self.region.conditional_cache_on_arguments(condition=cache_on)
313 region = self._region(wire)
314 @region.conditional_cache_on_arguments(condition=cache_on)
309 def _bulk_request(_repo_id, _rev, _pre_load):
315 def _bulk_request(_repo_id, _rev, _pre_load):
310 result = {}
316 result = {}
311 for attr in pre_load:
317 for attr in pre_load:
@@ -408,7 +414,8 b' class GitRemote(RemoteBase):'
408 @reraise_safe_exceptions
414 @reraise_safe_exceptions
409 def branch(self, wire, commit_id):
415 def branch(self, wire, commit_id):
410 cache_on, context_uid, repo_id = self._cache_on(wire)
416 cache_on, context_uid, repo_id = self._cache_on(wire)
411 @self.region.conditional_cache_on_arguments(condition=cache_on)
417 region = self._region(wire)
418 @region.conditional_cache_on_arguments(condition=cache_on)
412 def _branch(_context_uid, _repo_id, _commit_id):
419 def _branch(_context_uid, _repo_id, _commit_id):
413 regex = re.compile('^refs/heads')
420 regex = re.compile('^refs/heads')
414
421
@@ -423,7 +430,8 b' class GitRemote(RemoteBase):'
423 @reraise_safe_exceptions
430 @reraise_safe_exceptions
424 def commit_branches(self, wire, commit_id):
431 def commit_branches(self, wire, commit_id):
425 cache_on, context_uid, repo_id = self._cache_on(wire)
432 cache_on, context_uid, repo_id = self._cache_on(wire)
426 @self.region.conditional_cache_on_arguments(condition=cache_on)
433 region = self._region(wire)
434 @region.conditional_cache_on_arguments(condition=cache_on)
427 def _commit_branches(_context_uid, _repo_id, _commit_id):
435 def _commit_branches(_context_uid, _repo_id, _commit_id):
428 repo_init = self._factory.repo_libgit2(wire)
436 repo_init = self._factory.repo_libgit2(wire)
429 with repo_init as repo:
437 with repo_init as repo:
@@ -444,11 +452,22 b' class GitRemote(RemoteBase):'
444 # TODO: this is quite complex, check if that can be simplified
452 # TODO: this is quite complex, check if that can be simplified
445 @reraise_safe_exceptions
453 @reraise_safe_exceptions
446 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
454 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
455 # Defines the root tree
456 class _Root(object):
457 def __repr__(self):
458 return 'ROOT TREE'
459 ROOT = _Root()
460
447 repo = self._factory.repo(wire)
461 repo = self._factory.repo(wire)
448 object_store = repo.object_store
462 object_store = repo.object_store
449
463
450 # Create tree and populates it with blobs
464 # Create tree and populates it with blobs
451 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
465
466 if commit_tree and repo[commit_tree]:
467 git_commit = repo[commit_data['parents'][0]]
468 commit_tree = repo[git_commit.tree] # root tree
469 else:
470 commit_tree = objects.Tree()
452
471
453 for node in updated:
472 for node in updated:
454 # Compute subdirs if needed
473 # Compute subdirs if needed
@@ -507,21 +526,34 b' class GitRemote(RemoteBase):'
507
526
508 for node_path in removed:
527 for node_path in removed:
509 paths = node_path.split('/')
528 paths = node_path.split('/')
510 tree = commit_tree
529 tree = commit_tree # start with top-level
511 trees = [tree]
530 trees = [{'tree': tree, 'path': ROOT}]
512 # Traverse deep into the forest...
531 # Traverse deep into the forest...
532 # resolve final tree by iterating the path.
533 # e.g a/b/c.txt will get
534 # - root as tree then
535 # - 'a' as tree,
536 # - 'b' as tree,
537 # - stop at c as blob.
513 for path in paths:
538 for path in paths:
514 try:
539 try:
515 obj = repo[tree[path][1]]
540 obj = repo[tree[path][1]]
516 if isinstance(obj, objects.Tree):
541 if isinstance(obj, objects.Tree):
517 trees.append(obj)
542 trees.append({'tree': obj, 'path': path})
518 tree = obj
543 tree = obj
519 except KeyError:
544 except KeyError:
520 break
545 break
546 #PROBLEM:
547 """
548 We're not editing same reference tree object
549 """
521 # Cut down the blob and all rotten trees on the way back...
550 # Cut down the blob and all rotten trees on the way back...
522 for path, tree in reversed(list(zip(paths, trees))):
551 for path, tree_data in reversed(list(zip(paths, trees))):
523 del tree[path]
552 tree = tree_data['tree']
524 if tree:
553 tree.__delitem__(path)
554 # This operation edits the tree, we need to mark new commit back
555
556 if len(tree) > 0:
525 # This tree still has elements - don't remove it or any
557 # This tree still has elements - don't remove it or any
526 # of it's parents
558 # of it's parents
527 break
559 break
@@ -587,7 +619,7 b' class GitRemote(RemoteBase):'
587
619
588 if refs and not update_after:
620 if refs and not update_after:
589 # mikhail: explicitly set the head to the last ref.
621 # mikhail: explicitly set the head to the last ref.
590 repo['HEAD'] = remote_refs[refs[-1]]
622 repo["HEAD"] = remote_refs[refs[-1]]
591
623
592 if update_after:
624 if update_after:
593 # we want to checkout HEAD
625 # we want to checkout HEAD
@@ -689,7 +721,8 b' class GitRemote(RemoteBase):'
689 @reraise_safe_exceptions
721 @reraise_safe_exceptions
690 def get_object(self, wire, sha, maybe_unreachable=False):
722 def get_object(self, wire, sha, maybe_unreachable=False):
691 cache_on, context_uid, repo_id = self._cache_on(wire)
723 cache_on, context_uid, repo_id = self._cache_on(wire)
692 @self.region.conditional_cache_on_arguments(condition=cache_on)
724 region = self._region(wire)
725 @region.conditional_cache_on_arguments(condition=cache_on)
693 def _get_object(_context_uid, _repo_id, _sha):
726 def _get_object(_context_uid, _repo_id, _sha):
694 repo_init = self._factory.repo_libgit2(wire)
727 repo_init = self._factory.repo_libgit2(wire)
695 with repo_init as repo:
728 with repo_init as repo:
@@ -747,7 +780,8 b' class GitRemote(RemoteBase):'
747 @reraise_safe_exceptions
780 @reraise_safe_exceptions
748 def get_refs(self, wire):
781 def get_refs(self, wire):
749 cache_on, context_uid, repo_id = self._cache_on(wire)
782 cache_on, context_uid, repo_id = self._cache_on(wire)
750 @self.region.conditional_cache_on_arguments(condition=cache_on)
783 region = self._region(wire)
784 @region.conditional_cache_on_arguments(condition=cache_on)
751 def _get_refs(_context_uid, _repo_id):
785 def _get_refs(_context_uid, _repo_id):
752
786
753 repo_init = self._factory.repo_libgit2(wire)
787 repo_init = self._factory.repo_libgit2(wire)
@@ -761,7 +795,8 b' class GitRemote(RemoteBase):'
761 @reraise_safe_exceptions
795 @reraise_safe_exceptions
762 def get_branch_pointers(self, wire):
796 def get_branch_pointers(self, wire):
763 cache_on, context_uid, repo_id = self._cache_on(wire)
797 cache_on, context_uid, repo_id = self._cache_on(wire)
764 @self.region.conditional_cache_on_arguments(condition=cache_on)
798 region = self._region(wire)
799 @region.conditional_cache_on_arguments(condition=cache_on)
765 def _get_branch_pointers(_context_uid, _repo_id):
800 def _get_branch_pointers(_context_uid, _repo_id):
766
801
767 repo_init = self._factory.repo_libgit2(wire)
802 repo_init = self._factory.repo_libgit2(wire)
@@ -775,7 +810,8 b' class GitRemote(RemoteBase):'
775 @reraise_safe_exceptions
810 @reraise_safe_exceptions
776 def head(self, wire, show_exc=True):
811 def head(self, wire, show_exc=True):
777 cache_on, context_uid, repo_id = self._cache_on(wire)
812 cache_on, context_uid, repo_id = self._cache_on(wire)
778 @self.region.conditional_cache_on_arguments(condition=cache_on)
813 region = self._region(wire)
814 @region.conditional_cache_on_arguments(condition=cache_on)
779 def _head(_context_uid, _repo_id, _show_exc):
815 def _head(_context_uid, _repo_id, _show_exc):
780 repo_init = self._factory.repo_libgit2(wire)
816 repo_init = self._factory.repo_libgit2(wire)
781 with repo_init as repo:
817 with repo_init as repo:
@@ -800,7 +836,8 b' class GitRemote(RemoteBase):'
800 def revision(self, wire, rev):
836 def revision(self, wire, rev):
801
837
802 cache_on, context_uid, repo_id = self._cache_on(wire)
838 cache_on, context_uid, repo_id = self._cache_on(wire)
803 @self.region.conditional_cache_on_arguments(condition=cache_on)
839 region = self._region(wire)
840 @region.conditional_cache_on_arguments(condition=cache_on)
804 def _revision(_context_uid, _repo_id, _rev):
841 def _revision(_context_uid, _repo_id, _rev):
805 repo_init = self._factory.repo_libgit2(wire)
842 repo_init = self._factory.repo_libgit2(wire)
806 with repo_init as repo:
843 with repo_init as repo:
@@ -818,7 +855,8 b' class GitRemote(RemoteBase):'
818 @reraise_safe_exceptions
855 @reraise_safe_exceptions
819 def date(self, wire, commit_id):
856 def date(self, wire, commit_id):
820 cache_on, context_uid, repo_id = self._cache_on(wire)
857 cache_on, context_uid, repo_id = self._cache_on(wire)
821 @self.region.conditional_cache_on_arguments(condition=cache_on)
858 region = self._region(wire)
859 @region.conditional_cache_on_arguments(condition=cache_on)
822 def _date(_repo_id, _commit_id):
860 def _date(_repo_id, _commit_id):
823 repo_init = self._factory.repo_libgit2(wire)
861 repo_init = self._factory.repo_libgit2(wire)
824 with repo_init as repo:
862 with repo_init as repo:
@@ -837,7 +875,8 b' class GitRemote(RemoteBase):'
837 @reraise_safe_exceptions
875 @reraise_safe_exceptions
838 def author(self, wire, commit_id):
876 def author(self, wire, commit_id):
839 cache_on, context_uid, repo_id = self._cache_on(wire)
877 cache_on, context_uid, repo_id = self._cache_on(wire)
840 @self.region.conditional_cache_on_arguments(condition=cache_on)
878 region = self._region(wire)
879 @region.conditional_cache_on_arguments(condition=cache_on)
841 def _author(_repo_id, _commit_id):
880 def _author(_repo_id, _commit_id):
842 repo_init = self._factory.repo_libgit2(wire)
881 repo_init = self._factory.repo_libgit2(wire)
843 with repo_init as repo:
882 with repo_init as repo:
@@ -861,7 +900,8 b' class GitRemote(RemoteBase):'
861 @reraise_safe_exceptions
900 @reraise_safe_exceptions
862 def message(self, wire, commit_id):
901 def message(self, wire, commit_id):
863 cache_on, context_uid, repo_id = self._cache_on(wire)
902 cache_on, context_uid, repo_id = self._cache_on(wire)
864 @self.region.conditional_cache_on_arguments(condition=cache_on)
903 region = self._region(wire)
904 @region.conditional_cache_on_arguments(condition=cache_on)
865 def _message(_repo_id, _commit_id):
905 def _message(_repo_id, _commit_id):
866 repo_init = self._factory.repo_libgit2(wire)
906 repo_init = self._factory.repo_libgit2(wire)
867 with repo_init as repo:
907 with repo_init as repo:
@@ -872,7 +912,8 b' class GitRemote(RemoteBase):'
872 @reraise_safe_exceptions
912 @reraise_safe_exceptions
873 def parents(self, wire, commit_id):
913 def parents(self, wire, commit_id):
874 cache_on, context_uid, repo_id = self._cache_on(wire)
914 cache_on, context_uid, repo_id = self._cache_on(wire)
875 @self.region.conditional_cache_on_arguments(condition=cache_on)
915 region = self._region(wire)
916 @region.conditional_cache_on_arguments(condition=cache_on)
876 def _parents(_repo_id, _commit_id):
917 def _parents(_repo_id, _commit_id):
877 repo_init = self._factory.repo_libgit2(wire)
918 repo_init = self._factory.repo_libgit2(wire)
878 with repo_init as repo:
919 with repo_init as repo:
@@ -888,7 +929,8 b' class GitRemote(RemoteBase):'
888 @reraise_safe_exceptions
929 @reraise_safe_exceptions
889 def children(self, wire, commit_id):
930 def children(self, wire, commit_id):
890 cache_on, context_uid, repo_id = self._cache_on(wire)
931 cache_on, context_uid, repo_id = self._cache_on(wire)
891 @self.region.conditional_cache_on_arguments(condition=cache_on)
932 region = self._region(wire)
933 @region.conditional_cache_on_arguments(condition=cache_on)
892 def _children(_repo_id, _commit_id):
934 def _children(_repo_id, _commit_id):
893 output, __ = self.run_git_command(
935 output, __ = self.run_git_command(
894 wire, ['rev-list', '--all', '--children'])
936 wire, ['rev-list', '--all', '--children'])
@@ -947,7 +989,8 b' class GitRemote(RemoteBase):'
947 def tree_and_type_for_path(self, wire, commit_id, path):
989 def tree_and_type_for_path(self, wire, commit_id, path):
948
990
949 cache_on, context_uid, repo_id = self._cache_on(wire)
991 cache_on, context_uid, repo_id = self._cache_on(wire)
950 @self.region.conditional_cache_on_arguments(condition=cache_on)
992 region = self._region(wire)
993 @region.conditional_cache_on_arguments(condition=cache_on)
951 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
994 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
952 repo_init = self._factory.repo_libgit2(wire)
995 repo_init = self._factory.repo_libgit2(wire)
953
996
@@ -964,7 +1007,8 b' class GitRemote(RemoteBase):'
964 @reraise_safe_exceptions
1007 @reraise_safe_exceptions
965 def tree_items(self, wire, tree_id):
1008 def tree_items(self, wire, tree_id):
966 cache_on, context_uid, repo_id = self._cache_on(wire)
1009 cache_on, context_uid, repo_id = self._cache_on(wire)
967 @self.region.conditional_cache_on_arguments(condition=cache_on)
1010 region = self._region(wire)
1011 @region.conditional_cache_on_arguments(condition=cache_on)
968 def _tree_items(_repo_id, _tree_id):
1012 def _tree_items(_repo_id, _tree_id):
969
1013
970 repo_init = self._factory.repo_libgit2(wire)
1014 repo_init = self._factory.repo_libgit2(wire)
@@ -1065,7 +1109,8 b' class GitRemote(RemoteBase):'
1065 @reraise_safe_exceptions
1109 @reraise_safe_exceptions
1066 def node_history(self, wire, commit_id, path, limit):
1110 def node_history(self, wire, commit_id, path, limit):
1067 cache_on, context_uid, repo_id = self._cache_on(wire)
1111 cache_on, context_uid, repo_id = self._cache_on(wire)
1068 @self.region.conditional_cache_on_arguments(condition=cache_on)
1112 region = self._region(wire)
1113 @region.conditional_cache_on_arguments(condition=cache_on)
1069 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1114 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1070 # optimize for n==1, rev-list is much faster for that use-case
1115 # optimize for n==1, rev-list is much faster for that use-case
1071 if limit == 1:
1116 if limit == 1:
@@ -1107,7 +1152,8 b' class GitRemote(RemoteBase):'
1107 def get_all_commit_ids(self, wire):
1152 def get_all_commit_ids(self, wire):
1108
1153
1109 cache_on, context_uid, repo_id = self._cache_on(wire)
1154 cache_on, context_uid, repo_id = self._cache_on(wire)
1110 @self.region.conditional_cache_on_arguments(condition=cache_on)
1155 region = self._region(wire)
1156 @region.conditional_cache_on_arguments(condition=cache_on)
1111 def _get_all_commit_ids(_context_uid, _repo_id):
1157 def _get_all_commit_ids(_context_uid, _repo_id):
1112
1158
1113 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1159 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
@@ -1190,3 +1236,46 b' class GitRemote(RemoteBase):'
1190 'pre_version': get_git_pre_hook_version(path, bare),
1236 'pre_version': get_git_pre_hook_version(path, bare),
1191 'post_version': get_git_post_hook_version(path, bare),
1237 'post_version': get_git_post_hook_version(path, bare),
1192 }
1238 }
1239
1240 @reraise_safe_exceptions
1241 def set_head_ref(self, wire, head_name):
1242 log.debug('Setting refs/head to `%s`', head_name)
1243 cmd = ['symbolic-ref', '"HEAD"', '"refs/heads/%s"' % head_name]
1244 output, __ = self.run_git_command(wire, cmd)
1245 return [head_name] + output.splitlines()
1246
1247 @reraise_safe_exceptions
1248 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1249 archive_dir_name, commit_id):
1250
1251 def file_walker(_commit_id, path):
1252 repo_init = self._factory.repo_libgit2(wire)
1253
1254 with repo_init as repo:
1255 commit = repo[commit_id]
1256
1257 if path in ['', '/']:
1258 tree = commit.tree
1259 else:
1260 tree = commit.tree[path.rstrip('/')]
1261 tree_id = tree.id.hex
1262 try:
1263 tree = repo[tree_id]
1264 except KeyError:
1265 raise ObjectMissing('No tree with id: {}'.format(tree_id))
1266
1267 index = LibGit2Index.Index()
1268 index.read_tree(tree)
1269 file_iter = index
1270
1271 for fn in file_iter:
1272 file_path = fn.path
1273 mode = fn.mode
1274 is_link = stat.S_ISLNK(mode)
1275 if mode == pygit2.GIT_FILEMODE_COMMIT:
1276 log.debug('Skipping path %s as a commit node', file_path)
1277 continue
1278 yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw)
1279
1280 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1281 archive_dir_name, commit_id)
@@ -14,9 +14,10 b''
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17 import functools
18 import io
18 import io
19 import logging
19 import logging
20 import os
20 import stat
21 import stat
21 import urllib.request, urllib.parse, urllib.error
22 import urllib.request, urllib.parse, urllib.error
22 import urllib.request, urllib.error, urllib.parse
23 import urllib.request, urllib.error, urllib.parse
@@ -31,13 +32,14 b' from mercurial import repair'
31
32
32 import vcsserver
33 import vcsserver
33 from vcsserver import exceptions
34 from vcsserver import exceptions
34 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
35 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, archive_repo, ArchiveNode
35 from vcsserver.hgcompat import (
36 from vcsserver.hgcompat import (
36 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
37 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
37 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
38 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
38 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
39 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
39 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
40 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
40 RepoLookupError, InterventionRequired, RequirementError)
41 RepoLookupError, InterventionRequired, RequirementError,
42 alwaysmatcher, patternmatcher, hgutil)
41 from vcsserver.vcs_base import RemoteBase
43 from vcsserver.vcs_base import RemoteBase
42
44
43 log = logging.getLogger(__name__)
45 log = logging.getLogger(__name__)
@@ -205,25 +207,10 b' class HgRemote(RemoteBase):'
205 return False
207 return False
206
208
207 @reraise_safe_exceptions
209 @reraise_safe_exceptions
208 def archive_repo(self, archive_path, mtime, file_info, kind):
209 if kind == "tgz":
210 archiver = archival.tarit(archive_path, mtime, "gz")
211 elif kind == "tbz2":
212 archiver = archival.tarit(archive_path, mtime, "bz2")
213 elif kind == 'zip':
214 archiver = archival.zipit(archive_path, mtime)
215 else:
216 raise exceptions.ArchiveException()(
217 'Remote does not support: "%s".' % kind)
218
219 for f_path, f_mode, f_is_link, f_content in file_info:
220 archiver.addfile(f_path, f_mode, f_is_link, f_content)
221 archiver.done()
222
223 @reraise_safe_exceptions
224 def bookmarks(self, wire):
210 def bookmarks(self, wire):
225 cache_on, context_uid, repo_id = self._cache_on(wire)
211 cache_on, context_uid, repo_id = self._cache_on(wire)
226 @self.region.conditional_cache_on_arguments(condition=cache_on)
212 region = self._region(wire)
213 @region.conditional_cache_on_arguments(condition=cache_on)
227 def _bookmarks(_context_uid, _repo_id):
214 def _bookmarks(_context_uid, _repo_id):
228 repo = self._factory.repo(wire)
215 repo = self._factory.repo(wire)
229 return dict(repo._bookmarks)
216 return dict(repo._bookmarks)
@@ -233,7 +220,8 b' class HgRemote(RemoteBase):'
233 @reraise_safe_exceptions
220 @reraise_safe_exceptions
234 def branches(self, wire, normal, closed):
221 def branches(self, wire, normal, closed):
235 cache_on, context_uid, repo_id = self._cache_on(wire)
222 cache_on, context_uid, repo_id = self._cache_on(wire)
236 @self.region.conditional_cache_on_arguments(condition=cache_on)
223 region = self._region(wire)
224 @region.conditional_cache_on_arguments(condition=cache_on)
237 def _branches(_context_uid, _repo_id, _normal, _closed):
225 def _branches(_context_uid, _repo_id, _normal, _closed):
238 repo = self._factory.repo(wire)
226 repo = self._factory.repo(wire)
239 iter_branches = repo.branchmap().iterbranches()
227 iter_branches = repo.branchmap().iterbranches()
@@ -251,7 +239,8 b' class HgRemote(RemoteBase):'
251 @reraise_safe_exceptions
239 @reraise_safe_exceptions
252 def bulk_request(self, wire, commit_id, pre_load):
240 def bulk_request(self, wire, commit_id, pre_load):
253 cache_on, context_uid, repo_id = self._cache_on(wire)
241 cache_on, context_uid, repo_id = self._cache_on(wire)
254 @self.region.conditional_cache_on_arguments(condition=cache_on)
242 region = self._region(wire)
243 @region.conditional_cache_on_arguments(condition=cache_on)
255 def _bulk_request(_repo_id, _commit_id, _pre_load):
244 def _bulk_request(_repo_id, _commit_id, _pre_load):
256 result = {}
245 result = {}
257 for attr in pre_load:
246 for attr in pre_load:
@@ -268,7 +257,8 b' class HgRemote(RemoteBase):'
268 @reraise_safe_exceptions
257 @reraise_safe_exceptions
269 def ctx_branch(self, wire, commit_id):
258 def ctx_branch(self, wire, commit_id):
270 cache_on, context_uid, repo_id = self._cache_on(wire)
259 cache_on, context_uid, repo_id = self._cache_on(wire)
271 @self.region.conditional_cache_on_arguments(condition=cache_on)
260 region = self._region(wire)
261 @region.conditional_cache_on_arguments(condition=cache_on)
272 def _ctx_branch(_repo_id, _commit_id):
262 def _ctx_branch(_repo_id, _commit_id):
273 repo = self._factory.repo(wire)
263 repo = self._factory.repo(wire)
274 ctx = self._get_ctx(repo, commit_id)
264 ctx = self._get_ctx(repo, commit_id)
@@ -278,7 +268,8 b' class HgRemote(RemoteBase):'
278 @reraise_safe_exceptions
268 @reraise_safe_exceptions
279 def ctx_date(self, wire, commit_id):
269 def ctx_date(self, wire, commit_id):
280 cache_on, context_uid, repo_id = self._cache_on(wire)
270 cache_on, context_uid, repo_id = self._cache_on(wire)
281 @self.region.conditional_cache_on_arguments(condition=cache_on)
271 region = self._region(wire)
272 @region.conditional_cache_on_arguments(condition=cache_on)
282 def _ctx_date(_repo_id, _commit_id):
273 def _ctx_date(_repo_id, _commit_id):
283 repo = self._factory.repo(wire)
274 repo = self._factory.repo(wire)
284 ctx = self._get_ctx(repo, commit_id)
275 ctx = self._get_ctx(repo, commit_id)
@@ -294,7 +285,8 b' class HgRemote(RemoteBase):'
294 @reraise_safe_exceptions
285 @reraise_safe_exceptions
295 def ctx_files(self, wire, commit_id):
286 def ctx_files(self, wire, commit_id):
296 cache_on, context_uid, repo_id = self._cache_on(wire)
287 cache_on, context_uid, repo_id = self._cache_on(wire)
297 @self.region.conditional_cache_on_arguments(condition=cache_on)
288 region = self._region(wire)
289 @region.conditional_cache_on_arguments(condition=cache_on)
298 def _ctx_files(_repo_id, _commit_id):
290 def _ctx_files(_repo_id, _commit_id):
299 repo = self._factory.repo(wire)
291 repo = self._factory.repo(wire)
300 ctx = self._get_ctx(repo, commit_id)
292 ctx = self._get_ctx(repo, commit_id)
@@ -311,7 +303,8 b' class HgRemote(RemoteBase):'
311 @reraise_safe_exceptions
303 @reraise_safe_exceptions
312 def ctx_parents(self, wire, commit_id):
304 def ctx_parents(self, wire, commit_id):
313 cache_on, context_uid, repo_id = self._cache_on(wire)
305 cache_on, context_uid, repo_id = self._cache_on(wire)
314 @self.region.conditional_cache_on_arguments(condition=cache_on)
306 region = self._region(wire)
307 @region.conditional_cache_on_arguments(condition=cache_on)
315 def _ctx_parents(_repo_id, _commit_id):
308 def _ctx_parents(_repo_id, _commit_id):
316 repo = self._factory.repo(wire)
309 repo = self._factory.repo(wire)
317 ctx = self._get_ctx(repo, commit_id)
310 ctx = self._get_ctx(repo, commit_id)
@@ -323,7 +316,8 b' class HgRemote(RemoteBase):'
323 @reraise_safe_exceptions
316 @reraise_safe_exceptions
324 def ctx_children(self, wire, commit_id):
317 def ctx_children(self, wire, commit_id):
325 cache_on, context_uid, repo_id = self._cache_on(wire)
318 cache_on, context_uid, repo_id = self._cache_on(wire)
326 @self.region.conditional_cache_on_arguments(condition=cache_on)
319 region = self._region(wire)
320 @region.conditional_cache_on_arguments(condition=cache_on)
327 def _ctx_children(_repo_id, _commit_id):
321 def _ctx_children(_repo_id, _commit_id):
328 repo = self._factory.repo(wire)
322 repo = self._factory.repo(wire)
329 ctx = self._get_ctx(repo, commit_id)
323 ctx = self._get_ctx(repo, commit_id)
@@ -335,7 +329,8 b' class HgRemote(RemoteBase):'
335 @reraise_safe_exceptions
329 @reraise_safe_exceptions
336 def ctx_phase(self, wire, commit_id):
330 def ctx_phase(self, wire, commit_id):
337 cache_on, context_uid, repo_id = self._cache_on(wire)
331 cache_on, context_uid, repo_id = self._cache_on(wire)
338 @self.region.conditional_cache_on_arguments(condition=cache_on)
332 region = self._region(wire)
333 @region.conditional_cache_on_arguments(condition=cache_on)
339 def _ctx_phase(_context_uid, _repo_id, _commit_id):
334 def _ctx_phase(_context_uid, _repo_id, _commit_id):
340 repo = self._factory.repo(wire)
335 repo = self._factory.repo(wire)
341 ctx = self._get_ctx(repo, commit_id)
336 ctx = self._get_ctx(repo, commit_id)
@@ -346,7 +341,8 b' class HgRemote(RemoteBase):'
346 @reraise_safe_exceptions
341 @reraise_safe_exceptions
347 def ctx_obsolete(self, wire, commit_id):
342 def ctx_obsolete(self, wire, commit_id):
348 cache_on, context_uid, repo_id = self._cache_on(wire)
343 cache_on, context_uid, repo_id = self._cache_on(wire)
349 @self.region.conditional_cache_on_arguments(condition=cache_on)
344 region = self._region(wire)
345 @region.conditional_cache_on_arguments(condition=cache_on)
350 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
346 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
351 repo = self._factory.repo(wire)
347 repo = self._factory.repo(wire)
352 ctx = self._get_ctx(repo, commit_id)
348 ctx = self._get_ctx(repo, commit_id)
@@ -356,7 +352,8 b' class HgRemote(RemoteBase):'
356 @reraise_safe_exceptions
352 @reraise_safe_exceptions
357 def ctx_hidden(self, wire, commit_id):
353 def ctx_hidden(self, wire, commit_id):
358 cache_on, context_uid, repo_id = self._cache_on(wire)
354 cache_on, context_uid, repo_id = self._cache_on(wire)
359 @self.region.conditional_cache_on_arguments(condition=cache_on)
355 region = self._region(wire)
356 @region.conditional_cache_on_arguments(condition=cache_on)
360 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
357 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
361 repo = self._factory.repo(wire)
358 repo = self._factory.repo(wire)
362 ctx = self._get_ctx(repo, commit_id)
359 ctx = self._get_ctx(repo, commit_id)
@@ -469,7 +466,8 b' class HgRemote(RemoteBase):'
469 @reraise_safe_exceptions
466 @reraise_safe_exceptions
470 def node_history(self, wire, revision, path, limit):
467 def node_history(self, wire, revision, path, limit):
471 cache_on, context_uid, repo_id = self._cache_on(wire)
468 cache_on, context_uid, repo_id = self._cache_on(wire)
472 @self.region.conditional_cache_on_arguments(condition=cache_on)
469 region = self._region(wire)
470 @region.conditional_cache_on_arguments(condition=cache_on)
473 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
471 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
474 repo = self._factory.repo(wire)
472 repo = self._factory.repo(wire)
475
473
@@ -499,7 +497,8 b' class HgRemote(RemoteBase):'
499 @reraise_safe_exceptions
497 @reraise_safe_exceptions
500 def node_history_untill(self, wire, revision, path, limit):
498 def node_history_untill(self, wire, revision, path, limit):
501 cache_on, context_uid, repo_id = self._cache_on(wire)
499 cache_on, context_uid, repo_id = self._cache_on(wire)
502 @self.region.conditional_cache_on_arguments(condition=cache_on)
500 region = self._region(wire)
501 @region.conditional_cache_on_arguments(condition=cache_on)
503 def _node_history_until(_context_uid, _repo_id):
502 def _node_history_until(_context_uid, _repo_id):
504 repo = self._factory.repo(wire)
503 repo = self._factory.repo(wire)
505 ctx = self._get_ctx(repo, revision)
504 ctx = self._get_ctx(repo, revision)
@@ -537,7 +536,8 b' class HgRemote(RemoteBase):'
537 @reraise_safe_exceptions
536 @reraise_safe_exceptions
538 def fctx_flags(self, wire, commit_id, path):
537 def fctx_flags(self, wire, commit_id, path):
539 cache_on, context_uid, repo_id = self._cache_on(wire)
538 cache_on, context_uid, repo_id = self._cache_on(wire)
540 @self.region.conditional_cache_on_arguments(condition=cache_on)
539 region = self._region(wire)
540 @region.conditional_cache_on_arguments(condition=cache_on)
541 def _fctx_flags(_repo_id, _commit_id, _path):
541 def _fctx_flags(_repo_id, _commit_id, _path):
542 repo = self._factory.repo(wire)
542 repo = self._factory.repo(wire)
543 ctx = self._get_ctx(repo, commit_id)
543 ctx = self._get_ctx(repo, commit_id)
@@ -549,7 +549,8 b' class HgRemote(RemoteBase):'
549 @reraise_safe_exceptions
549 @reraise_safe_exceptions
550 def fctx_size(self, wire, commit_id, path):
550 def fctx_size(self, wire, commit_id, path):
551 cache_on, context_uid, repo_id = self._cache_on(wire)
551 cache_on, context_uid, repo_id = self._cache_on(wire)
552 @self.region.conditional_cache_on_arguments(condition=cache_on)
552 region = self._region(wire)
553 @region.conditional_cache_on_arguments(condition=cache_on)
553 def _fctx_size(_repo_id, _revision, _path):
554 def _fctx_size(_repo_id, _revision, _path):
554 repo = self._factory.repo(wire)
555 repo = self._factory.repo(wire)
555 ctx = self._get_ctx(repo, commit_id)
556 ctx = self._get_ctx(repo, commit_id)
@@ -560,7 +561,8 b' class HgRemote(RemoteBase):'
560 @reraise_safe_exceptions
561 @reraise_safe_exceptions
561 def get_all_commit_ids(self, wire, name):
562 def get_all_commit_ids(self, wire, name):
562 cache_on, context_uid, repo_id = self._cache_on(wire)
563 cache_on, context_uid, repo_id = self._cache_on(wire)
563 @self.region.conditional_cache_on_arguments(condition=cache_on)
564 region = self._region(wire)
565 @region.conditional_cache_on_arguments(condition=cache_on)
564 def _get_all_commit_ids(_context_uid, _repo_id, _name):
566 def _get_all_commit_ids(_context_uid, _repo_id, _name):
565 repo = self._factory.repo(wire)
567 repo = self._factory.repo(wire)
566 repo = repo.filtered(name)
568 repo = repo.filtered(name)
@@ -576,7 +578,8 b' class HgRemote(RemoteBase):'
576 @reraise_safe_exceptions
578 @reraise_safe_exceptions
577 def is_large_file(self, wire, commit_id, path):
579 def is_large_file(self, wire, commit_id, path):
578 cache_on, context_uid, repo_id = self._cache_on(wire)
580 cache_on, context_uid, repo_id = self._cache_on(wire)
579 @self.region.conditional_cache_on_arguments(condition=cache_on)
581 region = self._region(wire)
582 @region.conditional_cache_on_arguments(condition=cache_on)
580 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
583 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
581 return largefiles.lfutil.isstandin(path)
584 return largefiles.lfutil.isstandin(path)
582
585
@@ -586,7 +589,8 b' class HgRemote(RemoteBase):'
586 def is_binary(self, wire, revision, path):
589 def is_binary(self, wire, revision, path):
587 cache_on, context_uid, repo_id = self._cache_on(wire)
590 cache_on, context_uid, repo_id = self._cache_on(wire)
588
591
589 @self.region.conditional_cache_on_arguments(condition=cache_on)
592 region = self._region(wire)
593 @region.conditional_cache_on_arguments(condition=cache_on)
590 def _is_binary(_repo_id, _sha, _path):
594 def _is_binary(_repo_id, _sha, _path):
591 repo = self._factory.repo(wire)
595 repo = self._factory.repo(wire)
592 ctx = self._get_ctx(repo, revision)
596 ctx = self._get_ctx(repo, revision)
@@ -623,7 +627,9 b' class HgRemote(RemoteBase):'
623 @reraise_safe_exceptions
627 @reraise_safe_exceptions
624 def lookup(self, wire, revision, both):
628 def lookup(self, wire, revision, both):
625 cache_on, context_uid, repo_id = self._cache_on(wire)
629 cache_on, context_uid, repo_id = self._cache_on(wire)
626 @self.region.conditional_cache_on_arguments(condition=cache_on)
630
631 region = self._region(wire)
632 @region.conditional_cache_on_arguments(condition=cache_on)
627 def _lookup(_context_uid, _repo_id, _revision, _both):
633 def _lookup(_context_uid, _repo_id, _revision, _both):
628
634
629 repo = self._factory.repo(wire)
635 repo = self._factory.repo(wire)
@@ -681,7 +687,8 b' class HgRemote(RemoteBase):'
681 def rev_range(self, wire, commit_filter):
687 def rev_range(self, wire, commit_filter):
682 cache_on, context_uid, repo_id = self._cache_on(wire)
688 cache_on, context_uid, repo_id = self._cache_on(wire)
683
689
684 @self.region.conditional_cache_on_arguments(condition=cache_on)
690 region = self._region(wire)
691 @region.conditional_cache_on_arguments(condition=cache_on)
685 def _rev_range(_context_uid, _repo_id, _filter):
692 def _rev_range(_context_uid, _repo_id, _filter):
686 repo = self._factory.repo(wire)
693 repo = self._factory.repo(wire)
687 revisions = [rev for rev in revrange(repo, commit_filter)]
694 revisions = [rev for rev in revrange(repo, commit_filter)]
@@ -756,7 +763,8 b' class HgRemote(RemoteBase):'
756 @reraise_safe_exceptions
763 @reraise_safe_exceptions
757 def tags(self, wire):
764 def tags(self, wire):
758 cache_on, context_uid, repo_id = self._cache_on(wire)
765 cache_on, context_uid, repo_id = self._cache_on(wire)
759 @self.region.conditional_cache_on_arguments(condition=cache_on)
766 region = self._region(wire)
767 @region.conditional_cache_on_arguments(condition=cache_on)
760 def _tags(_context_uid, _repo_id):
768 def _tags(_context_uid, _repo_id):
761 repo = self._factory.repo(wire)
769 repo = self._factory.repo(wire)
762 return repo.tags()
770 return repo.tags()
@@ -1007,3 +1015,33 b' class HgRemote(RemoteBase):'
1007 'pre_version': vcsserver.__version__,
1015 'pre_version': vcsserver.__version__,
1008 'post_version': vcsserver.__version__,
1016 'post_version': vcsserver.__version__,
1009 }
1017 }
1018
1019 @reraise_safe_exceptions
1020 def set_head_ref(self, wire, head_name):
1021 pass
1022
1023 @reraise_safe_exceptions
1024 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1025 archive_dir_name, commit_id):
1026
1027 def file_walker(_commit_id, path):
1028 repo = self._factory.repo(wire)
1029 ctx = repo[_commit_id]
1030 is_root = path in ['', '/']
1031 if is_root:
1032 matcher = alwaysmatcher(badfn=None)
1033 else:
1034 matcher = patternmatcher('', [(b'glob', path+'/**', b'')], badfn=None)
1035 file_iter = ctx.manifest().walk(matcher)
1036
1037 for fn in file_iter:
1038 file_path = fn
1039 flags = ctx.flags(fn)
1040 mode = b'x' in flags and 0o755 or 0o644
1041 is_link = b'l' in flags
1042
1043 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
1044
1045 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1046 archive_dir_name, commit_id)
1047
@@ -38,7 +38,7 b' from mercurial import merge as hg_merge'
38 from mercurial import subrepo
38 from mercurial import subrepo
39 from mercurial import subrepoutil
39 from mercurial import subrepoutil
40 from mercurial import tags as hg_tag
40 from mercurial import tags as hg_tag
41
41 from mercurial import util as hgutil
42 from mercurial.commands import clone, nullid, pull
42 from mercurial.commands import clone, nullid, pull
43 from mercurial.context import memctx, memfilectx
43 from mercurial.context import memctx, memfilectx
44 from mercurial.error import (
44 from mercurial.error import (
@@ -46,7 +46,7 b' from mercurial.error import ('
46 RequirementError, ProgrammingError)
46 RequirementError, ProgrammingError)
47 from mercurial.hgweb import hgweb_mod
47 from mercurial.hgweb import hgweb_mod
48 from mercurial.localrepo import instance
48 from mercurial.localrepo import instance
49 from mercurial.match import match
49 from mercurial.match import match, alwaysmatcher, patternmatcher
50 from mercurial.mdiff import diffopts
50 from mercurial.mdiff import diffopts
51 from mercurial.node import bin, hex
51 from mercurial.node import bin, hex
52 from mercurial.encoding import tolocal
52 from mercurial.encoding import tolocal
@@ -67,7 +67,7 b' from mercurial.url import httpbasicauthh'
67 def get_ctx(repo, ref):
67 def get_ctx(repo, ref):
68 try:
68 try:
69 ctx = repo[ref]
69 ctx = repo[ref]
70 except ProgrammingError:
70 except (ProgrammingError, TypeError):
71 # we're unable to find the rev using a regular lookup, we fallback
71 # we're unable to find the rev using a regular lookup, we fallback
72 # to slower, but backward compat revsymbol usage
72 # to slower, but backward compat revsymbol usage
73 ctx = revsymbol(repo, ref)
73 ctx = revsymbol(repo, ref)
@@ -49,7 +49,7 b' class HooksHttpClient(object):'
49 try:
49 try:
50 connection.request('POST', '/', body)
50 connection.request('POST', '/', body)
51 except Exception:
51 except Exception:
52 log.error('Connection failed on %s', connection)
52 log.error('Hooks calling Connection failed on %s', connection.__dict__)
53 raise
53 raise
54 response = connection.getresponse()
54 response = connection.getresponse()
55
55
@@ -577,8 +577,8 b' def git_post_receive(unused_repo_path, r'
577 try:
577 try:
578 subprocessio.run_command(cmd, env=os.environ.copy())
578 subprocessio.run_command(cmd, env=os.environ.copy())
579 except Exception:
579 except Exception:
580 cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
580 cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', '"HEAD"',
581 'refs/heads/%s' % push_ref['name']]
581 '"refs/heads/%s"' % push_ref['name']]
582 print("Setting default branch to %s" % push_ref['name'])
582 print("Setting default branch to %s" % push_ref['name'])
583 subprocessio.run_command(cmd, env=os.environ.copy())
583 subprocessio.run_command(cmd, env=os.environ.copy())
584
584
@@ -21,21 +21,27 b' import base64'
21 import locale
21 import locale
22 import logging
22 import logging
23 import uuid
23 import uuid
24 import time
24 import wsgiref.util
25 import wsgiref.util
25 import traceback
26 import traceback
26 import tempfile
27 import tempfile
28 import psutil
29
27 from itertools import chain
30 from itertools import chain
28 from io import StringIO
31 from io import StringIO
29
32
30 import simplejson as json
33 import simplejson as json
31 import msgpack
34 import msgpack
32 import configparser
35 import configparser
36
33 from pyramid.config import Configurator
37 from pyramid.config import Configurator
34 from pyramid.settings import asbool, aslist
38 from pyramid.settings import asbool, aslist
35 from pyramid.wsgi import wsgiapp
39 from pyramid.wsgi import wsgiapp
36 from pyramid.response import Response
40 from pyramid.response import Response
37
41
42 from vcsserver.config.settings_maker import SettingsMaker
38 from vcsserver.utils import safe_int
43 from vcsserver.utils import safe_int
44 from vcsserver.lib.statsd_client import StatsdClient
39
45
40 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
41
47
@@ -49,6 +55,7 b' except locale.Error as e:'
49 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
55 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
50 os.environ['LC_ALL'] = 'C'
56 os.environ['LC_ALL'] = 'C'
51
57
58
52 import vcsserver
59 import vcsserver
53 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
60 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
54 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
61 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
@@ -98,38 +105,12 b' def _is_request_chunked(environ):'
98 return stream
105 return stream
99
106
100
107
101 def _int_setting(settings, name, default):
108 def log_max_fd():
102 settings[name] = int(settings.get(name, default))
109 try:
103 return settings[name]
110 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
104
111 log.info('Max file descriptors value: %s', maxfd)
105
112 except Exception:
106 def _bool_setting(settings, name, default):
113 pass
107 input_val = settings.get(name, default)
108 if isinstance(input_val, str):
109 input_val = input_val.encode('utf8')
110 settings[name] = asbool(input_val)
111 return settings[name]
112
113
114 def _list_setting(settings, name, default):
115 raw_value = settings.get(name, default)
116
117 # Otherwise we assume it uses pyramids space/newline separation.
118 settings[name] = aslist(raw_value)
119 return settings[name]
120
121
122 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
123 value = settings.get(name, default)
124
125 if default_when_empty and not value:
126 # use default value when value is empty
127 value = default
128
129 if lower:
130 value = value.lower()
131 settings[name] = value
132 return settings[name]
133
114
134
115
135 class VCS(object):
116 class VCS(object):
@@ -138,6 +119,8 b' class VCS(object):'
138 self.cache_config = cache_config
119 self.cache_config = cache_config
139 self._configure_locale()
120 self._configure_locale()
140
121
122 log_max_fd()
123
141 if GitFactory and GitRemote:
124 if GitFactory and GitRemote:
142 git_factory = GitFactory()
125 git_factory = GitFactory()
143 self._git_remote = GitRemote(git_factory)
126 self._git_remote = GitRemote(git_factory)
@@ -243,9 +226,11 b' class HTTPApplication(object):'
243 _use_echo_app = False
226 _use_echo_app = False
244
227
245 def __init__(self, settings=None, global_config=None):
228 def __init__(self, settings=None, global_config=None):
246 self._sanitize_settings_and_apply_defaults(settings)
247
229
248 self.config = Configurator(settings=settings)
230 self.config = Configurator(settings=settings)
231 # Init our statsd at very start
232 self.config.registry.statsd = StatsdClient.statsd
233
249 self.global_config = global_config
234 self.global_config = global_config
250 self.config.include('vcsserver.lib.rc_cache')
235 self.config.include('vcsserver.lib.rc_cache')
251
236
@@ -263,6 +248,7 b' class HTTPApplication(object):'
263 self.remote_wsgi = remote_wsgi_stub
248 self.remote_wsgi = remote_wsgi_stub
264
249
265 self._configure_settings(global_config, settings)
250 self._configure_settings(global_config, settings)
251
266 self._configure()
252 self._configure()
267
253
268 def _configure_settings(self, global_config, app_settings):
254 def _configure_settings(self, global_config, app_settings):
@@ -283,40 +269,6 b' class HTTPApplication(object):'
283 vcsserver.PYRAMID_SETTINGS = settings_merged
269 vcsserver.PYRAMID_SETTINGS = settings_merged
284 vcsserver.CONFIG = settings_merged
270 vcsserver.CONFIG = settings_merged
285
271
286 def _sanitize_settings_and_apply_defaults(self, settings):
287 temp_store = tempfile.gettempdir()
288 default_cache_dir = os.path.join(temp_store, 'rc_cache')
289
290 # save default, cache dir, and use it for all backends later.
291 default_cache_dir = _string_setting(
292 settings,
293 'cache_dir',
294 default_cache_dir, lower=False, default_when_empty=True)
295
296 # ensure we have our dir created
297 if not os.path.isdir(default_cache_dir):
298 os.makedirs(default_cache_dir, mode=0o755)
299
300 # exception store cache
301 _string_setting(
302 settings,
303 'exception_tracker.store_path',
304 temp_store, lower=False, default_when_empty=True)
305
306 # repo_object cache
307 _string_setting(
308 settings,
309 'rc_cache.repo_object.backend',
310 'dogpile.cache.rc.file_namespace', lower=False)
311 _int_setting(
312 settings,
313 'rc_cache.repo_object.expiration_time',
314 30 * 24 * 60 * 60)
315 _string_setting(
316 settings,
317 'rc_cache.repo_object.arguments.filename',
318 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
319
320 def _configure(self):
272 def _configure(self):
321 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
273 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
322
274
@@ -385,16 +337,22 b' class HTTPApplication(object):'
385 # NOTE(marcink): trading complexity for slight performance
337 # NOTE(marcink): trading complexity for slight performance
386 if log.isEnabledFor(logging.DEBUG):
338 if log.isEnabledFor(logging.DEBUG):
387 no_args_methods = [
339 no_args_methods = [
388 'archive_repo'
340
389 ]
341 ]
390 if method in no_args_methods:
342 if method in no_args_methods:
391 call_args = ''
343 call_args = ''
392 else:
344 else:
393 call_args = args[1:]
345 call_args = args[1:]
394
346
395 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
347 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
396 method, call_args, kwargs, context_uid, repo_state_uid)
348 method, call_args, kwargs, context_uid, repo_state_uid)
397
349
350 statsd = request.registry.statsd
351 if statsd:
352 statsd.incr(
353 'vcsserver_method_total', tags=[
354 "method:{}".format(method),
355 ])
398 return payload, remote, method, args, kwargs
356 return payload, remote, method, args, kwargs
399
357
400 def vcs_view(self, request):
358 def vcs_view(self, request):
@@ -431,7 +389,7 b' class HTTPApplication(object):'
431 should_store_exc = False
389 should_store_exc = False
432
390
433 if should_store_exc:
391 if should_store_exc:
434 store_exception(id(exc_info), exc_info)
392 store_exception(id(exc_info), exc_info, request_path=request.path)
435
393
436 tb_info = ''.join(
394 tb_info = ''.join(
437 traceback.format_exception(exc_type, exc_value, exc_traceback))
395 traceback.format_exception(exc_type, exc_value, exc_traceback))
@@ -450,6 +408,7 b' class HTTPApplication(object):'
450 'type': type_
408 'type': type_
451 }
409 }
452 }
410 }
411
453 try:
412 try:
454 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
413 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
455 except AttributeError:
414 except AttributeError:
@@ -678,6 +637,12 b' class HTTPApplication(object):'
678 log.error(
637 log.error(
679 'error occurred handling this request for path: %s, \n tb: %s',
638 'error occurred handling this request for path: %s, \n tb: %s',
680 request.path, traceback_info)
639 request.path, traceback_info)
640
641 statsd = request.registry.statsd
642 if statsd:
643 exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
644 statsd.incr('vcsserver_exception_total',
645 tags=["type:{}".format(exc_type)])
681 raise exception
646 raise exception
682
647
683
648
@@ -693,10 +658,83 b' class ResponseFilter(object):'
693 return self._start_response(status, headers, exc_info)
658 return self._start_response(status, headers, exc_info)
694
659
695
660
661 def sanitize_settings_and_apply_defaults(global_config, settings):
662 global_settings_maker = SettingsMaker(global_config)
663 settings_maker = SettingsMaker(settings)
664
665 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
666
667 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
668 settings_maker.enable_logging(logging_conf)
669
670 # Default includes, possible to change as a user
671 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
672 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
673
674 settings_maker.make_setting('__file__', global_config.get('__file__'))
675
676 settings_maker.make_setting('pyramid.default_locale_name', 'en')
677 settings_maker.make_setting('locale', 'en_US.UTF-8')
678
679 settings_maker.make_setting('core.binary_dir', '')
680
681 temp_store = tempfile.gettempdir()
682 default_cache_dir = os.path.join(temp_store, 'rc_cache')
683 # save default, cache dir, and use it for all backends later.
684 default_cache_dir = settings_maker.make_setting(
685 'cache_dir',
686 default=default_cache_dir, default_when_empty=True,
687 parser='dir:ensured')
688
689 # exception store cache
690 settings_maker.make_setting(
691 'exception_tracker.store_path',
692 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
693 parser='dir:ensured'
694 )
695
696 # repo_object cache defaults
697 settings_maker.make_setting(
698 'rc_cache.repo_object.backend',
699 default='dogpile.cache.rc.file_namespace',
700 parser='string')
701 settings_maker.make_setting(
702 'rc_cache.repo_object.expiration_time',
703 default=30 * 24 * 60 * 60, # 30days
704 parser='int')
705 settings_maker.make_setting(
706 'rc_cache.repo_object.arguments.filename',
707 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
708 parser='string')
709
710 # statsd
711 settings_maker.make_setting('statsd.enabled', False, parser='bool')
712 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
713 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
714 settings_maker.make_setting('statsd.statsd_prefix', '')
715 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
716
717 settings_maker.env_expand()
718
719
696 def main(global_config, **settings):
720 def main(global_config, **settings):
721 start_time = time.time()
722 log.info('Pyramid app config starting')
723
697 if MercurialFactory:
724 if MercurialFactory:
698 hgpatches.patch_largefiles_capabilities()
725 hgpatches.patch_largefiles_capabilities()
699 hgpatches.patch_subrepo_type_mapping()
726 hgpatches.patch_subrepo_type_mapping()
700
727
701 app = HTTPApplication(settings=settings, global_config=global_config)
728 # Fill in and sanitize the defaults & do ENV expansion
702 return app.wsgi_app()
729 sanitize_settings_and_apply_defaults(global_config, settings)
730
731 # init and bootstrap StatsdClient
732 StatsdClient.setup(settings)
733
734 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
735 total_time = time.time() - start_time
736 log.info('Pyramid app `%s` created and configured in %.2fs',
737 getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
738 return pyramid_app
739
740
@@ -66,7 +66,7 b' def get_exc_store():'
66 return _exc_store_path
66 return _exc_store_path
67
67
68
68
69 def _store_exception(exc_id, exc_info, prefix):
69 def _store_exception(exc_id, exc_info, prefix, request_path=''):
70 exc_type, exc_value, exc_traceback = exc_info
70 exc_type, exc_value, exc_traceback = exc_info
71
71
72 tb = ''.join(traceback.format_exception(
72 tb = ''.join(traceback.format_exception(
@@ -99,8 +99,13 b' def _store_exception(exc_id, exc_info, p'
99 f.write(exc_data)
99 f.write(exc_data)
100 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
100 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
101
101
102 log.error(
103 'error occurred handling this request.\n'
104 'Path: `%s`, tb: %s',
105 request_path, tb)
102
106
103 def store_exception(exc_id, exc_info, prefix=global_prefix):
107
108 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
104 """
109 """
105 Example usage::
110 Example usage::
106
111
@@ -109,7 +114,8 b' def store_exception(exc_id, exc_info, pr'
109 """
114 """
110
115
111 try:
116 try:
112 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
117 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix,
118 request_path=request_path)
113 except Exception:
119 except Exception:
114 log.exception('Failed to store exception `%s` information', exc_id)
120 log.exception('Failed to store exception `%s` information', exc_id)
115 # there's no way this can fail, it will crash server badly if it does.
121 # there's no way this can fail, it will crash server badly if it does.
@@ -38,7 +38,9 b' register_backend('
38 log = logging.getLogger(__name__)
38 log = logging.getLogger(__name__)
39
39
40 from . import region_meta
40 from . import region_meta
41 from .utils import (get_default_cache_settings, backend_key_generator, make_region)
41 from .utils import (
42 get_default_cache_settings, backend_key_generator, get_or_create_region,
43 clear_cache_namespace, make_region)
42
44
43
45
44 def configure_dogpile_cache(settings):
46 def configure_dogpile_cache(settings):
@@ -52,20 +54,25 b' def configure_dogpile_cache(settings):'
52 avail_regions = set()
54 avail_regions = set()
53 for key in rc_cache_data.keys():
55 for key in rc_cache_data.keys():
54 namespace_name = key.split('.', 1)[0]
56 namespace_name = key.split('.', 1)[0]
55 avail_regions.add(namespace_name)
57 if namespace_name in avail_regions:
56 log.debug('dogpile: found following cache regions: %s', avail_regions)
58 continue
57
59
58 # register them into namespace
60 avail_regions.add(namespace_name)
59 for region_name in avail_regions:
61 log.debug('dogpile: found following cache regions: %s', namespace_name)
62
60 new_region = make_region(
63 new_region = make_region(
61 name=region_name,
64 name=namespace_name,
62 function_key_generator=None
65 function_key_generator=None
63 )
66 )
64
67
65 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
68 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(namespace_name))
66 new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
69 new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
67 log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
70 if log.isEnabledFor(logging.DEBUG):
68 region_meta.dogpile_cache_regions[region_name] = new_region
71 region_args = dict(backend=new_region.actual_backend.__class__,
72 region_invalidator=new_region.region_invalidator.__class__)
73 log.debug('dogpile: registering a new region `%s` %s', namespace_name, region_args)
74
75 region_meta.dogpile_cache_regions[namespace_name] = new_region
69
76
70
77
71 def includeme(config):
78 def includeme(config):
@@ -29,7 +29,10 b' from dogpile.cache.backends import redis'
29 from dogpile.cache.backends.file import NO_VALUE, FileLock
29 from dogpile.cache.backends.file import NO_VALUE, FileLock
30 from dogpile.cache.util import memoized_property
30 from dogpile.cache.util import memoized_property
31
31
32 from pyramid.settings import asbool
33
32 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
34 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
35 from vcsserver.utils import safe_str, safe_unicode
33
36
34
37
35 _default_max_size = 1024
38 _default_max_size = 1024
@@ -124,7 +127,14 b' class FileNamespaceBackend(PickleSeriali'
124
127
125 def __init__(self, arguments):
128 def __init__(self, arguments):
126 arguments['lock_factory'] = CustomLockFactory
129 arguments['lock_factory'] = CustomLockFactory
127 super(FileNamespaceBackend, self).__init__(arguments)
130 db_file = arguments.get('filename')
131
132 log.debug('initialing %s DB in %s', self.__class__.__name__, db_file)
133 try:
134 super(FileNamespaceBackend, self).__init__(arguments)
135 except Exception:
136 log.exception('Failed to initialize db at: %s', db_file)
137 raise
128
138
129 def __repr__(self):
139 def __repr__(self):
130 return '{} `{}`'.format(self.__class__, self.filename)
140 return '{} `{}`'.format(self.__class__, self.filename)
@@ -141,13 +151,16 b' class FileNamespaceBackend(PickleSeriali'
141 return False
151 return False
142
152
143 with self._dbm_file(True) as dbm:
153 with self._dbm_file(True) as dbm:
144
154 try:
145 return filter(cond, dbm.keys())
155 return filter(cond, dbm.keys())
156 except Exception:
157 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
158 raise
146
159
147 def get_store(self):
160 def get_store(self):
148 return self.filename
161 return self.filename
149
162
150 def get(self, key):
163 def _dbm_get(self, key):
151 with self._dbm_file(False) as dbm:
164 with self._dbm_file(False) as dbm:
152 if hasattr(dbm, 'get'):
165 if hasattr(dbm, 'get'):
153 value = dbm.get(key, NO_VALUE)
166 value = dbm.get(key, NO_VALUE)
@@ -161,6 +174,13 b' class FileNamespaceBackend(PickleSeriali'
161 value = self._loads(value)
174 value = self._loads(value)
162 return value
175 return value
163
176
177 def get(self, key):
178 try:
179 return self._dbm_get(key)
180 except Exception:
181 log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
182 raise
183
164 def set(self, key, value):
184 def set(self, key, value):
165 with self._dbm_file(True) as dbm:
185 with self._dbm_file(True) as dbm:
166 dbm[key] = self._dumps(value)
186 dbm[key] = self._dumps(value)
@@ -172,6 +192,16 b' class FileNamespaceBackend(PickleSeriali'
172
192
173
193
174 class BaseRedisBackend(redis_backend.RedisBackend):
194 class BaseRedisBackend(redis_backend.RedisBackend):
195 key_prefix = ''
196
197 def __init__(self, arguments):
198 super(BaseRedisBackend, self).__init__(arguments)
199 self._lock_timeout = self.lock_timeout
200 self._lock_auto_renewal = asbool(arguments.pop("lock_auto_renewal", True))
201
202 if self._lock_auto_renewal and not self._lock_timeout:
203 # set default timeout for auto_renewal
204 self._lock_timeout = 30
175
205
176 def _create_client(self):
206 def _create_client(self):
177 args = {}
207 args = {}
@@ -234,11 +264,10 b' class BaseRedisBackend(redis_backend.Red'
234 pipe.execute()
264 pipe.execute()
235
265
236 def get_mutex(self, key):
266 def get_mutex(self, key):
237 u = redis_backend.u
238 if self.distributed_lock:
267 if self.distributed_lock:
239 lock_key = u('_lock_{0}').format(key)
268 lock_key = u'_lock_{0}'.format(safe_unicode(key))
240 log.debug('Trying to acquire Redis lock for key %s', lock_key)
269 return get_mutex_lock(self.client, lock_key, self._lock_timeout,
241 return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
270 auto_renewal=self._lock_auto_renewal)
242 else:
271 else:
243 return None
272 return None
244
273
@@ -251,3 +280,50 b' class RedisPickleBackend(PickleSerialize'
251 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
280 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
252 key_prefix = 'redis_msgpack_backend'
281 key_prefix = 'redis_msgpack_backend'
253 pass
282 pass
283
284
285 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
286 import redis_lock
287
288 class _RedisLockWrapper(object):
289 """LockWrapper for redis_lock"""
290
291 @classmethod
292 def get_lock(cls):
293 return redis_lock.Lock(
294 redis_client=client,
295 name=lock_key,
296 expire=lock_timeout,
297 auto_renewal=auto_renewal,
298 strict=True,
299 )
300
301 def __repr__(self):
302 return "{}:{}".format(self.__class__.__name__, lock_key)
303
304 def __str__(self):
305 return "{}:{}".format(self.__class__.__name__, lock_key)
306
307 def __init__(self):
308 self.lock = self.get_lock()
309 self.lock_key = lock_key
310
311 def acquire(self, wait=True):
312 log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
313 try:
314 acquired = self.lock.acquire(wait)
315 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
316 return acquired
317 except redis_lock.AlreadyAcquired:
318 return False
319 except redis_lock.AlreadyStarted:
320 # refresh thread exists, but it also means we acquired the lock
321 return True
322
323 def release(self):
324 try:
325 self.lock.release()
326 except redis_lock.NotAcquired:
327 pass
328
329 return _RedisLockWrapper()
@@ -16,15 +16,16 b''
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import time
19 import logging
20 import logging
20 import functools
21 import functools
21 from decorator import decorate
22
22
23 from dogpile.cache import CacheRegion
23 from dogpile.cache import CacheRegion
24 from dogpile.cache.util import compat
24 from dogpile.cache.util import compat
25
25
26 from vcsserver.utils import safe_str, sha1
26 from vcsserver.utils import safe_str, sha1
27
27
28 from vcsserver.lib.rc_cache import region_meta
28
29
29 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
30
31
@@ -48,18 +49,77 b' class RhodeCodeCacheRegion(CacheRegion):'
48 if function_key_generator is None:
49 if function_key_generator is None:
49 function_key_generator = self.function_key_generator
50 function_key_generator = self.function_key_generator
50
51
52 # workaround for py2 and cython problems, this block should be removed
53 # once we've migrated to py3
54 if 'cython' == 'cython':
55 def decorator(fn):
56 if to_str is compat.string_type:
57 # backwards compatible
58 key_generator = function_key_generator(namespace, fn)
59 else:
60 key_generator = function_key_generator(namespace, fn, to_str=to_str)
61
62 @functools.wraps(fn)
63 def decorate(*arg, **kw):
64 key = key_generator(*arg, **kw)
65
66 @functools.wraps(fn)
67 def creator():
68 return fn(*arg, **kw)
69
70 if not condition:
71 return creator()
72
73 timeout = expiration_time() if expiration_time_is_callable \
74 else expiration_time
75
76 return self.get_or_create(key, creator, timeout, should_cache_fn)
77
78 def invalidate(*arg, **kw):
79 key = key_generator(*arg, **kw)
80 self.delete(key)
81
82 def set_(value, *arg, **kw):
83 key = key_generator(*arg, **kw)
84 self.set(key, value)
85
86 def get(*arg, **kw):
87 key = key_generator(*arg, **kw)
88 return self.get(key)
89
90 def refresh(*arg, **kw):
91 key = key_generator(*arg, **kw)
92 value = fn(*arg, **kw)
93 self.set(key, value)
94 return value
95
96 decorate.set = set_
97 decorate.invalidate = invalidate
98 decorate.refresh = refresh
99 decorate.get = get
100 decorate.original = fn
101 decorate.key_generator = key_generator
102 decorate.__wrapped__ = fn
103
104 return decorate
105 return decorator
106
51 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
107 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
52
108
53 if not condition:
109 if not condition:
54 log.debug('Calling un-cached func:%s', user_func.func_name)
110 log.debug('Calling un-cached method:%s', user_func.func_name)
55 return user_func(*arg, **kw)
111 start = time.time()
112 result = user_func(*arg, **kw)
113 total = time.time() - start
114 log.debug('un-cached method:%s took %.4fs', user_func.func_name, total)
115 return result
56
116
57 key = key_generator(*arg, **kw)
117 key = key_generator(*arg, **kw)
58
118
59 timeout = expiration_time() if expiration_time_is_callable \
119 timeout = expiration_time() if expiration_time_is_callable \
60 else expiration_time
120 else expiration_time
61
121
62 log.debug('Calling cached fn:%s', user_func.func_name)
122 log.debug('Calling cached method:`%s`', user_func.func_name)
63 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
123 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
64
124
65 def cache_decorator(user_func):
125 def cache_decorator(user_func):
@@ -98,8 +158,7 b' class RhodeCodeCacheRegion(CacheRegion):'
98 user_func.original = user_func
158 user_func.original = user_func
99
159
100 # Use `decorate` to preserve the signature of :param:`user_func`.
160 # Use `decorate` to preserve the signature of :param:`user_func`.
101
161 return decorator.decorate(user_func, functools.partial(
102 return decorate(user_func, functools.partial(
103 get_or_create_for_user_func, key_generator))
162 get_or_create_for_user_func, key_generator))
104
163
105 return cache_decorator
164 return cache_decorator
@@ -151,3 +210,54 b' def key_generator(backend, namespace, fn'
151 return final_key
210 return final_key
152
211
153 return generate_key
212 return generate_key
213
214
215 def get_or_create_region(region_name, region_namespace=None):
216 from vcsserver.lib.rc_cache.backends import FileNamespaceBackend
217 region_obj = region_meta.dogpile_cache_regions.get(region_name)
218 if not region_obj:
219 raise EnvironmentError(
220 'Region `{}` not in configured: {}.'.format(
221 region_name, region_meta.dogpile_cache_regions.keys()))
222
223 region_uid_name = '{}:{}'.format(region_name, region_namespace)
224 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
225 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
226 if region_exist:
227 log.debug('Using already configured region: %s', region_namespace)
228 return region_exist
229 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
230 expiration_time = region_obj.expiration_time
231
232 if not os.path.isdir(cache_dir):
233 os.makedirs(cache_dir)
234 new_region = make_region(
235 name=region_uid_name,
236 function_key_generator=backend_key_generator(region_obj.actual_backend)
237 )
238 namespace_filename = os.path.join(
239 cache_dir, "{}.cache.dbm".format(region_namespace))
240 # special type that allows 1db per namespace
241 new_region.configure(
242 backend='dogpile.cache.rc.file_namespace',
243 expiration_time=expiration_time,
244 arguments={"filename": namespace_filename}
245 )
246
247 # create and save in region caches
248 log.debug('configuring new region: %s', region_uid_name)
249 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
250
251 return region_obj
252
253
254 def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
255 region = get_or_create_region(cache_region, cache_namespace_uid)
256 cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
257 num_delete_keys = len(cache_keys)
258 if invalidate:
259 region.invalidate(hard=False)
260 else:
261 if num_delete_keys:
262 region.delete_multi(cache_keys)
263 return num_delete_keys
@@ -19,6 +19,7 b''
19
19
20 import os
20 import os
21 import subprocess
21 import subprocess
22 import time
22 from urllib.error import URLError
23 from urllib.error import URLError
23 import urllib.parse
24 import urllib.parse
24 import logging
25 import logging
@@ -35,7 +36,9 b' import svn.fs'
35 import svn.repos
36 import svn.repos
36
37
37 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 from vcsserver.base import RepoFactory, raise_from_original
39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
40 from vcsserver.exceptions import NoContentException
41 from vcsserver.utils import safe_str
39 from vcsserver.vcs_base import RemoteBase
42 from vcsserver.vcs_base import RemoteBase
40
43
41 log = logging.getLogger(__name__)
44 log = logging.getLogger(__name__)
@@ -198,7 +201,8 b' class SvnRemote(RemoteBase):'
198 def revision_properties(self, wire, revision):
201 def revision_properties(self, wire, revision):
199
202
200 cache_on, context_uid, repo_id = self._cache_on(wire)
203 cache_on, context_uid, repo_id = self._cache_on(wire)
201 @self.region.conditional_cache_on_arguments(condition=cache_on)
204 region = self._region(wire)
205 @region.conditional_cache_on_arguments(condition=cache_on)
202 def _revision_properties(_repo_id, _revision):
206 def _revision_properties(_repo_id, _revision):
203 repo = self._factory.repo(wire)
207 repo = self._factory.repo(wire)
204 fs_ptr = svn.repos.fs(repo)
208 fs_ptr = svn.repos.fs(repo)
@@ -252,7 +256,8 b' class SvnRemote(RemoteBase):'
252 @reraise_safe_exceptions
256 @reraise_safe_exceptions
253 def node_history(self, wire, path, revision, limit):
257 def node_history(self, wire, path, revision, limit):
254 cache_on, context_uid, repo_id = self._cache_on(wire)
258 cache_on, context_uid, repo_id = self._cache_on(wire)
255 @self.region.conditional_cache_on_arguments(condition=cache_on)
259 region = self._region(wire)
260 @region.conditional_cache_on_arguments(condition=cache_on)
256 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
261 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
257 cross_copies = False
262 cross_copies = False
258 repo = self._factory.repo(wire)
263 repo = self._factory.repo(wire)
@@ -273,7 +278,8 b' class SvnRemote(RemoteBase):'
273
278
274 def node_properties(self, wire, path, revision):
279 def node_properties(self, wire, path, revision):
275 cache_on, context_uid, repo_id = self._cache_on(wire)
280 cache_on, context_uid, repo_id = self._cache_on(wire)
276 @self.region.conditional_cache_on_arguments(condition=cache_on)
281 region = self._region(wire)
282 @region.conditional_cache_on_arguments(condition=cache_on)
277 def _node_properties(_repo_id, _path, _revision):
283 def _node_properties(_repo_id, _path, _revision):
278 repo = self._factory.repo(wire)
284 repo = self._factory.repo(wire)
279 fsobj = svn.repos.fs(repo)
285 fsobj = svn.repos.fs(repo)
@@ -282,7 +288,7 b' class SvnRemote(RemoteBase):'
282 return _node_properties(repo_id, path, revision)
288 return _node_properties(repo_id, path, revision)
283
289
284 def file_annotate(self, wire, path, revision):
290 def file_annotate(self, wire, path, revision):
285 abs_path = 'file://' + urllib.request.pathname2url(
291 abs_path = 'file://' + urllib.pathname2url(
286 vcspath.join(wire['path'], path))
292 vcspath.join(wire['path'], path))
287 file_uri = svn.core.svn_path_canonicalize(abs_path)
293 file_uri = svn.core.svn_path_canonicalize(abs_path)
288
294
@@ -311,7 +317,8 b' class SvnRemote(RemoteBase):'
311 def get_node_type(self, wire, path, revision=None):
317 def get_node_type(self, wire, path, revision=None):
312
318
313 cache_on, context_uid, repo_id = self._cache_on(wire)
319 cache_on, context_uid, repo_id = self._cache_on(wire)
314 @self.region.conditional_cache_on_arguments(condition=cache_on)
320 region = self._region(wire)
321 @region.conditional_cache_on_arguments(condition=cache_on)
315 def _get_node_type(_repo_id, _path, _revision):
322 def _get_node_type(_repo_id, _path, _revision):
316 repo = self._factory.repo(wire)
323 repo = self._factory.repo(wire)
317 fs_ptr = svn.repos.fs(repo)
324 fs_ptr = svn.repos.fs(repo)
@@ -325,7 +332,8 b' class SvnRemote(RemoteBase):'
325 def get_nodes(self, wire, path, revision=None):
332 def get_nodes(self, wire, path, revision=None):
326
333
327 cache_on, context_uid, repo_id = self._cache_on(wire)
334 cache_on, context_uid, repo_id = self._cache_on(wire)
328 @self.region.conditional_cache_on_arguments(condition=cache_on)
335 region = self._region(wire)
336 @region.conditional_cache_on_arguments(condition=cache_on)
329 def _get_nodes(_repo_id, _path, _revision):
337 def _get_nodes(_repo_id, _path, _revision):
330 repo = self._factory.repo(wire)
338 repo = self._factory.repo(wire)
331 fsobj = svn.repos.fs(repo)
339 fsobj = svn.repos.fs(repo)
@@ -352,7 +360,8 b' class SvnRemote(RemoteBase):'
352 def get_file_size(self, wire, path, revision=None):
360 def get_file_size(self, wire, path, revision=None):
353
361
354 cache_on, context_uid, repo_id = self._cache_on(wire)
362 cache_on, context_uid, repo_id = self._cache_on(wire)
355 @self.region.conditional_cache_on_arguments(condition=cache_on)
363 region = self._region(wire)
364 @region.conditional_cache_on_arguments(condition=cache_on)
356 def _get_file_size(_repo_id, _path, _revision):
365 def _get_file_size(_repo_id, _path, _revision):
357 repo = self._factory.repo(wire)
366 repo = self._factory.repo(wire)
358 fsobj = svn.repos.fs(repo)
367 fsobj = svn.repos.fs(repo)
@@ -467,7 +476,8 b' class SvnRemote(RemoteBase):'
467 def is_binary(self, wire, rev, path):
476 def is_binary(self, wire, rev, path):
468 cache_on, context_uid, repo_id = self._cache_on(wire)
477 cache_on, context_uid, repo_id = self._cache_on(wire)
469
478
470 @self.region.conditional_cache_on_arguments(condition=cache_on)
479 region = self._region(wire)
480 @region.conditional_cache_on_arguments(condition=cache_on)
471 def _is_binary(_repo_id, _rev, _path):
481 def _is_binary(_repo_id, _rev, _path):
472 raw_bytes = self.get_file_content(wire, path, rev)
482 raw_bytes = self.get_file_content(wire, path, rev)
473 return raw_bytes and '\0' in raw_bytes
483 return raw_bytes and '\0' in raw_bytes
@@ -481,9 +491,7 b' class SvnRemote(RemoteBase):'
481 if path and os.path.isdir(path):
491 if path and os.path.isdir(path):
482 opts['cwd'] = path
492 opts['cwd'] = path
483
493
484 safe_call = False
494 safe_call = opts.pop('_safe', False)
485 if '_safe' in opts:
486 safe_call = True
487
495
488 svnenv = os.environ.copy()
496 svnenv = os.environ.copy()
489 svnenv.update(opts.pop('extra_env', {}))
497 svnenv.update(opts.pop('extra_env', {}))
@@ -496,15 +504,15 b' class SvnRemote(RemoteBase):'
496
504
497 return ''.join(p), ''.join(p.error)
505 return ''.join(p), ''.join(p.error)
498 except (EnvironmentError, OSError) as err:
506 except (EnvironmentError, OSError) as err:
507 if safe_call:
508 return '', safe_str(err).strip()
509 else:
499 cmd = ' '.join(cmd) # human friendly CMD
510 cmd = ' '.join(cmd) # human friendly CMD
500 tb_err = ("Couldn't run svn command (%s).\n"
511 tb_err = ("Couldn't run svn command (%s).\n"
501 "Original error was:%s\n"
512 "Original error was:%s\n"
502 "Call options:%s\n"
513 "Call options:%s\n"
503 % (cmd, err, _opts))
514 % (cmd, err, _opts))
504 log.exception(tb_err)
515 log.exception(tb_err)
505 if safe_call:
506 return '', err
507 else:
508 raise exceptions.VcsException()(tb_err)
516 raise exceptions.VcsException()(tb_err)
509
517
510 @reraise_safe_exceptions
518 @reraise_safe_exceptions
@@ -528,6 +536,74 b' class SvnRemote(RemoteBase):'
528 'post_version': get_svn_post_hook_version(repo_path),
536 'post_version': get_svn_post_hook_version(repo_path),
529 }
537 }
530
538
539 @reraise_safe_exceptions
540 def set_head_ref(self, wire, head_name):
541 pass
542
543 @reraise_safe_exceptions
544 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
545 archive_dir_name, commit_id):
546
547 def walk_tree(root, root_dir, _commit_id):
548 """
549 Special recursive svn repo walker
550 """
551
552 filemode_default = 0o100644
553 filemode_executable = 0o100755
554
555 file_iter = svn.fs.dir_entries(root, root_dir)
556 for f_name in file_iter:
557 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
558
559 if f_type == 'dir':
560 # return only DIR, and then all entries in that dir
561 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
562 new_root = os.path.join(root_dir, f_name)
563 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
564 yield _f_name, _f_data, _f_type
565 else:
566 f_path = os.path.join(root_dir, f_name).rstrip('/')
567 prop_list = svn.fs.node_proplist(root, f_path)
568
569 f_mode = filemode_default
570 if prop_list.get('svn:executable'):
571 f_mode = filemode_executable
572
573 f_is_link = False
574 if prop_list.get('svn:special'):
575 f_is_link = True
576
577 data = {
578 'is_link': f_is_link,
579 'mode': f_mode,
580 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
581 }
582
583 yield f_path, data, f_type
584
585 def file_walker(_commit_id, path):
586 repo = self._factory.repo(wire)
587 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
588
589 def no_content():
590 raise NoContentException()
591
592 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
593 file_path = f_name
594
595 if f_type == 'dir':
596 mode = f_data['mode']
597 yield ArchiveNode(file_path, mode, False, no_content)
598 else:
599 mode = f_data['mode']
600 is_link = f_data['is_link']
601 data_stream = f_data['content_stream']
602 yield ArchiveNode(file_path, mode, is_link, data_stream)
603
604 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
605 archive_dir_name, commit_id)
606
531
607
532 class SvnDiffer(object):
608 class SvnDiffer(object):
533 """
609 """
@@ -16,13 +16,12 b''
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import socket
18 import socket
19
20 import pytest
19 import pytest
21
20
22
21
23 def pytest_addoption(parser):
22 def pytest_addoption(parser):
24 parser.addoption(
23 parser.addoption(
25 '--repeat', type=int, default=100,
24 '--perf-repeat-vcs', type=int, default=100,
26 help="Number of repetitions in performance tests.")
25 help="Number of repetitions in performance tests.")
27
26
28
27
@@ -34,7 +33,7 b' def repeat(request):'
34 Slower calls may divide it by 10 or 100. It is chosen in a way so that the
33 Slower calls may divide it by 10 or 100. It is chosen in a way so that the
35 tests are not too slow in our default test suite.
34 tests are not too slow in our default test suite.
36 """
35 """
37 return request.config.getoption('--repeat')
36 return request.config.getoption('--perf-repeat-vcs')
38
37
39
38
40 @pytest.fixture(scope='session')
39 @pytest.fixture(scope='session')
@@ -113,7 +113,7 b' class TestReraiseSafeExceptions(object):'
113
113
114 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
114 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
115 for method_name, method in methods:
115 for method_name, method in methods:
116 if not method_name.startswith('_'):
116 if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']:
117 assert method.im_func.__code__ == decorator.__code__
117 assert method.im_func.__code__ == decorator.__code__
118
118
119 @pytest.mark.parametrize('side_effect, expected_type', [
119 @pytest.mark.parametrize('side_effect, expected_type', [
@@ -50,7 +50,7 b' class TestReraiseSafeExceptions(object):'
50 methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
50 methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
51 decorator = hg.reraise_safe_exceptions(None)
51 decorator = hg.reraise_safe_exceptions(None)
52 for method_name, method in methods:
52 for method_name, method in methods:
53 if not method_name.startswith('_'):
53 if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']:
54 assert method.im_func.__code__ == decorator.__code__
54 assert method.im_func.__code__ == decorator.__code__
55
55
56 @pytest.mark.parametrize('side_effect, expected_type', [
56 @pytest.mark.parametrize('side_effect, expected_type', [
@@ -14,7 +14,10 b' def vcs_app():'
14 'dev.use_echo_app': 'true',
14 'dev.use_echo_app': 'true',
15 'locale': 'en_US.UTF-8',
15 'locale': 'en_US.UTF-8',
16 }
16 }
17 vcs_app = main({}, **stub_settings)
17 stub_global_conf = {
18 '__file__': ''
19 }
20 vcs_app = main(stub_global_conf, **stub_settings)
18 app = webtest.TestApp(vcs_app)
21 app = webtest.TestApp(vcs_app)
19 return app
22 return app
20
23
@@ -25,7 +25,7 b' from vcsserver.base import obfuscate_qs'
25 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
25 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
28 http_main.main({})
28 http_main.main({'__file__': ''})
29 patch_largefiles_capabilities.assert_called_once_with()
29 patch_largefiles_capabilities.assert_called_once_with()
30
30
31
31
@@ -35,7 +35,7 b' def test_applies_largefiles_patch(patch_'
35 'vcsserver.hgpatches.patch_largefiles_capabilities',
35 'vcsserver.hgpatches.patch_largefiles_capabilities',
36 mock.Mock(side_effect=Exception("Must not be called")))
36 mock.Mock(side_effect=Exception("Must not be called")))
37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
38 http_main.main({})
38 http_main.main({'__file__': ''})
39
39
40
40
41 @pytest.mark.parametrize('given, expected', [
41 @pytest.mark.parametrize('given, expected', [
@@ -25,15 +25,23 b' from vcsserver.utils import safe_str'
25 log = logging.getLogger(__name__)
25 log = logging.getLogger(__name__)
26
26
27
27
28 def get_access_path(request):
28 def get_access_path(environ):
29 environ = request.environ
29 path = environ.get('PATH_INFO')
30 return environ.get('PATH_INFO')
30 return path
31
31
32
32
33 def get_user_agent(environ):
33 def get_user_agent(environ):
34 return environ.get('HTTP_USER_AGENT')
34 return environ.get('HTTP_USER_AGENT')
35
35
36
36
37 def get_vcs_method(environ):
38 return environ.get('HTTP_X_RC_METHOD')
39
40
41 def get_vcs_repo(environ):
42 return environ.get('HTTP_X_RC_REPO_NAME')
43
44
37 class RequestWrapperTween(object):
45 class RequestWrapperTween(object):
38 def __init__(self, handler, registry):
46 def __init__(self, handler, registry):
39 self.handler = handler
47 self.handler = handler
@@ -43,17 +51,52 b' class RequestWrapperTween(object):'
43
51
44 def __call__(self, request):
52 def __call__(self, request):
45 start = time.time()
53 start = time.time()
54 log.debug('Starting request time measurement')
55 response = None
56
57 ua = get_user_agent(request.environ)
58 vcs_method = get_vcs_method(request.environ)
59 repo_name = get_vcs_repo(request.environ)
60
46 try:
61 try:
47 response = self.handler(request)
62 response = self.handler(request)
48 finally:
63 finally:
49 end = time.time()
50 total = end - start
51 count = request.request_count()
64 count = request.request_count()
52 _ver_ = vcsserver.__version__
65 _ver_ = vcsserver.__version__
66 _path = safe_str(get_access_path(request.environ))
67 ip = '127.0.0.1'
68 match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
69 resp_code = getattr(response, 'status_code', 'UNDEFINED')
70
71 total = time.time() - start
72
73 _view_path = "{}/{}@{}".format(_path, vcs_method, repo_name)
53 log.info(
74 log.info(
54 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
75 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
55 count, '127.0.0.1', request.environ.get('REQUEST_METHOD'),
76 count, ip, request.environ.get('REQUEST_METHOD'),
56 safe_str(get_access_path(request)), total, get_user_agent(request.environ), _ver_)
77 _view_path, total, ua, _ver_,
78 extra={"time": total, "ver": _ver_, "code": resp_code,
79 "path": _path, "view_name": match_route, "user_agent": ua,
80 "vcs_method": vcs_method, "repo_name": repo_name}
81 )
82
83 statsd = request.registry.statsd
84 if statsd:
85 match_route = request.matched_route.name if request.matched_route else _path
86 elapsed_time_ms = round(1000.0 * total) # use ms only
87 statsd.timing(
88 "vcsserver_req_timing.histogram", elapsed_time_ms,
89 tags=[
90 "view_name:{}".format(match_route),
91 "code:{}".format(resp_code)
92 ],
93 use_decimals=False
94 )
95 statsd.incr(
96 "vcsserver_req_total", tags=[
97 "view_name:{}".format(match_route),
98 "code:{}".format(resp_code)
99 ])
57
100
58 return response
101 return response
59
102
@@ -15,13 +15,15 b''
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 from vcsserver.lib import rc_cache
18
19
19 class RemoteBase(object):
20 class RemoteBase(object):
20 EMPTY_COMMIT = '0' * 40
21 EMPTY_COMMIT = '0' * 40
21
22
22 @property
23 def _region(self, wire):
23 def region(self):
24 cache_repo_id = wire.get('cache_repo_id', '')
24 return self._factory._cache_region
25 cache_namespace_uid = 'cache_repo.{}'.format(cache_repo_id)
26 return rc_cache.get_or_create_region('repo_object', cache_namespace_uid)
25
27
26 def _cache_on(self, wire):
28 def _cache_on(self, wire):
27 context = wire.get('context', '')
29 context = wire.get('context', '')
@@ -30,3 +32,15 b' class RemoteBase(object):'
30 cache = wire.get('cache', True)
32 cache = wire.get('cache', True)
31 cache_on = context and cache
33 cache_on = context and cache
32 return cache_on, context_uid, repo_id
34 return cache_on, context_uid, repo_id
35
36 def vcsserver_invalidate_cache(self, wire, delete):
37 from vcsserver.lib import rc_cache
38 repo_id = wire.get('repo_id', '')
39 cache_repo_id = wire.get('cache_repo_id', '')
40 cache_namespace_uid = 'cache_repo.{}'.format(cache_repo_id)
41
42 if delete:
43 rc_cache.clear_cache_namespace(
44 'repo_object', cache_namespace_uid, invalidate=True)
45
46 return {'invalidated': {'repo_id': repo_id, 'delete': delete}}
General Comments 0
You need to be logged in to leave comments. Login now