##// END OF EJS Templates
merge: with lataest default branch
super-admin -
r1041:e1e68d19 merge python3
parent child Browse files
Show More
@@ -0,0 +1,16 b''
1 [DEFAULT]
2 done = false
3
4 [task:bump_version]
5 done = true
6
7 [task:fixes_on_stable]
8 done = true
9
10 [task:pip2nix_generated]
11 done = true
12
13 [release]
14 state = prepared
15 version = 4.27.1
16
@@ -0,0 +1,53 b''
1 ; #####################
2 ; LOGGING CONFIGURATION
3 ; #####################
4 ; Logging template, used for configure the logging
5 ; some variables here are replaced by RhodeCode to default values
6
7 [loggers]
8 keys = root, vcsserver
9
10 [handlers]
11 keys = console
12
13 [formatters]
14 keys = generic, json
15
16 ; #######
17 ; LOGGERS
18 ; #######
19 [logger_root]
20 level = NOTSET
21 handlers = console
22
23 [logger_vcsserver]
24 level = $RC_LOGGING_LEVEL
25 handlers =
26 qualname = vcsserver
27 propagate = 1
28
29 ; ########
30 ; HANDLERS
31 ; ########
32
33 [handler_console]
34 class = StreamHandler
35 args = (sys.stderr, )
36 level = $RC_LOGGING_LEVEL
37 ; To enable JSON formatted logs replace generic with json
38 ; This allows sending properly formatted logs to grafana loki or elasticsearch
39 #formatter = json
40 #formatter = generic
41 formatter = $RC_LOGGING_FORMATTER
42
43 ; ##########
44 ; FORMATTERS
45 ; ##########
46
47 [formatter_generic]
48 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
49 datefmt = %Y-%m-%d %H:%M:%S
50
51 [formatter_json]
52 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
53 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -0,0 +1,10 b''
1 diff -rup configparser-4.0.2-orig/pyproject.toml configparser-4.0.2/pyproject.toml
2 --- configparser-4.0.2-orig/pyproject.toml 2021-03-22 21:28:11.000000000 +0100
3 +++ configparser-4.0.2/pyproject.toml 2021-03-22 21:28:11.000000000 +0100
4 @@ -1,5 +1,5 @@
5 [build-system]
6 -requires = ["setuptools>=40.7", "wheel", "setuptools_scm>=1.15"]
7 +requires = ["setuptools<=42.0", "wheel", "setuptools_scm<6.0.0"]
8 build-backend = "setuptools.build_meta"
9
10 [tool.black]
@@ -0,0 +1,7 b''
1 diff -rup importlib-metadata-1.6.0-orig/yproject.toml importlib-metadata-1.6.0/pyproject.toml
2 --- importlib-metadata-1.6.0-orig/yproject.toml 2021-03-22 22:10:33.000000000 +0100
3 +++ importlib-metadata-1.6.0/pyproject.toml 2021-03-22 22:11:09.000000000 +0100
4 @@ -1,3 +1,3 @@
5 [build-system]
6 -requires = ["setuptools>=30.3", "wheel", "setuptools_scm"]
7 +requires = ["setuptools<42.0", "wheel", "setuptools_scm<6.0.0"]
@@ -0,0 +1,12 b''
1 diff -rup pytest-4.6.9-orig/setup.py pytest-4.6.9/setup.py
2 --- pytest-4.6.9-orig/setup.py 2018-04-10 10:23:04.000000000 +0200
3 +++ pytest-4.6.9/setup.py 2018-04-10 10:23:34.000000000 +0200
4 @@ -24,7 +24,7 @@ def main():
5 def main():
6 setup(
7 use_scm_version={"write_to": "src/_pytest/_version.py"},
8 - setup_requires=["setuptools-scm", "setuptools>=40.0"],
9 + setup_requires=["setuptools-scm<6.0.0", "setuptools<=42.0"],
10 package_dir={"": "src"},
11 # fmt: off
12 extras_require={ No newline at end of file
@@ -0,0 +1,10 b''
1 diff -rup zip-1.2.0-orig/pyproject.toml zip-1.2.0/pyproject.toml
2 --- zip-1.2.0-orig/pyproject.toml 2021-03-23 10:55:37.000000000 +0100
3 +++ zip-1.2.0/pyproject.toml 2021-03-23 10:56:05.000000000 +0100
4 @@ -1,5 +1,5 @@
5 [build-system]
6 -requires = ["setuptools>=34.4", "wheel", "setuptools_scm>=1.15"]
7 +requires = ["setuptools<42.0", "wheel", "setuptools_scm<6.0.0"]
8 build-backend = "setuptools.build_meta"
9
10 [tool.black]
1 NO CONTENT: new file 100644
@@ -0,0 +1,207 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2020 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import os
22 import textwrap
23 import string
24 import functools
25 import logging
26 import tempfile
27 import logging.config
28 log = logging.getLogger(__name__)
29
30 # skip keys, that are set here, so we don't double process those
31 set_keys = {
32 '__file__': ''
33 }
34
35
36 def str2bool(_str):
37 """
38 returns True/False value from given string, it tries to translate the
39 string into boolean
40
41 :param _str: string value to translate into boolean
42 :rtype: boolean
43 :returns: boolean from given string
44 """
45 if _str is None:
46 return False
47 if _str in (True, False):
48 return _str
49 _str = str(_str).strip().lower()
50 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
51
52
53 def aslist(obj, sep=None, strip=True):
54 """
55 Returns given string separated by sep as list
56
57 :param obj:
58 :param sep:
59 :param strip:
60 """
61 if isinstance(obj, (basestring,)):
62 if obj in ['', ""]:
63 return []
64
65 lst = obj.split(sep)
66 if strip:
67 lst = [v.strip() for v in lst]
68 return lst
69 elif isinstance(obj, (list, tuple)):
70 return obj
71 elif obj is None:
72 return []
73 else:
74 return [obj]
75
76
77 class SettingsMaker(object):
78
79 def __init__(self, app_settings):
80 self.settings = app_settings
81
82 @classmethod
83 def _bool_func(cls, input_val):
84 if isinstance(input_val, unicode):
85 input_val = input_val.encode('utf8')
86 return str2bool(input_val)
87
88 @classmethod
89 def _int_func(cls, input_val):
90 return int(input_val)
91
92 @classmethod
93 def _list_func(cls, input_val, sep=','):
94 return aslist(input_val, sep=sep)
95
96 @classmethod
97 def _string_func(cls, input_val, lower=True):
98 if lower:
99 input_val = input_val.lower()
100 return input_val
101
102 @classmethod
103 def _float_func(cls, input_val):
104 return float(input_val)
105
106 @classmethod
107 def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
108
109 # ensure we have our dir created
110 if not os.path.isdir(input_val) and ensure_dir:
111 os.makedirs(input_val, mode=mode)
112
113 if not os.path.isdir(input_val):
114 raise Exception('Dir at {} does not exist'.format(input_val))
115 return input_val
116
117 @classmethod
118 def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
119 dirname = os.path.dirname(input_val)
120 cls._dir_func(dirname, ensure_dir=ensure_dir)
121 return input_val
122
123 @classmethod
124 def _key_transformator(cls, key):
125 return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
126
127 def maybe_env_key(self, key):
128 # now maybe we have this KEY in env, search and use the value with higher priority.
129 transformed_key = self._key_transformator(key)
130 envvar_value = os.environ.get(transformed_key)
131 if envvar_value:
132 log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
133
134 return envvar_value
135
136 def env_expand(self):
137 replaced = {}
138 for k, v in self.settings.items():
139 if k not in set_keys:
140 envvar_value = self.maybe_env_key(k)
141 if envvar_value:
142 replaced[k] = envvar_value
143 set_keys[k] = envvar_value
144
145 # replace ALL keys updated
146 self.settings.update(replaced)
147
148 def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
149 """
150 Helper to enable debug on running instance
151 :return:
152 """
153
154 if not str2bool(self.settings.get('logging.autoconfigure')):
155 log.info('logging configuration based on main .ini file')
156 return
157
158 if logging_conf is None:
159 logging_conf = self.settings.get('logging.logging_conf_file') or ''
160
161 if not os.path.isfile(logging_conf):
162 log.error('Unable to setup logging based on %s, '
163 'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
164 return
165
166 with open(logging_conf, 'rb') as f:
167 ini_template = textwrap.dedent(f.read())
168 ini_template = string.Template(ini_template).safe_substitute(
169 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
170 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
171 )
172
173 with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
174 log.info('Saved Temporary LOGGING config at %s', f.name)
175 f.write(ini_template)
176
177 logging.config.fileConfig(f.name)
178 os.remove(f.name)
179
180 def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
181 input_val = self.settings.get(key, default)
182
183 if default_when_empty and not input_val:
184 # use default value when value is set in the config but it is empty
185 input_val = default
186
187 parser_func = {
188 'bool': self._bool_func,
189 'int': self._int_func,
190 'list': self._list_func,
191 'list:newline': functools.partial(self._list_func, sep='/n'),
192 'list:spacesep': functools.partial(self._list_func, sep=' '),
193 'string': functools.partial(self._string_func, lower=lower),
194 'dir': self._dir_func,
195 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
196 'file': self._file_path_func,
197 'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
198 None: lambda i: i
199 }[parser]
200
201 envvar_value = self.maybe_env_key(key)
202 if envvar_value:
203 input_val = envvar_value
204 set_keys[key] = input_val
205
206 self.settings[key] = parser_func(input_val)
207 return self.settings[key]
@@ -0,0 +1,26 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 # This package contains non rhodecode licensed packages that are
19 # vendored for various reasons
20
21 import os
22 import sys
23
24 vendor_dir = os.path.abspath(os.path.dirname(__file__))
25
26 sys.path.append(vendor_dir)
@@ -0,0 +1,243 b''
1 '''
2 This library is provided to allow standard python logging
3 to output log data as JSON formatted strings
4 '''
5 import logging
6 import json
7 import re
8 from datetime import date, datetime, time, tzinfo, timedelta
9 import traceback
10 import importlib
11
12 from inspect import istraceback
13
14 from collections import OrderedDict
15
16
17 def _inject_req_id(record, *args, **kwargs):
18 return record
19
20
21 ExceptionAwareFormatter = logging.Formatter
22
23
24 ZERO = timedelta(0)
25 HOUR = timedelta(hours=1)
26
27
28 class UTC(tzinfo):
29 """UTC"""
30
31 def utcoffset(self, dt):
32 return ZERO
33
34 def tzname(self, dt):
35 return "UTC"
36
37 def dst(self, dt):
38 return ZERO
39
40 utc = UTC()
41
42
43 # skip natural LogRecord attributes
44 # http://docs.python.org/library/logging.html#logrecord-attributes
45 RESERVED_ATTRS = (
46 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
47 'funcName', 'levelname', 'levelno', 'lineno', 'module',
48 'msecs', 'message', 'msg', 'name', 'pathname', 'process',
49 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName')
50
51
52 def merge_record_extra(record, target, reserved):
53 """
54 Merges extra attributes from LogRecord object into target dictionary
55
56 :param record: logging.LogRecord
57 :param target: dict to update
58 :param reserved: dict or list with reserved keys to skip
59 """
60 for key, value in record.__dict__.items():
61 # this allows to have numeric keys
62 if (key not in reserved
63 and not (hasattr(key, "startswith")
64 and key.startswith('_'))):
65 target[key] = value
66 return target
67
68
69 class JsonEncoder(json.JSONEncoder):
70 """
71 A custom encoder extending the default JSONEncoder
72 """
73
74 def default(self, obj):
75 if isinstance(obj, (date, datetime, time)):
76 return self.format_datetime_obj(obj)
77
78 elif istraceback(obj):
79 return ''.join(traceback.format_tb(obj)).strip()
80
81 elif type(obj) == Exception \
82 or isinstance(obj, Exception) \
83 or type(obj) == type:
84 return str(obj)
85
86 try:
87 return super(JsonEncoder, self).default(obj)
88
89 except TypeError:
90 try:
91 return str(obj)
92
93 except Exception:
94 return None
95
96 def format_datetime_obj(self, obj):
97 return obj.isoformat()
98
99
100 class JsonFormatter(ExceptionAwareFormatter):
101 """
102 A custom formatter to format logging records as json strings.
103 Extra values will be formatted as str() if not supported by
104 json default encoder
105 """
106
107 def __init__(self, *args, **kwargs):
108 """
109 :param json_default: a function for encoding non-standard objects
110 as outlined in http://docs.python.org/2/library/json.html
111 :param json_encoder: optional custom encoder
112 :param json_serializer: a :meth:`json.dumps`-compatible callable
113 that will be used to serialize the log record.
114 :param json_indent: an optional :meth:`json.dumps`-compatible numeric value
115 that will be used to customize the indent of the output json.
116 :param prefix: an optional string prefix added at the beginning of
117 the formatted string
118 :param json_indent: indent parameter for json.dumps
119 :param json_ensure_ascii: ensure_ascii parameter for json.dumps
120 :param reserved_attrs: an optional list of fields that will be skipped when
121 outputting json log record. Defaults to all log record attributes:
122 http://docs.python.org/library/logging.html#logrecord-attributes
123 :param timestamp: an optional string/boolean field to add a timestamp when
124 outputting the json log record. If string is passed, timestamp will be added
125 to log record using string as key. If True boolean is passed, timestamp key
126 will be "timestamp". Defaults to False/off.
127 """
128 self.json_default = self._str_to_fn(kwargs.pop("json_default", None))
129 self.json_encoder = self._str_to_fn(kwargs.pop("json_encoder", None))
130 self.json_serializer = self._str_to_fn(kwargs.pop("json_serializer", json.dumps))
131 self.json_indent = kwargs.pop("json_indent", None)
132 self.json_ensure_ascii = kwargs.pop("json_ensure_ascii", True)
133 self.prefix = kwargs.pop("prefix", "")
134 reserved_attrs = kwargs.pop("reserved_attrs", RESERVED_ATTRS)
135 self.reserved_attrs = dict(zip(reserved_attrs, reserved_attrs))
136 self.timestamp = kwargs.pop("timestamp", True)
137
138 # super(JsonFormatter, self).__init__(*args, **kwargs)
139 logging.Formatter.__init__(self, *args, **kwargs)
140 if not self.json_encoder and not self.json_default:
141 self.json_encoder = JsonEncoder
142
143 self._required_fields = self.parse()
144 self._skip_fields = dict(zip(self._required_fields,
145 self._required_fields))
146 self._skip_fields.update(self.reserved_attrs)
147
148 def _str_to_fn(self, fn_as_str):
149 """
150 If the argument is not a string, return whatever was passed in.
151 Parses a string such as package.module.function, imports the module
152 and returns the function.
153
154 :param fn_as_str: The string to parse. If not a string, return it.
155 """
156 if not isinstance(fn_as_str, str):
157 return fn_as_str
158
159 path, _, function = fn_as_str.rpartition('.')
160 module = importlib.import_module(path)
161 return getattr(module, function)
162
163 def parse(self):
164 """
165 Parses format string looking for substitutions
166
167 This method is responsible for returning a list of fields (as strings)
168 to include in all log messages.
169 """
170 standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE)
171 return standard_formatters.findall(self._fmt)
172
173 def add_fields(self, log_record, record, message_dict):
174 """
175 Override this method to implement custom logic for adding fields.
176 """
177 for field in self._required_fields:
178 log_record[field] = record.__dict__.get(field)
179 log_record.update(message_dict)
180 merge_record_extra(record, log_record, reserved=self._skip_fields)
181
182 if self.timestamp:
183 key = self.timestamp if type(self.timestamp) == str else 'timestamp'
184 log_record[key] = datetime.fromtimestamp(record.created, tz=utc)
185
186 def process_log_record(self, log_record):
187 """
188 Override this method to implement custom logic
189 on the possibly ordered dictionary.
190 """
191 return log_record
192
193 def jsonify_log_record(self, log_record):
194 """Returns a json string of the log record."""
195 return self.json_serializer(log_record,
196 default=self.json_default,
197 cls=self.json_encoder,
198 indent=self.json_indent,
199 ensure_ascii=self.json_ensure_ascii)
200
201 def serialize_log_record(self, log_record):
202 """Returns the final representation of the log record."""
203 return "%s%s" % (self.prefix, self.jsonify_log_record(log_record))
204
205 def format(self, record):
206 """Formats a log record and serializes to json"""
207 message_dict = {}
208 # FIXME: logging.LogRecord.msg and logging.LogRecord.message in typeshed
209 # are always type of str. We shouldn't need to override that.
210 if isinstance(record.msg, dict):
211 message_dict = record.msg
212 record.message = None
213 else:
214 record.message = record.getMessage()
215 # only format time if needed
216 if "asctime" in self._required_fields:
217 record.asctime = self.formatTime(record, self.datefmt)
218
219 # Display formatted exception, but allow overriding it in the
220 # user-supplied dict.
221 if record.exc_info and not message_dict.get('exc_info'):
222 message_dict['exc_info'] = self.formatException(record.exc_info)
223 if not message_dict.get('exc_info') and record.exc_text:
224 message_dict['exc_info'] = record.exc_text
225 # Display formatted record of stack frames
226 # default format is a string returned from :func:`traceback.print_stack`
227 try:
228 if record.stack_info and not message_dict.get('stack_info'):
229 message_dict['stack_info'] = self.formatStack(record.stack_info)
230 except AttributeError:
231 # Python2.7 doesn't have stack_info.
232 pass
233
234 try:
235 log_record = OrderedDict()
236 except NameError:
237 log_record = {}
238
239 _inject_req_id(record, with_prefix=False)
240 self.add_fields(log_record, record, message_dict)
241 log_record = self.process_log_record(log_record)
242
243 return self.serialize_log_record(log_record)
@@ -0,0 +1,390 b''
1 import sys
2 import threading
3 import weakref
4 from base64 import b64encode
5 from logging import getLogger
6 from os import urandom
7
8 from redis import StrictRedis
9
10 __version__ = '3.7.0'
11
12 loggers = {
13 k: getLogger("vcsserver." + ".".join((__name__, k)))
14 for k in [
15 "acquire",
16 "refresh.thread.start",
17 "refresh.thread.stop",
18 "refresh.thread.exit",
19 "refresh.start",
20 "refresh.shutdown",
21 "refresh.exit",
22 "release",
23 ]
24 }
25
26 PY3 = sys.version_info[0] == 3
27
28 if PY3:
29 text_type = str
30 binary_type = bytes
31 else:
32 text_type = unicode # noqa
33 binary_type = str
34
35
36 # Check if the id match. If not, return an error code.
37 UNLOCK_SCRIPT = b"""
38 if redis.call("get", KEYS[1]) ~= ARGV[1] then
39 return 1
40 else
41 redis.call("del", KEYS[2])
42 redis.call("lpush", KEYS[2], 1)
43 redis.call("pexpire", KEYS[2], ARGV[2])
44 redis.call("del", KEYS[1])
45 return 0
46 end
47 """
48
49 # Covers both cases when key doesn't exist and doesn't equal to lock's id
50 EXTEND_SCRIPT = b"""
51 if redis.call("get", KEYS[1]) ~= ARGV[1] then
52 return 1
53 elseif redis.call("ttl", KEYS[1]) < 0 then
54 return 2
55 else
56 redis.call("expire", KEYS[1], ARGV[2])
57 return 0
58 end
59 """
60
61 RESET_SCRIPT = b"""
62 redis.call('del', KEYS[2])
63 redis.call('lpush', KEYS[2], 1)
64 redis.call('pexpire', KEYS[2], ARGV[2])
65 return redis.call('del', KEYS[1])
66 """
67
68 RESET_ALL_SCRIPT = b"""
69 local locks = redis.call('keys', 'lock:*')
70 local signal
71 for _, lock in pairs(locks) do
72 signal = 'lock-signal:' .. string.sub(lock, 6)
73 redis.call('del', signal)
74 redis.call('lpush', signal, 1)
75 redis.call('expire', signal, 1)
76 redis.call('del', lock)
77 end
78 return #locks
79 """
80
81
82 class AlreadyAcquired(RuntimeError):
83 pass
84
85
86 class NotAcquired(RuntimeError):
87 pass
88
89
90 class AlreadyStarted(RuntimeError):
91 pass
92
93
94 class TimeoutNotUsable(RuntimeError):
95 pass
96
97
98 class InvalidTimeout(RuntimeError):
99 pass
100
101
102 class TimeoutTooLarge(RuntimeError):
103 pass
104
105
106 class NotExpirable(RuntimeError):
107 pass
108
109
110 class Lock(object):
111 """
112 A Lock context manager implemented via redis SETNX/BLPOP.
113 """
114 unlock_script = None
115 extend_script = None
116 reset_script = None
117 reset_all_script = None
118
119 def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
120 """
121 :param redis_client:
122 An instance of :class:`~StrictRedis`.
123 :param name:
124 The name (redis key) the lock should have.
125 :param expire:
126 The lock expiry time in seconds. If left at the default (None)
127 the lock will not expire.
128 :param id:
129 The ID (redis value) the lock should have. A random value is
130 generated when left at the default.
131
132 Note that if you specify this then the lock is marked as "held". Acquires
133 won't be possible.
134 :param auto_renewal:
135 If set to ``True``, Lock will automatically renew the lock so that it
136 doesn't expire for as long as the lock is held (acquire() called
137 or running in a context manager).
138
139 Implementation note: Renewal will happen using a daemon thread with
140 an interval of ``expire*2/3``. If wishing to use a different renewal
141 time, subclass Lock, call ``super().__init__()`` then set
142 ``self._lock_renewal_interval`` to your desired interval.
143 :param strict:
144 If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
145 :param signal_expire:
146 Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
147 """
148 if strict and not isinstance(redis_client, StrictRedis):
149 raise ValueError("redis_client must be instance of StrictRedis. "
150 "Use strict=False if you know what you're doing.")
151 if auto_renewal and expire is None:
152 raise ValueError("Expire may not be None when auto_renewal is set")
153
154 self._client = redis_client
155
156 if expire:
157 expire = int(expire)
158 if expire < 0:
159 raise ValueError("A negative expire is not acceptable.")
160 else:
161 expire = None
162 self._expire = expire
163
164 self._signal_expire = signal_expire
165 if id is None:
166 self._id = b64encode(urandom(18)).decode('ascii')
167 elif isinstance(id, binary_type):
168 try:
169 self._id = id.decode('ascii')
170 except UnicodeDecodeError:
171 self._id = b64encode(id).decode('ascii')
172 elif isinstance(id, text_type):
173 self._id = id
174 else:
175 raise TypeError("Incorrect type for `id`. Must be bytes/str not %s." % type(id))
176 self._name = 'lock:' + name
177 self._signal = 'lock-signal:' + name
178 self._lock_renewal_interval = (float(expire) * 2 / 3
179 if auto_renewal
180 else None)
181 self._lock_renewal_thread = None
182
183 self.register_scripts(redis_client)
184
185 @classmethod
186 def register_scripts(cls, redis_client):
187 global reset_all_script
188 if reset_all_script is None:
189 reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
190 cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
191 cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
192 cls.reset_script = redis_client.register_script(RESET_SCRIPT)
193 cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
194
195 @property
196 def _held(self):
197 return self.id == self.get_owner_id()
198
199 def reset(self):
200 """
201 Forcibly deletes the lock. Use this with care.
202 """
203 self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
204
205 @property
206 def id(self):
207 return self._id
208
209 def get_owner_id(self):
210 owner_id = self._client.get(self._name)
211 if isinstance(owner_id, binary_type):
212 owner_id = owner_id.decode('ascii', 'replace')
213 return owner_id
214
215 def acquire(self, blocking=True, timeout=None):
216 """
217 :param blocking:
218 Boolean value specifying whether lock should be blocking or not.
219 :param timeout:
220 An integer value specifying the maximum number of seconds to block.
221 """
222 logger = loggers["acquire"]
223
224 logger.debug("Getting blocking: %s acquire on %r ...", blocking, self._name)
225
226 if self._held:
227 owner_id = self.get_owner_id()
228 raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
229
230 if not blocking and timeout is not None:
231 raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
232
233 if timeout:
234 timeout = int(timeout)
235 if timeout < 0:
236 raise InvalidTimeout("Timeout (%d) cannot be less than or equal to 0" % timeout)
237
238 if self._expire and not self._lock_renewal_interval and timeout > self._expire:
239 raise TimeoutTooLarge("Timeout (%d) cannot be greater than expire (%d)" % (timeout, self._expire))
240
241 busy = True
242 blpop_timeout = timeout or self._expire or 0
243 timed_out = False
244 while busy:
245 busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
246 if busy:
247 if timed_out:
248 return False
249 elif blocking:
250 timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
251 else:
252 logger.warning("Failed to get %r.", self._name)
253 return False
254
255 logger.debug("Got lock for %r.", self._name)
256 if self._lock_renewal_interval is not None:
257 self._start_lock_renewer()
258 return True
259
260 def extend(self, expire=None):
261 """Extends expiration time of the lock.
262
263 :param expire:
264 New expiration time. If ``None`` - `expire` provided during
265 lock initialization will be taken.
266 """
267 if expire:
268 expire = int(expire)
269 if expire < 0:
270 raise ValueError("A negative expire is not acceptable.")
271 elif self._expire is not None:
272 expire = self._expire
273 else:
274 raise TypeError(
275 "To extend a lock 'expire' must be provided as an "
276 "argument to extend() method or at initialization time."
277 )
278
279 error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
280 if error == 1:
281 raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
282 elif error == 2:
283 raise NotExpirable("Lock %s has no assigned expiration time" % self._name)
284 elif error:
285 raise RuntimeError("Unsupported error code %s from EXTEND script" % error)
286
287 @staticmethod
288 def _lock_renewer(lockref, interval, stop):
289 """
290 Renew the lock key in redis every `interval` seconds for as long
291 as `self._lock_renewal_thread.should_exit` is False.
292 """
293 while not stop.wait(timeout=interval):
294 loggers["refresh.thread.start"].debug("Refreshing lock")
295 lock = lockref()
296 if lock is None:
297 loggers["refresh.thread.stop"].debug(
298 "The lock no longer exists, stopping lock refreshing"
299 )
300 break
301 lock.extend(expire=lock._expire)
302 del lock
303 loggers["refresh.thread.exit"].debug("Exit requested, stopping lock refreshing")
304
305 def _start_lock_renewer(self):
306 """
307 Starts the lock refresher thread.
308 """
309 if self._lock_renewal_thread is not None:
310 raise AlreadyStarted("Lock refresh thread already started")
311
312 loggers["refresh.start"].debug(
313 "Starting thread to refresh lock every %s seconds",
314 self._lock_renewal_interval
315 )
316 self._lock_renewal_stop = threading.Event()
317 self._lock_renewal_thread = threading.Thread(
318 group=None,
319 target=self._lock_renewer,
320 kwargs={'lockref': weakref.ref(self),
321 'interval': self._lock_renewal_interval,
322 'stop': self._lock_renewal_stop}
323 )
324 self._lock_renewal_thread.setDaemon(True)
325 self._lock_renewal_thread.start()
326
327 def _stop_lock_renewer(self):
328 """
329 Stop the lock renewer.
330
331 This signals the renewal thread and waits for its exit.
332 """
333 if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
334 return
335 loggers["refresh.shutdown"].debug("Signalling the lock refresher to stop")
336 self._lock_renewal_stop.set()
337 self._lock_renewal_thread.join()
338 self._lock_renewal_thread = None
339 loggers["refresh.exit"].debug("Lock refresher has stopped")
340
341 def __enter__(self):
342 acquired = self.acquire(blocking=True)
343 assert acquired, "Lock wasn't acquired, but blocking=True"
344 return self
345
346 def __exit__(self, exc_type=None, exc_value=None, traceback=None):
347 self.release()
348
349 def release(self):
350 """Releases the lock, that was acquired with the same object.
351
352 .. note::
353
354 If you want to release a lock that you acquired in a different place you have two choices:
355
356 * Use ``Lock("name", id=id_from_other_place).release()``
357 * Use ``Lock("name").reset()``
358 """
359 if self._lock_renewal_thread is not None:
360 self._stop_lock_renewer()
361 loggers["release"].debug("Releasing %r.", self._name)
362 error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
363 if error == 1:
364 raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
365 elif error:
366 raise RuntimeError("Unsupported error code %s from EXTEND script." % error)
367
368 def locked(self):
369 """
370 Return true if the lock is acquired.
371
372 Checks that lock with same name already exists. This method returns true, even if
373 lock have another id.
374 """
375 return self._client.exists(self._name) == 1
376
377
378 reset_all_script = None
379
380
381 def reset_all(redis_client):
382 """
383 Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
384
385 :param redis_client:
386 An instance of :class:`~StrictRedis`.
387 """
388 Lock.register_scripts(redis_client)
389
390 reset_all_script(client=redis_client) # noqa
@@ -0,0 +1,52 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import logging
4
5 from .stream import TCPStatsClient, UnixSocketStatsClient # noqa
6 from .udp import StatsClient # noqa
7
8 HOST = 'localhost'
9 PORT = 8125
10 IPV6 = False
11 PREFIX = None
12 MAXUDPSIZE = 512
13
14 log = logging.getLogger('rhodecode.statsd')
15
16
17 def statsd_config(config, prefix='statsd.'):
18 _config = {}
19 for key in config.keys():
20 if key.startswith(prefix):
21 _config[key[len(prefix):]] = config[key]
22 return _config
23
24
25 def client_from_config(configuration, prefix='statsd.', **kwargs):
26 from pyramid.settings import asbool
27
28 _config = statsd_config(configuration, prefix)
29 statsd_enabled = asbool(_config.pop('enabled', False))
30 if not statsd_enabled:
31 log.debug('statsd client not enabled by statsd.enabled = flag, skipping...')
32 return
33
34 host = _config.pop('statsd_host', HOST)
35 port = _config.pop('statsd_port', PORT)
36 prefix = _config.pop('statsd_prefix', PREFIX)
37 maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
38 ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
39 log.debug('configured statsd client %s:%s', host, port)
40
41 try:
42 client = StatsClient(
43 host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
44 except Exception:
45 log.exception('StatsD is enabled, but failed to connect to statsd server, fallback: disable statsd')
46 client = None
47
48 return client
49
50
51 def get_statsd_client(request):
52 return client_from_config(request.registry.settings)
@@ -0,0 +1,156 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import re
4 import random
5 from collections import deque
6 from datetime import timedelta
7 from repoze.lru import lru_cache
8
9 from .timer import Timer
10
11 TAG_INVALID_CHARS_RE = re.compile(
12 r"[^\w\d_\-:/\.]",
13 #re.UNICODE
14 )
15 TAG_INVALID_CHARS_SUBS = "_"
16
17 # we save and expose methods called by statsd for discovery
18 buckets_dict = {
19
20 }
21
22
23 @lru_cache(maxsize=500)
24 def _normalize_tags_with_cache(tag_list):
25 return [TAG_INVALID_CHARS_RE.sub(TAG_INVALID_CHARS_SUBS, tag) for tag in tag_list]
26
27
28 def normalize_tags(tag_list):
29 # We have to turn our input tag list into a non-mutable tuple for it to
30 # be hashable (and thus usable) by the @lru_cache decorator.
31 return _normalize_tags_with_cache(tuple(tag_list))
32
33
34 class StatsClientBase(object):
35 """A Base class for various statsd clients."""
36
37 def close(self):
38 """Used to close and clean up any underlying resources."""
39 raise NotImplementedError()
40
41 def _send(self):
42 raise NotImplementedError()
43
44 def pipeline(self):
45 raise NotImplementedError()
46
47 def timer(self, stat, rate=1, tags=None, auto_send=True):
48 """
49 statsd = StatsdClient.statsd
50 with statsd.timer('bucket_name', auto_send=True) as tmr:
51 # This block will be timed.
52 for i in xrange(0, 100000):
53 i ** 2
54 # you can access time here...
55 elapsed_ms = tmr.ms
56 """
57 return Timer(self, stat, rate, tags, auto_send=auto_send)
58
59 def timing(self, stat, delta, rate=1, tags=None, use_decimals=True):
60 """
61 Send new timing information.
62
63 `delta` can be either a number of milliseconds or a timedelta.
64 """
65 if isinstance(delta, timedelta):
66 # Convert timedelta to number of milliseconds.
67 delta = delta.total_seconds() * 1000.
68 if use_decimals:
69 fmt = '%0.6f|ms'
70 else:
71 fmt = '%s|ms'
72 self._send_stat(stat, fmt % delta, rate, tags)
73
74 def incr(self, stat, count=1, rate=1, tags=None):
75 """Increment a stat by `count`."""
76 self._send_stat(stat, '%s|c' % count, rate, tags)
77
78 def decr(self, stat, count=1, rate=1, tags=None):
79 """Decrement a stat by `count`."""
80 self.incr(stat, -count, rate, tags)
81
82 def gauge(self, stat, value, rate=1, delta=False, tags=None):
83 """Set a gauge value."""
84 if value < 0 and not delta:
85 if rate < 1:
86 if random.random() > rate:
87 return
88 with self.pipeline() as pipe:
89 pipe._send_stat(stat, '0|g', 1)
90 pipe._send_stat(stat, '%s|g' % value, 1)
91 else:
92 prefix = '+' if delta and value >= 0 else ''
93 self._send_stat(stat, '%s%s|g' % (prefix, value), rate, tags)
94
95 def set(self, stat, value, rate=1):
96 """Set a set value."""
97 self._send_stat(stat, '%s|s' % value, rate)
98
99 def histogram(self, stat, value, rate=1, tags=None):
100 """Set a histogram"""
101 self._send_stat(stat, '%s|h' % value, rate, tags)
102
103 def _send_stat(self, stat, value, rate, tags=None):
104 self._after(self._prepare(stat, value, rate, tags))
105
106 def _prepare(self, stat, value, rate, tags=None):
107 global buckets_dict
108 buckets_dict[stat] = 1
109
110 if rate < 1:
111 if random.random() > rate:
112 return
113 value = '%s|@%s' % (value, rate)
114
115 if self._prefix:
116 stat = '%s.%s' % (self._prefix, stat)
117
118 res = '%s:%s%s' % (
119 stat,
120 value,
121 ("|#" + ",".join(normalize_tags(tags))) if tags else "",
122 )
123 return res
124
125 def _after(self, data):
126 if data:
127 self._send(data)
128
129
130 class PipelineBase(StatsClientBase):
131
132 def __init__(self, client):
133 self._client = client
134 self._prefix = client._prefix
135 self._stats = deque()
136
137 def _send(self):
138 raise NotImplementedError()
139
140 def _after(self, data):
141 if data is not None:
142 self._stats.append(data)
143
144 def __enter__(self):
145 return self
146
147 def __exit__(self, typ, value, tb):
148 self.send()
149
150 def send(self):
151 if not self._stats:
152 return
153 self._send()
154
155 def pipeline(self):
156 return self.__class__(self)
@@ -0,0 +1,75 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import socket
4
5 from .base import StatsClientBase, PipelineBase
6
7
8 class StreamPipeline(PipelineBase):
9 def _send(self):
10 self._client._after('\n'.join(self._stats))
11 self._stats.clear()
12
13
14 class StreamClientBase(StatsClientBase):
15 def connect(self):
16 raise NotImplementedError()
17
18 def close(self):
19 if self._sock and hasattr(self._sock, 'close'):
20 self._sock.close()
21 self._sock = None
22
23 def reconnect(self):
24 self.close()
25 self.connect()
26
27 def pipeline(self):
28 return StreamPipeline(self)
29
30 def _send(self, data):
31 """Send data to statsd."""
32 if not self._sock:
33 self.connect()
34 self._do_send(data)
35
36 def _do_send(self, data):
37 self._sock.sendall(data.encode('ascii') + b'\n')
38
39
40 class TCPStatsClient(StreamClientBase):
41 """TCP version of StatsClient."""
42
43 def __init__(self, host='localhost', port=8125, prefix=None,
44 timeout=None, ipv6=False):
45 """Create a new client."""
46 self._host = host
47 self._port = port
48 self._ipv6 = ipv6
49 self._timeout = timeout
50 self._prefix = prefix
51 self._sock = None
52
53 def connect(self):
54 fam = socket.AF_INET6 if self._ipv6 else socket.AF_INET
55 family, _, _, _, addr = socket.getaddrinfo(
56 self._host, self._port, fam, socket.SOCK_STREAM)[0]
57 self._sock = socket.socket(family, socket.SOCK_STREAM)
58 self._sock.settimeout(self._timeout)
59 self._sock.connect(addr)
60
61
62 class UnixSocketStatsClient(StreamClientBase):
63 """Unix domain socket version of StatsClient."""
64
65 def __init__(self, socket_path, prefix=None, timeout=None):
66 """Create a new client."""
67 self._socket_path = socket_path
68 self._timeout = timeout
69 self._prefix = prefix
70 self._sock = None
71
72 def connect(self):
73 self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
74 self._sock.settimeout(self._timeout)
75 self._sock.connect(self._socket_path)
@@ -0,0 +1,75 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import functools
4
5 # Use timer that's not susceptible to time of day adjustments.
6 try:
7 # perf_counter is only present on Py3.3+
8 from time import perf_counter as time_now
9 except ImportError:
10 # fall back to using time
11 from time import time as time_now
12
13
14 def safe_wraps(wrapper, *args, **kwargs):
15 """Safely wraps partial functions."""
16 while isinstance(wrapper, functools.partial):
17 wrapper = wrapper.func
18 return functools.wraps(wrapper, *args, **kwargs)
19
20
21 class Timer(object):
22 """A context manager/decorator for statsd.timing()."""
23
24 def __init__(self, client, stat, rate=1, tags=None, use_decimals=True, auto_send=True):
25 self.client = client
26 self.stat = stat
27 self.rate = rate
28 self.tags = tags
29 self.ms = None
30 self._sent = False
31 self._start_time = None
32 self.use_decimals = use_decimals
33 self.auto_send = auto_send
34
35 def __call__(self, f):
36 """Thread-safe timing function decorator."""
37 @safe_wraps(f)
38 def _wrapped(*args, **kwargs):
39 start_time = time_now()
40 try:
41 return f(*args, **kwargs)
42 finally:
43 elapsed_time_ms = 1000.0 * (time_now() - start_time)
44 self.client.timing(self.stat, elapsed_time_ms, self.rate, self.tags, self.use_decimals)
45 self._sent = True
46 return _wrapped
47
48 def __enter__(self):
49 return self.start()
50
51 def __exit__(self, typ, value, tb):
52 self.stop(send=self.auto_send)
53
54 def start(self):
55 self.ms = None
56 self._sent = False
57 self._start_time = time_now()
58 return self
59
60 def stop(self, send=True):
61 if self._start_time is None:
62 raise RuntimeError('Timer has not started.')
63 dt = time_now() - self._start_time
64 self.ms = 1000.0 * dt # Convert to milliseconds.
65 if send:
66 self.send()
67 return self
68
69 def send(self):
70 if self.ms is None:
71 raise RuntimeError('No data recorded.')
72 if self._sent:
73 raise RuntimeError('Already sent data.')
74 self._sent = True
75 self.client.timing(self.stat, self.ms, self.rate, self.tags, self.use_decimals)
@@ -0,0 +1,55 b''
1 from __future__ import absolute_import, division, unicode_literals
2
3 import socket
4
5 from .base import StatsClientBase, PipelineBase
6
7
8 class Pipeline(PipelineBase):
9
10 def __init__(self, client):
11 super(Pipeline, self).__init__(client)
12 self._maxudpsize = client._maxudpsize
13
14 def _send(self):
15 data = self._stats.popleft()
16 while self._stats:
17 # Use popleft to preserve the order of the stats.
18 stat = self._stats.popleft()
19 if len(stat) + len(data) + 1 >= self._maxudpsize:
20 self._client._after(data)
21 data = stat
22 else:
23 data += '\n' + stat
24 self._client._after(data)
25
26
27 class StatsClient(StatsClientBase):
28 """A client for statsd."""
29
30 def __init__(self, host='localhost', port=8125, prefix=None,
31 maxudpsize=512, ipv6=False):
32 """Create a new client."""
33 fam = socket.AF_INET6 if ipv6 else socket.AF_INET
34 family, _, _, _, addr = socket.getaddrinfo(
35 host, port, fam, socket.SOCK_DGRAM)[0]
36 self._addr = addr
37 self._sock = socket.socket(family, socket.SOCK_DGRAM)
38 self._prefix = prefix
39 self._maxudpsize = maxudpsize
40
41 def _send(self, data):
42 """Send data to statsd."""
43 try:
44 self._sock.sendto(data.encode('ascii'), self._addr)
45 except (socket.error, RuntimeError):
46 # No time for love, Dr. Jones!
47 pass
48
49 def close(self):
50 if self._sock and hasattr(self._sock, 'close'):
51 self._sock.close()
52 self._sock = None
53
54 def pipeline(self):
55 return Pipeline(self)
@@ -0,0 +1,49 b''
1 from vcsserver.lib._vendor.statsd import client_from_config
2
3
4 class StatsdClientNotInitialised(Exception):
5 pass
6
7
8 class _Singleton(type):
9 """A metaclass that creates a Singleton base class when called."""
10
11 _instances = {}
12
13 def __call__(cls, *args, **kwargs):
14 if cls not in cls._instances:
15 cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs)
16 return cls._instances[cls]
17
18
19 class Singleton(_Singleton("SingletonMeta", (object,), {})):
20 pass
21
22
23 class StatsdClientClass(Singleton):
24 setup_run = False
25 statsd_client = None
26 statsd = None
27
28 def __getattribute__(self, name):
29
30 if name.startswith("statsd"):
31 if self.setup_run:
32 return super(StatsdClientClass, self).__getattribute__(name)
33 else:
34 return None
35 #raise StatsdClientNotInitialised("requested key was %s" % name)
36
37 return super(StatsdClientClass, self).__getattribute__(name)
38
39 def setup(self, settings):
40 """
41 Initialize the client
42 """
43 statsd = client_from_config(settings)
44 self.statsd = statsd
45 self.statsd_client = statsd
46 self.setup_run = True
47
48
49 StatsdClient = StatsdClientClass()
@@ -3,4 +3,3 b' current_version = 5.0.0'
3 3 message = release: Bump version {current_version} to {new_version}
4 4
5 5 [bumpversion:file:vcsserver/VERSION]
6
@@ -61,3 +61,22 b' ba6a6dc9ecd7fd8b1dcd6eb0c4ee0210e897c426'
61 61 17bc818b41bcf6883b9ff0da31f01d8c2a5d0781 v4.18.1
62 62 1e9f12aa01f82c335abc9017efe94ce1c30b52ba v4.18.2
63 63 f4cc6b3c5680bdf4541d7d442fbb7086640fb547 v4.18.3
64 5dc0277e4f77bd4cc3042d99625bb5d3ba480c8c v4.19.0
65 3a815eeb1b1efa340dda9b81a8da3cf24a7d605b v4.19.1
66 8841da3680fba841e5a54ebccd8ca56c078f7553 v4.19.2
67 4b0dec7fd80b1ca38e5073e5e562a5a450f73669 v4.19.3
68 1485aa75ffe1b1ec48352dce7b7492d92f85e95f v4.20.0
69 5b740274011766ef2f73803cc196d081e1e7f1d4 v4.20.1
70 5a7835234e2c45e8fb8184c60f548a64b5842af8 v4.21.0
71 26af88343015f8b89d5a66f92bc7547c51fcf0df v4.22.0
72 cf54e5f700fe5dc50af1a1bdf5197c18cf52105f v4.23.0
73 179d989bcfe02c6227f9f6aa9236cbbe1c14c400 v4.23.1
74 383aee8b1652affaa26aefe336a89ee366b2b26d v4.23.2
75 bc1a8141cc51fc23c455ebc50c6609c810b46f8d v4.24.0
76 530a1c03caabc806ea1ef34605f8f67f18c70e55 v4.24.1
77 5908ae65cee1043982e1b26d7b618af5fcfebbb3 v4.25.0
78 cce8bcdf75090d5943a1e9706fe5212d7b5d1fa1 v4.25.1
79 8610c4bf846c63bbc95d3ddfb53fadaaa9c7aa42 v4.25.2
80 d46b7d1be72c76c9f9aaeab6a342951d54459f49 v4.26.0
81 6fba0daab1e20a9e18fb70fa59bd21753e0a5b90 v4.27.0
82 6195da4fc454087173918ae59cae946289458676 v4.27.1
@@ -31,7 +31,7 b' asyncore_use_poll = true'
31 31 ; GUNICORN APPLICATION SERVER
32 32 ; ###########################
33 33
34 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
34 ; run with gunicorn --paste rhodecode.ini
35 35
36 36 ; Module to use, this setting shouldn't be changed
37 37 #use = egg:gunicorn#main
@@ -86,7 +86,7 b' asyncore_use_poll = true'
86 86 ; serving requests. Workers still alive after the timeout (starting from the
87 87 ; receipt of the restart signal) are force killed.
88 88 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
89 #graceful_timeout = 3600
89 #graceful_timeout = 21600
90 90
91 91 # The number of seconds to wait for requests on a Keep-Alive connection.
92 92 # Generally set in the 1-5 seconds range.
@@ -110,6 +110,17 b' asyncore_use_poll = true'
110 110 [app:main]
111 111 ; The %(here)s variable will be replaced with the absolute path of parent directory
112 112 ; of this file
113 ; Each option in the app:main can be override by an environmental variable
114 ;
115 ;To override an option:
116 ;
117 ;RC_<KeyName>
118 ;Everything should be uppercase, . and - should be replaced by _.
119 ;For example, if you have these configuration settings:
120 ;rc_cache.repo_object.backend = foo
121 ;can be overridden by
122 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
123
113 124 use = egg:rhodecode-vcsserver
114 125
115 126
@@ -133,13 +144,13 b' debugtoolbar.exclude_prefixes ='
133 144 ; #################
134 145
135 146 ; Pyramid default locales, we need this to be set
136 pyramid.default_locale_name = en
147 #pyramid.default_locale_name = en
137 148
138 149 ; default locale used by VCS systems
139 locale = en_US.UTF-8
150 #locale = en_US.UTF-8
140 151
141 152 ; path to binaries for vcsserver, it should be set by the installer
142 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
153 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
143 154 ; it can also be a path to nix-build output in case of development
144 155 core.binary_dir = ""
145 156
@@ -153,21 +164,21 b' core.binary_dir = ""'
153 164
154 165 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
155 166 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
156 cache_dir = %(here)s/data
167 #cache_dir = %(here)s/data
157 168
158 169 ; ***************************************
159 170 ; `repo_object` cache, default file based
160 171 ; ***************************************
161 172
162 173 ; `repo_object` cache settings for vcs methods for repositories
163 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
174 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
164 175
165 176 ; cache auto-expires after N seconds
166 177 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
167 rc_cache.repo_object.expiration_time = 2592000
178 #rc_cache.repo_object.expiration_time = 2592000
168 179
169 180 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
170 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
181 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
171 182
172 183 ; ***********************************************************
173 184 ; `repo_object` cache with redis backend
@@ -191,10 +202,32 b' rc_cache.repo_object.expiration_time = 2'
191 202 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
192 203 #rc_cache.repo_object.arguments.distributed_lock = true
193 204
205 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
206 #rc_cache.repo_object.arguments.lock_auto_renewal = true
207
208 ; Statsd client config, this is used to send metrics to statsd
209 ; We recommend setting statsd_exported and scrape them using Promethues
210 #statsd.enabled = false
211 #statsd.statsd_host = 0.0.0.0
212 #statsd.statsd_port = 8125
213 #statsd.statsd_prefix =
214 #statsd.statsd_ipv6 = false
215
216 ; configure logging automatically at server startup set to false
217 ; to use the below custom logging config.
218 ; RC_LOGGING_FORMATTER
219 ; RC_LOGGING_LEVEL
220 ; env variables can control the settings for logging in case of autoconfigure
221
222 #logging.autoconfigure = true
223
224 ; specify your own custom logging config file to configure logging
225 #logging.logging_conf_file = /path/to/custom_logging.ini
194 226
195 227 ; #####################
196 228 ; LOGGING CONFIGURATION
197 229 ; #####################
230
198 231 [loggers]
199 232 keys = root, vcsserver
200 233
@@ -202,7 +235,7 b' keys = root, vcsserver'
202 235 keys = console
203 236
204 237 [formatters]
205 keys = generic
238 keys = generic, json
206 239
207 240 ; #######
208 241 ; LOGGERS
@@ -217,7 +250,6 b' handlers ='
217 250 qualname = vcsserver
218 251 propagate = 1
219 252
220
221 253 ; ########
222 254 ; HANDLERS
223 255 ; ########
@@ -226,6 +258,8 b' propagate = 1'
226 258 class = StreamHandler
227 259 args = (sys.stderr, )
228 260 level = DEBUG
261 ; To enable JSON formatted logs replace 'generic' with 'json'
262 ; This allows sending properly formatted logs to grafana loki or elasticsearch
229 263 formatter = generic
230 264
231 265 ; ##########
@@ -235,3 +269,7 b' formatter = generic'
235 269 [formatter_generic]
236 270 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
237 271 datefmt = %Y-%m-%d %H:%M:%S
272
273 [formatter_json]
274 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
275 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -11,6 +11,7 b' import time'
11 11 import threading
12 12 import traceback
13 13 import random
14 import socket
14 15 from gunicorn.glogging import Logger
15 16
16 17
@@ -29,9 +30,15 b" accesslog = '-'"
29 30 worker_tmp_dir = None
30 31 tmp_upload_dir = None
31 32
33 #reuse_port = True
34
32 35 # Custom log format
36 #access_log_format = (
37 # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
38
39 # loki format for easier parsing in grafana
33 40 access_log_format = (
34 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
41 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
35 42
36 43 # self adjust workers based on CPU count
37 44 # workers = get_workers()
@@ -90,9 +97,12 b' def post_fork(server, worker):'
90 97 if conf.has_option(section, 'memory_usage_recovery_threshold'):
91 98 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
92 99
93 worker._memory_max_usage = _memory_max_usage
94 worker._memory_usage_check_interval = _memory_usage_check_interval
95 worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
100 worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
101 or _memory_max_usage)
102 worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
103 or _memory_usage_check_interval)
104 worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
105 or _memory_usage_recovery_threshold)
96 106
97 107 # register memory last check time, with some random offset so we don't recycle all
98 108 # at once
@@ -242,6 +252,74 b' def post_request(worker, req, environ, r'
242 252 _check_memory_usage(worker)
243 253
244 254
255 def _filter_proxy(ip):
256 """
257 Passed in IP addresses in HEADERS can be in a special format of multiple
258 ips. Those comma separated IPs are passed from various proxies in the
259 chain of request processing. The left-most being the original client.
260 We only care about the first IP which came from the org. client.
261
262 :param ip: ip string from headers
263 """
264 if ',' in ip:
265 _ips = ip.split(',')
266 _first_ip = _ips[0].strip()
267 return _first_ip
268 return ip
269
270
271 def _filter_port(ip):
272 """
273 Removes a port from ip, there are 4 main cases to handle here.
274 - ipv4 eg. 127.0.0.1
275 - ipv6 eg. ::1
276 - ipv4+port eg. 127.0.0.1:8080
277 - ipv6+port eg. [::1]:8080
278
279 :param ip:
280 """
281 def is_ipv6(ip_addr):
282 if hasattr(socket, 'inet_pton'):
283 try:
284 socket.inet_pton(socket.AF_INET6, ip_addr)
285 except socket.error:
286 return False
287 else:
288 return False
289 return True
290
291 if ':' not in ip: # must be ipv4 pure ip
292 return ip
293
294 if '[' in ip and ']' in ip: # ipv6 with port
295 return ip.split(']')[0][1:].lower()
296
297 # must be ipv6 or ipv4 with port
298 if is_ipv6(ip):
299 return ip
300 else:
301 ip, _port = ip.split(':')[:2] # means ipv4+port
302 return ip
303
304
305 def get_ip_addr(environ):
306 proxy_key = 'HTTP_X_REAL_IP'
307 proxy_key2 = 'HTTP_X_FORWARDED_FOR'
308 def_key = 'REMOTE_ADDR'
309 _filters = lambda x: _filter_port(_filter_proxy(x))
310
311 ip = environ.get(proxy_key)
312 if ip:
313 return _filters(ip)
314
315 ip = environ.get(proxy_key2)
316 if ip:
317 return _filters(ip)
318
319 ip = environ.get(def_key, '0.0.0.0')
320 return _filters(ip)
321
322
245 323 class RhodeCodeLogger(Logger):
246 324 """
247 325 Custom Logger that allows some customization that gunicorn doesn't allow
@@ -258,5 +336,58 b' class RhodeCodeLogger(Logger):'
258 336 msecs = int((now - long(now)) * 1000)
259 337 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
260 338
339 def atoms(self, resp, req, environ, request_time):
340 """ Gets atoms for log formatting.
341 """
342 status = resp.status
343 if isinstance(status, str):
344 status = status.split(None, 1)[0]
345 atoms = {
346 'h': get_ip_addr(environ),
347 'l': '-',
348 'u': self._get_user(environ) or '-',
349 't': self.now(),
350 'r': "%s %s %s" % (environ['REQUEST_METHOD'],
351 environ['RAW_URI'],
352 environ["SERVER_PROTOCOL"]),
353 's': status,
354 'm': environ.get('REQUEST_METHOD'),
355 'U': environ.get('PATH_INFO'),
356 'q': environ.get('QUERY_STRING'),
357 'H': environ.get('SERVER_PROTOCOL'),
358 'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
359 'B': getattr(resp, 'sent', None),
360 'f': environ.get('HTTP_REFERER', '-'),
361 'a': environ.get('HTTP_USER_AGENT', '-'),
362 'T': request_time.seconds,
363 'D': (request_time.seconds * 1000000) + request_time.microseconds,
364 'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
365 'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
366 'p': "<%s>" % os.getpid()
367 }
368
369 # add request headers
370 if hasattr(req, 'headers'):
371 req_headers = req.headers
372 else:
373 req_headers = req
374
375 if hasattr(req_headers, "items"):
376 req_headers = req_headers.items()
377
378 atoms.update({"{%s}i" % k.lower(): v for k, v in req_headers})
379
380 resp_headers = resp.headers
381 if hasattr(resp_headers, "items"):
382 resp_headers = resp_headers.items()
383
384 # add response headers
385 atoms.update({"{%s}o" % k.lower(): v for k, v in resp_headers})
386
387 # add environ variables
388 environ_variables = environ.items()
389 atoms.update({"{%s}e" % k.lower(): v for k, v in environ_variables})
390
391 return atoms
261 392
262 393 logger_class = RhodeCodeLogger
@@ -14,7 +14,7 b' port = 9900'
14 14 ; GUNICORN APPLICATION SERVER
15 15 ; ###########################
16 16
17 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
17 ; run with gunicorn --paste rhodecode.ini
18 18
19 19 ; Module to use, this setting shouldn't be changed
20 20 use = egg:gunicorn#main
@@ -69,7 +69,7 b' limit_request_field_size = 0'
69 69 ; serving requests. Workers still alive after the timeout (starting from the
70 70 ; receipt of the restart signal) are force killed.
71 71 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
72 graceful_timeout = 3600
72 graceful_timeout = 21600
73 73
74 74 # The number of seconds to wait for requests on a Keep-Alive connection.
75 75 # Generally set in the 1-5 seconds range.
@@ -93,16 +93,27 b' memory_usage_recovery_threshold = 0.8'
93 93 [app:main]
94 94 ; The %(here)s variable will be replaced with the absolute path of parent directory
95 95 ; of this file
96 ; Each option in the app:main can be override by an environmental variable
97 ;
98 ;To override an option:
99 ;
100 ;RC_<KeyName>
101 ;Everything should be uppercase, . and - should be replaced by _.
102 ;For example, if you have these configuration settings:
103 ;rc_cache.repo_object.backend = foo
104 ;can be overridden by
105 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
106
96 107 use = egg:rhodecode-vcsserver
97 108
98 109 ; Pyramid default locales, we need this to be set
99 pyramid.default_locale_name = en
110 #pyramid.default_locale_name = en
100 111
101 112 ; default locale used by VCS systems
102 locale = en_US.UTF-8
113 #locale = en_US.UTF-8
103 114
104 115 ; path to binaries for vcsserver, it should be set by the installer
105 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
116 ; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
106 117 ; it can also be a path to nix-build output in case of development
107 118 core.binary_dir = ""
108 119
@@ -116,21 +127,21 b' core.binary_dir = ""'
116 127
117 128 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
118 129 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
119 cache_dir = %(here)s/data
130 #cache_dir = %(here)s/data
120 131
121 132 ; ***************************************
122 133 ; `repo_object` cache, default file based
123 134 ; ***************************************
124 135
125 136 ; `repo_object` cache settings for vcs methods for repositories
126 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
137 #rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
127 138
128 139 ; cache auto-expires after N seconds
129 140 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
130 rc_cache.repo_object.expiration_time = 2592000
141 #rc_cache.repo_object.expiration_time = 2592000
131 142
132 143 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
133 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
144 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
134 145
135 146 ; ***********************************************************
136 147 ; `repo_object` cache with redis backend
@@ -154,10 +165,32 b' rc_cache.repo_object.expiration_time = 2'
154 165 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
155 166 #rc_cache.repo_object.arguments.distributed_lock = true
156 167
168 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
169 #rc_cache.repo_object.arguments.lock_auto_renewal = true
170
171 ; Statsd client config, this is used to send metrics to statsd
172 ; We recommend setting statsd_exported and scrape them using Promethues
173 #statsd.enabled = false
174 #statsd.statsd_host = 0.0.0.0
175 #statsd.statsd_port = 8125
176 #statsd.statsd_prefix =
177 #statsd.statsd_ipv6 = false
178
179 ; configure logging automatically at server startup set to false
180 ; to use the below custom logging config.
181 ; RC_LOGGING_FORMATTER
182 ; RC_LOGGING_LEVEL
183 ; env variables can control the settings for logging in case of autoconfigure
184
185 #logging.autoconfigure = true
186
187 ; specify your own custom logging config file to configure logging
188 #logging.logging_conf_file = /path/to/custom_logging.ini
157 189
158 190 ; #####################
159 191 ; LOGGING CONFIGURATION
160 192 ; #####################
193
161 194 [loggers]
162 195 keys = root, vcsserver
163 196
@@ -165,7 +198,7 b' keys = root, vcsserver'
165 198 keys = console
166 199
167 200 [formatters]
168 keys = generic
201 keys = generic, json
169 202
170 203 ; #######
171 204 ; LOGGERS
@@ -175,12 +208,11 b' level = NOTSET'
175 208 handlers = console
176 209
177 210 [logger_vcsserver]
178 level = DEBUG
211 level = INFO
179 212 handlers =
180 213 qualname = vcsserver
181 214 propagate = 1
182 215
183
184 216 ; ########
185 217 ; HANDLERS
186 218 ; ########
@@ -189,6 +221,8 b' propagate = 1'
189 221 class = StreamHandler
190 222 args = (sys.stderr, )
191 223 level = INFO
224 ; To enable JSON formatted logs replace 'generic' with 'json'
225 ; This allows sending properly formatted logs to grafana loki or elasticsearch
192 226 formatter = generic
193 227
194 228 ; ##########
@@ -198,3 +232,7 b' formatter = generic'
198 232 [formatter_generic]
199 233 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
200 234 datefmt = %Y-%m-%d %H:%M:%S
235
236 [formatter_json]
237 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
238 class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
@@ -1,4 +1,5 b''
1 1 # -*- coding: utf-8 -*-
2
2 3 # RhodeCode VCSServer provides access to different vcs backends via network.
3 4 # Copyright (C) 2014-2019 RodeCode GmbH
4 5 #
@@ -20,20 +21,47 b''
20 21 from setuptools import setup, find_packages
21 22
22 23 import os
24 import re
23 25 import sys
24 26 import pkgutil
25 27 import platform
26 28 import codecs
27 29
28 try: # for pip >= 10
30 import pip
31
32 pip_major_version = int(pip.__version__.split(".")[0])
33 if pip_major_version >= 20:
29 34 from pip._internal.req import parse_requirements
30 except ImportError: # for pip <= 9.0.3
35 from pip._internal.network.session import PipSession
36 elif pip_major_version >= 10:
37 from pip._internal.req import parse_requirements
38 from pip._internal.download import PipSession
39 else:
31 40 from pip.req import parse_requirements
41 from pip.download import PipSession
42
32 43
33 try: # for pip >= 10
34 from pip._internal.download import PipSession
35 except ImportError: # for pip <= 9.0.3
36 from pip.download import PipSession
44 def get_package_name(req_object):
45 package_name = None
46 try:
47 from pip._internal.req.constructors import install_req_from_parsed_requirement
48 except ImportError:
49 install_req_from_parsed_requirement = None
50
51 # In 20.1 of pip, the requirements object changed
52 if hasattr(req_object, 'req'):
53 package_name = req_object.req.name
54
55 if package_name is None:
56 if install_req_from_parsed_requirement:
57 package = install_req_from_parsed_requirement(req_object)
58 package_name = package.req.name
59
60 if package_name is None:
61 # fallback for older pip
62 package_name = re.split('===|<=|!=|==|>=|~=|<|>', req_object.requirement)[0]
63
64 return package_name
37 65
38 66
39 67 if sys.version_info < (2, 7):
@@ -61,14 +89,15 b' def _get_requirements(req_filename, excl'
61 89 parsed = parse_requirements(os.path.join(here, req_filename))
62 90
63 91 requirements = []
64 for ir in parsed:
65 if ir.req and ir.name not in exclude:
66 requirements.append(str(ir.req))
92 for int_req in parsed:
93 req_name = get_package_name(int_req)
94 if req_name not in exclude:
95 requirements.append(req_name)
67 96 return requirements + extras
68 97
69 98
70 99 # requirements extract
71 setup_requirements = ['pytest-runner']
100 setup_requirements = []
72 101 install_requirements = _get_requirements(
73 102 'requirements.txt', exclude=['setuptools'])
74 103 test_requirements = _get_requirements(
@@ -14,13 +14,18 b''
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17 import os
18 18 import sys
19 19 import traceback
20 20 import logging
21 21 import urllib.parse
22 22
23 23 from vcsserver.lib.rc_cache import region_meta
24
25 from vcsserver import exceptions
26 from vcsserver.exceptions import NoContentException
27 from vcsserver.hgcompat import (archival)
28
24 29 log = logging.getLogger(__name__)
25 30
26 31
@@ -74,3 +79,54 b' def raise_from_original(new_type):'
74 79 raise new_exc.with_traceback(exc_traceback)
75 80 finally:
76 81 del exc_traceback
82
83
84 class ArchiveNode(object):
85 def __init__(self, path, mode, is_link, raw_bytes):
86 self.path = path
87 self.mode = mode
88 self.is_link = is_link
89 self.raw_bytes = raw_bytes
90
91
92 def archive_repo(walker, archive_dest_path, kind, mtime, archive_at_path,
93 archive_dir_name, commit_id, write_metadata=True, extra_metadata=None):
94 """
95 walker should be a file walker, for example:
96 def walker():
97 for file_info in files:
98 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
99 """
100 extra_metadata = extra_metadata or {}
101
102 if kind == "tgz":
103 archiver = archival.tarit(archive_dest_path, mtime, "gz")
104 elif kind == "tbz2":
105 archiver = archival.tarit(archive_dest_path, mtime, "bz2")
106 elif kind == 'zip':
107 archiver = archival.zipit(archive_dest_path, mtime)
108 else:
109 raise exceptions.ArchiveException()(
110 'Remote does not support: "%s" archive type.' % kind)
111
112 for f in walker(commit_id, archive_at_path):
113 f_path = os.path.join(archive_dir_name, f.path.lstrip('/'))
114 try:
115 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
116 except NoContentException:
117 # NOTE(marcink): this is a special case for SVN so we can create "empty"
118 # directories which arent supported by archiver
119 archiver.addfile(os.path.join(f_path, '.dir'), f.mode, f.is_link, '')
120
121 if write_metadata:
122 metadata = dict([
123 ('commit_id', commit_id),
124 ('mtime', mtime),
125 ])
126 metadata.update(extra_metadata)
127
128 meta = ["%s:%s" % (f_name, value) for f_name, value in metadata.items()]
129 f_path = os.path.join(archive_dir_name, '.archival.txt')
130 archiver.addfile(f_path, 0o644, False, '\n'.join(meta))
131
132 return archiver.done()
@@ -119,3 +119,7 b' class HTTPRepoBranchProtected(HTTPForbid'
119 119
120 120 class RefNotFoundException(KeyError):
121 121 pass
122
123
124 class NoContentException(ValueError):
125 pass
@@ -1,4 +1,4 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
@@ -29,6 +29,7 b' from functools import wraps'
29 29 import more_itertools
30 30 import pygit2
31 31 from pygit2 import Repository as LibGit2Repo
32 from pygit2 import index as LibGit2Index
32 33 from dulwich import index, objects
33 34 from dulwich.client import HttpGitClient, LocalGitClient
34 35 from dulwich.errors import (
@@ -40,7 +41,7 b' from dulwich.server import update_server'
40 41
41 42 from vcsserver import exceptions, settings, subprocessio
42 43 from vcsserver.utils import safe_str, safe_int, safe_unicode
43 from vcsserver.base import RepoFactory, obfuscate_qs
44 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
44 45 from vcsserver.hgcompat import (
45 46 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
46 47 from vcsserver.git_lfs.lib import LFSOidStore
@@ -184,7 +185,8 b' class GitRemote(RemoteBase):'
184 185 @reraise_safe_exceptions
185 186 def assert_correct_path(self, wire):
186 187 cache_on, context_uid, repo_id = self._cache_on(wire)
187 @self.region.conditional_cache_on_arguments(condition=cache_on)
188 region = self._region(wire)
189 @region.conditional_cache_on_arguments(condition=cache_on)
188 190 def _assert_correct_path(_context_uid, _repo_id):
189 191 try:
190 192 repo_init = self._factory.repo_libgit2(wire)
@@ -216,7 +218,8 b' class GitRemote(RemoteBase):'
216 218 @reraise_safe_exceptions
217 219 def blob_raw_length(self, wire, sha):
218 220 cache_on, context_uid, repo_id = self._cache_on(wire)
219 @self.region.conditional_cache_on_arguments(condition=cache_on)
221 region = self._region(wire)
222 @region.conditional_cache_on_arguments(condition=cache_on)
220 223 def _blob_raw_length(_repo_id, _sha):
221 224
222 225 repo_init = self._factory.repo_libgit2(wire)
@@ -247,7 +250,8 b' class GitRemote(RemoteBase):'
247 250 def is_large_file(self, wire, commit_id):
248 251 cache_on, context_uid, repo_id = self._cache_on(wire)
249 252
250 @self.region.conditional_cache_on_arguments(condition=cache_on)
253 region = self._region(wire)
254 @region.conditional_cache_on_arguments(condition=cache_on)
251 255 def _is_large_file(_repo_id, _sha):
252 256 repo_init = self._factory.repo_libgit2(wire)
253 257 with repo_init as repo:
@@ -263,7 +267,8 b' class GitRemote(RemoteBase):'
263 267 def is_binary(self, wire, tree_id):
264 268 cache_on, context_uid, repo_id = self._cache_on(wire)
265 269
266 @self.region.conditional_cache_on_arguments(condition=cache_on)
270 region = self._region(wire)
271 @region.conditional_cache_on_arguments(condition=cache_on)
267 272 def _is_binary(_repo_id, _tree_id):
268 273 repo_init = self._factory.repo_libgit2(wire)
269 274 with repo_init as repo:
@@ -305,7 +310,8 b' class GitRemote(RemoteBase):'
305 310 @reraise_safe_exceptions
306 311 def bulk_request(self, wire, rev, pre_load):
307 312 cache_on, context_uid, repo_id = self._cache_on(wire)
308 @self.region.conditional_cache_on_arguments(condition=cache_on)
313 region = self._region(wire)
314 @region.conditional_cache_on_arguments(condition=cache_on)
309 315 def _bulk_request(_repo_id, _rev, _pre_load):
310 316 result = {}
311 317 for attr in pre_load:
@@ -408,7 +414,8 b' class GitRemote(RemoteBase):'
408 414 @reraise_safe_exceptions
409 415 def branch(self, wire, commit_id):
410 416 cache_on, context_uid, repo_id = self._cache_on(wire)
411 @self.region.conditional_cache_on_arguments(condition=cache_on)
417 region = self._region(wire)
418 @region.conditional_cache_on_arguments(condition=cache_on)
412 419 def _branch(_context_uid, _repo_id, _commit_id):
413 420 regex = re.compile('^refs/heads')
414 421
@@ -423,7 +430,8 b' class GitRemote(RemoteBase):'
423 430 @reraise_safe_exceptions
424 431 def commit_branches(self, wire, commit_id):
425 432 cache_on, context_uid, repo_id = self._cache_on(wire)
426 @self.region.conditional_cache_on_arguments(condition=cache_on)
433 region = self._region(wire)
434 @region.conditional_cache_on_arguments(condition=cache_on)
427 435 def _commit_branches(_context_uid, _repo_id, _commit_id):
428 436 repo_init = self._factory.repo_libgit2(wire)
429 437 with repo_init as repo:
@@ -444,11 +452,22 b' class GitRemote(RemoteBase):'
444 452 # TODO: this is quite complex, check if that can be simplified
445 453 @reraise_safe_exceptions
446 454 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
455 # Defines the root tree
456 class _Root(object):
457 def __repr__(self):
458 return 'ROOT TREE'
459 ROOT = _Root()
460
447 461 repo = self._factory.repo(wire)
448 462 object_store = repo.object_store
449 463
450 464 # Create tree and populates it with blobs
451 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
465
466 if commit_tree and repo[commit_tree]:
467 git_commit = repo[commit_data['parents'][0]]
468 commit_tree = repo[git_commit.tree] # root tree
469 else:
470 commit_tree = objects.Tree()
452 471
453 472 for node in updated:
454 473 # Compute subdirs if needed
@@ -507,21 +526,34 b' class GitRemote(RemoteBase):'
507 526
508 527 for node_path in removed:
509 528 paths = node_path.split('/')
510 tree = commit_tree
511 trees = [tree]
529 tree = commit_tree # start with top-level
530 trees = [{'tree': tree, 'path': ROOT}]
512 531 # Traverse deep into the forest...
532 # resolve final tree by iterating the path.
533 # e.g a/b/c.txt will get
534 # - root as tree then
535 # - 'a' as tree,
536 # - 'b' as tree,
537 # - stop at c as blob.
513 538 for path in paths:
514 539 try:
515 540 obj = repo[tree[path][1]]
516 541 if isinstance(obj, objects.Tree):
517 trees.append(obj)
542 trees.append({'tree': obj, 'path': path})
518 543 tree = obj
519 544 except KeyError:
520 545 break
546 #PROBLEM:
547 """
548 We're not editing same reference tree object
549 """
521 550 # Cut down the blob and all rotten trees on the way back...
522 for path, tree in reversed(list(zip(paths, trees))):
523 del tree[path]
524 if tree:
551 for path, tree_data in reversed(list(zip(paths, trees))):
552 tree = tree_data['tree']
553 tree.__delitem__(path)
554 # This operation edits the tree, we need to mark new commit back
555
556 if len(tree) > 0:
525 557 # This tree still has elements - don't remove it or any
526 558 # of it's parents
527 559 break
@@ -587,7 +619,7 b' class GitRemote(RemoteBase):'
587 619
588 620 if refs and not update_after:
589 621 # mikhail: explicitly set the head to the last ref.
590 repo['HEAD'] = remote_refs[refs[-1]]
622 repo["HEAD"] = remote_refs[refs[-1]]
591 623
592 624 if update_after:
593 625 # we want to checkout HEAD
@@ -689,7 +721,8 b' class GitRemote(RemoteBase):'
689 721 @reraise_safe_exceptions
690 722 def get_object(self, wire, sha, maybe_unreachable=False):
691 723 cache_on, context_uid, repo_id = self._cache_on(wire)
692 @self.region.conditional_cache_on_arguments(condition=cache_on)
724 region = self._region(wire)
725 @region.conditional_cache_on_arguments(condition=cache_on)
693 726 def _get_object(_context_uid, _repo_id, _sha):
694 727 repo_init = self._factory.repo_libgit2(wire)
695 728 with repo_init as repo:
@@ -747,7 +780,8 b' class GitRemote(RemoteBase):'
747 780 @reraise_safe_exceptions
748 781 def get_refs(self, wire):
749 782 cache_on, context_uid, repo_id = self._cache_on(wire)
750 @self.region.conditional_cache_on_arguments(condition=cache_on)
783 region = self._region(wire)
784 @region.conditional_cache_on_arguments(condition=cache_on)
751 785 def _get_refs(_context_uid, _repo_id):
752 786
753 787 repo_init = self._factory.repo_libgit2(wire)
@@ -761,7 +795,8 b' class GitRemote(RemoteBase):'
761 795 @reraise_safe_exceptions
762 796 def get_branch_pointers(self, wire):
763 797 cache_on, context_uid, repo_id = self._cache_on(wire)
764 @self.region.conditional_cache_on_arguments(condition=cache_on)
798 region = self._region(wire)
799 @region.conditional_cache_on_arguments(condition=cache_on)
765 800 def _get_branch_pointers(_context_uid, _repo_id):
766 801
767 802 repo_init = self._factory.repo_libgit2(wire)
@@ -775,7 +810,8 b' class GitRemote(RemoteBase):'
775 810 @reraise_safe_exceptions
776 811 def head(self, wire, show_exc=True):
777 812 cache_on, context_uid, repo_id = self._cache_on(wire)
778 @self.region.conditional_cache_on_arguments(condition=cache_on)
813 region = self._region(wire)
814 @region.conditional_cache_on_arguments(condition=cache_on)
779 815 def _head(_context_uid, _repo_id, _show_exc):
780 816 repo_init = self._factory.repo_libgit2(wire)
781 817 with repo_init as repo:
@@ -800,7 +836,8 b' class GitRemote(RemoteBase):'
800 836 def revision(self, wire, rev):
801 837
802 838 cache_on, context_uid, repo_id = self._cache_on(wire)
803 @self.region.conditional_cache_on_arguments(condition=cache_on)
839 region = self._region(wire)
840 @region.conditional_cache_on_arguments(condition=cache_on)
804 841 def _revision(_context_uid, _repo_id, _rev):
805 842 repo_init = self._factory.repo_libgit2(wire)
806 843 with repo_init as repo:
@@ -818,7 +855,8 b' class GitRemote(RemoteBase):'
818 855 @reraise_safe_exceptions
819 856 def date(self, wire, commit_id):
820 857 cache_on, context_uid, repo_id = self._cache_on(wire)
821 @self.region.conditional_cache_on_arguments(condition=cache_on)
858 region = self._region(wire)
859 @region.conditional_cache_on_arguments(condition=cache_on)
822 860 def _date(_repo_id, _commit_id):
823 861 repo_init = self._factory.repo_libgit2(wire)
824 862 with repo_init as repo:
@@ -837,7 +875,8 b' class GitRemote(RemoteBase):'
837 875 @reraise_safe_exceptions
838 876 def author(self, wire, commit_id):
839 877 cache_on, context_uid, repo_id = self._cache_on(wire)
840 @self.region.conditional_cache_on_arguments(condition=cache_on)
878 region = self._region(wire)
879 @region.conditional_cache_on_arguments(condition=cache_on)
841 880 def _author(_repo_id, _commit_id):
842 881 repo_init = self._factory.repo_libgit2(wire)
843 882 with repo_init as repo:
@@ -861,7 +900,8 b' class GitRemote(RemoteBase):'
861 900 @reraise_safe_exceptions
862 901 def message(self, wire, commit_id):
863 902 cache_on, context_uid, repo_id = self._cache_on(wire)
864 @self.region.conditional_cache_on_arguments(condition=cache_on)
903 region = self._region(wire)
904 @region.conditional_cache_on_arguments(condition=cache_on)
865 905 def _message(_repo_id, _commit_id):
866 906 repo_init = self._factory.repo_libgit2(wire)
867 907 with repo_init as repo:
@@ -872,7 +912,8 b' class GitRemote(RemoteBase):'
872 912 @reraise_safe_exceptions
873 913 def parents(self, wire, commit_id):
874 914 cache_on, context_uid, repo_id = self._cache_on(wire)
875 @self.region.conditional_cache_on_arguments(condition=cache_on)
915 region = self._region(wire)
916 @region.conditional_cache_on_arguments(condition=cache_on)
876 917 def _parents(_repo_id, _commit_id):
877 918 repo_init = self._factory.repo_libgit2(wire)
878 919 with repo_init as repo:
@@ -888,7 +929,8 b' class GitRemote(RemoteBase):'
888 929 @reraise_safe_exceptions
889 930 def children(self, wire, commit_id):
890 931 cache_on, context_uid, repo_id = self._cache_on(wire)
891 @self.region.conditional_cache_on_arguments(condition=cache_on)
932 region = self._region(wire)
933 @region.conditional_cache_on_arguments(condition=cache_on)
892 934 def _children(_repo_id, _commit_id):
893 935 output, __ = self.run_git_command(
894 936 wire, ['rev-list', '--all', '--children'])
@@ -947,7 +989,8 b' class GitRemote(RemoteBase):'
947 989 def tree_and_type_for_path(self, wire, commit_id, path):
948 990
949 991 cache_on, context_uid, repo_id = self._cache_on(wire)
950 @self.region.conditional_cache_on_arguments(condition=cache_on)
992 region = self._region(wire)
993 @region.conditional_cache_on_arguments(condition=cache_on)
951 994 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
952 995 repo_init = self._factory.repo_libgit2(wire)
953 996
@@ -964,7 +1007,8 b' class GitRemote(RemoteBase):'
964 1007 @reraise_safe_exceptions
965 1008 def tree_items(self, wire, tree_id):
966 1009 cache_on, context_uid, repo_id = self._cache_on(wire)
967 @self.region.conditional_cache_on_arguments(condition=cache_on)
1010 region = self._region(wire)
1011 @region.conditional_cache_on_arguments(condition=cache_on)
968 1012 def _tree_items(_repo_id, _tree_id):
969 1013
970 1014 repo_init = self._factory.repo_libgit2(wire)
@@ -1065,7 +1109,8 b' class GitRemote(RemoteBase):'
1065 1109 @reraise_safe_exceptions
1066 1110 def node_history(self, wire, commit_id, path, limit):
1067 1111 cache_on, context_uid, repo_id = self._cache_on(wire)
1068 @self.region.conditional_cache_on_arguments(condition=cache_on)
1112 region = self._region(wire)
1113 @region.conditional_cache_on_arguments(condition=cache_on)
1069 1114 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1070 1115 # optimize for n==1, rev-list is much faster for that use-case
1071 1116 if limit == 1:
@@ -1107,7 +1152,8 b' class GitRemote(RemoteBase):'
1107 1152 def get_all_commit_ids(self, wire):
1108 1153
1109 1154 cache_on, context_uid, repo_id = self._cache_on(wire)
1110 @self.region.conditional_cache_on_arguments(condition=cache_on)
1155 region = self._region(wire)
1156 @region.conditional_cache_on_arguments(condition=cache_on)
1111 1157 def _get_all_commit_ids(_context_uid, _repo_id):
1112 1158
1113 1159 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
@@ -1190,3 +1236,46 b' class GitRemote(RemoteBase):'
1190 1236 'pre_version': get_git_pre_hook_version(path, bare),
1191 1237 'post_version': get_git_post_hook_version(path, bare),
1192 1238 }
1239
1240 @reraise_safe_exceptions
1241 def set_head_ref(self, wire, head_name):
1242 log.debug('Setting refs/head to `%s`', head_name)
1243 cmd = ['symbolic-ref', '"HEAD"', '"refs/heads/%s"' % head_name]
1244 output, __ = self.run_git_command(wire, cmd)
1245 return [head_name] + output.splitlines()
1246
1247 @reraise_safe_exceptions
1248 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1249 archive_dir_name, commit_id):
1250
1251 def file_walker(_commit_id, path):
1252 repo_init = self._factory.repo_libgit2(wire)
1253
1254 with repo_init as repo:
1255 commit = repo[commit_id]
1256
1257 if path in ['', '/']:
1258 tree = commit.tree
1259 else:
1260 tree = commit.tree[path.rstrip('/')]
1261 tree_id = tree.id.hex
1262 try:
1263 tree = repo[tree_id]
1264 except KeyError:
1265 raise ObjectMissing('No tree with id: {}'.format(tree_id))
1266
1267 index = LibGit2Index.Index()
1268 index.read_tree(tree)
1269 file_iter = index
1270
1271 for fn in file_iter:
1272 file_path = fn.path
1273 mode = fn.mode
1274 is_link = stat.S_ISLNK(mode)
1275 if mode == pygit2.GIT_FILEMODE_COMMIT:
1276 log.debug('Skipping path %s as a commit node', file_path)
1277 continue
1278 yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw)
1279
1280 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1281 archive_dir_name, commit_id)
@@ -14,9 +14,10 b''
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17 import functools
18 18 import io
19 19 import logging
20 import os
20 21 import stat
21 22 import urllib.request, urllib.parse, urllib.error
22 23 import urllib.request, urllib.error, urllib.parse
@@ -31,13 +32,14 b' from mercurial import repair'
31 32
32 33 import vcsserver
33 34 from vcsserver import exceptions
34 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
35 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, archive_repo, ArchiveNode
35 36 from vcsserver.hgcompat import (
36 37 archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
37 38 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
38 39 makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
39 40 patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
40 RepoLookupError, InterventionRequired, RequirementError)
41 RepoLookupError, InterventionRequired, RequirementError,
42 alwaysmatcher, patternmatcher, hgutil)
41 43 from vcsserver.vcs_base import RemoteBase
42 44
43 45 log = logging.getLogger(__name__)
@@ -205,25 +207,10 b' class HgRemote(RemoteBase):'
205 207 return False
206 208
207 209 @reraise_safe_exceptions
208 def archive_repo(self, archive_path, mtime, file_info, kind):
209 if kind == "tgz":
210 archiver = archival.tarit(archive_path, mtime, "gz")
211 elif kind == "tbz2":
212 archiver = archival.tarit(archive_path, mtime, "bz2")
213 elif kind == 'zip':
214 archiver = archival.zipit(archive_path, mtime)
215 else:
216 raise exceptions.ArchiveException()(
217 'Remote does not support: "%s".' % kind)
218
219 for f_path, f_mode, f_is_link, f_content in file_info:
220 archiver.addfile(f_path, f_mode, f_is_link, f_content)
221 archiver.done()
222
223 @reraise_safe_exceptions
224 210 def bookmarks(self, wire):
225 211 cache_on, context_uid, repo_id = self._cache_on(wire)
226 @self.region.conditional_cache_on_arguments(condition=cache_on)
212 region = self._region(wire)
213 @region.conditional_cache_on_arguments(condition=cache_on)
227 214 def _bookmarks(_context_uid, _repo_id):
228 215 repo = self._factory.repo(wire)
229 216 return dict(repo._bookmarks)
@@ -233,7 +220,8 b' class HgRemote(RemoteBase):'
233 220 @reraise_safe_exceptions
234 221 def branches(self, wire, normal, closed):
235 222 cache_on, context_uid, repo_id = self._cache_on(wire)
236 @self.region.conditional_cache_on_arguments(condition=cache_on)
223 region = self._region(wire)
224 @region.conditional_cache_on_arguments(condition=cache_on)
237 225 def _branches(_context_uid, _repo_id, _normal, _closed):
238 226 repo = self._factory.repo(wire)
239 227 iter_branches = repo.branchmap().iterbranches()
@@ -251,7 +239,8 b' class HgRemote(RemoteBase):'
251 239 @reraise_safe_exceptions
252 240 def bulk_request(self, wire, commit_id, pre_load):
253 241 cache_on, context_uid, repo_id = self._cache_on(wire)
254 @self.region.conditional_cache_on_arguments(condition=cache_on)
242 region = self._region(wire)
243 @region.conditional_cache_on_arguments(condition=cache_on)
255 244 def _bulk_request(_repo_id, _commit_id, _pre_load):
256 245 result = {}
257 246 for attr in pre_load:
@@ -268,7 +257,8 b' class HgRemote(RemoteBase):'
268 257 @reraise_safe_exceptions
269 258 def ctx_branch(self, wire, commit_id):
270 259 cache_on, context_uid, repo_id = self._cache_on(wire)
271 @self.region.conditional_cache_on_arguments(condition=cache_on)
260 region = self._region(wire)
261 @region.conditional_cache_on_arguments(condition=cache_on)
272 262 def _ctx_branch(_repo_id, _commit_id):
273 263 repo = self._factory.repo(wire)
274 264 ctx = self._get_ctx(repo, commit_id)
@@ -278,7 +268,8 b' class HgRemote(RemoteBase):'
278 268 @reraise_safe_exceptions
279 269 def ctx_date(self, wire, commit_id):
280 270 cache_on, context_uid, repo_id = self._cache_on(wire)
281 @self.region.conditional_cache_on_arguments(condition=cache_on)
271 region = self._region(wire)
272 @region.conditional_cache_on_arguments(condition=cache_on)
282 273 def _ctx_date(_repo_id, _commit_id):
283 274 repo = self._factory.repo(wire)
284 275 ctx = self._get_ctx(repo, commit_id)
@@ -294,7 +285,8 b' class HgRemote(RemoteBase):'
294 285 @reraise_safe_exceptions
295 286 def ctx_files(self, wire, commit_id):
296 287 cache_on, context_uid, repo_id = self._cache_on(wire)
297 @self.region.conditional_cache_on_arguments(condition=cache_on)
288 region = self._region(wire)
289 @region.conditional_cache_on_arguments(condition=cache_on)
298 290 def _ctx_files(_repo_id, _commit_id):
299 291 repo = self._factory.repo(wire)
300 292 ctx = self._get_ctx(repo, commit_id)
@@ -311,7 +303,8 b' class HgRemote(RemoteBase):'
311 303 @reraise_safe_exceptions
312 304 def ctx_parents(self, wire, commit_id):
313 305 cache_on, context_uid, repo_id = self._cache_on(wire)
314 @self.region.conditional_cache_on_arguments(condition=cache_on)
306 region = self._region(wire)
307 @region.conditional_cache_on_arguments(condition=cache_on)
315 308 def _ctx_parents(_repo_id, _commit_id):
316 309 repo = self._factory.repo(wire)
317 310 ctx = self._get_ctx(repo, commit_id)
@@ -323,7 +316,8 b' class HgRemote(RemoteBase):'
323 316 @reraise_safe_exceptions
324 317 def ctx_children(self, wire, commit_id):
325 318 cache_on, context_uid, repo_id = self._cache_on(wire)
326 @self.region.conditional_cache_on_arguments(condition=cache_on)
319 region = self._region(wire)
320 @region.conditional_cache_on_arguments(condition=cache_on)
327 321 def _ctx_children(_repo_id, _commit_id):
328 322 repo = self._factory.repo(wire)
329 323 ctx = self._get_ctx(repo, commit_id)
@@ -335,7 +329,8 b' class HgRemote(RemoteBase):'
335 329 @reraise_safe_exceptions
336 330 def ctx_phase(self, wire, commit_id):
337 331 cache_on, context_uid, repo_id = self._cache_on(wire)
338 @self.region.conditional_cache_on_arguments(condition=cache_on)
332 region = self._region(wire)
333 @region.conditional_cache_on_arguments(condition=cache_on)
339 334 def _ctx_phase(_context_uid, _repo_id, _commit_id):
340 335 repo = self._factory.repo(wire)
341 336 ctx = self._get_ctx(repo, commit_id)
@@ -346,7 +341,8 b' class HgRemote(RemoteBase):'
346 341 @reraise_safe_exceptions
347 342 def ctx_obsolete(self, wire, commit_id):
348 343 cache_on, context_uid, repo_id = self._cache_on(wire)
349 @self.region.conditional_cache_on_arguments(condition=cache_on)
344 region = self._region(wire)
345 @region.conditional_cache_on_arguments(condition=cache_on)
350 346 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
351 347 repo = self._factory.repo(wire)
352 348 ctx = self._get_ctx(repo, commit_id)
@@ -356,7 +352,8 b' class HgRemote(RemoteBase):'
356 352 @reraise_safe_exceptions
357 353 def ctx_hidden(self, wire, commit_id):
358 354 cache_on, context_uid, repo_id = self._cache_on(wire)
359 @self.region.conditional_cache_on_arguments(condition=cache_on)
355 region = self._region(wire)
356 @region.conditional_cache_on_arguments(condition=cache_on)
360 357 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
361 358 repo = self._factory.repo(wire)
362 359 ctx = self._get_ctx(repo, commit_id)
@@ -469,7 +466,8 b' class HgRemote(RemoteBase):'
469 466 @reraise_safe_exceptions
470 467 def node_history(self, wire, revision, path, limit):
471 468 cache_on, context_uid, repo_id = self._cache_on(wire)
472 @self.region.conditional_cache_on_arguments(condition=cache_on)
469 region = self._region(wire)
470 @region.conditional_cache_on_arguments(condition=cache_on)
473 471 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
474 472 repo = self._factory.repo(wire)
475 473
@@ -499,7 +497,8 b' class HgRemote(RemoteBase):'
499 497 @reraise_safe_exceptions
500 498 def node_history_untill(self, wire, revision, path, limit):
501 499 cache_on, context_uid, repo_id = self._cache_on(wire)
502 @self.region.conditional_cache_on_arguments(condition=cache_on)
500 region = self._region(wire)
501 @region.conditional_cache_on_arguments(condition=cache_on)
503 502 def _node_history_until(_context_uid, _repo_id):
504 503 repo = self._factory.repo(wire)
505 504 ctx = self._get_ctx(repo, revision)
@@ -537,7 +536,8 b' class HgRemote(RemoteBase):'
537 536 @reraise_safe_exceptions
538 537 def fctx_flags(self, wire, commit_id, path):
539 538 cache_on, context_uid, repo_id = self._cache_on(wire)
540 @self.region.conditional_cache_on_arguments(condition=cache_on)
539 region = self._region(wire)
540 @region.conditional_cache_on_arguments(condition=cache_on)
541 541 def _fctx_flags(_repo_id, _commit_id, _path):
542 542 repo = self._factory.repo(wire)
543 543 ctx = self._get_ctx(repo, commit_id)
@@ -549,7 +549,8 b' class HgRemote(RemoteBase):'
549 549 @reraise_safe_exceptions
550 550 def fctx_size(self, wire, commit_id, path):
551 551 cache_on, context_uid, repo_id = self._cache_on(wire)
552 @self.region.conditional_cache_on_arguments(condition=cache_on)
552 region = self._region(wire)
553 @region.conditional_cache_on_arguments(condition=cache_on)
553 554 def _fctx_size(_repo_id, _revision, _path):
554 555 repo = self._factory.repo(wire)
555 556 ctx = self._get_ctx(repo, commit_id)
@@ -560,7 +561,8 b' class HgRemote(RemoteBase):'
560 561 @reraise_safe_exceptions
561 562 def get_all_commit_ids(self, wire, name):
562 563 cache_on, context_uid, repo_id = self._cache_on(wire)
563 @self.region.conditional_cache_on_arguments(condition=cache_on)
564 region = self._region(wire)
565 @region.conditional_cache_on_arguments(condition=cache_on)
564 566 def _get_all_commit_ids(_context_uid, _repo_id, _name):
565 567 repo = self._factory.repo(wire)
566 568 repo = repo.filtered(name)
@@ -576,7 +578,8 b' class HgRemote(RemoteBase):'
576 578 @reraise_safe_exceptions
577 579 def is_large_file(self, wire, commit_id, path):
578 580 cache_on, context_uid, repo_id = self._cache_on(wire)
579 @self.region.conditional_cache_on_arguments(condition=cache_on)
581 region = self._region(wire)
582 @region.conditional_cache_on_arguments(condition=cache_on)
580 583 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
581 584 return largefiles.lfutil.isstandin(path)
582 585
@@ -586,7 +589,8 b' class HgRemote(RemoteBase):'
586 589 def is_binary(self, wire, revision, path):
587 590 cache_on, context_uid, repo_id = self._cache_on(wire)
588 591
589 @self.region.conditional_cache_on_arguments(condition=cache_on)
592 region = self._region(wire)
593 @region.conditional_cache_on_arguments(condition=cache_on)
590 594 def _is_binary(_repo_id, _sha, _path):
591 595 repo = self._factory.repo(wire)
592 596 ctx = self._get_ctx(repo, revision)
@@ -623,7 +627,9 b' class HgRemote(RemoteBase):'
623 627 @reraise_safe_exceptions
624 628 def lookup(self, wire, revision, both):
625 629 cache_on, context_uid, repo_id = self._cache_on(wire)
626 @self.region.conditional_cache_on_arguments(condition=cache_on)
630
631 region = self._region(wire)
632 @region.conditional_cache_on_arguments(condition=cache_on)
627 633 def _lookup(_context_uid, _repo_id, _revision, _both):
628 634
629 635 repo = self._factory.repo(wire)
@@ -681,7 +687,8 b' class HgRemote(RemoteBase):'
681 687 def rev_range(self, wire, commit_filter):
682 688 cache_on, context_uid, repo_id = self._cache_on(wire)
683 689
684 @self.region.conditional_cache_on_arguments(condition=cache_on)
690 region = self._region(wire)
691 @region.conditional_cache_on_arguments(condition=cache_on)
685 692 def _rev_range(_context_uid, _repo_id, _filter):
686 693 repo = self._factory.repo(wire)
687 694 revisions = [rev for rev in revrange(repo, commit_filter)]
@@ -756,7 +763,8 b' class HgRemote(RemoteBase):'
756 763 @reraise_safe_exceptions
757 764 def tags(self, wire):
758 765 cache_on, context_uid, repo_id = self._cache_on(wire)
759 @self.region.conditional_cache_on_arguments(condition=cache_on)
766 region = self._region(wire)
767 @region.conditional_cache_on_arguments(condition=cache_on)
760 768 def _tags(_context_uid, _repo_id):
761 769 repo = self._factory.repo(wire)
762 770 return repo.tags()
@@ -1007,3 +1015,33 b' class HgRemote(RemoteBase):'
1007 1015 'pre_version': vcsserver.__version__,
1008 1016 'post_version': vcsserver.__version__,
1009 1017 }
1018
1019 @reraise_safe_exceptions
1020 def set_head_ref(self, wire, head_name):
1021 pass
1022
1023 @reraise_safe_exceptions
1024 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
1025 archive_dir_name, commit_id):
1026
1027 def file_walker(_commit_id, path):
1028 repo = self._factory.repo(wire)
1029 ctx = repo[_commit_id]
1030 is_root = path in ['', '/']
1031 if is_root:
1032 matcher = alwaysmatcher(badfn=None)
1033 else:
1034 matcher = patternmatcher('', [(b'glob', path+'/**', b'')], badfn=None)
1035 file_iter = ctx.manifest().walk(matcher)
1036
1037 for fn in file_iter:
1038 file_path = fn
1039 flags = ctx.flags(fn)
1040 mode = b'x' in flags and 0o755 or 0o644
1041 is_link = b'l' in flags
1042
1043 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
1044
1045 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
1046 archive_dir_name, commit_id)
1047
@@ -38,7 +38,7 b' from mercurial import merge as hg_merge'
38 38 from mercurial import subrepo
39 39 from mercurial import subrepoutil
40 40 from mercurial import tags as hg_tag
41
41 from mercurial import util as hgutil
42 42 from mercurial.commands import clone, nullid, pull
43 43 from mercurial.context import memctx, memfilectx
44 44 from mercurial.error import (
@@ -46,7 +46,7 b' from mercurial.error import ('
46 46 RequirementError, ProgrammingError)
47 47 from mercurial.hgweb import hgweb_mod
48 48 from mercurial.localrepo import instance
49 from mercurial.match import match
49 from mercurial.match import match, alwaysmatcher, patternmatcher
50 50 from mercurial.mdiff import diffopts
51 51 from mercurial.node import bin, hex
52 52 from mercurial.encoding import tolocal
@@ -67,7 +67,7 b' from mercurial.url import httpbasicauthh'
67 67 def get_ctx(repo, ref):
68 68 try:
69 69 ctx = repo[ref]
70 except ProgrammingError:
70 except (ProgrammingError, TypeError):
71 71 # we're unable to find the rev using a regular lookup, we fallback
72 72 # to slower, but backward compat revsymbol usage
73 73 ctx = revsymbol(repo, ref)
@@ -49,7 +49,7 b' class HooksHttpClient(object):'
49 49 try:
50 50 connection.request('POST', '/', body)
51 51 except Exception:
52 log.error('Connection failed on %s', connection)
52 log.error('Hooks calling Connection failed on %s', connection.__dict__)
53 53 raise
54 54 response = connection.getresponse()
55 55
@@ -577,8 +577,8 b' def git_post_receive(unused_repo_path, r'
577 577 try:
578 578 subprocessio.run_command(cmd, env=os.environ.copy())
579 579 except Exception:
580 cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
581 'refs/heads/%s' % push_ref['name']]
580 cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', '"HEAD"',
581 '"refs/heads/%s"' % push_ref['name']]
582 582 print("Setting default branch to %s" % push_ref['name'])
583 583 subprocessio.run_command(cmd, env=os.environ.copy())
584 584
@@ -21,21 +21,27 b' import base64'
21 21 import locale
22 22 import logging
23 23 import uuid
24 import time
24 25 import wsgiref.util
25 26 import traceback
26 27 import tempfile
28 import psutil
29
27 30 from itertools import chain
28 31 from io import StringIO
29 32
30 33 import simplejson as json
31 34 import msgpack
32 35 import configparser
36
33 37 from pyramid.config import Configurator
34 38 from pyramid.settings import asbool, aslist
35 39 from pyramid.wsgi import wsgiapp
36 40 from pyramid.response import Response
37 41
42 from vcsserver.config.settings_maker import SettingsMaker
38 43 from vcsserver.utils import safe_int
44 from vcsserver.lib.statsd_client import StatsdClient
39 45
40 46 log = logging.getLogger(__name__)
41 47
@@ -49,6 +55,7 b' except locale.Error as e:'
49 55 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
50 56 os.environ['LC_ALL'] = 'C'
51 57
58
52 59 import vcsserver
53 60 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
54 61 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
@@ -98,38 +105,12 b' def _is_request_chunked(environ):'
98 105 return stream
99 106
100 107
101 def _int_setting(settings, name, default):
102 settings[name] = int(settings.get(name, default))
103 return settings[name]
104
105
106 def _bool_setting(settings, name, default):
107 input_val = settings.get(name, default)
108 if isinstance(input_val, str):
109 input_val = input_val.encode('utf8')
110 settings[name] = asbool(input_val)
111 return settings[name]
112
113
114 def _list_setting(settings, name, default):
115 raw_value = settings.get(name, default)
116
117 # Otherwise we assume it uses pyramids space/newline separation.
118 settings[name] = aslist(raw_value)
119 return settings[name]
120
121
122 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
123 value = settings.get(name, default)
124
125 if default_when_empty and not value:
126 # use default value when value is empty
127 value = default
128
129 if lower:
130 value = value.lower()
131 settings[name] = value
132 return settings[name]
108 def log_max_fd():
109 try:
110 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
111 log.info('Max file descriptors value: %s', maxfd)
112 except Exception:
113 pass
133 114
134 115
135 116 class VCS(object):
@@ -138,6 +119,8 b' class VCS(object):'
138 119 self.cache_config = cache_config
139 120 self._configure_locale()
140 121
122 log_max_fd()
123
141 124 if GitFactory and GitRemote:
142 125 git_factory = GitFactory()
143 126 self._git_remote = GitRemote(git_factory)
@@ -243,9 +226,11 b' class HTTPApplication(object):'
243 226 _use_echo_app = False
244 227
245 228 def __init__(self, settings=None, global_config=None):
246 self._sanitize_settings_and_apply_defaults(settings)
247 229
248 230 self.config = Configurator(settings=settings)
231 # Init our statsd at very start
232 self.config.registry.statsd = StatsdClient.statsd
233
249 234 self.global_config = global_config
250 235 self.config.include('vcsserver.lib.rc_cache')
251 236
@@ -263,6 +248,7 b' class HTTPApplication(object):'
263 248 self.remote_wsgi = remote_wsgi_stub
264 249
265 250 self._configure_settings(global_config, settings)
251
266 252 self._configure()
267 253
268 254 def _configure_settings(self, global_config, app_settings):
@@ -283,40 +269,6 b' class HTTPApplication(object):'
283 269 vcsserver.PYRAMID_SETTINGS = settings_merged
284 270 vcsserver.CONFIG = settings_merged
285 271
286 def _sanitize_settings_and_apply_defaults(self, settings):
287 temp_store = tempfile.gettempdir()
288 default_cache_dir = os.path.join(temp_store, 'rc_cache')
289
290 # save default, cache dir, and use it for all backends later.
291 default_cache_dir = _string_setting(
292 settings,
293 'cache_dir',
294 default_cache_dir, lower=False, default_when_empty=True)
295
296 # ensure we have our dir created
297 if not os.path.isdir(default_cache_dir):
298 os.makedirs(default_cache_dir, mode=0o755)
299
300 # exception store cache
301 _string_setting(
302 settings,
303 'exception_tracker.store_path',
304 temp_store, lower=False, default_when_empty=True)
305
306 # repo_object cache
307 _string_setting(
308 settings,
309 'rc_cache.repo_object.backend',
310 'dogpile.cache.rc.file_namespace', lower=False)
311 _int_setting(
312 settings,
313 'rc_cache.repo_object.expiration_time',
314 30 * 24 * 60 * 60)
315 _string_setting(
316 settings,
317 'rc_cache.repo_object.arguments.filename',
318 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
319
320 272 def _configure(self):
321 273 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
322 274
@@ -385,16 +337,22 b' class HTTPApplication(object):'
385 337 # NOTE(marcink): trading complexity for slight performance
386 338 if log.isEnabledFor(logging.DEBUG):
387 339 no_args_methods = [
388 'archive_repo'
340
389 341 ]
390 342 if method in no_args_methods:
391 343 call_args = ''
392 344 else:
393 345 call_args = args[1:]
394 346
395 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
347 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
396 348 method, call_args, kwargs, context_uid, repo_state_uid)
397 349
350 statsd = request.registry.statsd
351 if statsd:
352 statsd.incr(
353 'vcsserver_method_total', tags=[
354 "method:{}".format(method),
355 ])
398 356 return payload, remote, method, args, kwargs
399 357
400 358 def vcs_view(self, request):
@@ -431,7 +389,7 b' class HTTPApplication(object):'
431 389 should_store_exc = False
432 390
433 391 if should_store_exc:
434 store_exception(id(exc_info), exc_info)
392 store_exception(id(exc_info), exc_info, request_path=request.path)
435 393
436 394 tb_info = ''.join(
437 395 traceback.format_exception(exc_type, exc_value, exc_traceback))
@@ -450,6 +408,7 b' class HTTPApplication(object):'
450 408 'type': type_
451 409 }
452 410 }
411
453 412 try:
454 413 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
455 414 except AttributeError:
@@ -678,6 +637,12 b' class HTTPApplication(object):'
678 637 log.error(
679 638 'error occurred handling this request for path: %s, \n tb: %s',
680 639 request.path, traceback_info)
640
641 statsd = request.registry.statsd
642 if statsd:
643 exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
644 statsd.incr('vcsserver_exception_total',
645 tags=["type:{}".format(exc_type)])
681 646 raise exception
682 647
683 648
@@ -693,10 +658,83 b' class ResponseFilter(object):'
693 658 return self._start_response(status, headers, exc_info)
694 659
695 660
661 def sanitize_settings_and_apply_defaults(global_config, settings):
662 global_settings_maker = SettingsMaker(global_config)
663 settings_maker = SettingsMaker(settings)
664
665 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
666
667 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
668 settings_maker.enable_logging(logging_conf)
669
670 # Default includes, possible to change as a user
671 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
672 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
673
674 settings_maker.make_setting('__file__', global_config.get('__file__'))
675
676 settings_maker.make_setting('pyramid.default_locale_name', 'en')
677 settings_maker.make_setting('locale', 'en_US.UTF-8')
678
679 settings_maker.make_setting('core.binary_dir', '')
680
681 temp_store = tempfile.gettempdir()
682 default_cache_dir = os.path.join(temp_store, 'rc_cache')
683 # save default, cache dir, and use it for all backends later.
684 default_cache_dir = settings_maker.make_setting(
685 'cache_dir',
686 default=default_cache_dir, default_when_empty=True,
687 parser='dir:ensured')
688
689 # exception store cache
690 settings_maker.make_setting(
691 'exception_tracker.store_path',
692 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
693 parser='dir:ensured'
694 )
695
696 # repo_object cache defaults
697 settings_maker.make_setting(
698 'rc_cache.repo_object.backend',
699 default='dogpile.cache.rc.file_namespace',
700 parser='string')
701 settings_maker.make_setting(
702 'rc_cache.repo_object.expiration_time',
703 default=30 * 24 * 60 * 60, # 30days
704 parser='int')
705 settings_maker.make_setting(
706 'rc_cache.repo_object.arguments.filename',
707 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
708 parser='string')
709
710 # statsd
711 settings_maker.make_setting('statsd.enabled', False, parser='bool')
712 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
713 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
714 settings_maker.make_setting('statsd.statsd_prefix', '')
715 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
716
717 settings_maker.env_expand()
718
719
696 720 def main(global_config, **settings):
721 start_time = time.time()
722 log.info('Pyramid app config starting')
723
697 724 if MercurialFactory:
698 725 hgpatches.patch_largefiles_capabilities()
699 726 hgpatches.patch_subrepo_type_mapping()
700 727
701 app = HTTPApplication(settings=settings, global_config=global_config)
702 return app.wsgi_app()
728 # Fill in and sanitize the defaults & do ENV expansion
729 sanitize_settings_and_apply_defaults(global_config, settings)
730
731 # init and bootstrap StatsdClient
732 StatsdClient.setup(settings)
733
734 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
735 total_time = time.time() - start_time
736 log.info('Pyramid app `%s` created and configured in %.2fs',
737 getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
738 return pyramid_app
739
740
@@ -66,7 +66,7 b' def get_exc_store():'
66 66 return _exc_store_path
67 67
68 68
69 def _store_exception(exc_id, exc_info, prefix):
69 def _store_exception(exc_id, exc_info, prefix, request_path=''):
70 70 exc_type, exc_value, exc_traceback = exc_info
71 71
72 72 tb = ''.join(traceback.format_exception(
@@ -99,8 +99,13 b' def _store_exception(exc_id, exc_info, p'
99 99 f.write(exc_data)
100 100 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
101 101
102 log.error(
103 'error occurred handling this request.\n'
104 'Path: `%s`, tb: %s',
105 request_path, tb)
102 106
103 def store_exception(exc_id, exc_info, prefix=global_prefix):
107
108 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
104 109 """
105 110 Example usage::
106 111
@@ -109,7 +114,8 b' def store_exception(exc_id, exc_info, pr'
109 114 """
110 115
111 116 try:
112 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
117 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix,
118 request_path=request_path)
113 119 except Exception:
114 120 log.exception('Failed to store exception `%s` information', exc_id)
115 121 # there's no way this can fail, it will crash server badly if it does.
@@ -38,7 +38,9 b' register_backend('
38 38 log = logging.getLogger(__name__)
39 39
40 40 from . import region_meta
41 from .utils import (get_default_cache_settings, backend_key_generator, make_region)
41 from .utils import (
42 get_default_cache_settings, backend_key_generator, get_or_create_region,
43 clear_cache_namespace, make_region)
42 44
43 45
44 46 def configure_dogpile_cache(settings):
@@ -52,20 +54,25 b' def configure_dogpile_cache(settings):'
52 54 avail_regions = set()
53 55 for key in rc_cache_data.keys():
54 56 namespace_name = key.split('.', 1)[0]
55 avail_regions.add(namespace_name)
56 log.debug('dogpile: found following cache regions: %s', avail_regions)
57 if namespace_name in avail_regions:
58 continue
57 59
58 # register them into namespace
59 for region_name in avail_regions:
60 avail_regions.add(namespace_name)
61 log.debug('dogpile: found following cache regions: %s', namespace_name)
62
60 63 new_region = make_region(
61 name=region_name,
64 name=namespace_name,
62 65 function_key_generator=None
63 66 )
64 67
65 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
68 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(namespace_name))
66 69 new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
67 log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
68 region_meta.dogpile_cache_regions[region_name] = new_region
70 if log.isEnabledFor(logging.DEBUG):
71 region_args = dict(backend=new_region.actual_backend.__class__,
72 region_invalidator=new_region.region_invalidator.__class__)
73 log.debug('dogpile: registering a new region `%s` %s', namespace_name, region_args)
74
75 region_meta.dogpile_cache_regions[namespace_name] = new_region
69 76
70 77
71 78 def includeme(config):
@@ -29,7 +29,10 b' from dogpile.cache.backends import redis'
29 29 from dogpile.cache.backends.file import NO_VALUE, FileLock
30 30 from dogpile.cache.util import memoized_property
31 31
32 from pyramid.settings import asbool
33
32 34 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
35 from vcsserver.utils import safe_str, safe_unicode
33 36
34 37
35 38 _default_max_size = 1024
@@ -124,7 +127,14 b' class FileNamespaceBackend(PickleSeriali'
124 127
125 128 def __init__(self, arguments):
126 129 arguments['lock_factory'] = CustomLockFactory
130 db_file = arguments.get('filename')
131
132 log.debug('initialing %s DB in %s', self.__class__.__name__, db_file)
133 try:
127 134 super(FileNamespaceBackend, self).__init__(arguments)
135 except Exception:
136 log.exception('Failed to initialize db at: %s', db_file)
137 raise
128 138
129 139 def __repr__(self):
130 140 return '{} `{}`'.format(self.__class__, self.filename)
@@ -141,13 +151,16 b' class FileNamespaceBackend(PickleSeriali'
141 151 return False
142 152
143 153 with self._dbm_file(True) as dbm:
144
154 try:
145 155 return filter(cond, dbm.keys())
156 except Exception:
157 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
158 raise
146 159
147 160 def get_store(self):
148 161 return self.filename
149 162
150 def get(self, key):
163 def _dbm_get(self, key):
151 164 with self._dbm_file(False) as dbm:
152 165 if hasattr(dbm, 'get'):
153 166 value = dbm.get(key, NO_VALUE)
@@ -161,6 +174,13 b' class FileNamespaceBackend(PickleSeriali'
161 174 value = self._loads(value)
162 175 return value
163 176
177 def get(self, key):
178 try:
179 return self._dbm_get(key)
180 except Exception:
181 log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
182 raise
183
164 184 def set(self, key, value):
165 185 with self._dbm_file(True) as dbm:
166 186 dbm[key] = self._dumps(value)
@@ -172,6 +192,16 b' class FileNamespaceBackend(PickleSeriali'
172 192
173 193
174 194 class BaseRedisBackend(redis_backend.RedisBackend):
195 key_prefix = ''
196
197 def __init__(self, arguments):
198 super(BaseRedisBackend, self).__init__(arguments)
199 self._lock_timeout = self.lock_timeout
200 self._lock_auto_renewal = asbool(arguments.pop("lock_auto_renewal", True))
201
202 if self._lock_auto_renewal and not self._lock_timeout:
203 # set default timeout for auto_renewal
204 self._lock_timeout = 30
175 205
176 206 def _create_client(self):
177 207 args = {}
@@ -234,11 +264,10 b' class BaseRedisBackend(redis_backend.Red'
234 264 pipe.execute()
235 265
236 266 def get_mutex(self, key):
237 u = redis_backend.u
238 267 if self.distributed_lock:
239 lock_key = u('_lock_{0}').format(key)
240 log.debug('Trying to acquire Redis lock for key %s', lock_key)
241 return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
268 lock_key = u'_lock_{0}'.format(safe_unicode(key))
269 return get_mutex_lock(self.client, lock_key, self._lock_timeout,
270 auto_renewal=self._lock_auto_renewal)
242 271 else:
243 272 return None
244 273
@@ -251,3 +280,50 b' class RedisPickleBackend(PickleSerialize'
251 280 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
252 281 key_prefix = 'redis_msgpack_backend'
253 282 pass
283
284
285 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
286 import redis_lock
287
288 class _RedisLockWrapper(object):
289 """LockWrapper for redis_lock"""
290
291 @classmethod
292 def get_lock(cls):
293 return redis_lock.Lock(
294 redis_client=client,
295 name=lock_key,
296 expire=lock_timeout,
297 auto_renewal=auto_renewal,
298 strict=True,
299 )
300
301 def __repr__(self):
302 return "{}:{}".format(self.__class__.__name__, lock_key)
303
304 def __str__(self):
305 return "{}:{}".format(self.__class__.__name__, lock_key)
306
307 def __init__(self):
308 self.lock = self.get_lock()
309 self.lock_key = lock_key
310
311 def acquire(self, wait=True):
312 log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
313 try:
314 acquired = self.lock.acquire(wait)
315 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
316 return acquired
317 except redis_lock.AlreadyAcquired:
318 return False
319 except redis_lock.AlreadyStarted:
320 # refresh thread exists, but it also means we acquired the lock
321 return True
322
323 def release(self):
324 try:
325 self.lock.release()
326 except redis_lock.NotAcquired:
327 pass
328
329 return _RedisLockWrapper()
@@ -16,15 +16,16 b''
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 import time
19 20 import logging
20 21 import functools
21 from decorator import decorate
22 22
23 23 from dogpile.cache import CacheRegion
24 24 from dogpile.cache.util import compat
25 25
26 26 from vcsserver.utils import safe_str, sha1
27 27
28 from vcsserver.lib.rc_cache import region_meta
28 29
29 30 log = logging.getLogger(__name__)
30 31
@@ -48,18 +49,77 b' class RhodeCodeCacheRegion(CacheRegion):'
48 49 if function_key_generator is None:
49 50 function_key_generator = self.function_key_generator
50 51
52 # workaround for py2 and cython problems, this block should be removed
53 # once we've migrated to py3
54 if 'cython' == 'cython':
55 def decorator(fn):
56 if to_str is compat.string_type:
57 # backwards compatible
58 key_generator = function_key_generator(namespace, fn)
59 else:
60 key_generator = function_key_generator(namespace, fn, to_str=to_str)
61
62 @functools.wraps(fn)
63 def decorate(*arg, **kw):
64 key = key_generator(*arg, **kw)
65
66 @functools.wraps(fn)
67 def creator():
68 return fn(*arg, **kw)
69
70 if not condition:
71 return creator()
72
73 timeout = expiration_time() if expiration_time_is_callable \
74 else expiration_time
75
76 return self.get_or_create(key, creator, timeout, should_cache_fn)
77
78 def invalidate(*arg, **kw):
79 key = key_generator(*arg, **kw)
80 self.delete(key)
81
82 def set_(value, *arg, **kw):
83 key = key_generator(*arg, **kw)
84 self.set(key, value)
85
86 def get(*arg, **kw):
87 key = key_generator(*arg, **kw)
88 return self.get(key)
89
90 def refresh(*arg, **kw):
91 key = key_generator(*arg, **kw)
92 value = fn(*arg, **kw)
93 self.set(key, value)
94 return value
95
96 decorate.set = set_
97 decorate.invalidate = invalidate
98 decorate.refresh = refresh
99 decorate.get = get
100 decorate.original = fn
101 decorate.key_generator = key_generator
102 decorate.__wrapped__ = fn
103
104 return decorate
105 return decorator
106
51 107 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
52 108
53 109 if not condition:
54 log.debug('Calling un-cached func:%s', user_func.func_name)
55 return user_func(*arg, **kw)
110 log.debug('Calling un-cached method:%s', user_func.func_name)
111 start = time.time()
112 result = user_func(*arg, **kw)
113 total = time.time() - start
114 log.debug('un-cached method:%s took %.4fs', user_func.func_name, total)
115 return result
56 116
57 117 key = key_generator(*arg, **kw)
58 118
59 119 timeout = expiration_time() if expiration_time_is_callable \
60 120 else expiration_time
61 121
62 log.debug('Calling cached fn:%s', user_func.func_name)
122 log.debug('Calling cached method:`%s`', user_func.func_name)
63 123 return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
64 124
65 125 def cache_decorator(user_func):
@@ -98,8 +158,7 b' class RhodeCodeCacheRegion(CacheRegion):'
98 158 user_func.original = user_func
99 159
100 160 # Use `decorate` to preserve the signature of :param:`user_func`.
101
102 return decorate(user_func, functools.partial(
161 return decorator.decorate(user_func, functools.partial(
103 162 get_or_create_for_user_func, key_generator))
104 163
105 164 return cache_decorator
@@ -151,3 +210,54 b' def key_generator(backend, namespace, fn'
151 210 return final_key
152 211
153 212 return generate_key
213
214
215 def get_or_create_region(region_name, region_namespace=None):
216 from vcsserver.lib.rc_cache.backends import FileNamespaceBackend
217 region_obj = region_meta.dogpile_cache_regions.get(region_name)
218 if not region_obj:
219 raise EnvironmentError(
220 'Region `{}` not in configured: {}.'.format(
221 region_name, region_meta.dogpile_cache_regions.keys()))
222
223 region_uid_name = '{}:{}'.format(region_name, region_namespace)
224 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
225 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
226 if region_exist:
227 log.debug('Using already configured region: %s', region_namespace)
228 return region_exist
229 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
230 expiration_time = region_obj.expiration_time
231
232 if not os.path.isdir(cache_dir):
233 os.makedirs(cache_dir)
234 new_region = make_region(
235 name=region_uid_name,
236 function_key_generator=backend_key_generator(region_obj.actual_backend)
237 )
238 namespace_filename = os.path.join(
239 cache_dir, "{}.cache.dbm".format(region_namespace))
240 # special type that allows 1db per namespace
241 new_region.configure(
242 backend='dogpile.cache.rc.file_namespace',
243 expiration_time=expiration_time,
244 arguments={"filename": namespace_filename}
245 )
246
247 # create and save in region caches
248 log.debug('configuring new region: %s', region_uid_name)
249 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
250
251 return region_obj
252
253
254 def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
255 region = get_or_create_region(cache_region, cache_namespace_uid)
256 cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
257 num_delete_keys = len(cache_keys)
258 if invalidate:
259 region.invalidate(hard=False)
260 else:
261 if num_delete_keys:
262 region.delete_multi(cache_keys)
263 return num_delete_keys
@@ -19,6 +19,7 b''
19 19
20 20 import os
21 21 import subprocess
22 import time
22 23 from urllib.error import URLError
23 24 import urllib.parse
24 25 import logging
@@ -35,7 +36,9 b' import svn.fs'
35 36 import svn.repos
36 37
37 38 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 from vcsserver.base import RepoFactory, raise_from_original
39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
40 from vcsserver.exceptions import NoContentException
41 from vcsserver.utils import safe_str
39 42 from vcsserver.vcs_base import RemoteBase
40 43
41 44 log = logging.getLogger(__name__)
@@ -198,7 +201,8 b' class SvnRemote(RemoteBase):'
198 201 def revision_properties(self, wire, revision):
199 202
200 203 cache_on, context_uid, repo_id = self._cache_on(wire)
201 @self.region.conditional_cache_on_arguments(condition=cache_on)
204 region = self._region(wire)
205 @region.conditional_cache_on_arguments(condition=cache_on)
202 206 def _revision_properties(_repo_id, _revision):
203 207 repo = self._factory.repo(wire)
204 208 fs_ptr = svn.repos.fs(repo)
@@ -252,7 +256,8 b' class SvnRemote(RemoteBase):'
252 256 @reraise_safe_exceptions
253 257 def node_history(self, wire, path, revision, limit):
254 258 cache_on, context_uid, repo_id = self._cache_on(wire)
255 @self.region.conditional_cache_on_arguments(condition=cache_on)
259 region = self._region(wire)
260 @region.conditional_cache_on_arguments(condition=cache_on)
256 261 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
257 262 cross_copies = False
258 263 repo = self._factory.repo(wire)
@@ -273,7 +278,8 b' class SvnRemote(RemoteBase):'
273 278
274 279 def node_properties(self, wire, path, revision):
275 280 cache_on, context_uid, repo_id = self._cache_on(wire)
276 @self.region.conditional_cache_on_arguments(condition=cache_on)
281 region = self._region(wire)
282 @region.conditional_cache_on_arguments(condition=cache_on)
277 283 def _node_properties(_repo_id, _path, _revision):
278 284 repo = self._factory.repo(wire)
279 285 fsobj = svn.repos.fs(repo)
@@ -282,7 +288,7 b' class SvnRemote(RemoteBase):'
282 288 return _node_properties(repo_id, path, revision)
283 289
284 290 def file_annotate(self, wire, path, revision):
285 abs_path = 'file://' + urllib.request.pathname2url(
291 abs_path = 'file://' + urllib.pathname2url(
286 292 vcspath.join(wire['path'], path))
287 293 file_uri = svn.core.svn_path_canonicalize(abs_path)
288 294
@@ -311,7 +317,8 b' class SvnRemote(RemoteBase):'
311 317 def get_node_type(self, wire, path, revision=None):
312 318
313 319 cache_on, context_uid, repo_id = self._cache_on(wire)
314 @self.region.conditional_cache_on_arguments(condition=cache_on)
320 region = self._region(wire)
321 @region.conditional_cache_on_arguments(condition=cache_on)
315 322 def _get_node_type(_repo_id, _path, _revision):
316 323 repo = self._factory.repo(wire)
317 324 fs_ptr = svn.repos.fs(repo)
@@ -325,7 +332,8 b' class SvnRemote(RemoteBase):'
325 332 def get_nodes(self, wire, path, revision=None):
326 333
327 334 cache_on, context_uid, repo_id = self._cache_on(wire)
328 @self.region.conditional_cache_on_arguments(condition=cache_on)
335 region = self._region(wire)
336 @region.conditional_cache_on_arguments(condition=cache_on)
329 337 def _get_nodes(_repo_id, _path, _revision):
330 338 repo = self._factory.repo(wire)
331 339 fsobj = svn.repos.fs(repo)
@@ -352,7 +360,8 b' class SvnRemote(RemoteBase):'
352 360 def get_file_size(self, wire, path, revision=None):
353 361
354 362 cache_on, context_uid, repo_id = self._cache_on(wire)
355 @self.region.conditional_cache_on_arguments(condition=cache_on)
363 region = self._region(wire)
364 @region.conditional_cache_on_arguments(condition=cache_on)
356 365 def _get_file_size(_repo_id, _path, _revision):
357 366 repo = self._factory.repo(wire)
358 367 fsobj = svn.repos.fs(repo)
@@ -467,7 +476,8 b' class SvnRemote(RemoteBase):'
467 476 def is_binary(self, wire, rev, path):
468 477 cache_on, context_uid, repo_id = self._cache_on(wire)
469 478
470 @self.region.conditional_cache_on_arguments(condition=cache_on)
479 region = self._region(wire)
480 @region.conditional_cache_on_arguments(condition=cache_on)
471 481 def _is_binary(_repo_id, _rev, _path):
472 482 raw_bytes = self.get_file_content(wire, path, rev)
473 483 return raw_bytes and '\0' in raw_bytes
@@ -481,9 +491,7 b' class SvnRemote(RemoteBase):'
481 491 if path and os.path.isdir(path):
482 492 opts['cwd'] = path
483 493
484 safe_call = False
485 if '_safe' in opts:
486 safe_call = True
494 safe_call = opts.pop('_safe', False)
487 495
488 496 svnenv = os.environ.copy()
489 497 svnenv.update(opts.pop('extra_env', {}))
@@ -496,15 +504,15 b' class SvnRemote(RemoteBase):'
496 504
497 505 return ''.join(p), ''.join(p.error)
498 506 except (EnvironmentError, OSError) as err:
507 if safe_call:
508 return '', safe_str(err).strip()
509 else:
499 510 cmd = ' '.join(cmd) # human friendly CMD
500 511 tb_err = ("Couldn't run svn command (%s).\n"
501 512 "Original error was:%s\n"
502 513 "Call options:%s\n"
503 514 % (cmd, err, _opts))
504 515 log.exception(tb_err)
505 if safe_call:
506 return '', err
507 else:
508 516 raise exceptions.VcsException()(tb_err)
509 517
510 518 @reraise_safe_exceptions
@@ -528,6 +536,74 b' class SvnRemote(RemoteBase):'
528 536 'post_version': get_svn_post_hook_version(repo_path),
529 537 }
530 538
539 @reraise_safe_exceptions
540 def set_head_ref(self, wire, head_name):
541 pass
542
543 @reraise_safe_exceptions
544 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
545 archive_dir_name, commit_id):
546
547 def walk_tree(root, root_dir, _commit_id):
548 """
549 Special recursive svn repo walker
550 """
551
552 filemode_default = 0o100644
553 filemode_executable = 0o100755
554
555 file_iter = svn.fs.dir_entries(root, root_dir)
556 for f_name in file_iter:
557 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
558
559 if f_type == 'dir':
560 # return only DIR, and then all entries in that dir
561 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
562 new_root = os.path.join(root_dir, f_name)
563 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
564 yield _f_name, _f_data, _f_type
565 else:
566 f_path = os.path.join(root_dir, f_name).rstrip('/')
567 prop_list = svn.fs.node_proplist(root, f_path)
568
569 f_mode = filemode_default
570 if prop_list.get('svn:executable'):
571 f_mode = filemode_executable
572
573 f_is_link = False
574 if prop_list.get('svn:special'):
575 f_is_link = True
576
577 data = {
578 'is_link': f_is_link,
579 'mode': f_mode,
580 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
581 }
582
583 yield f_path, data, f_type
584
585 def file_walker(_commit_id, path):
586 repo = self._factory.repo(wire)
587 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
588
589 def no_content():
590 raise NoContentException()
591
592 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
593 file_path = f_name
594
595 if f_type == 'dir':
596 mode = f_data['mode']
597 yield ArchiveNode(file_path, mode, False, no_content)
598 else:
599 mode = f_data['mode']
600 is_link = f_data['is_link']
601 data_stream = f_data['content_stream']
602 yield ArchiveNode(file_path, mode, is_link, data_stream)
603
604 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
605 archive_dir_name, commit_id)
606
531 607
532 608 class SvnDiffer(object):
533 609 """
@@ -16,13 +16,12 b''
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import socket
19
20 19 import pytest
21 20
22 21
23 22 def pytest_addoption(parser):
24 23 parser.addoption(
25 '--repeat', type=int, default=100,
24 '--perf-repeat-vcs', type=int, default=100,
26 25 help="Number of repetitions in performance tests.")
27 26
28 27
@@ -34,7 +33,7 b' def repeat(request):'
34 33 Slower calls may divide it by 10 or 100. It is chosen in a way so that the
35 34 tests are not too slow in our default test suite.
36 35 """
37 return request.config.getoption('--repeat')
36 return request.config.getoption('--perf-repeat-vcs')
38 37
39 38
40 39 @pytest.fixture(scope='session')
@@ -113,7 +113,7 b' class TestReraiseSafeExceptions(object):'
113 113
114 114 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
115 115 for method_name, method in methods:
116 if not method_name.startswith('_'):
116 if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']:
117 117 assert method.im_func.__code__ == decorator.__code__
118 118
119 119 @pytest.mark.parametrize('side_effect, expected_type', [
@@ -50,7 +50,7 b' class TestReraiseSafeExceptions(object):'
50 50 methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
51 51 decorator = hg.reraise_safe_exceptions(None)
52 52 for method_name, method in methods:
53 if not method_name.startswith('_'):
53 if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']:
54 54 assert method.im_func.__code__ == decorator.__code__
55 55
56 56 @pytest.mark.parametrize('side_effect, expected_type', [
@@ -14,7 +14,10 b' def vcs_app():'
14 14 'dev.use_echo_app': 'true',
15 15 'locale': 'en_US.UTF-8',
16 16 }
17 vcs_app = main({}, **stub_settings)
17 stub_global_conf = {
18 '__file__': ''
19 }
20 vcs_app = main(stub_global_conf, **stub_settings)
18 21 app = webtest.TestApp(vcs_app)
19 22 return app
20 23
@@ -25,7 +25,7 b' from vcsserver.base import obfuscate_qs'
25 25 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
26 26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
27 27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
28 http_main.main({})
28 http_main.main({'__file__': ''})
29 29 patch_largefiles_capabilities.assert_called_once_with()
30 30
31 31
@@ -35,7 +35,7 b' def test_applies_largefiles_patch(patch_'
35 35 'vcsserver.hgpatches.patch_largefiles_capabilities',
36 36 mock.Mock(side_effect=Exception("Must not be called")))
37 37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
38 http_main.main({})
38 http_main.main({'__file__': ''})
39 39
40 40
41 41 @pytest.mark.parametrize('given, expected', [
@@ -25,15 +25,23 b' from vcsserver.utils import safe_str'
25 25 log = logging.getLogger(__name__)
26 26
27 27
28 def get_access_path(request):
29 environ = request.environ
30 return environ.get('PATH_INFO')
28 def get_access_path(environ):
29 path = environ.get('PATH_INFO')
30 return path
31 31
32 32
33 33 def get_user_agent(environ):
34 34 return environ.get('HTTP_USER_AGENT')
35 35
36 36
37 def get_vcs_method(environ):
38 return environ.get('HTTP_X_RC_METHOD')
39
40
41 def get_vcs_repo(environ):
42 return environ.get('HTTP_X_RC_REPO_NAME')
43
44
37 45 class RequestWrapperTween(object):
38 46 def __init__(self, handler, registry):
39 47 self.handler = handler
@@ -43,17 +51,52 b' class RequestWrapperTween(object):'
43 51
44 52 def __call__(self, request):
45 53 start = time.time()
54 log.debug('Starting request time measurement')
55 response = None
56
57 ua = get_user_agent(request.environ)
58 vcs_method = get_vcs_method(request.environ)
59 repo_name = get_vcs_repo(request.environ)
60
46 61 try:
47 62 response = self.handler(request)
48 63 finally:
49 end = time.time()
50 total = end - start
51 64 count = request.request_count()
52 65 _ver_ = vcsserver.__version__
66 _path = safe_str(get_access_path(request.environ))
67 ip = '127.0.0.1'
68 match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
69 resp_code = getattr(response, 'status_code', 'UNDEFINED')
70
71 total = time.time() - start
72
73 _view_path = "{}/{}@{}".format(_path, vcs_method, repo_name)
53 74 log.info(
54 75 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
55 count, '127.0.0.1', request.environ.get('REQUEST_METHOD'),
56 safe_str(get_access_path(request)), total, get_user_agent(request.environ), _ver_)
76 count, ip, request.environ.get('REQUEST_METHOD'),
77 _view_path, total, ua, _ver_,
78 extra={"time": total, "ver": _ver_, "code": resp_code,
79 "path": _path, "view_name": match_route, "user_agent": ua,
80 "vcs_method": vcs_method, "repo_name": repo_name}
81 )
82
83 statsd = request.registry.statsd
84 if statsd:
85 match_route = request.matched_route.name if request.matched_route else _path
86 elapsed_time_ms = round(1000.0 * total) # use ms only
87 statsd.timing(
88 "vcsserver_req_timing.histogram", elapsed_time_ms,
89 tags=[
90 "view_name:{}".format(match_route),
91 "code:{}".format(resp_code)
92 ],
93 use_decimals=False
94 )
95 statsd.incr(
96 "vcsserver_req_total", tags=[
97 "view_name:{}".format(match_route),
98 "code:{}".format(resp_code)
99 ])
57 100
58 101 return response
59 102
@@ -15,13 +15,15 b''
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 from vcsserver.lib import rc_cache
18 19
19 20 class RemoteBase(object):
20 21 EMPTY_COMMIT = '0' * 40
21 22
22 @property
23 def region(self):
24 return self._factory._cache_region
23 def _region(self, wire):
24 cache_repo_id = wire.get('cache_repo_id', '')
25 cache_namespace_uid = 'cache_repo.{}'.format(cache_repo_id)
26 return rc_cache.get_or_create_region('repo_object', cache_namespace_uid)
25 27
26 28 def _cache_on(self, wire):
27 29 context = wire.get('context', '')
@@ -30,3 +32,15 b' class RemoteBase(object):'
30 32 cache = wire.get('cache', True)
31 33 cache_on = context and cache
32 34 return cache_on, context_uid, repo_id
35
36 def vcsserver_invalidate_cache(self, wire, delete):
37 from vcsserver.lib import rc_cache
38 repo_id = wire.get('repo_id', '')
39 cache_repo_id = wire.get('cache_repo_id', '')
40 cache_namespace_uid = 'cache_repo.{}'.format(cache_repo_id)
41
42 if delete:
43 rc_cache.clear_cache_namespace(
44 'repo_object', cache_namespace_uid, invalidate=True)
45
46 return {'invalidated': {'repo_id': repo_id, 'delete': delete}}
General Comments 0
You need to be logged in to leave comments. Login now