##// END OF EJS Templates
feat(ssh-wrapper-speedup): major rewrite of code to address imports problem with ssh-wrapper-v2...
super-admin -
r5325:359b5cac default
parent child Browse files
Show More
@@ -0,0 +1,198 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import os
20 import tempfile
21 import logging
22
23 from pyramid.settings import asbool
24
25 from rhodecode.config.settings_maker import SettingsMaker
26 from rhodecode.config import utils as config_utils
27
28 log = logging.getLogger(__name__)
29
30
31 def sanitize_settings_and_apply_defaults(global_config, settings):
32 """
33 Applies settings defaults and does all type conversion.
34
35 We would move all settings parsing and preparation into this place, so that
36 we have only one place left which deals with this part. The remaining parts
37 of the application would start to rely fully on well-prepared settings.
38
39 This piece would later be split up per topic to avoid a big fat monster
40 function.
41 """
42 jn = os.path.join
43
44 global_settings_maker = SettingsMaker(global_config)
45 global_settings_maker.make_setting('debug', default=False, parser='bool')
46 debug_enabled = asbool(global_config.get('debug'))
47
48 settings_maker = SettingsMaker(settings)
49
50 settings_maker.make_setting(
51 'logging.autoconfigure',
52 default=False,
53 parser='bool')
54
55 logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
56 settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
57
58 # Default includes, possible to change as a user
59 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
60 log.debug(
61 "Using the following pyramid.includes: %s",
62 pyramid_includes)
63
64 settings_maker.make_setting('rhodecode.edition', 'Community Edition')
65 settings_maker.make_setting('rhodecode.edition_id', 'CE')
66
67 if 'mako.default_filters' not in settings:
68 # set custom default filters if we don't have it defined
69 settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
70 settings['mako.default_filters'] = 'h_filter'
71
72 if 'mako.directories' not in settings:
73 mako_directories = settings.setdefault('mako.directories', [
74 # Base templates of the original application
75 'rhodecode:templates',
76 ])
77 log.debug(
78 "Using the following Mako template directories: %s",
79 mako_directories)
80
81 # NOTE(marcink): fix redis requirement for schema of connection since 3.X
82 if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
83 raw_url = settings['beaker.session.url']
84 if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
85 settings['beaker.session.url'] = 'redis://' + raw_url
86
87 settings_maker.make_setting('__file__', global_config.get('__file__'))
88
89 # TODO: johbo: Re-think this, usually the call to config.include
90 # should allow to pass in a prefix.
91 settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
92
93 # Sanitize generic settings.
94 settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
95 settings_maker.make_setting('is_test', False, parser='bool')
96 settings_maker.make_setting('gzip_responses', False, parser='bool')
97
98 # statsd
99 settings_maker.make_setting('statsd.enabled', False, parser='bool')
100 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
101 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
102 settings_maker.make_setting('statsd.statsd_prefix', '')
103 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
104
105 settings_maker.make_setting('vcs.svn.compatible_version', '')
106 settings_maker.make_setting('vcs.hooks.protocol', 'http')
107 settings_maker.make_setting('vcs.hooks.host', '*')
108 settings_maker.make_setting('vcs.scm_app_implementation', 'http')
109 settings_maker.make_setting('vcs.server', '')
110 settings_maker.make_setting('vcs.server.protocol', 'http')
111 settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
112 settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
113 settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
114 settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
115 settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
116 settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
117
118 settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
119
120 # Support legacy values of vcs.scm_app_implementation. Legacy
121 # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
122 # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
123 scm_app_impl = settings['vcs.scm_app_implementation']
124 if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
125 settings['vcs.scm_app_implementation'] = 'http'
126
127 settings_maker.make_setting('appenlight', False, parser='bool')
128
129 temp_store = tempfile.gettempdir()
130 tmp_cache_dir = jn(temp_store, 'rc_cache')
131
132 # save default, cache dir, and use it for all backends later.
133 default_cache_dir = settings_maker.make_setting(
134 'cache_dir',
135 default=tmp_cache_dir, default_when_empty=True,
136 parser='dir:ensured')
137
138 # exception store cache
139 settings_maker.make_setting(
140 'exception_tracker.store_path',
141 default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
142 parser='dir:ensured'
143 )
144
145 settings_maker.make_setting(
146 'celerybeat-schedule.path',
147 default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
148 parser='file:ensured'
149 )
150
151 settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
152 settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
153
154 # sessions, ensure file since no-value is memory
155 settings_maker.make_setting('beaker.session.type', 'file')
156 settings_maker.make_setting('beaker.session.data_dir', jn(default_cache_dir, 'session_data'))
157
158 # cache_general
159 settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
160 settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
161 settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
162
163 # cache_perms
164 settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
165 settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
166 settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
167
168 # cache_repo
169 settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
170 settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
171 settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
172
173 # cache_license
174 settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
175 settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
176 settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
177
178 # cache_repo_longterm memory, 96H
179 settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
180 settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
181 settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
182
183 # sql_cache_short
184 settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
185 settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
186 settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
187
188 # archive_cache
189 settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
190 settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
191 settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
192
193 settings_maker.env_expand()
194
195 # configure instance id
196 config_utils.set_instance_id(settings)
197
198 return settings
@@ -0,0 +1,38 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import urllib.parse
20
21 from rhodecode.lib.vcs import CurlSession
22 from rhodecode.lib.ext_json import json
23
24
25 def call_service_api(service_api_host, service_api_token, api_url, payload):
26
27 payload.update({
28 'id': 'service',
29 'auth_token': service_api_token
30 })
31
32 service_api_url = urllib.parse.urljoin(service_api_host, api_url)
33 response = CurlSession().post(service_api_url, json.dumps(payload))
34
35 if response.status_code != 200:
36 raise Exception(f"Service API at {service_api_url} responded with error: {response.status_code}")
37
38 return json.loads(response.content)['result']
@@ -0,0 +1,40 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 import os
19
20
21 def get_config(ini_path, **kwargs):
22 import configparser
23 parser = configparser.ConfigParser(**kwargs)
24 parser.read(ini_path)
25 return parser
26
27
28 def get_app_config_lightweight(ini_path):
29 parser = get_config(ini_path)
30 parser.set('app:main', 'here', os.getcwd())
31 parser.set('app:main', '__file__', ini_path)
32 return dict(parser.items('app:main'))
33
34
35 def get_app_config(ini_path):
36 """
37 This loads the app context and provides a heavy type iniliaziation of config
38 """
39 from paste.deploy.loadwsgi import appconfig
40 return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
@@ -0,0 +1,17 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
@@ -0,0 +1,115 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 import os
19 import time
20 import logging
21 import tempfile
22
23 from rhodecode.lib.config_utils import get_config
24 from rhodecode.lib.ext_json import json
25
26 log = logging.getLogger(__name__)
27
28
29 class BaseHooksCallbackDaemon:
30 """
31 Basic context manager for actions that don't require some extra
32 """
33 def __init__(self):
34 pass
35
36 def __enter__(self):
37 log.debug('Running `%s` callback daemon', self.__class__.__name__)
38 return self
39
40 def __exit__(self, exc_type, exc_val, exc_tb):
41 log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
42
43
44 class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
45
46 def __init__(self, module):
47 super().__init__()
48 self.hooks_module = module
49
50
51 def get_txn_id_data_path(txn_id):
52 import rhodecode
53
54 root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
55 final_dir = os.path.join(root, 'svn_txn_id')
56
57 if not os.path.isdir(final_dir):
58 os.makedirs(final_dir)
59 return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
60
61
62 def store_txn_id_data(txn_id, data_dict):
63 if not txn_id:
64 log.warning('Cannot store txn_id because it is empty')
65 return
66
67 path = get_txn_id_data_path(txn_id)
68 try:
69 with open(path, 'wb') as f:
70 f.write(json.dumps(data_dict))
71 except Exception:
72 log.exception('Failed to write txn_id metadata')
73
74
75 def get_txn_id_from_store(txn_id):
76 """
77 Reads txn_id from store and if present returns the data for callback manager
78 """
79 path = get_txn_id_data_path(txn_id)
80 try:
81 with open(path, 'rb') as f:
82 return json.loads(f.read())
83 except Exception:
84 return {}
85
86
87 def prepare_callback_daemon(extras, protocol, host, txn_id=None):
88 txn_details = get_txn_id_from_store(txn_id)
89 port = txn_details.get('port', 0)
90 match protocol:
91 case 'http':
92 from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
93 callback_daemon = HttpHooksCallbackDaemon(
94 txn_id=txn_id, host=host, port=port)
95 case 'celery':
96 from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
97 callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
98 case 'local':
99 from rhodecode.lib.hook_daemon.hook_module import Hooks
100 callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
101 case _:
102 log.error('Unsupported callback daemon protocol "%s"', protocol)
103 raise Exception('Unsupported callback daemon protocol.')
104
105 extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
106 extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
107 extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
108 extras['hooks_protocol'] = protocol
109 extras['time'] = time.time()
110
111 # register txn_id
112 extras['txn_id'] = txn_id
113 log.debug('Prepared a callback daemon: %s',
114 callback_daemon.__class__.__name__)
115 return callback_daemon, extras
@@ -0,0 +1,30 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
20
21
22 class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
23 """
24 Context manger for achieving a compatibility with celery backend
25 """
26
27 def __init__(self, config):
28 # TODO: replace this with settings bootstrapped...
29 self.task_queue = config.get('app:main', 'celery.broker_url')
30 self.task_backend = config.get('app:main', 'celery.result_backend')
@@ -0,0 +1,104 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import logging
20 import traceback
21
22 from rhodecode.model import meta
23
24 from rhodecode.lib import hooks_base
25 from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
26 from rhodecode.lib.utils2 import AttributeDict
27
28 log = logging.getLogger(__name__)
29
30
31 class Hooks(object):
32 """
33 Exposes the hooks for remote callbacks
34 """
35 def __init__(self, request=None, log_prefix=''):
36 self.log_prefix = log_prefix
37 self.request = request
38
39 def repo_size(self, extras):
40 log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
41 return self._call_hook(hooks_base.repo_size, extras)
42
43 def pre_pull(self, extras):
44 log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
45 return self._call_hook(hooks_base.pre_pull, extras)
46
47 def post_pull(self, extras):
48 log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
49 return self._call_hook(hooks_base.post_pull, extras)
50
51 def pre_push(self, extras):
52 log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
53 return self._call_hook(hooks_base.pre_push, extras)
54
55 def post_push(self, extras):
56 log.debug("%sCalled post_push of %s object", self.log_prefix, self)
57 return self._call_hook(hooks_base.post_push, extras)
58
59 def _call_hook(self, hook, extras):
60 extras = AttributeDict(extras)
61 _server_url = extras['server_url']
62
63 extras.request = self.request
64
65 try:
66 result = hook(extras)
67 if result is None:
68 raise Exception(f'Failed to obtain hook result from func: {hook}')
69 except HTTPBranchProtected as handled_error:
70 # Those special cases don't need error reporting. It's a case of
71 # locked repo or protected branch
72 result = AttributeDict({
73 'status': handled_error.code,
74 'output': handled_error.explanation
75 })
76 except (HTTPLockedRC, Exception) as error:
77 # locked needs different handling since we need to also
78 # handle PULL operations
79 exc_tb = ''
80 if not isinstance(error, HTTPLockedRC):
81 exc_tb = traceback.format_exc()
82 log.exception('%sException when handling hook %s', self.log_prefix, hook)
83 error_args = error.args
84 return {
85 'status': 128,
86 'output': '',
87 'exception': type(error).__name__,
88 'exception_traceback': exc_tb,
89 'exception_args': error_args,
90 }
91 finally:
92 meta.Session.remove()
93
94 log.debug('%sGot hook call response %s', self.log_prefix, result)
95 return {
96 'status': result.status,
97 'output': result.output,
98 }
99
100 def __enter__(self):
101 return self
102
103 def __exit__(self, exc_type, exc_val, exc_tb):
104 pass
@@ -0,0 +1,280 b''
1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 #
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
6 #
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
11 #
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
19 import os
20 import logging
21 import traceback
22 import threading
23 import socket
24 import msgpack
25 import gevent
26
27 from http.server import BaseHTTPRequestHandler
28 from socketserver import TCPServer
29
30 from rhodecode.model import meta
31 from rhodecode.lib.ext_json import json
32 from rhodecode.lib import rc_cache
33 from rhodecode.lib.hook_daemon.base import get_txn_id_data_path
34 from rhodecode.lib.hook_daemon.hook_module import Hooks
35
36 log = logging.getLogger(__name__)
37
38
39 class HooksHttpHandler(BaseHTTPRequestHandler):
40
41 JSON_HOOKS_PROTO = 'json.v1'
42 MSGPACK_HOOKS_PROTO = 'msgpack.v1'
43 # starting with RhodeCode 5.0.0 MsgPack is the default, prior it used json
44 DEFAULT_HOOKS_PROTO = MSGPACK_HOOKS_PROTO
45
46 @classmethod
47 def serialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
48 if proto == cls.MSGPACK_HOOKS_PROTO:
49 return msgpack.packb(data)
50 return json.dumps(data)
51
52 @classmethod
53 def deserialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
54 if proto == cls.MSGPACK_HOOKS_PROTO:
55 return msgpack.unpackb(data)
56 return json.loads(data)
57
58 def do_POST(self):
59 hooks_proto, method, extras = self._read_request()
60 log.debug('Handling HooksHttpHandler %s with %s proto', method, hooks_proto)
61
62 txn_id = getattr(self.server, 'txn_id', None)
63 if txn_id:
64 log.debug('Computing TXN_ID based on `%s`:`%s`',
65 extras['repository'], extras['txn_id'])
66 computed_txn_id = rc_cache.utils.compute_key_from_params(
67 extras['repository'], extras['txn_id'])
68 if txn_id != computed_txn_id:
69 raise Exception(
70 'TXN ID fail: expected {} got {} instead'.format(
71 txn_id, computed_txn_id))
72
73 request = getattr(self.server, 'request', None)
74 try:
75 hooks = Hooks(request=request, log_prefix='HOOKS: {} '.format(self.server.server_address))
76 result = self._call_hook_method(hooks, method, extras)
77
78 except Exception as e:
79 exc_tb = traceback.format_exc()
80 result = {
81 'exception': e.__class__.__name__,
82 'exception_traceback': exc_tb,
83 'exception_args': e.args
84 }
85 self._write_response(hooks_proto, result)
86
87 def _read_request(self):
88 length = int(self.headers['Content-Length'])
89 # respect sent headers, fallback to OLD proto for compatability
90 hooks_proto = self.headers.get('rc-hooks-protocol') or self.JSON_HOOKS_PROTO
91 if hooks_proto == self.MSGPACK_HOOKS_PROTO:
92 # support for new vcsserver msgpack based protocol hooks
93 body = self.rfile.read(length)
94 data = self.deserialize_data(body)
95 else:
96 body = self.rfile.read(length)
97 data = self.deserialize_data(body)
98
99 return hooks_proto, data['method'], data['extras']
100
101 def _write_response(self, hooks_proto, result):
102 self.send_response(200)
103 if hooks_proto == self.MSGPACK_HOOKS_PROTO:
104 self.send_header("Content-type", "application/msgpack")
105 self.end_headers()
106 data = self.serialize_data(result)
107 self.wfile.write(data)
108 else:
109 self.send_header("Content-type", "text/json")
110 self.end_headers()
111 data = self.serialize_data(result)
112 self.wfile.write(data)
113
114 def _call_hook_method(self, hooks, method, extras):
115 try:
116 result = getattr(hooks, method)(extras)
117 finally:
118 meta.Session.remove()
119 return result
120
121 def log_message(self, format, *args):
122 """
123 This is an overridden method of BaseHTTPRequestHandler which logs using
124 a logging library instead of writing directly to stderr.
125 """
126
127 message = format % args
128
129 log.debug(
130 "HOOKS: client=%s - - [%s] %s", self.client_address,
131 self.log_date_time_string(), message)
132
133
134 class ThreadedHookCallbackDaemon(object):
135
136 _callback_thread = None
137 _daemon = None
138 _done = False
139 use_gevent = False
140
141 def __init__(self, txn_id=None, host=None, port=None):
142 self._prepare(txn_id=txn_id, host=host, port=port)
143 if self.use_gevent:
144 self._run_func = self._run_gevent
145 self._stop_func = self._stop_gevent
146 else:
147 self._run_func = self._run
148 self._stop_func = self._stop
149
150 def __enter__(self):
151 log.debug('Running `%s` callback daemon', self.__class__.__name__)
152 self._run_func()
153 return self
154
155 def __exit__(self, exc_type, exc_val, exc_tb):
156 log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
157 self._stop_func()
158
159 def _prepare(self, txn_id=None, host=None, port=None):
160 raise NotImplementedError()
161
162 def _run(self):
163 raise NotImplementedError()
164
165 def _stop(self):
166 raise NotImplementedError()
167
168 def _run_gevent(self):
169 raise NotImplementedError()
170
171 def _stop_gevent(self):
172 raise NotImplementedError()
173
174
175 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
176 """
177 Context manager which will run a callback daemon in a background thread.
178 """
179
180 hooks_uri = None
181
182 # From Python docs: Polling reduces our responsiveness to a shutdown
183 # request and wastes cpu at all other times.
184 POLL_INTERVAL = 0.01
185
186 use_gevent = False
187
188 @property
189 def _hook_prefix(self):
190 return 'HOOKS: {} '.format(self.hooks_uri)
191
192 def get_hostname(self):
193 return socket.gethostname() or '127.0.0.1'
194
195 def get_available_port(self, min_port=20000, max_port=65535):
196 from rhodecode.lib.utils2 import get_available_port as _get_port
197 return _get_port(min_port, max_port)
198
199 def _prepare(self, txn_id=None, host=None, port=None):
200 from pyramid.threadlocal import get_current_request
201
202 if not host or host == "*":
203 host = self.get_hostname()
204 if not port:
205 port = self.get_available_port()
206
207 server_address = (host, port)
208 self.hooks_uri = '{}:{}'.format(host, port)
209 self.txn_id = txn_id
210 self._done = False
211
212 log.debug(
213 "%s Preparing HTTP callback daemon registering hook object: %s",
214 self._hook_prefix, HooksHttpHandler)
215
216 self._daemon = TCPServer(server_address, HooksHttpHandler)
217 # inject transaction_id for later verification
218 self._daemon.txn_id = self.txn_id
219
220 # pass the WEB app request into daemon
221 self._daemon.request = get_current_request()
222
223 def _run(self):
224 log.debug("Running thread-based loop of callback daemon in background")
225 callback_thread = threading.Thread(
226 target=self._daemon.serve_forever,
227 kwargs={'poll_interval': self.POLL_INTERVAL})
228 callback_thread.daemon = True
229 callback_thread.start()
230 self._callback_thread = callback_thread
231
232 def _run_gevent(self):
233 log.debug("Running gevent-based loop of callback daemon in background")
234 # create a new greenlet for the daemon's serve_forever method
235 callback_greenlet = gevent.spawn(
236 self._daemon.serve_forever,
237 poll_interval=self.POLL_INTERVAL)
238
239 # store reference to greenlet
240 self._callback_greenlet = callback_greenlet
241
242 # switch to this greenlet
243 gevent.sleep(0.01)
244
245 def _stop(self):
246 log.debug("Waiting for background thread to finish.")
247 self._daemon.shutdown()
248 self._callback_thread.join()
249 self._daemon = None
250 self._callback_thread = None
251 if self.txn_id:
252 txn_id_file = get_txn_id_data_path(self.txn_id)
253 log.debug('Cleaning up TXN ID %s', txn_id_file)
254 if os.path.isfile(txn_id_file):
255 os.remove(txn_id_file)
256
257 log.debug("Background thread done.")
258
259 def _stop_gevent(self):
260 log.debug("Waiting for background greenlet to finish.")
261
262 # if greenlet exists and is running
263 if self._callback_greenlet and not self._callback_greenlet.dead:
264 # shutdown daemon if it exists
265 if self._daemon:
266 self._daemon.shutdown()
267
268 # kill the greenlet
269 self._callback_greenlet.kill()
270
271 self._daemon = None
272 self._callback_greenlet = None
273
274 if self.txn_id:
275 txn_id_file = get_txn_id_data_path(self.txn_id)
276 log.debug('Cleaning up TXN ID %s', txn_id_file)
277 if os.path.isfile(txn_id_file):
278 os.remove(txn_id_file)
279
280 log.debug("Background greenlet done.")
@@ -20,12 +20,11 b' import os'
20 20 import re
21 21 import logging
22 22 import datetime
23 import configparser
24 23 from sqlalchemy import Table
25 24
26 from rhodecode.lib.utils import call_service_api
25 from rhodecode.lib.api_utils import call_service_api
27 26 from rhodecode.lib.utils2 import AttributeDict
28 from rhodecode.model.scm import ScmModel
27 from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
29 28
30 29 from .hg import MercurialServer
31 30 from .git import GitServer
@@ -39,7 +38,7 b' class SshWrapper(object):'
39 38 svn_cmd_pat = re.compile(r'^svnserve -t')
40 39
41 40 def __init__(self, command, connection_info, mode,
42 user, user_id, key_id: int, shell, ini_path: str, env):
41 user, user_id, key_id: int, shell, ini_path: str, settings, env):
43 42 self.command = command
44 43 self.connection_info = connection_info
45 44 self.mode = mode
@@ -49,15 +48,9 b' class SshWrapper(object):'
49 48 self.shell = shell
50 49 self.ini_path = ini_path
51 50 self.env = env
52
53 self.config = self.parse_config(ini_path)
51 self.settings = settings
54 52 self.server_impl = None
55 53
56 def parse_config(self, config_path):
57 parser = configparser.ConfigParser()
58 parser.read(config_path)
59 return parser
60
61 54 def update_key_access_time(self, key_id):
62 55 from rhodecode.model.meta import raw_query_executor, Base
63 56
@@ -162,6 +155,9 b' class SshWrapper(object):'
162 155 return vcs_type, repo_name, mode
163 156
164 157 def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
158 # TODO: remove this once we have .ini defined access path...
159 from rhodecode.model.scm import ScmModel
160
165 161 store = ScmModel().repos_path
166 162
167 163 check_branch_perms = False
@@ -186,7 +182,7 b' class SshWrapper(object):'
186 182 server = MercurialServer(
187 183 store=store, ini_path=self.ini_path,
188 184 repo_name=repo, user=user,
189 user_permissions=permissions, config=self.config, env=self.env)
185 user_permissions=permissions, settings=self.settings, env=self.env)
190 186 self.server_impl = server
191 187 return server.run(tunnel_extras=extras)
192 188
@@ -194,7 +190,7 b' class SshWrapper(object):'
194 190 server = GitServer(
195 191 store=store, ini_path=self.ini_path,
196 192 repo_name=repo, repo_mode=mode, user=user,
197 user_permissions=permissions, config=self.config, env=self.env)
193 user_permissions=permissions, settings=self.settings, env=self.env)
198 194 self.server_impl = server
199 195 return server.run(tunnel_extras=extras)
200 196
@@ -202,7 +198,7 b' class SshWrapper(object):'
202 198 server = SubversionServer(
203 199 store=store, ini_path=self.ini_path,
204 200 repo_name=None, user=user,
205 user_permissions=permissions, config=self.config, env=self.env)
201 user_permissions=permissions, settings=self.settings, env=self.env)
206 202 self.server_impl = server
207 203 return server.run(tunnel_extras=extras)
208 204
@@ -269,6 +265,35 b' class SshWrapperStandalone(SshWrapper):'
269 265 New version of SshWrapper designed to be depended only on service API
270 266 """
271 267 repos_path = None
268 service_api_host: str
269 service_api_token: str
270 api_url: str
271
272 def __init__(self, command, connection_info, mode,
273 user, user_id, key_id: int, shell, ini_path: str, settings, env):
274
275 # validate our settings for making a standalone calls
276 try:
277 self.service_api_host = settings['app.service_api.host']
278 self.service_api_token = settings['app.service_api.token']
279 except KeyError:
280 raise ImproperlyConfiguredError(
281 "app.service_api.host or app.service_api.token are missing. "
282 "Please ensure that app.service_api.host and app.service_api.token are "
283 "defined inside of .ini configuration file."
284 )
285
286 try:
287 self.api_url = settings['rhodecode.api.url']
288 except KeyError:
289 raise ImproperlyConfiguredError(
290 "rhodecode.api.url is missing. "
291 "Please ensure that rhodecode.api.url is "
292 "defined inside of .ini configuration file."
293 )
294
295 super(SshWrapperStandalone, self).__init__(
296 command, connection_info, mode, user, user_id, key_id, shell, ini_path, settings, env)
272 297
273 298 @staticmethod
274 299 def parse_user_related_data(user_data):
@@ -301,7 +326,7 b' class SshWrapperStandalone(SshWrapper):'
301 326 exit_code = 1
302 327
303 328 elif scm_detected:
304 data = call_service_api(self.ini_path, {
329 data = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
305 330 "method": "service_get_data_for_ssh_wrapper",
306 331 "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
307 332 })
@@ -339,7 +364,7 b' class SshWrapperStandalone(SshWrapper):'
339 364 if repo_name.startswith('_'):
340 365 org_repo_name = repo_name
341 366 log.debug('translating UID repo %s', org_repo_name)
342 by_id_match = call_service_api(self.ini_path, {
367 by_id_match = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
343 368 'method': 'service_get_repo_name_by_id',
344 369 "args": {"repo_id": repo_name}
345 370 })
@@ -375,17 +400,17 b' class SshWrapperStandalone(SshWrapper):'
375 400 server = MercurialServer(
376 401 store=store, ini_path=self.ini_path,
377 402 repo_name=repo, user=user,
378 user_permissions=permissions, config=self.config, env=self.env)
403 user_permissions=permissions, settings=self.settings, env=self.env)
379 404 case 'git':
380 405 server = GitServer(
381 406 store=store, ini_path=self.ini_path,
382 407 repo_name=repo, repo_mode=mode, user=user,
383 user_permissions=permissions, config=self.config, env=self.env)
408 user_permissions=permissions, settings=self.settings, env=self.env)
384 409 case 'svn':
385 410 server = SubversionServer(
386 411 store=store, ini_path=self.ini_path,
387 412 repo_name=None, user=user,
388 user_permissions=permissions, config=self.config, env=self.env)
413 user_permissions=permissions, settings=self.settings, env=self.env)
389 414 case _:
390 415 raise Exception(f'Unrecognised VCS: {vcs}')
391 416 self.server_impl = server
@@ -20,27 +20,27 b' import os'
20 20 import sys
21 21 import logging
22 22
23 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
23 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
24 24 from rhodecode.lib.ext_json import sjson as json
25 25 from rhodecode.lib.vcs.conf import settings as vcs_settings
26 from rhodecode.lib.utils import call_service_api
27 from rhodecode.model.scm import ScmModel
26 from rhodecode.lib.api_utils import call_service_api
28 27
29 28 log = logging.getLogger(__name__)
30 29
31 30
32 class VcsServer(object):
31 class SSHVcsServer(object):
33 32 repo_user_agent = None # set in child classes
34 33 _path = None # set executable path for hg/git/svn binary
35 34 backend = None # set in child classes
36 35 tunnel = None # subprocess handling tunnel
36 settings = None # parsed settings module
37 37 write_perms = ['repository.admin', 'repository.write']
38 38 read_perms = ['repository.read', 'repository.admin', 'repository.write']
39 39
40 def __init__(self, user, user_permissions, config, env):
40 def __init__(self, user, user_permissions, settings, env):
41 41 self.user = user
42 42 self.user_permissions = user_permissions
43 self.config = config
43 self.settings = settings
44 44 self.env = env
45 45 self.stdin = sys.stdin
46 46
@@ -59,9 +59,14 b' class VcsServer(object):'
59 59 # Todo: Leave only "celery" case after transition.
60 60 match self.hooks_protocol:
61 61 case 'http':
62 from rhodecode.model.scm import ScmModel
62 63 ScmModel().mark_for_invalidation(repo_name)
63 64 case 'celery':
64 call_service_api(self.ini_path, {
65 service_api_host = self.settings['app.service_api.host']
66 service_api_token = self.settings['app.service_api.token']
67 api_url = self.settings['rhodecode.api.url']
68
69 call_service_api(service_api_host, service_api_token, api_url, {
65 70 "method": "service_mark_for_invalidation",
66 71 "args": {"repo_name": repo_name}
67 72 })
@@ -118,7 +123,7 b' class VcsServer(object):'
118 123 'server_url': None,
119 124 'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
120 125 'hooks': ['push', 'pull'],
121 'hooks_module': 'rhodecode.lib.hooks_daemon',
126 'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
122 127 'is_shadow_repo': False,
123 128 'detect_force_push': False,
124 129 'check_branch_perms': False,
@@ -156,7 +161,7 b' class VcsServer(object):'
156 161 return exit_code, action == "push"
157 162
158 163 def run(self, tunnel_extras=None):
159 self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
164 self.hooks_protocol = self.settings['vcs.hooks.protocol']
160 165 tunnel_extras = tunnel_extras or {}
161 166 extras = {}
162 167 extras.update(tunnel_extras)
@@ -21,7 +21,7 b' import logging'
21 21 import subprocess
22 22
23 23 from vcsserver import hooks
24 from .base import VcsServer
24 from .base import SSHVcsServer
25 25
26 26 log = logging.getLogger(__name__)
27 27
@@ -70,19 +70,17 b' class GitTunnelWrapper(object):'
70 70 return result
71 71
72 72
73 class GitServer(VcsServer):
73 class GitServer(SSHVcsServer):
74 74 backend = 'git'
75 75 repo_user_agent = 'git'
76 76
77 def __init__(self, store, ini_path, repo_name, repo_mode,
78 user, user_permissions, config, env):
79 super().\
80 __init__(user, user_permissions, config, env)
77 def __init__(self, store, ini_path, repo_name, repo_mode, user, user_permissions, settings, env):
78 super().__init__(user, user_permissions, settings, env)
81 79
82 80 self.store = store
83 81 self.ini_path = ini_path
84 82 self.repo_name = repo_name
85 self._path = self.git_path = config.get('app:main', 'ssh.executable.git')
83 self._path = self.git_path = settings['ssh.executable.git']
86 84
87 85 self.repo_mode = repo_mode
88 86 self.tunnel = GitTunnelWrapper(server=self)
@@ -22,10 +22,10 b' import logging'
22 22 import tempfile
23 23 import textwrap
24 24 import collections
25 from .base import VcsServer
26 from rhodecode.lib.utils import call_service_api
27 from rhodecode.model.db import RhodeCodeUi
28 from rhodecode.model.settings import VcsSettingsModel
25
26 from .base import SSHVcsServer
27
28 from rhodecode.lib.api_utils import call_service_api
29 29
30 30 log = logging.getLogger(__name__)
31 31
@@ -57,7 +57,7 b' class MercurialTunnelWrapper(object):'
57 57 # cleanup custom hgrc file
58 58 if os.path.isfile(hgrc_custom):
59 59 with open(hgrc_custom, 'wb') as f:
60 f.write('')
60 f.write(b'')
61 61 log.debug('Cleanup custom hgrc file under %s', hgrc_custom)
62 62
63 63 # write temp
@@ -94,29 +94,34 b' class MercurialTunnelWrapper(object):'
94 94 self.remove_configs()
95 95
96 96
97 class MercurialServer(VcsServer):
97 class MercurialServer(SSHVcsServer):
98 98 backend = 'hg'
99 99 repo_user_agent = 'mercurial'
100 100 cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
101 101
102 def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):
103 super().__init__(user, user_permissions, config, env)
102 def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
103 super().__init__(user, user_permissions, settings, env)
104 104
105 105 self.store = store
106 106 self.ini_path = ini_path
107 107 self.repo_name = repo_name
108 self._path = self.hg_path = config.get('app:main', 'ssh.executable.hg')
108 self._path = self.hg_path = settings['ssh.executable.hg']
109 109 self.tunnel = MercurialTunnelWrapper(server=self)
110 110
111 111 def config_to_hgrc(self, repo_name):
112 112 # Todo: once transition is done only call to service api should exist
113 113 if self.hooks_protocol == 'celery':
114 data = call_service_api(self.ini_path, {
114 service_api_host = self.settings['app.service_api.host']
115 service_api_token = self.settings['app.service_api.token']
116 api_url = self.settings['rhodecode.api.url']
117 data = call_service_api(service_api_host, service_api_token, api_url, {
115 118 "method": "service_config_to_hgrc",
116 119 "args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
117 120 })
118 121 return data['flags']
119
122 else:
123 from rhodecode.model.db import RhodeCodeUi
124 from rhodecode.model.settings import VcsSettingsModel
120 125 ui_sections = collections.defaultdict(list)
121 126 ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
122 127
@@ -25,7 +25,7 b' import tempfile'
25 25 from subprocess import Popen, PIPE
26 26 import urllib.parse
27 27
28 from .base import VcsServer
28 from .base import SSHVcsServer
29 29
30 30 log = logging.getLogger(__name__)
31 31
@@ -218,20 +218,18 b' class SubversionTunnelWrapper(object):'
218 218 return self.return_code
219 219
220 220
221 class SubversionServer(VcsServer):
221 class SubversionServer(SSHVcsServer):
222 222 backend = 'svn'
223 223 repo_user_agent = 'svn'
224 224
225 def __init__(self, store, ini_path, repo_name,
226 user, user_permissions, config, env):
227 super()\
228 .__init__(user, user_permissions, config, env)
225 def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
226 super().__init__(user, user_permissions, settings, env)
229 227 self.store = store
230 228 self.ini_path = ini_path
231 229 # NOTE(dan): repo_name at this point is empty,
232 230 # this is set later in .run() based from parsed input stream
233 231 self.repo_name = repo_name
234 self._path = self.svn_path = config.get('app:main', 'ssh.executable.svn')
232 self._path = self.svn_path = settings['ssh.executable.svn']
235 233
236 234 self.tunnel = SubversionTunnelWrapper(server=self)
237 235
@@ -31,7 +31,6 b' from .utils import setup_custom_logging'
31 31 log = logging.getLogger(__name__)
32 32
33 33
34
35 34 @click.command()
36 35 @click.argument('ini_path', type=click.Path(exists=True))
37 36 @click.option(
@@ -55,11 +54,12 b' def main(ini_path, mode, user, user_id, '
55 54 connection_info = os.environ.get('SSH_CONNECTION', '')
56 55 time_start = time.time()
57 56 with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env:
57 settings = env['registry'].settings
58 58 statsd = StatsdClient.statsd
59 59 try:
60 60 ssh_wrapper = SshWrapper(
61 61 command, connection_info, mode,
62 user, user_id, key_id, shell, ini_path, env)
62 user, user_id, key_id, shell, ini_path, settings, env)
63 63 except Exception:
64 64 log.exception('Failed to execute SshWrapper')
65 65 sys.exit(-5)
@@ -16,6 +16,14 b''
16 16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 18
19 """
20 WARNING: be really carefully with changing ANY imports in this file
21 # This script is to mean as really fast executable, doing some imports here that would yield an import chain change
22 # can affect execution times...
23 # This can be easily debugged using such command::
24 # time PYTHONPROFILEIMPORTTIME=1 rc-ssh-wrapper-v2 --debug --mode=test .dev/dev.ini
25 """
26
19 27 import os
20 28 import sys
21 29 import time
@@ -23,9 +31,12 b' import logging'
23 31
24 32 import click
25 33
34 from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
26 35 from rhodecode.lib.statsd_client import StatsdClient
36 from rhodecode.lib.config_utils import get_app_config_lightweight
37
38 from .utils import setup_custom_logging
27 39 from .backends import SshWrapperStandalone
28 from .utils import setup_custom_logging
29 40
30 41 log = logging.getLogger(__name__)
31 42
@@ -42,6 +53,8 b' log = logging.getLogger(__name__)'
42 53 @click.option('--shell', '-s', is_flag=True, help='Allow Shell')
43 54 @click.option('--debug', is_flag=True, help='Enabled detailed output logging')
44 55 def main(ini_path, mode, user, user_id, key_id, shell, debug):
56
57 time_start = time.time()
45 58 setup_custom_logging(ini_path, debug)
46 59
47 60 command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
@@ -50,21 +63,30 b' def main(ini_path, mode, user, user_id, '
50 63 'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
51 64 'Please make sure this is set and available during execution '
52 65 'of this script.')
66
67 # initialize settings and get defaults
68 settings = get_app_config_lightweight(ini_path)
69 settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, settings)
70
71 # init and bootstrap StatsdClient
72 StatsdClient.setup(settings)
73 statsd = StatsdClient.statsd
74
75 try:
53 76 connection_info = os.environ.get('SSH_CONNECTION', '')
54 time_start = time.time()
55 77 env = {'RC_CMD_SSH_WRAPPER': '1'}
56 statsd = StatsdClient.statsd
57 try:
58 78 ssh_wrapper = SshWrapperStandalone(
59 79 command, connection_info, mode,
60 user, user_id, key_id, shell, ini_path, env)
80 user, user_id, key_id, shell, ini_path, settings, env)
61 81 except Exception:
62 82 log.exception('Failed to execute SshWrapper')
63 83 sys.exit(-5)
84
64 85 return_code = ssh_wrapper.wrap()
65 86 operation_took = time.time() - time_start
66 87 if statsd:
67 88 operation_took_ms = round(1000.0 * operation_took)
68 89 statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
69 90 use_decimals=False)
91
70 92 sys.exit(return_code)
@@ -17,11 +17,11 b''
17 17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 18
19 19 import logging
20 from pyramid.paster import setup_logging
21 20
22 21
23 22 def setup_custom_logging(ini_path, debug):
24 23 if debug:
24 from pyramid.paster import setup_logging # Lazy import
25 25 # enabled rhodecode.ini controlled logging setup
26 26 setup_logging(ini_path)
27 27 else:
@@ -52,7 +52,10 b' def dummy_env():'
52 52
53 53
54 54 def plain_dummy_user():
55 return AttributeDict(username='test_user')
55 return AttributeDict(
56 user_id=1,
57 username='test_user'
58 )
56 59
57 60
58 61 @pytest.fixture()
@@ -65,4 +68,4 b' def ssh_wrapper(app, dummy_conf_file, du'
65 68 conn_info = '127.0.0.1 22 10.0.0.1 443'
66 69 return SshWrapper(
67 70 'random command', conn_info, 'auto', 'admin', '1', key_id='1',
68 shell=False, ini_path=dummy_conf_file, env=dummy_env)
71 shell=False, ini_path=dummy_conf_file, settings={}, env=dummy_env)
@@ -25,6 +25,7 b' from rhodecode.apps.ssh_support.lib.back'
25 25 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
26 26 from rhodecode.lib.ext_json import json
27 27
28
28 29 class GitServerCreator(object):
29 30 root = '/tmp/repo/path/'
30 31 git_path = '/usr/local/bin/git'
@@ -39,10 +40,7 b' class GitServerCreator(object):'
39 40 user = plain_dummy_user()
40 41
41 42 def __init__(self):
42 def config_get(part, key):
43 return self.config_data.get(part, {}).get(key)
44 self.config_mock = mock.Mock()
45 self.config_mock.get = mock.Mock(side_effect=config_get)
43 pass
46 44
47 45 def create(self, **kwargs):
48 46 parameters = {
@@ -54,7 +52,7 b' class GitServerCreator(object):'
54 52 'user_permissions': {
55 53 self.repo_name: 'repository.admin'
56 54 },
57 'config': self.config_mock,
55 'settings': self.config_data['app:main'],
58 56 'env': plain_dummy_env()
59 57 }
60 58 parameters.update(kwargs)
@@ -142,7 +140,7 b' class TestGitServer(object):'
142 140 'server_url': None,
143 141 'hooks': ['push', 'pull'],
144 142 'is_shadow_repo': False,
145 'hooks_module': 'rhodecode.lib.hooks_daemon',
143 'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
146 144 'check_branch_perms': False,
147 145 'detect_force_push': False,
148 146 'user_agent': u'git/ssh-user-agent',
@@ -38,10 +38,7 b' class MercurialServerCreator(object):'
38 38 user = plain_dummy_user()
39 39
40 40 def __init__(self):
41 def config_get(part, key):
42 return self.config_data.get(part, {}).get(key)
43 self.config_mock = mock.Mock()
44 self.config_mock.get = mock.Mock(side_effect=config_get)
41 pass
45 42
46 43 def create(self, **kwargs):
47 44 parameters = {
@@ -52,7 +49,7 b' class MercurialServerCreator(object):'
52 49 'user_permissions': {
53 50 'test_hg': 'repository.admin'
54 51 },
55 'config': self.config_mock,
52 'settings': self.config_data['app:main'],
56 53 'env': plain_dummy_env()
57 54 }
58 55 parameters.update(kwargs)
@@ -36,10 +36,7 b' class SubversionServerCreator(object):'
36 36 user = plain_dummy_user()
37 37
38 38 def __init__(self):
39 def config_get(part, key):
40 return self.config_data.get(part, {}).get(key)
41 self.config_mock = mock.Mock()
42 self.config_mock.get = mock.Mock(side_effect=config_get)
39 pass
43 40
44 41 def create(self, **kwargs):
45 42 parameters = {
@@ -50,7 +47,7 b' class SubversionServerCreator(object):'
50 47 'user_permissions': {
51 48 self.repo_name: 'repository.admin'
52 49 },
53 'config': self.config_mock,
50 'settings': self.config_data['app:main'],
54 51 'env': plain_dummy_env()
55 52 }
56 53
@@ -28,10 +28,6 b' class TestSSHWrapper(object):'
28 28 permissions={}, branch_permissions={})
29 29 assert str(exc_info.value) == 'Unrecognised VCS: microsoft-tfs'
30 30
31 def test_parse_config(self, ssh_wrapper):
32 config = ssh_wrapper.parse_config(ssh_wrapper.ini_path)
33 assert config
34
35 31 def test_get_connection_info(self, ssh_wrapper):
36 32 conn_info = ssh_wrapper.get_connection_info()
37 33 assert {'client_ip': '127.0.0.1',
@@ -19,7 +19,7 b''
19 19 import os
20 20 import sys
21 21 import collections
22 import tempfile
22
23 23 import time
24 24 import logging.config
25 25
@@ -32,14 +32,13 b' from pyramid.httpexceptions import ('
32 32 HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
33 33 from pyramid.renderers import render_to_response
34 34
35 from rhodecode import api
36 35 from rhodecode.model import meta
37 36 from rhodecode.config import patches
38 from rhodecode.config import utils as config_utils
39 from rhodecode.config.settings_maker import SettingsMaker
37
40 38 from rhodecode.config.environment import load_pyramid_environment
41 39
42 40 import rhodecode.events
41 from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
43 42 from rhodecode.lib.middleware.vcs import VCSMiddleware
44 43 from rhodecode.lib.request import Request
45 44 from rhodecode.lib.vcs import VCSCommunicationError
@@ -465,173 +464,3 b' def wrap_app_in_wsgi_middlewares(pyramid'
465 464 log.debug('Request processing finalized: %.4fs', total)
466 465
467 466 return pyramid_app_with_cleanup
468
469
470 def sanitize_settings_and_apply_defaults(global_config, settings):
471 """
472 Applies settings defaults and does all type conversion.
473
474 We would move all settings parsing and preparation into this place, so that
475 we have only one place left which deals with this part. The remaining parts
476 of the application would start to rely fully on well prepared settings.
477
478 This piece would later be split up per topic to avoid a big fat monster
479 function.
480 """
481 jn = os.path.join
482
483 global_settings_maker = SettingsMaker(global_config)
484 global_settings_maker.make_setting('debug', default=False, parser='bool')
485 debug_enabled = asbool(global_config.get('debug'))
486
487 settings_maker = SettingsMaker(settings)
488
489 settings_maker.make_setting(
490 'logging.autoconfigure',
491 default=False,
492 parser='bool')
493
494 logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
495 settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
496
497 # Default includes, possible to change as a user
498 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
499 log.debug(
500 "Using the following pyramid.includes: %s",
501 pyramid_includes)
502
503 settings_maker.make_setting('rhodecode.edition', 'Community Edition')
504 settings_maker.make_setting('rhodecode.edition_id', 'CE')
505
506 if 'mako.default_filters' not in settings:
507 # set custom default filters if we don't have it defined
508 settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
509 settings['mako.default_filters'] = 'h_filter'
510
511 if 'mako.directories' not in settings:
512 mako_directories = settings.setdefault('mako.directories', [
513 # Base templates of the original application
514 'rhodecode:templates',
515 ])
516 log.debug(
517 "Using the following Mako template directories: %s",
518 mako_directories)
519
520 # NOTE(marcink): fix redis requirement for schema of connection since 3.X
521 if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
522 raw_url = settings['beaker.session.url']
523 if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
524 settings['beaker.session.url'] = 'redis://' + raw_url
525
526 settings_maker.make_setting('__file__', global_config.get('__file__'))
527
528 # TODO: johbo: Re-think this, usually the call to config.include
529 # should allow to pass in a prefix.
530 settings_maker.make_setting('rhodecode.api.url', api.DEFAULT_URL)
531
532 # Sanitize generic settings.
533 settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
534 settings_maker.make_setting('is_test', False, parser='bool')
535 settings_maker.make_setting('gzip_responses', False, parser='bool')
536
537 # statsd
538 settings_maker.make_setting('statsd.enabled', False, parser='bool')
539 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
540 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
541 settings_maker.make_setting('statsd.statsd_prefix', '')
542 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
543
544 settings_maker.make_setting('vcs.svn.compatible_version', '')
545 settings_maker.make_setting('vcs.hooks.protocol', 'http')
546 settings_maker.make_setting('vcs.hooks.host', '*')
547 settings_maker.make_setting('vcs.scm_app_implementation', 'http')
548 settings_maker.make_setting('vcs.server', '')
549 settings_maker.make_setting('vcs.server.protocol', 'http')
550 settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
551 settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
552 settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
553 settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
554 settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
555 settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
556
557 settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
558
559 # Support legacy values of vcs.scm_app_implementation. Legacy
560 # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
561 # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
562 scm_app_impl = settings['vcs.scm_app_implementation']
563 if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
564 settings['vcs.scm_app_implementation'] = 'http'
565
566 settings_maker.make_setting('appenlight', False, parser='bool')
567
568 temp_store = tempfile.gettempdir()
569 tmp_cache_dir = jn(temp_store, 'rc_cache')
570
571 # save default, cache dir, and use it for all backends later.
572 default_cache_dir = settings_maker.make_setting(
573 'cache_dir',
574 default=tmp_cache_dir, default_when_empty=True,
575 parser='dir:ensured')
576
577 # exception store cache
578 settings_maker.make_setting(
579 'exception_tracker.store_path',
580 default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
581 parser='dir:ensured'
582 )
583
584 settings_maker.make_setting(
585 'celerybeat-schedule.path',
586 default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
587 parser='file:ensured'
588 )
589
590 settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
591 settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
592
593 # sessions, ensure file since no-value is memory
594 settings_maker.make_setting('beaker.session.type', 'file')
595 settings_maker.make_setting('beaker.session.data_dir', jn(default_cache_dir, 'session_data'))
596
597 # cache_general
598 settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
599 settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
600 settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
601
602 # cache_perms
603 settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
604 settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
605 settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
606
607 # cache_repo
608 settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
609 settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
610 settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
611
612 # cache_license
613 settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
614 settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
615 settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
616
617 # cache_repo_longterm memory, 96H
618 settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
619 settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
620 settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
621
622 # sql_cache_short
623 settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
624 settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
625 settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
626
627 # archive_cache
628 settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
629 settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
630 settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
631
632 settings_maker.env_expand()
633
634 # configure instance id
635 config_utils.set_instance_id(settings)
636
637 return settings
@@ -19,8 +19,6 b''
19 19 import os
20 20 import platform
21 21
22 from rhodecode.model import init_model
23
24 22
25 23 def configure_vcs(config):
26 24 """
@@ -44,6 +42,7 b' def configure_vcs(config):'
44 42
45 43 def initialize_database(config):
46 44 from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
45 from rhodecode.model import init_model
47 46 engine = engine_from_config(config, 'sqlalchemy.db1.')
48 47 init_model(engine, encryption_key=get_encryption_key(config))
49 48
@@ -567,7 +567,7 b' def add_events_routes(config):'
567 567
568 568
569 569 def bootstrap_config(request, registry_name='RcTestRegistry'):
570 from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
570 from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
571 571 import pyramid.testing
572 572 registry = pyramid.testing.Registry(registry_name)
573 573
@@ -34,8 +34,7 b' from rhodecode.lib.utils import is_valid'
34 34 from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
35 35 from rhodecode.lib.type_utils import str2bool
36 36 from rhodecode.lib.ext_json import json
37 from rhodecode.lib.hooks_daemon import store_txn_id_data
38
37 from rhodecode.lib.hook_daemon.base import store_txn_id_data
39 38
40 39 log = logging.getLogger(__name__)
41 40
@@ -45,7 +45,7 b' from rhodecode.lib.auth import AuthUser,'
45 45 from rhodecode.lib.base import (
46 46 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
47 47 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
48 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
48 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
49 49 from rhodecode.lib.middleware import appenlight
50 50 from rhodecode.lib.middleware.utils import scm_app_http
51 51 from rhodecode.lib.str_utils import safe_bytes
@@ -20,24 +20,14 b''
20 20
21 21 import os
22 22 import configparser
23
24 from rhodecode.lib.config_utils import get_config
23 25 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging # pragma: no cover
24 26
25 from rhodecode.lib.request import Request
26
27
28 def get_config(ini_path, **kwargs):
29 parser = configparser.ConfigParser(**kwargs)
30 parser.read(ini_path)
31 return parser
32
33
34 def get_app_config(ini_path):
35 from paste.deploy.loadwsgi import appconfig
36 return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
37
38 27
39 28 def bootstrap(config_uri, options=None, env=None):
40 29 from rhodecode.lib.utils2 import AttributeDict
30 from rhodecode.lib.request import Request
41 31
42 32 if env:
43 33 os.environ.update(env)
@@ -20,7 +20,8 b' import logging'
20 20 import click
21 21 import pyramid.paster
22 22
23 from rhodecode.lib.pyramid_utils import bootstrap, get_app_config
23 from rhodecode.lib.pyramid_utils import bootstrap
24 from rhodecode.lib.config_utils import get_app_config
24 25 from rhodecode.lib.db_manage import DbManage
25 26 from rhodecode.lib.utils2 import get_encryption_key
26 27 from rhodecode.model.db import Session
@@ -32,11 +32,9 b' import socket'
32 32 import tempfile
33 33 import traceback
34 34 import tarfile
35 import urllib.parse
36 import warnings
35
37 36 from functools import wraps
38 37 from os.path import join as jn
39 from configparser import NoOptionError
40 38
41 39 import paste
42 40 import pkg_resources
@@ -55,9 +53,6 b' from rhodecode.model import meta'
55 53 from rhodecode.model.db import (
56 54 Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
57 55 from rhodecode.model.meta import Session
58 from rhodecode.lib.pyramid_utils import get_config
59 from rhodecode.lib.vcs import CurlSession
60 from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
61 56
62 57
63 58 log = logging.getLogger(__name__)
@@ -827,33 +822,3 b' def send_test_email(recipients, email_bo'
827 822 email_body = email_body_plaintext = email_body
828 823 subject = f'SUBJECT FROM: {socket.gethostname()}'
829 824 tasks.send_email(recipients, subject, email_body_plaintext, email_body)
830
831
832 def call_service_api(ini_path, payload):
833 config = get_config(ini_path)
834 try:
835 host = config.get('app:main', 'app.service_api.host')
836 except NoOptionError:
837 raise ImproperlyConfiguredError(
838 "app.service_api.host is missing. "
839 "Please ensure that app.service_api.host and app.service_api.token are "
840 "defined inside of .ini configuration file."
841 )
842 try:
843 api_url = config.get('app:main', 'rhodecode.api.url')
844 except NoOptionError:
845 from rhodecode import api
846 log.debug('Cannot find rhodecode.api.url, setting API URL TO Default value')
847 api_url = api.DEFAULT_URL
848
849 payload.update({
850 'id': 'service',
851 'auth_token': config.get('app:main', 'app.service_api.token')
852 })
853
854 response = CurlSession().post(urllib.parse.urljoin(host, api_url), json.dumps(payload))
855
856 if response.status_code != 200:
857 raise Exception("Service API responded with error")
858
859 return json.loads(response.content)['result']
@@ -38,7 +38,7 b' from rhodecode.translation import lazy_u'
38 38 from rhodecode.lib import helpers as h, hooks_utils, diffs
39 39 from rhodecode.lib import audit_logger
40 40 from collections import OrderedDict
41 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
41 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
42 42 from rhodecode.lib.ext_json import sjson as json
43 43 from rhodecode.lib.markup_renderer import (
44 44 DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
@@ -19,7 +19,7 b''
19 19
20 20 import pytest
21 21
22 from rhodecode.lib.pyramid_utils import get_app_config
22 from rhodecode.lib.config_utils import get_app_config
23 23 from rhodecode.tests.fixture import TestINI
24 24 from rhodecode.tests.server_utils import RcVCSServer
25 25
@@ -174,7 +174,7 b' def http_environ():'
174 174
175 175 @pytest.fixture(scope='session')
176 176 def baseapp(ini_config, vcsserver, http_environ_session):
177 from rhodecode.lib.pyramid_utils import get_app_config
177 from rhodecode.lib.config_utils import get_app_config
178 178 from rhodecode.config.middleware import make_pyramid_app
179 179
180 180 log.info("Using the RhodeCode configuration:{}".format(ini_config))
@@ -25,17 +25,20 b' import msgpack'
25 25 import pytest
26 26 import tempfile
27 27
28 from rhodecode.lib import hooks_daemon
28 from rhodecode.lib.hook_daemon import http_hooks_deamon
29 from rhodecode.lib.hook_daemon import celery_hooks_deamon
30 from rhodecode.lib.hook_daemon import hook_module
31 from rhodecode.lib.hook_daemon import base as hook_base
29 32 from rhodecode.lib.str_utils import safe_bytes
30 33 from rhodecode.tests.utils import assert_message_in_log
31 34 from rhodecode.lib.ext_json import json
32 35
33 test_proto = hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
36 test_proto = http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
34 37
35 38
36 39 class TestHooks(object):
37 40 def test_hooks_can_be_used_as_a_context_processor(self):
38 hooks = hooks_daemon.Hooks()
41 hooks = hook_module.Hooks()
39 42 with hooks as return_value:
40 43 pass
41 44 assert hooks == return_value
@@ -52,10 +55,10 b' class TestHooksHttpHandler(object):'
52 55 }
53 56 request = self._generate_post_request(data)
54 57 hooks_patcher = mock.patch.object(
55 hooks_daemon.Hooks, data['method'], create=True, return_value=1)
58 hook_module.Hooks, data['method'], create=True, return_value=1)
56 59
57 60 with hooks_patcher as hooks_mock:
58 handler = hooks_daemon.HooksHttpHandler
61 handler = http_hooks_deamon.HooksHttpHandler
59 62 handler.DEFAULT_HOOKS_PROTO = test_proto
60 63 handler.wbufsize = 10240
61 64 MockServer(handler, request)
@@ -73,21 +76,21 b' class TestHooksHttpHandler(object):'
73 76
74 77 # patching our _read to return test method and proto used
75 78 read_patcher = mock.patch.object(
76 hooks_daemon.HooksHttpHandler, '_read_request',
79 http_hooks_deamon.HooksHttpHandler, '_read_request',
77 80 return_value=(test_proto, rpc_method, extras))
78 81
79 82 # patch Hooks instance to return hook_result data on 'test' call
80 83 hooks_patcher = mock.patch.object(
81 hooks_daemon.Hooks, rpc_method, create=True,
84 hook_module.Hooks, rpc_method, create=True,
82 85 return_value=hook_result)
83 86
84 87 with read_patcher, hooks_patcher:
85 handler = hooks_daemon.HooksHttpHandler
88 handler = http_hooks_deamon.HooksHttpHandler
86 89 handler.DEFAULT_HOOKS_PROTO = test_proto
87 90 handler.wbufsize = 10240
88 91 server = MockServer(handler, request)
89 92
90 expected_result = hooks_daemon.HooksHttpHandler.serialize_data(hook_result)
93 expected_result = http_hooks_deamon.HooksHttpHandler.serialize_data(hook_result)
91 94
92 95 server.request.output_stream.seek(0)
93 96 assert server.request.output_stream.readlines()[-1] == expected_result
@@ -97,15 +100,15 b' class TestHooksHttpHandler(object):'
97 100 rpc_method = 'test'
98 101
99 102 read_patcher = mock.patch.object(
100 hooks_daemon.HooksHttpHandler, '_read_request',
103 http_hooks_deamon.HooksHttpHandler, '_read_request',
101 104 return_value=(test_proto, rpc_method, {}))
102 105
103 106 hooks_patcher = mock.patch.object(
104 hooks_daemon.Hooks, rpc_method, create=True,
107 hook_module.Hooks, rpc_method, create=True,
105 108 side_effect=Exception('Test exception'))
106 109
107 110 with read_patcher, hooks_patcher:
108 handler = hooks_daemon.HooksHttpHandler
111 handler = http_hooks_deamon.HooksHttpHandler
109 112 handler.DEFAULT_HOOKS_PROTO = test_proto
110 113 handler.wbufsize = 10240
111 114 server = MockServer(handler, request)
@@ -113,7 +116,7 b' class TestHooksHttpHandler(object):'
113 116 server.request.output_stream.seek(0)
114 117 data = server.request.output_stream.readlines()
115 118 msgpack_data = b''.join(data[5:])
116 org_exc = hooks_daemon.HooksHttpHandler.deserialize_data(msgpack_data)
119 org_exc = http_hooks_deamon.HooksHttpHandler.deserialize_data(msgpack_data)
117 120 expected_result = {
118 121 'exception': 'Exception',
119 122 'exception_traceback': org_exc['exception_traceback'],
@@ -123,8 +126,7 b' class TestHooksHttpHandler(object):'
123 126
124 127 def test_log_message_writes_to_debug_log(self, caplog):
125 128 ip_port = ('0.0.0.0', 8888)
126 handler = hooks_daemon.HooksHttpHandler(
127 MockRequest('POST /'), ip_port, mock.Mock())
129 handler = http_hooks_deamon.HooksHttpHandler(MockRequest('POST /'), ip_port, mock.Mock())
128 130 fake_date = '1/Nov/2015 00:00:00'
129 131 date_patcher = mock.patch.object(
130 132 handler, 'log_date_time_string', return_value=fake_date)
@@ -136,10 +138,10 b' class TestHooksHttpHandler(object):'
136 138
137 139 assert_message_in_log(
138 140 caplog.records, expected_message,
139 levelno=logging.DEBUG, module='hooks_daemon')
141 levelno=logging.DEBUG, module='http_hooks_deamon')
140 142
141 143 def _generate_post_request(self, data, proto=test_proto):
142 if proto == hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
144 if proto == http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
143 145 payload = msgpack.packb(data)
144 146 else:
145 147 payload = json.dumps(data)
@@ -151,18 +153,18 b' class TestHooksHttpHandler(object):'
151 153 class ThreadedHookCallbackDaemon(object):
152 154 def test_constructor_calls_prepare(self):
153 155 prepare_daemon_patcher = mock.patch.object(
154 hooks_daemon.ThreadedHookCallbackDaemon, '_prepare')
156 http_hooks_deamon.ThreadedHookCallbackDaemon, '_prepare')
155 157 with prepare_daemon_patcher as prepare_daemon_mock:
156 hooks_daemon.ThreadedHookCallbackDaemon()
158 http_hooks_deamon.ThreadedHookCallbackDaemon()
157 159 prepare_daemon_mock.assert_called_once_with()
158 160
159 161 def test_run_is_called_on_context_start(self):
160 162 patchers = mock.patch.multiple(
161 hooks_daemon.ThreadedHookCallbackDaemon,
163 http_hooks_deamon.ThreadedHookCallbackDaemon,
162 164 _run=mock.DEFAULT, _prepare=mock.DEFAULT, __exit__=mock.DEFAULT)
163 165
164 166 with patchers as mocks:
165 daemon = hooks_daemon.ThreadedHookCallbackDaemon()
167 daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
166 168 with daemon as daemon_context:
167 169 pass
168 170 mocks['_run'].assert_called_once_with()
@@ -170,11 +172,11 b' class ThreadedHookCallbackDaemon(object)'
170 172
171 173 def test_stop_is_called_on_context_exit(self):
172 174 patchers = mock.patch.multiple(
173 hooks_daemon.ThreadedHookCallbackDaemon,
175 http_hooks_deamon.ThreadedHookCallbackDaemon,
174 176 _run=mock.DEFAULT, _prepare=mock.DEFAULT, _stop=mock.DEFAULT)
175 177
176 178 with patchers as mocks:
177 daemon = hooks_daemon.ThreadedHookCallbackDaemon()
179 daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
178 180 with daemon as daemon_context:
179 181 assert mocks['_stop'].call_count == 0
180 182
@@ -185,46 +187,47 b' class ThreadedHookCallbackDaemon(object)'
185 187 class TestHttpHooksCallbackDaemon(object):
186 188 def test_hooks_callback_generates_new_port(self, caplog):
187 189 with caplog.at_level(logging.DEBUG):
188 daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
190 daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
189 191 assert daemon._daemon.server_address == ('127.0.0.1', 8881)
190 192
191 193 with caplog.at_level(logging.DEBUG):
192 daemon = hooks_daemon.HttpHooksCallbackDaemon(host=None, port=None)
194 daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host=None, port=None)
193 195 assert daemon._daemon.server_address[1] in range(0, 66000)
194 196 assert daemon._daemon.server_address[0] != '127.0.0.1'
195 197
196 198 def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
197 199 with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
198 daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
200 daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
199 201 assert daemon._daemon == tcp_server
200 202
201 203 _, port = tcp_server.server_address
202 204
203 205 msg = f"HOOKS: 127.0.0.1:{port} Preparing HTTP callback daemon registering " \
204 f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
206 f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
205 207 assert_message_in_log(
206 caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
208 caplog.records, msg, levelno=logging.DEBUG, module='http_hooks_deamon')
207 209
208 210 def test_prepare_inits_hooks_uri_and_logs_it(
209 211 self, tcp_server, caplog):
210 212 with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
211 daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
213 daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
212 214
213 215 _, port = tcp_server.server_address
214 216 expected_uri = '{}:{}'.format('127.0.0.1', port)
215 217 assert daemon.hooks_uri == expected_uri
216 218
217 219 msg = f"HOOKS: 127.0.0.1:{port} Preparing HTTP callback daemon registering " \
218 f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
220 f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
221
219 222 assert_message_in_log(
220 223 caplog.records, msg,
221 levelno=logging.DEBUG, module='hooks_daemon')
224 levelno=logging.DEBUG, module='http_hooks_deamon')
222 225
223 226 def test_run_creates_a_thread(self, tcp_server):
224 227 thread = mock.Mock()
225 228
226 229 with self._tcp_patcher(tcp_server):
227 daemon = hooks_daemon.HttpHooksCallbackDaemon()
230 daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
228 231
229 232 with self._thread_patcher(thread) as thread_mock:
230 233 daemon._run()
@@ -238,7 +241,7 b' class TestHttpHooksCallbackDaemon(object'
238 241 def test_run_logs(self, tcp_server, caplog):
239 242
240 243 with self._tcp_patcher(tcp_server):
241 daemon = hooks_daemon.HttpHooksCallbackDaemon()
244 daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
242 245
243 246 with self._thread_patcher(mock.Mock()), caplog.at_level(logging.DEBUG):
244 247 daemon._run()
@@ -246,13 +249,13 b' class TestHttpHooksCallbackDaemon(object'
246 249 assert_message_in_log(
247 250 caplog.records,
248 251 'Running thread-based loop of callback daemon in background',
249 levelno=logging.DEBUG, module='hooks_daemon')
252 levelno=logging.DEBUG, module='http_hooks_deamon')
250 253
251 254 def test_stop_cleans_up_the_connection(self, tcp_server, caplog):
252 255 thread = mock.Mock()
253 256
254 257 with self._tcp_patcher(tcp_server):
255 daemon = hooks_daemon.HttpHooksCallbackDaemon()
258 daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
256 259
257 260 with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
258 261 with daemon:
@@ -266,18 +269,19 b' class TestHttpHooksCallbackDaemon(object'
266 269
267 270 assert_message_in_log(
268 271 caplog.records, 'Waiting for background thread to finish.',
269 levelno=logging.DEBUG, module='hooks_daemon')
272 levelno=logging.DEBUG, module='http_hooks_deamon')
270 273
271 274 def _tcp_patcher(self, tcp_server):
272 275 return mock.patch.object(
273 hooks_daemon, 'TCPServer', return_value=tcp_server)
276 http_hooks_deamon, 'TCPServer', return_value=tcp_server)
274 277
275 278 def _thread_patcher(self, thread):
276 279 return mock.patch.object(
277 hooks_daemon.threading, 'Thread', return_value=thread)
280 http_hooks_deamon.threading, 'Thread', return_value=thread)
278 281
279 282
280 283 class TestPrepareHooksDaemon(object):
284
281 285 @pytest.mark.parametrize('protocol', ('celery',))
282 286 def test_returns_celery_hooks_callback_daemon_when_celery_protocol_specified(
283 287 self, protocol):
@@ -286,12 +290,12 b' class TestPrepareHooksDaemon(object):'
286 290 "celery.result_backend = redis://redis/0")
287 291 temp_file.flush()
288 292 expected_extras = {'config': temp_file.name}
289 callback, extras = hooks_daemon.prepare_callback_daemon(
293 callback, extras = hook_base.prepare_callback_daemon(
290 294 expected_extras, protocol=protocol, host='')
291 assert isinstance(callback, hooks_daemon.CeleryHooksCallbackDaemon)
295 assert isinstance(callback, celery_hooks_deamon.CeleryHooksCallbackDaemon)
292 296
293 297 @pytest.mark.parametrize('protocol, expected_class', (
294 ('http', hooks_daemon.HttpHooksCallbackDaemon),
298 ('http', http_hooks_deamon.HttpHooksCallbackDaemon),
295 299 ))
296 300 def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
297 301 self, protocol, expected_class):
@@ -302,7 +306,7 b' class TestPrepareHooksDaemon(object):'
302 306 'task_backend': '',
303 307 'task_queue': ''
304 308 }
305 callback, extras = hooks_daemon.prepare_callback_daemon(
309 callback, extras = hook_base.prepare_callback_daemon(
306 310 expected_extras.copy(), protocol=protocol, host='127.0.0.1',
307 311 txn_id='txnid2')
308 312 assert isinstance(callback, expected_class)
@@ -321,7 +325,7 b' class TestPrepareHooksDaemon(object):'
321 325 'hooks_protocol': protocol.lower()
322 326 }
323 327 with pytest.raises(Exception):
324 callback, extras = hooks_daemon.prepare_callback_daemon(
328 callback, extras = hook_base.prepare_callback_daemon(
325 329 expected_extras.copy(),
326 330 protocol=protocol, host='127.0.0.1')
327 331
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now