##// END OF EJS Templates
core: removed pyro4 from Enterprise code. Fixes #5198
marcink -
r1409:c1ce56be default
parent child
Show More
@@ -539,18 +539,15 vcs.server = localhost:9900
539
539
540 ## Web server connectivity protocol, responsible for web based VCS operatations
540 ## Web server connectivity protocol, responsible for web based VCS operatations
541 ## Available protocols are:
541 ## Available protocols are:
542 ## `pyro4` - use pyro4 server
543 ## `http` - use http-rpc backend (default)
542 ## `http` - use http-rpc backend (default)
544 vcs.server.protocol = http
543 vcs.server.protocol = http
545
544
546 ## Push/Pull operations protocol, available options are:
545 ## Push/Pull operations protocol, available options are:
547 ## `pyro4` - use pyro4 server
548 ## `http` - use http-rpc backend (default)
546 ## `http` - use http-rpc backend (default)
549 ##
547 ##
550 vcs.scm_app_implementation = http
548 vcs.scm_app_implementation = http
551
549
552 ## Push/Pull operations hooks protocol, available options are:
550 ## Push/Pull operations hooks protocol, available options are:
553 ## `pyro4` - use pyro4 server
554 ## `http` - use http-rpc backend (default)
551 ## `http` - use http-rpc backend (default)
555 vcs.hooks.protocol = http
552 vcs.hooks.protocol = http
556
553
@@ -666,7 +663,7 formatter = color_formatter_sql
666 ################
663 ################
667
664
668 [formatter_generic]
665 [formatter_generic]
669 class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
666 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
670 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
667 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
671 datefmt = %Y-%m-%d %H:%M:%S
668 datefmt = %Y-%m-%d %H:%M:%S
672
669
@@ -508,18 +508,15 vcs.server = localhost:9900
508
508
509 ## Web server connectivity protocol, responsible for web based VCS operatations
509 ## Web server connectivity protocol, responsible for web based VCS operatations
510 ## Available protocols are:
510 ## Available protocols are:
511 ## `pyro4` - use pyro4 server
512 ## `http` - use http-rpc backend (default)
511 ## `http` - use http-rpc backend (default)
513 vcs.server.protocol = http
512 vcs.server.protocol = http
514
513
515 ## Push/Pull operations protocol, available options are:
514 ## Push/Pull operations protocol, available options are:
516 ## `pyro4` - use pyro4 server
517 ## `http` - use http-rpc backend (default)
515 ## `http` - use http-rpc backend (default)
518 ##
516 ##
519 vcs.scm_app_implementation = http
517 vcs.scm_app_implementation = http
520
518
521 ## Push/Pull operations hooks protocol, available options are:
519 ## Push/Pull operations hooks protocol, available options are:
522 ## `pyro4` - use pyro4 server
523 ## `http` - use http-rpc backend (default)
520 ## `http` - use http-rpc backend (default)
524 vcs.hooks.protocol = http
521 vcs.hooks.protocol = http
525
522
@@ -635,7 +632,7 formatter = generic
635 ################
632 ################
636
633
637 [formatter_generic]
634 [formatter_generic]
638 class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
635 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
639 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
636 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
640 datefmt = %Y-%m-%d %H:%M:%S
637 datefmt = %Y-%m-%d %H:%M:%S
641
638
@@ -17,7 +17,6 implemented in :mod:`rhodecode.lib.middl
17 .. toctree::
17 .. toctree::
18 :maxdepth: 2
18 :maxdepth: 2
19
19
20 http-transition
21 middleware
20 middleware
22 vcsserver
21 vcsserver
23 subversion
22 subversion
@@ -47,7 +47,7 the ``debug`` level.
47 ### LOGGING CONFIGURATION ####
47 ### LOGGING CONFIGURATION ####
48 ################################
48 ################################
49 [loggers]
49 [loggers]
50 keys = root, routes, rhodecode, sqlalchemy, beaker, pyro4, templates
50 keys = root, routes, rhodecode, sqlalchemy, beaker, templates
51
51
52 [handlers]
52 [handlers]
53 keys = console, console_sql, file, file_rotating
53 keys = console, console_sql, file, file_rotating
@@ -75,12 +75,6 the ``debug`` level.
75 qualname = beaker.container
75 qualname = beaker.container
76 propagate = 1
76 propagate = 1
77
77
78 [logger_pyro4]
79 level = DEBUG
80 handlers =
81 qualname = Pyro4
82 propagate = 1
83
84 [logger_templates]
78 [logger_templates]
85 level = INFO
79 level = INFO
86 handlers =
80 handlers =
@@ -232,7 +232,7 For a more detailed explanation of the l
232 ### LOGGING CONFIGURATION ####
232 ### LOGGING CONFIGURATION ####
233 ################################
233 ################################
234 [loggers]
234 [loggers]
235 keys = root, vcsserver, pyro4, beaker
235 keys = root, vcsserver, beaker
236
236
237 [handlers]
237 [handlers]
238 keys = console
238 keys = console
@@ -259,12 +259,6 For a more detailed explanation of the l
259 qualname = beaker
259 qualname = beaker
260 propagate = 1
260 propagate = 1
261
261
262 [logger_pyro4]
263 level = DEBUG
264 handlers =
265 qualname = Pyro4
266 propagate = 1
267
268
262
269 ##############
263 ##############
270 ## HANDLERS ##
264 ## HANDLERS ##
@@ -197,19 +197,6
197 license = [ pkgs.lib.licenses.bsdOriginal ];
197 license = [ pkgs.lib.licenses.bsdOriginal ];
198 };
198 };
199 };
199 };
200 Pyro4 = super.buildPythonPackage {
201 name = "Pyro4-4.41";
202 buildInputs = with self; [];
203 doCheck = false;
204 propagatedBuildInputs = with self; [serpent];
205 src = fetchurl {
206 url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz";
207 md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c";
208 };
209 meta = {
210 license = [ pkgs.lib.licenses.mit ];
211 };
212 };
213 Routes = super.buildPythonPackage {
200 Routes = super.buildPythonPackage {
214 name = "Routes-1.13";
201 name = "Routes-1.13";
215 buildInputs = with self; [];
202 buildInputs = with self; [];
@@ -1501,7 +1488,7
1501 name = "rhodecode-enterprise-ce-4.7.0";
1488 name = "rhodecode-enterprise-ce-4.7.0";
1502 buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage cssselect lxml configobj];
1489 buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage cssselect lxml configobj];
1503 doCheck = true;
1490 doCheck = true;
1504 propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
1491 propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
1505 src = ./.;
1492 src = ./.;
1506 meta = {
1493 meta = {
1507 license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
1494 license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
@@ -1520,19 +1507,6
1520 license = [ { fullName = "AGPLv3 and Proprietary"; } ];
1507 license = [ { fullName = "AGPLv3 and Proprietary"; } ];
1521 };
1508 };
1522 };
1509 };
1523 serpent = super.buildPythonPackage {
1524 name = "serpent-1.15";
1525 buildInputs = with self; [];
1526 doCheck = false;
1527 propagatedBuildInputs = with self; [];
1528 src = fetchurl {
1529 url = "https://pypi.python.org/packages/7b/38/b2b27673a882ff2ea5871bb3e3e6b496ebbaafd1612e51990ffb158b9254/serpent-1.15.tar.gz";
1530 md5 = "e27b1aad5c218e16442f52abb7c7053a";
1531 };
1532 meta = {
1533 license = [ pkgs.lib.licenses.mit ];
1534 };
1535 };
1536 setproctitle = super.buildPythonPackage {
1510 setproctitle = super.buildPythonPackage {
1537 name = "setproctitle-1.1.8";
1511 name = "setproctitle-1.1.8";
1538 buildInputs = with self; [];
1512 buildInputs = with self; [];
@@ -2,7 +2,6
2 testpaths = ./rhodecode
2 testpaths = ./rhodecode
3 pylons_config = rhodecode/tests/rhodecode.ini
3 pylons_config = rhodecode/tests/rhodecode.ini
4 vcsserver_protocol = http
4 vcsserver_protocol = http
5 vcsserver_config_pyro4 = rhodecode/tests/vcsserver_pyro4.ini
6 vcsserver_config_http = rhodecode/tests/vcsserver_http.ini
5 vcsserver_config_http = rhodecode/tests/vcsserver_http.ini
7 norecursedirs = tests/scripts
6 norecursedirs = tests/scripts
8 addopts = -k "not _BaseTest"
7 addopts = -k "not _BaseTest"
@@ -123,9 +123,5 https://code.rhodecode.com/rhodecode-too
123 ## appenlight
123 ## appenlight
124 appenlight-client==0.6.14
124 appenlight-client==0.6.14
125
125
126 # Pyro/Deprecated TODO(Marcink): remove in 4.7 release.
127 Pyro4==4.41
128 serpent==1.15
129
130 ## test related requirements
126 ## test related requirements
131 -r requirements_test.txt
127 -r requirements_test.txt
@@ -184,7 +184,6 def load_pyramid_environment(global_conf
184 protocol=utils.get_vcs_server_protocol(settings),
184 protocol=utils.get_vcs_server_protocol(settings),
185 log_level=settings['vcs.server.log_level'])
185 log_level=settings['vcs.server.log_level'])
186
186
187 utils.configure_pyro4(settings)
188 utils.configure_vcs(settings)
187 utils.configure_vcs(settings)
189 if vcs_server_enabled:
188 if vcs_server_enabled:
190 connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
189 connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
@@ -40,10 +40,7
40 },
40 },
41 "python2.7-Pylons-1.0.1-patch1": {
41 "python2.7-Pylons-1.0.1-patch1": {
42 "BSD 4-clause \"Original\" or \"Old\" License": "http://spdx.org/licenses/BSD-4-Clause"
42 "BSD 4-clause \"Original\" or \"Old\" License": "http://spdx.org/licenses/BSD-4-Clause"
43 },
43 },
44 "python2.7-Pyro4-4.35": {
45 "MIT License": "http://spdx.org/licenses/MIT"
46 },
47 "python2.7-Routes-1.13": {
44 "python2.7-Routes-1.13": {
48 "BSD 4-clause \"Original\" or \"Old\" License": "http://spdx.org/licenses/BSD-4-Clause"
45 "BSD 4-clause \"Original\" or \"Old\" License": "http://spdx.org/licenses/BSD-4-Clause"
49 },
46 },
@@ -214,10 +211,7
214 },
211 },
215 "python2.7-requests-2.9.1": {
212 "python2.7-requests-2.9.1": {
216 "Apache License 2.0": "http://spdx.org/licenses/Apache-2.0"
213 "Apache License 2.0": "http://spdx.org/licenses/Apache-2.0"
217 },
214 },
218 "python2.7-serpent-1.12": {
219 "MIT License": "http://spdx.org/licenses/MIT"
220 },
221 "python2.7-setuptools-19.4": {
215 "python2.7-setuptools-19.4": {
222 "Python Software Foundation License version 2": "http://spdx.org/licenses/Python-2.0",
216 "Python Software Foundation License version 2": "http://spdx.org/licenses/Python-2.0",
223 "Zope Public License 2.0": "http://spdx.org/licenses/ZPL-2.0"
217 "Zope Public License 2.0": "http://spdx.org/licenses/ZPL-2.0"
@@ -20,29 +20,11
20
20
21 import os
21 import os
22 import shlex
22 import shlex
23 import Pyro4
24 import platform
23 import platform
25
24
26 from rhodecode.model import init_model
25 from rhodecode.model import init_model
27
26
28
27
29 def configure_pyro4(config):
30 """
31 Configure Pyro4 based on `config`.
32
33 This will mainly set the different configuration parameters of the Pyro4
34 library based on the settings in our INI files. The Pyro4 documentation
35 lists more details about the specific settings and their meaning.
36 """
37 Pyro4.config.COMMTIMEOUT = float(config['vcs.connection_timeout'])
38 Pyro4.config.SERIALIZER = 'pickle'
39 Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
40
41 # Note: We need server configuration in the WSGI processes
42 # because we provide a callback server in certain vcs operations.
43 Pyro4.config.SERVERTYPE = "multiplex"
44 Pyro4.config.POLLTIMEOUT = 0.01
45
46
28
47 def configure_vcs(config):
29 def configure_vcs(config):
48 """
30 """
@@ -226,7 +226,6 def vcsconnection(func):
226 for alias in rhodecode.BACKENDS.keys():
226 for alias in rhodecode.BACKENDS.keys():
227 if alias not in backends:
227 if alias not in backends:
228 del rhodecode.BACKENDS[alias]
228 del rhodecode.BACKENDS[alias]
229 utils.configure_pyro4(settings)
230 utils.configure_vcs(settings)
229 utils.configure_vcs(settings)
231 connect_vcs(
230 connect_vcs(
232 settings['vcs.server'],
231 settings['vcs.server'],
@@ -26,7 +26,6 from BaseHTTPServer import BaseHTTPReque
26 from SocketServer import TCPServer
26 from SocketServer import TCPServer
27 from routes.util import URLGenerator
27 from routes.util import URLGenerator
28
28
29 import Pyro4
30 import pylons
29 import pylons
31 import rhodecode
30 import rhodecode
32
31
@@ -73,7 +72,7 class HooksHttpHandler(BaseHTTPRequestHa
73
72
74 def log_message(self, format, *args):
73 def log_message(self, format, *args):
75 """
74 """
76 This is an overriden method of BaseHTTPRequestHandler which logs using
75 This is an overridden method of BaseHTTPRequestHandler which logs using
77 logging library instead of writing directly to stderr.
76 logging library instead of writing directly to stderr.
78 """
77 """
79
78
@@ -123,38 +122,6 class ThreadedHookCallbackDaemon(object)
123 raise NotImplementedError()
122 raise NotImplementedError()
124
123
125
124
126 class Pyro4HooksCallbackDaemon(ThreadedHookCallbackDaemon):
127 """
128 Context manager which will run a callback daemon in a background thread.
129 """
130
131 hooks_uri = None
132
133 def _prepare(self):
134 log.debug("Preparing callback daemon and registering hook object")
135 self._daemon = Pyro4.Daemon()
136 hooks_interface = Hooks()
137 self.hooks_uri = str(self._daemon.register(hooks_interface))
138 log.debug("Hooks uri is: %s", self.hooks_uri)
139
140 def _run(self):
141 log.debug("Running event loop of callback daemon in background thread")
142 callback_thread = threading.Thread(
143 target=self._daemon.requestLoop,
144 kwargs={'loopCondition': lambda: not self._done})
145 callback_thread.daemon = True
146 callback_thread.start()
147 self._callback_thread = callback_thread
148
149 def _stop(self):
150 log.debug("Waiting for background thread to finish.")
151 self._done = True
152 self._callback_thread.join()
153 self._daemon.close()
154 self._daemon = None
155 self._callback_thread = None
156
157
158 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
125 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
159 """
126 """
160 Context manager which will run a callback daemon in a background thread.
127 Context manager which will run a callback daemon in a background thread.
@@ -202,9 +169,7 def prepare_callback_daemon(extras, prot
202 callback_daemon = DummyHooksCallbackDaemon()
169 callback_daemon = DummyHooksCallbackDaemon()
203 extras['hooks_module'] = callback_daemon.hooks_module
170 extras['hooks_module'] = callback_daemon.hooks_module
204 else:
171 else:
205 if protocol == 'pyro4':
172 if protocol == 'http':
206 callback_daemon = Pyro4HooksCallbackDaemon()
207 elif protocol == 'http':
208 callback_daemon = HttpHooksCallbackDaemon()
173 callback_daemon = HttpHooksCallbackDaemon()
209 else:
174 else:
210 log.error('Unsupported callback daemon protocol "%s"', protocol)
175 log.error('Unsupported callback daemon protocol "%s"', protocol)
@@ -221,27 +186,22 class Hooks(object):
221 Exposes the hooks for remote call backs
186 Exposes the hooks for remote call backs
222 """
187 """
223
188
224 @Pyro4.callback
225 def repo_size(self, extras):
189 def repo_size(self, extras):
226 log.debug("Called repo_size of Hooks object")
190 log.debug("Called repo_size of Hooks object")
227 return self._call_hook(hooks_base.repo_size, extras)
191 return self._call_hook(hooks_base.repo_size, extras)
228
192
229 @Pyro4.callback
230 def pre_pull(self, extras):
193 def pre_pull(self, extras):
231 log.debug("Called pre_pull of Hooks object")
194 log.debug("Called pre_pull of Hooks object")
232 return self._call_hook(hooks_base.pre_pull, extras)
195 return self._call_hook(hooks_base.pre_pull, extras)
233
196
234 @Pyro4.callback
235 def post_pull(self, extras):
197 def post_pull(self, extras):
236 log.debug("Called post_pull of Hooks object")
198 log.debug("Called post_pull of Hooks object")
237 return self._call_hook(hooks_base.post_pull, extras)
199 return self._call_hook(hooks_base.post_pull, extras)
238
200
239 @Pyro4.callback
240 def pre_push(self, extras):
201 def pre_push(self, extras):
241 log.debug("Called pre_push of Hooks object")
202 log.debug("Called pre_push of Hooks object")
242 return self._call_hook(hooks_base.pre_push, extras)
203 return self._call_hook(hooks_base.pre_push, extras)
243
204
244 @Pyro4.callback
245 def post_push(self, extras):
205 def post_push(self, extras):
246 log.debug("Called post_push of Hooks object")
206 log.debug("Called post_push of Hooks object")
247 return self._call_hook(hooks_base.post_push, extras)
207 return self._call_hook(hooks_base.post_push, extras)
@@ -67,9 +67,9 def format_sql(sql):
67 return sql
67 return sql
68
68
69
69
70 class Pyro4AwareFormatter(logging.Formatter):
70 class ExceptionAwareFormatter(logging.Formatter):
71 """
71 """
72 Extended logging formatter which prints out Pyro4 remote tracebacks.
72 Extended logging formatter which prints out remote tracebacks.
73 """
73 """
74
74
75 def formatException(self, ei):
75 def formatException(self, ei):
@@ -86,10 +86,12 class Pyro4AwareFormatter(logging.Format
86
86
87 try:
87 try:
88 if ex_type is not None and ex_value is None and ex_tb is None:
88 if ex_type is not None and ex_value is None and ex_tb is None:
89 # possible old (3.x) call syntax where caller is only providing exception object
89 # possible old (3.x) call syntax where caller is only
90 # providing exception object
90 if type(ex_type) is not type:
91 if type(ex_type) is not type:
91 raise TypeError(
92 raise TypeError(
92 "invalid argument: ex_type should be an exception type, or just supply no arguments at all")
93 "invalid argument: ex_type should be an exception "
94 "type, or just supply no arguments at all")
93 if ex_type is None and ex_tb is None:
95 if ex_type is None and ex_tb is None:
94 ex_type, ex_value, ex_tb = sys.exc_info()
96 ex_type, ex_value, ex_tb = sys.exc_info()
95
97
@@ -105,7 +107,7 class Pyro4AwareFormatter(logging.Format
105 return local_tb
107 return local_tb
106
108
107
109
108 class ColorFormatter(Pyro4AwareFormatter):
110 class ColorFormatter(ExceptionAwareFormatter):
109
111
110 def format(self, record):
112 def format(self, record):
111 """
113 """
@@ -134,3 +136,6 class ColorFormatterSql(logging.Formatte
134
136
135 colored_record = ''.join([start, def_record, end])
137 colored_record = ''.join([start, def_record, end])
136 return colored_record
138 return colored_record
139
140 # marcink: needs to stay with this name for backward .ini compatability
141 Pyro4AwareFormatter = ExceptionAwareFormatter
@@ -42,7 +42,7 from rhodecode.lib.exceptions import (
42 NotAllowedToCreateUserError)
42 NotAllowedToCreateUserError)
43 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
43 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
44 from rhodecode.lib.middleware import appenlight
44 from rhodecode.lib.middleware import appenlight
45 from rhodecode.lib.middleware.utils import scm_app, scm_app_http
45 from rhodecode.lib.middleware.utils import scm_app_http
46 from rhodecode.lib.utils import (
46 from rhodecode.lib.utils import (
47 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
47 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
@@ -177,9 +177,6 class SimpleVCS(object):
177 if custom_implementation == 'http':
177 if custom_implementation == 'http':
178 log.info('Using HTTP implementation of scm app.')
178 log.info('Using HTTP implementation of scm app.')
179 scm_app_impl = scm_app_http
179 scm_app_impl = scm_app_http
180 elif custom_implementation == 'pyro4':
181 log.info('Using Pyro implementation of scm app.')
182 scm_app_impl = scm_app
183 else:
180 else:
184 log.info('Using custom implementation of scm_app: "{}"'.format(
181 log.info('Using custom implementation of scm_app: "{}"'.format(
185 custom_implementation))
182 custom_implementation))
@@ -512,7 +509,7 class SimpleVCS(object):
512 raise NotImplementedError()
509 raise NotImplementedError()
513
510
514 def _create_config(self, extras, repo_name):
511 def _create_config(self, extras, repo_name):
515 """Create a Pyro safe config representation."""
512 """Create a safe config representation."""
516 raise NotImplementedError()
513 raise NotImplementedError()
517
514
518 def _prepare_callback_daemon(self, extras):
515 def _prepare_callback_daemon(self, extras):
@@ -42,7 +42,7 def create_git_wsgi_app(repo_path, repo_
42 """
42 """
43 factory = GIT_REMOTE_WSGI
43 factory = GIT_REMOTE_WSGI
44 if not factory:
44 if not factory:
45 log.error('Pyro server has not been initialized yet')
45 log.error('VCSServer has not been initialized yet')
46
46
47 return wsgi_app_caller_client.RemoteAppCaller(
47 return wsgi_app_caller_client.RemoteAppCaller(
48 factory, repo_path, repo_name, config)
48 factory, repo_path, repo_name, config)
@@ -57,7 +57,7 def create_hg_wsgi_app(repo_path, repo_n
57 """
57 """
58 factory = HG_REMOTE_WSGI
58 factory = HG_REMOTE_WSGI
59 if not factory:
59 if not factory:
60 log.error('Pyro server has not been initialized yet')
60 log.error('VCSServer has not been initialized yet')
61
61
62 return wsgi_app_caller_client.RemoteAppCaller(
62 return wsgi_app_caller_client.RemoteAppCaller(
63 factory, repo_path, repo_name, config)
63 factory, repo_path, repo_name, config)
@@ -24,8 +24,6 Utility to call a WSGI app wrapped in a
24
24
25 import logging
25 import logging
26
26
27 from Pyro4.errors import ConnectionClosedError
28
29
27
30 log = logging.getLogger(__name__)
28 log = logging.getLogger(__name__)
31
29
@@ -83,14 +81,8 class RemoteAppCaller(object):
83 input_data = environ['wsgi.input'].read()
81 input_data = environ['wsgi.input'].read()
84 clean_environ = _get_clean_environ(environ)
82 clean_environ = _get_clean_environ(environ)
85
83
86 try:
84 data, status, headers = self._remote_wsgi.handle(
87 data, status, headers = self._remote_wsgi.handle(
85 clean_environ, input_data, *self._args, **self._kwargs)
88 clean_environ, input_data, *self._args, **self._kwargs)
89 except ConnectionClosedError:
90 log.debug('Remote Pyro Server ConnectionClosedError')
91 self._remote_wsgi._pyroReconnect(tries=15)
92 data, status, headers = self._remote_wsgi.handle(
93 clean_environ, input_data, *self._args, **self._kwargs)
94
86
95 log.debug("Got result from proxy, returning to WSGI container")
87 log.debug("Got result from proxy, returning to WSGI container")
96 start_response(status, headers)
88 start_response(status, headers)
@@ -216,8 +216,6 class VCSMiddleware(object):
216 vcs_handler.SCM):
216 vcs_handler.SCM):
217 return HTTPNotFound()(environ, start_response)
217 return HTTPNotFound()(environ, start_response)
218
218
219 # TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
220 # Remove once we fully switched to the HTTP backend.
221 environ['REPO_NAME'] = vcs_handler.url_repo_name
219 environ['REPO_NAME'] = vcs_handler.url_repo_name
222
220
223 # register repo config back to the handler
221 # register repo config back to the handler
@@ -40,8 +40,6 import time
40 import urlparse
40 import urlparse
41 from cStringIO import StringIO
41 from cStringIO import StringIO
42
42
43 import Pyro4
44 from Pyro4.errors import CommunicationError
45
43
46 from rhodecode.lib.vcs.conf import settings
44 from rhodecode.lib.vcs.conf import settings
47 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
45 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
@@ -72,36 +70,6 def get_version():
72 return '.'.join((str(each) for each in VERSION[:3]))
70 return '.'.join((str(each) for each in VERSION[:3]))
73
71
74
72
75 def connect_pyro4(server_and_port):
76 from rhodecode.lib.vcs import connection, client
77 from rhodecode.lib.middleware.utils import scm_app
78
79 git_remote = client.RequestScopeProxyFactory(
80 settings.pyro_remote(settings.PYRO_GIT, server_and_port))
81 hg_remote = client.RequestScopeProxyFactory(
82 settings.pyro_remote(settings.PYRO_HG, server_and_port))
83 svn_remote = client.RequestScopeProxyFactory(
84 settings.pyro_remote(settings.PYRO_SVN, server_and_port))
85
86 connection.Git = client.RepoMaker(proxy_factory=git_remote)
87 connection.Hg = client.RepoMaker(proxy_factory=hg_remote)
88 connection.Svn = client.RepoMaker(proxy_factory=svn_remote)
89
90 scm_app.GIT_REMOTE_WSGI = Pyro4.Proxy(
91 settings.pyro_remote(
92 settings.PYRO_GIT_REMOTE_WSGI, server_and_port))
93 scm_app.HG_REMOTE_WSGI = Pyro4.Proxy(
94 settings.pyro_remote(
95 settings.PYRO_HG_REMOTE_WSGI, server_and_port))
96
97 @atexit.register
98 def free_connection_resources():
99 connection.Git = None
100 connection.Hg = None
101 connection.Svn = None
102 connection.Service = None
103
104
105 def connect_http(server_and_port):
73 def connect_http(server_and_port):
106 from rhodecode.lib.vcs import connection, client_http
74 from rhodecode.lib.vcs import connection, client_http
107 from rhodecode.lib.middleware.utils import scm_app
75 from rhodecode.lib.middleware.utils import scm_app
@@ -135,11 +103,9 def connect_vcs(server_and_port, protoco
135 Initializes the connection to the vcs server.
103 Initializes the connection to the vcs server.
136
104
137 :param server_and_port: str, e.g. "localhost:9900"
105 :param server_and_port: str, e.g. "localhost:9900"
138 :param protocol: str, "pyro4" or "http"
106 :param protocol: str or "http"
139 """
107 """
140 if protocol == 'pyro4':
108 if protocol == 'http':
141 connect_pyro4(server_and_port)
142 elif protocol == 'http':
143 connect_http(server_and_port)
109 connect_http(server_and_port)
144 else:
110 else:
145 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
111 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
@@ -154,31 +120,10 def start_vcs_server(server_and_port, pr
154 log.info('Starting VCSServer as a sub process with %s protocol', protocol)
120 log.info('Starting VCSServer as a sub process with %s protocol', protocol)
155 if protocol == 'http':
121 if protocol == 'http':
156 return _start_http_vcs_server(server_and_port, log_level)
122 return _start_http_vcs_server(server_and_port, log_level)
157 elif protocol == 'pyro4':
158 return _start_pyro4_vcs_server(server_and_port, log_level)
159 else:
123 else:
160 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
124 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
161
125
162
126
163 def _start_pyro4_vcs_server(server_and_port, log_level=None):
164 _try_to_shutdown_running_server(server_and_port, protocol='pyro4')
165 host, port = server_and_port.rsplit(":", 1)
166 host = host.strip('[]')
167 args = [
168 'vcsserver', '--port', port, '--host', host, '--locale', 'en_US.UTF-8',
169 '--threadpool', '32']
170 if log_level:
171 args += ['--log-level', log_level]
172 proc = subprocess32.Popen(args)
173
174 def cleanup_server_process():
175 proc.kill()
176 atexit.register(cleanup_server_process)
177
178 server = create_vcsserver_proxy(server_and_port, protocol='pyro4')
179 _wait_until_vcs_server_is_reachable(server)
180
181
182 def _start_http_vcs_server(server_and_port, log_level=None):
127 def _start_http_vcs_server(server_and_port, log_level=None):
183 # TODO: mikhail: shutdown if an http server already runs
128 # TODO: mikhail: shutdown if an http server already runs
184
129
@@ -202,7 +147,7 def _wait_until_vcs_server_is_reachable(
202 try:
147 try:
203 server.ping()
148 server.ping()
204 return
149 return
205 except (VCSCommunicationError, CommunicationError, pycurl.error):
150 except (VCSCommunicationError, pycurl.error):
206 log.debug('VCSServer not started yet, retry to connect.')
151 log.debug('VCSServer not started yet, retry to connect.')
207 time.sleep(0.5)
152 time.sleep(0.5)
208 raise Exception(
153 raise Exception(
@@ -214,7 +159,7 def _try_to_shutdown_running_server(serv
214 server = create_vcsserver_proxy(server_and_port, protocol)
159 server = create_vcsserver_proxy(server_and_port, protocol)
215 try:
160 try:
216 server.shutdown()
161 server.shutdown()
217 except (CommunicationError, pycurl.error):
162 except pycurl.error:
218 return
163 return
219
164
220 # TODO: Not sure why this is important, but without it the following start
165 # TODO: Not sure why this is important, but without it the following start
@@ -224,20 +169,12 def _try_to_shutdown_running_server(serv
224
169
225
170
226 def create_vcsserver_proxy(server_and_port, protocol):
171 def create_vcsserver_proxy(server_and_port, protocol):
227 if protocol == 'pyro4':
172 if protocol == 'http':
228 return _create_vcsserver_proxy_pyro4(server_and_port)
229 elif protocol == 'http':
230 return _create_vcsserver_proxy_http(server_and_port)
173 return _create_vcsserver_proxy_http(server_and_port)
231 else:
174 else:
232 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
175 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
233
176
234
177
235 def _create_vcsserver_proxy_pyro4(server_and_port):
236 server = Pyro4.Proxy(
237 settings.pyro_remote(settings.PYRO_VCSSERVER, server_and_port))
238 return server
239
240
241 def _create_vcsserver_proxy_http(server_and_port):
178 def _create_vcsserver_proxy_http(server_and_port):
242 from rhodecode.lib.vcs import client_http
179 from rhodecode.lib.vcs import client_http
243
180
@@ -20,13 +20,6
20
20
21 """
21 """
22 Client for the VCSServer implemented based on HTTP.
22 Client for the VCSServer implemented based on HTTP.
23
24
25 Status
26 ------
27
28 This client implementation shall eventually replace the Pyro4 based
29 implementation.
30 """
23 """
31
24
32 import copy
25 import copy
@@ -52,27 +52,6 ARCHIVE_SPECS = {
52 HOOKS_PROTOCOL = None
52 HOOKS_PROTOCOL = None
53 HOOKS_DIRECT_CALLS = False
53 HOOKS_DIRECT_CALLS = False
54
54
55 PYRO_PORT = 9900
56
57 PYRO_GIT = 'git_remote'
58 PYRO_HG = 'hg_remote'
59 PYRO_SVN = 'svn_remote'
60 PYRO_VCSSERVER = 'vcs_server'
61 PYRO_GIT_REMOTE_WSGI = 'git_remote_wsgi'
62 PYRO_HG_REMOTE_WSGI = 'hg_remote_wsgi'
63
64 PYRO_RECONNECT_TRIES = 15
65 """
66 How many retries to reconnect will be performed if the connection was lost.
67
68 Each try takes 2s. Doing 15 means that we will give it up to 30s for a
69 connection to be re-established.
70 """
71
72
73 def pyro_remote(object_id, server_and_port):
74 return "PYRO:%s@%s" % (object_id, server_and_port)
75
76
55
77 def available_aliases():
56 def available_aliases():
78 """
57 """
@@ -30,10 +30,6 class VCSCommunicationError(Exception):
30 pass
30 pass
31
31
32
32
33 class PyroVCSCommunicationError(VCSCommunicationError):
34 pass
35
36
37 class HttpVCSCommunicationError(VCSCommunicationError):
33 class HttpVCSCommunicationError(VCSCommunicationError):
38 pass
34 pass
39
35
@@ -326,8 +326,6 class TestShadowRepoExposure(object):
326
326
327
327
328 @pytest.mark.usefixtures('db')
328 @pytest.mark.usefixtures('db')
329 @mock.patch.multiple(
330 'Pyro4.config', SERVERTYPE='multiplex', POLLTIMEOUT=0.01)
331 class TestGenerateVcsResponse:
329 class TestGenerateVcsResponse:
332
330
333 def test_ensures_that_start_response_is_called_early_enough(self):
331 def test_ensures_that_start_response_is_called_early_enough(self):
@@ -38,12 +38,10 def test_vcs_unavailable_returns_vcs_err
38
38
39 # Depending on the used VCSServer protocol we have to patch a different
39 # Depending on the used VCSServer protocol we have to patch a different
40 # RemoteRepo class to raise an exception. For the test it doesn't matter
40 # RemoteRepo class to raise an exception. For the test it doesn't matter
41 # if http or pyro4 is used, it just requires the exception to be raised.
41 # if http is used, it just requires the exception to be raised.
42 vcs_protocol = app_settings['vcs.server.protocol']
42 vcs_protocol = app_settings['vcs.server.protocol']
43 if vcs_protocol == 'http':
43 if vcs_protocol == 'http':
44 from rhodecode.lib.vcs.client_http import RemoteRepo
44 from rhodecode.lib.vcs.client_http import RemoteRepo
45 elif vcs_protocol == 'pyro4':
46 from rhodecode.lib.vcs.client import RemoteRepo
47 else:
45 else:
48 pytest.fail('Unknown VCS server protocol: "{}"'.format(vcs_protocol))
46 pytest.fail('Unknown VCS server protocol: "{}"'.format(vcs_protocol))
49
47