# HG changeset patch # User Marcin Kuzminski # Date 2017-04-19 19:35:08 # Node ID 28f265744157487bc60632fa7b79dbb914b223a3 # Parent f13fe8ba380698614d9435603ed51c6b9ace2bf5 core: finished removal of pyro4. backend is no longer available to be used. We rely purely on http implementation. diff --git a/configs/development_http.ini b/configs/development_http.ini --- a/configs/development_http.ini +++ b/configs/development_http.ini @@ -32,7 +32,7 @@ use = egg:waitress#main ### LOGGING CONFIGURATION #### ################################ [loggers] -keys = root, vcsserver, pyro4, beaker +keys = root, vcsserver, beaker [handlers] keys = console @@ -59,12 +59,6 @@ handlers = qualname = beaker propagate = 1 -[logger_pyro4] -level = DEBUG -handlers = -qualname = Pyro4 -propagate = 1 - ############## ## HANDLERS ## diff --git a/configs/development_pyro4.ini b/configs/development_pyro4.ini deleted file mode 100644 --- a/configs/development_pyro4.ini +++ /dev/null @@ -1,79 +0,0 @@ -################################################################################ -# RhodeCode VCSServer - configuration # -# # -################################################################################ - -[DEFAULT] -host = 127.0.0.1 -port = 9900 -locale = en_US.UTF-8 -# number of worker threads, this should be set based on a formula threadpool=N*6 -# where N is number of RhodeCode Enterprise workers, eg. running 2 instances -# 8 gunicorn workers each would be 2 * 8 * 6 = 96, threadpool_size = 96 -threadpool_size = 96 -timeout = 0 - -# cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -# cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, pyro4, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - -[logger_pyro4] -level = DEBUG -handlers = -qualname = Pyro4 -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/production_http.ini b/configs/production_http.ini --- a/configs/production_http.ini +++ b/configs/production_http.ini @@ -56,7 +56,7 @@ beaker.cache.repo_object.enabled = true ### LOGGING CONFIGURATION #### ################################ [loggers] -keys = root, vcsserver, pyro4, beaker +keys = root, vcsserver, beaker [handlers] keys = console @@ -83,12 +83,6 @@ handlers = qualname = beaker propagate = 1 -[logger_pyro4] -level = DEBUG -handlers = -qualname = Pyro4 -propagate = 1 - ############## ## HANDLERS ## diff --git a/configs/production_pyro4.ini b/configs/production_pyro4.ini deleted file mode 100644 --- a/configs/production_pyro4.ini +++ /dev/null @@ -1,79 +0,0 @@ -################################################################################ -# RhodeCode VCSServer - configuration # -# # -################################################################################ - -[DEFAULT] -host = 127.0.0.1 -port = 9900 -locale = en_US.UTF-8 -# number of worker threads, this should be set based on a formula threadpool=N*6 -# where N is number of RhodeCode Enterprise workers, eg. running 2 instances -# 8 gunicorn workers each would be 2 * 8 * 6 = 96, threadpool_size = 96 -threadpool_size = 96 -timeout = 0 - -# cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -# cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, pyro4, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - -[logger_pyro4] -level = DEBUG -handlers = -qualname = Pyro4 -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -67,19 +67,6 @@ license = [ pkgs.lib.licenses.mit ]; }; }; - Pyro4 = super.buildPythonPackage { - name = "Pyro4-4.41"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [serpent]; - src = fetchurl { - url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz"; - md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; WebOb = super.buildPythonPackage { name = "WebOb-1.3.1"; buildInputs = with self; []; @@ -591,25 +578,12 @@ name = "rhodecode-vcsserver-4.8.0"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; doCheck = true; - propagatedBuildInputs = with self; [Beaker configobj decorator dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress Pyro4 serpent pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; + propagatedBuildInputs = with self; [Beaker configobj decorator dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; src = ./.; meta = { license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ]; }; }; - serpent = super.buildPythonPackage { - name = "serpent-1.15"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; []; - src = fetchurl { - url = "https://pypi.python.org/packages/7b/38/b2b27673a882ff2ea5871bb3e3e6b496ebbaafd1612e51990ffb158b9254/serpent-1.15.tar.gz"; - md5 = "e27b1aad5c218e16442f52abb7c7053a"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; setuptools = super.buildPythonPackage { name = "setuptools-30.1.0"; buildInputs = with self; []; diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -35,9 +35,5 @@ greenlet==0.4.10 gunicorn==19.6.0 waitress==1.0.1 -# Pyro/Deprecated TODO(Marcink): remove in 4.7 release. -Pyro4==4.41 -serpent==1.15 - ## test related requirements -r requirements_test.txt diff --git a/test.ini b/test.ini --- a/test.ini +++ b/test.ini @@ -26,7 +26,7 @@ beaker.cache.repo_object.enabled = true ### LOGGING CONFIGURATION #### ################################ [loggers] -keys = root, vcsserver, pyro4, beaker +keys = root, vcsserver, beaker [handlers] keys = console @@ -53,12 +53,6 @@ handlers = qualname = beaker propagate = 1 -[logger_pyro4] -level = DEBUG -handlers = -qualname = Pyro4 -propagate = 1 - ############## ## HANDLERS ## diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -298,7 +298,7 @@ class HgRemote(object): ctx = repo[revision] status = repo[ctx.p1().node()].status(other=ctx.node()) # object of status (odd, custom named tuple in mercurial) is not - # correctly serializable via Pyro, we make it a list, as the underling + # correctly serializable, we make it a list, as the underling # API expects this to be a list return list(status) diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -30,7 +30,6 @@ from httplib import HTTPConnection import mercurial.scmutil import mercurial.node -import Pyro4 import simplejson as json from vcsserver import exceptions @@ -68,15 +67,6 @@ class HooksDummyClient(object): return getattr(hooks, hook_name)(extras) -class HooksPyro4Client(object): - def __init__(self, hooks_uri): - self.hooks_uri = hooks_uri - - def __call__(self, hook_name, extras): - with Pyro4.Proxy(self.hooks_uri) as hooks: - return getattr(hooks, hook_name)(extras) - - class RemoteMessageWriter(object): """Writer base class.""" def write(message): @@ -126,11 +116,7 @@ def _handle_exception(result): def _get_hooks_client(extras): if 'hooks_uri' in extras: protocol = extras.get('hooks_protocol') - return ( - HooksHttpClient(extras['hooks_uri']) - if protocol == 'http' - else HooksPyro4Client(extras['hooks_uri']) - ) + return HooksHttpClient(extras['hooks_uri']) else: return HooksDummyClient(extras['hooks_module']) diff --git a/vcsserver/main.py b/vcsserver/main.py deleted file mode 100644 --- a/vcsserver/main.py +++ /dev/null @@ -1,508 +0,0 @@ -# RhodeCode VCSServer provides access to different vcs backends via network. -# Copyright (C) 2014-2017 RodeCode GmbH -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -import atexit -import locale -import logging -import optparse -import os -import textwrap -import threading -import sys - -import configobj -import Pyro4 -from beaker.cache import CacheManager -from beaker.util import parse_cache_config_options - -try: - from vcsserver.git import GitFactory, GitRemote -except ImportError: - GitFactory = None - GitRemote = None -try: - from vcsserver.hg import MercurialFactory, HgRemote -except ImportError: - MercurialFactory = None - HgRemote = None -try: - from vcsserver.svn import SubversionFactory, SvnRemote -except ImportError: - SubversionFactory = None - SvnRemote = None - -from server import VcsServer -from vcsserver import hgpatches, remote_wsgi, settings -from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub - -log = logging.getLogger(__name__) - -HERE = os.path.dirname(os.path.abspath(__file__)) -SERVER_RUNNING_FILE = None - - -# HOOKS - inspired by gunicorn # - -def when_ready(server): - """ - Called just after the server is started. - """ - - def _remove_server_running_file(): - if os.path.isfile(SERVER_RUNNING_FILE): - os.remove(SERVER_RUNNING_FILE) - - # top up to match to level location - if SERVER_RUNNING_FILE: - with open(SERVER_RUNNING_FILE, 'wb') as f: - f.write(str(os.getpid())) - # register cleanup of that file when server exits - atexit.register(_remove_server_running_file) - - -class LazyWriter(object): - """ - File-like object that opens a file lazily when it is first written - to. - """ - - def __init__(self, filename, mode='w'): - self.filename = filename - self.fileobj = None - self.lock = threading.Lock() - self.mode = mode - - def open(self): - if self.fileobj is None: - with self.lock: - self.fileobj = open(self.filename, self.mode) - return self.fileobj - - def close(self): - fileobj = self.fileobj - if fileobj is not None: - fileobj.close() - - def __del__(self): - self.close() - - def write(self, text): - fileobj = self.open() - fileobj.write(text) - fileobj.flush() - - def writelines(self, text): - fileobj = self.open() - fileobj.writelines(text) - fileobj.flush() - - def flush(self): - self.open().flush() - - -class Application(object): - """ - Represents the vcs server application. - - This object is responsible to initialize the application and all needed - libraries. After that it hooks together the different objects and provides - them a way to access things like configuration. - """ - - def __init__( - self, host, port=None, locale='', threadpool_size=None, - timeout=None, cache_config=None, remote_wsgi_=None): - - self.host = host - self.port = int(port) or settings.PYRO_PORT - self.threadpool_size = ( - int(threadpool_size) if threadpool_size else None) - self.locale = locale - self.timeout = timeout - self.cache_config = cache_config - self.remote_wsgi = remote_wsgi_ or remote_wsgi - - def init(self): - """ - Configure and hook together all relevant objects. - """ - self._configure_locale() - self._configure_pyro() - self._initialize_cache() - self._create_daemon_and_remote_objects(host=self.host, port=self.port) - - def run(self): - """ - Start the main loop of the application. - """ - - if hasattr(os, 'getpid'): - log.info('Starting %s in PID %i.', __name__, os.getpid()) - else: - log.info('Starting %s.', __name__) - if SERVER_RUNNING_FILE: - log.info('PID file written as %s', SERVER_RUNNING_FILE) - else: - log.info('No PID file written by default.') - when_ready(self) - try: - self._pyrodaemon.requestLoop( - loopCondition=lambda: not self._vcsserver._shutdown) - finally: - self._pyrodaemon.shutdown() - - def _configure_locale(self): - if self.locale: - log.info('Settings locale: `LC_ALL` to %s' % self.locale) - else: - log.info( - 'Configuring locale subsystem based on environment variables') - - try: - # If self.locale is the empty string, then the locale - # module will use the environment variables. See the - # documentation of the package `locale`. - locale.setlocale(locale.LC_ALL, self.locale) - - language_code, encoding = locale.getlocale() - log.info( - 'Locale set to language code "%s" with encoding "%s".', - language_code, encoding) - except locale.Error: - log.exception( - 'Cannot set locale, not configuring the locale system') - - def _configure_pyro(self): - if self.threadpool_size is not None: - log.info("Threadpool size set to %s", self.threadpool_size) - Pyro4.config.THREADPOOL_SIZE = self.threadpool_size - if self.timeout not in (None, 0, 0.0, '0'): - log.info("Timeout for RPC calls set to %s seconds", self.timeout) - Pyro4.config.COMMTIMEOUT = float(self.timeout) - Pyro4.config.SERIALIZER = 'pickle' - Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle') - Pyro4.config.SOCK_REUSE = True - # Uncomment the next line when you need to debug remote errors - # Pyro4.config.DETAILED_TRACEBACK = True - - def _initialize_cache(self): - cache_config = parse_cache_config_options(self.cache_config) - log.info('Initializing beaker cache: %s' % cache_config) - self.cache = CacheManager(**cache_config) - - def _create_daemon_and_remote_objects(self, host='localhost', - port=settings.PYRO_PORT): - daemon = Pyro4.Daemon(host=host, port=port) - - self._vcsserver = VcsServer() - uri = daemon.register( - self._vcsserver, objectId=settings.PYRO_VCSSERVER) - log.info("Object registered = %s", uri) - - if GitFactory and GitRemote: - git_repo_cache = self.cache.get_cache_region('git', region='repo_object') - git_factory = GitFactory(git_repo_cache) - self._git_remote = GitRemote(git_factory) - uri = daemon.register(self._git_remote, objectId=settings.PYRO_GIT) - log.info("Object registered = %s", uri) - else: - log.info("Git client import failed") - - if MercurialFactory and HgRemote: - hg_repo_cache = self.cache.get_cache_region('hg', region='repo_object') - hg_factory = MercurialFactory(hg_repo_cache) - self._hg_remote = HgRemote(hg_factory) - uri = daemon.register(self._hg_remote, objectId=settings.PYRO_HG) - log.info("Object registered = %s", uri) - else: - log.info("Mercurial client import failed") - - if SubversionFactory and SvnRemote: - svn_repo_cache = self.cache.get_cache_region('svn', region='repo_object') - svn_factory = SubversionFactory(svn_repo_cache) - self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory) - uri = daemon.register(self._svn_remote, objectId=settings.PYRO_SVN) - log.info("Object registered = %s", uri) - else: - log.info("Subversion client import failed") - - self._git_remote_wsgi = self.remote_wsgi.GitRemoteWsgi() - uri = daemon.register(self._git_remote_wsgi, - objectId=settings.PYRO_GIT_REMOTE_WSGI) - log.info("Object registered = %s", uri) - - self._hg_remote_wsgi = self.remote_wsgi.HgRemoteWsgi() - uri = daemon.register(self._hg_remote_wsgi, - objectId=settings.PYRO_HG_REMOTE_WSGI) - log.info("Object registered = %s", uri) - - self._pyrodaemon = daemon - - -class VcsServerCommand(object): - - usage = '%prog' - description = """ - Runs the VCS server - """ - default_verbosity = 1 - - parser = optparse.OptionParser( - usage, - description=textwrap.dedent(description) - ) - parser.add_option( - '--host', - type="str", - dest="host", - ) - parser.add_option( - '--port', - type="int", - dest="port" - ) - parser.add_option( - '--running-file', - dest='running_file', - metavar='RUNNING_FILE', - help="Create a running file after the server is initalized with " - "stored PID of process" - ) - parser.add_option( - '--locale', - dest='locale', - help="Allows to set the locale, e.g. en_US.UTF-8", - default="" - ) - parser.add_option( - '--log-file', - dest='log_file', - metavar='LOG_FILE', - help="Save output to the given log file (redirects stdout)" - ) - parser.add_option( - '--log-level', - dest="log_level", - metavar="LOG_LEVEL", - help="use LOG_LEVEL to set log level " - "(debug,info,warning,error,critical)" - ) - parser.add_option( - '--threadpool', - dest='threadpool_size', - type='int', - help="Set the size of the threadpool used to communicate with the " - "WSGI workers. This should be at least 6 times the number of " - "WSGI worker processes." - ) - parser.add_option( - '--timeout', - dest='timeout', - type='float', - help="Set the timeout for RPC communication in seconds." - ) - parser.add_option( - '--config', - dest='config_file', - type='string', - help="Configuration file for vcsserver." - ) - - def __init__(self, argv, quiet=False): - self.options, self.args = self.parser.parse_args(argv[1:]) - if quiet: - self.options.verbose = 0 - - def _get_file_config(self): - ini_conf = {} - conf = configobj.ConfigObj(self.options.config_file) - if 'DEFAULT' in conf: - ini_conf = conf['DEFAULT'] - - return ini_conf - - def _show_config(self, vcsserver_config): - order = [ - 'config_file', - 'host', - 'port', - 'log_file', - 'log_level', - 'locale', - 'threadpool_size', - 'timeout', - 'cache_config', - ] - - def sorter(k): - return dict([(y, x) for x, y in enumerate(order)]).get(k) - - _config = [] - for k in sorted(vcsserver_config.keys(), key=sorter): - v = vcsserver_config[k] - # construct padded key for display eg %-20s % = key: val - k_formatted = ('%-'+str(len(max(order, key=len))+1)+'s') % (k+':') - _config.append(' * %s %s' % (k_formatted, v)) - log.info('\n[vcsserver configuration]:\n'+'\n'.join(_config)) - - def _get_vcsserver_configuration(self): - _defaults = { - 'config_file': None, - 'git_path': 'git', - 'host': 'localhost', - 'port': settings.PYRO_PORT, - 'log_file': None, - 'log_level': 'debug', - 'locale': None, - 'threadpool_size': 16, - 'timeout': None, - - # Development support - 'dev.use_echo_app': False, - - # caches, baker style config - 'beaker.cache.regions': 'repo_object', - 'beaker.cache.repo_object.expire': '10', - 'beaker.cache.repo_object.type': 'memory', - } - config = {} - config.update(_defaults) - # overwrite defaults with one loaded from file - config.update(self._get_file_config()) - - # overwrite with self.option which has the top priority - for k, v in self.options.__dict__.items(): - if v or v == 0: - config[k] = v - - # clear all "extra" keys if they are somehow passed, - # we only want defaults, so any extra stuff from self.options is cleared - # except beaker stuff which needs to be dynamic - for k in [k for k in config.copy().keys() if not k.startswith('beaker.cache.')]: - if k not in _defaults: - del config[k] - - # group together the cache into one key. - # Needed further for beaker lib configuration - _k = {} - for k in [k for k in config.copy() if k.startswith('beaker.cache.')]: - _k[k] = config.pop(k) - config['cache_config'] = _k - - return config - - def out(self, msg): # pragma: no cover - if self.options.verbose > 0: - print(msg) - - def run(self): # pragma: no cover - vcsserver_config = self._get_vcsserver_configuration() - - # Ensure the log file is writeable - if vcsserver_config['log_file']: - stdout_log = self._configure_logfile() - else: - stdout_log = None - - # set PID file with running lock - if self.options.running_file: - global SERVER_RUNNING_FILE - SERVER_RUNNING_FILE = self.options.running_file - - # configure logging, and logging based on configuration file - self._configure_logging(level=vcsserver_config['log_level'], - stream=stdout_log) - if self.options.config_file: - if not os.path.isfile(self.options.config_file): - raise OSError('File %s does not exist' % - self.options.config_file) - - self._configure_file_logging(self.options.config_file) - - self._configure_settings(vcsserver_config) - - # display current configuration of vcsserver - self._show_config(vcsserver_config) - - if not vcsserver_config['dev.use_echo_app']: - remote_wsgi_mod = remote_wsgi - else: - log.warning("Using EchoApp for VCS endpoints.") - remote_wsgi_mod = remote_wsgi_stub - - app = Application( - host=vcsserver_config['host'], - port=vcsserver_config['port'], - locale=vcsserver_config['locale'], - threadpool_size=vcsserver_config['threadpool_size'], - timeout=vcsserver_config['timeout'], - cache_config=vcsserver_config['cache_config'], - remote_wsgi_=remote_wsgi_mod) - app.init() - app.run() - - def _configure_logging(self, level, stream=None): - _format = ( - '%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s') - levels = { - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL, - } - try: - level = levels[level] - except KeyError: - raise AttributeError( - 'Invalid log level please use one of %s' % (levels.keys(),)) - logging.basicConfig(format=_format, stream=stream, level=level) - logging.getLogger('Pyro4').setLevel(level) - - def _configure_file_logging(self, config): - import logging.config - try: - logging.config.fileConfig(config) - except Exception as e: - log.warning('Failed to configure logging based on given ' - 'config file. Error: %s' % e) - - def _configure_logfile(self): - try: - writeable_log_file = open(self.options.log_file, 'a') - except IOError as ioe: - msg = 'Error: Unable to write to log file: %s' % ioe - raise ValueError(msg) - writeable_log_file.close() - stdout_log = LazyWriter(self.options.log_file, 'a') - sys.stdout = stdout_log - sys.stderr = stdout_log - return stdout_log - - def _configure_settings(self, config): - """ - Configure the settings module based on the given `config`. - """ - settings.GIT_EXECUTABLE = config['git_path'] - - -def main(argv=sys.argv, quiet=False): - if MercurialFactory: - hgpatches.patch_largefiles_capabilities() - hgpatches.patch_subrepo_type_mapping() - command = VcsServerCommand(argv, quiet=quiet) - return command.run() diff --git a/vcsserver/settings.py b/vcsserver/settings.py --- a/vcsserver/settings.py +++ b/vcsserver/settings.py @@ -15,16 +15,5 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -PYRO_PORT = 9900 - -PYRO_GIT = 'git_remote' -PYRO_HG = 'hg_remote' -PYRO_SVN = 'svn_remote' -PYRO_VCSSERVER = 'vcs_server' -PYRO_GIT_REMOTE_WSGI = 'git_remote_wsgi' -PYRO_HG_REMOTE_WSGI = 'hg_remote_wsgi' - WIRE_ENCODING = 'UTF-8' - GIT_EXECUTABLE = 'git' diff --git a/vcsserver/tests/test_hooks.py b/vcsserver/tests/test_hooks.py --- a/vcsserver/tests/test_hooks.py +++ b/vcsserver/tests/test_hooks.py @@ -29,46 +29,6 @@ import simplejson as json from vcsserver import hooks -class HooksStub(object): - """ - Simulates a Proy4.Proxy object. - - Will always return `result`, no matter which hook has been called on it. - """ - - def __init__(self, result): - self._result = result - - def __call__(self, hooks_uri): - return self - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - pass - - def __getattr__(self, name): - return mock.Mock(return_value=self._result) - - -@contextlib.contextmanager -def mock_hook_response( - status=0, output='', exception=None, exception_args=None): - response = { - 'status': status, - 'output': output, - } - if exception: - response.update({ - 'exception': exception, - 'exception_args': exception_args, - }) - - with mock.patch('Pyro4.Proxy', HooksStub(response)): - yield - - def get_hg_ui(extras=None): """Create a Config object with a valid RC_SCM_DATA entry.""" extras = extras or {} @@ -89,126 +49,6 @@ def get_hg_ui(extras=None): return hg_ui -def test_call_hook_no_error(capsys): - extras = { - 'hooks_uri': 'fake_hook_uri', - } - expected_output = 'My mock outptut' - writer = mock.Mock() - - with mock_hook_response(status=1, output=expected_output): - hooks._call_hook('hook_name', extras, writer) - - out, err = capsys.readouterr() - - writer.write.assert_called_with(expected_output) - assert err == '' - - -def test_call_hook_with_exception(capsys): - extras = { - 'hooks_uri': 'fake_hook_uri', - } - expected_output = 'My mock outptut' - writer = mock.Mock() - - with mock_hook_response(status=1, output=expected_output, - exception='TypeError', - exception_args=('Mock exception', )): - with pytest.raises(Exception) as excinfo: - hooks._call_hook('hook_name', extras, writer) - - assert excinfo.type == Exception - assert 'Mock exception' in str(excinfo.value) - - out, err = capsys.readouterr() - - writer.write.assert_called_with(expected_output) - assert err == '' - - -def test_call_hook_with_locked_exception(capsys): - extras = { - 'hooks_uri': 'fake_hook_uri', - } - expected_output = 'My mock outptut' - writer = mock.Mock() - - with mock_hook_response(status=1, output=expected_output, - exception='HTTPLockedRC', - exception_args=('message',)): - with pytest.raises(Exception) as excinfo: - hooks._call_hook('hook_name', extras, writer) - - assert excinfo.value._vcs_kind == 'repo_locked' - assert 'message' == str(excinfo.value) - - out, err = capsys.readouterr() - - writer.write.assert_called_with(expected_output) - assert err == '' - - -def test_call_hook_with_stdout(): - extras = { - 'hooks_uri': 'fake_hook_uri', - } - expected_output = 'My mock outptut' - - stdout = io.BytesIO() - with mock_hook_response(status=1, output=expected_output): - hooks._call_hook('hook_name', extras, stdout) - - assert stdout.getvalue() == expected_output - - -def test_repo_size(): - hg_ui = get_hg_ui() - - with mock_hook_response(status=1): - assert hooks.repo_size(hg_ui, None) == 1 - - -def test_pre_pull(): - hg_ui = get_hg_ui() - - with mock_hook_response(status=1): - assert hooks.pre_pull(hg_ui, None) == 1 - - -def test_post_pull(): - hg_ui = get_hg_ui() - - with mock_hook_response(status=1): - assert hooks.post_pull(hg_ui, None) == 1 - - -def test_pre_push(): - hg_ui = get_hg_ui() - - with mock_hook_response(status=1): - assert hooks.pre_push(hg_ui, None) == 1 - - -def test_post_push(): - hg_ui = get_hg_ui() - - with mock_hook_response(status=1): - with mock.patch('vcsserver.hooks._rev_range_hash', return_value=[]): - assert hooks.post_push(hg_ui, None, None) == 1 - - -def test_git_pre_receive(): - extras = { - 'hooks': ['push'], - 'hooks_uri': 'fake_hook_uri', - } - with mock_hook_response(status=1): - response = hooks.git_pre_receive(None, None, - {'RC_SCM_DATA': json.dumps(extras)}) - assert response == 1 - - def test_git_pre_receive_is_disabled(): extras = {'hooks': ['pull']} response = hooks.git_pre_receive(None, None, @@ -217,18 +57,6 @@ def test_git_pre_receive_is_disabled(): assert response == 0 -def test_git_post_receive_no_subprocess_call(): - extras = { - 'hooks': ['push'], - 'hooks_uri': 'fake_hook_uri', - } - # Setting revision_lines to '' avoid all subprocess_calls - with mock_hook_response(status=1): - response = hooks.git_post_receive(None, '', - {'RC_SCM_DATA': json.dumps(extras)}) - assert response == 1 - - def test_git_post_receive_is_disabled(): extras = {'hooks': ['pull']} response = hooks.git_post_receive(None, '', @@ -279,122 +107,16 @@ def test_repo_size_exception_does_not_af assert result == status -@mock.patch('vcsserver.hooks._run_command') -def test_git_post_receive_first_commit_sub_branch(cmd_mock): - def cmd_mock_returns(args): - if args == ['git', 'show', 'HEAD']: - raise - if args == ['git', 'for-each-ref', '--format=%(refname)', - 'refs/heads/*']: - return 'refs/heads/test-branch2/sub-branch' - if args == ['git', 'log', '--reverse', '--pretty=format:%H', '--', - '9695eef57205c17566a3ae543be187759b310bb7', '--not', - 'refs/heads/test-branch2/sub-branch']: - return '' - - cmd_mock.side_effect = cmd_mock_returns - - extras = { - 'hooks': ['push'], - 'hooks_uri': 'fake_hook_uri' - } - rev_lines = ['0000000000000000000000000000000000000000 ' - '9695eef57205c17566a3ae543be187759b310bb7 ' - 'refs/heads/feature/sub-branch\n'] - with mock_hook_response(status=0): - response = hooks.git_post_receive(None, rev_lines, - {'RC_SCM_DATA': json.dumps(extras)}) - - calls = [ - mock.call(['git', 'show', 'HEAD']), - mock.call(['git', 'symbolic-ref', 'HEAD', - 'refs/heads/feature/sub-branch']), - ] - cmd_mock.assert_has_calls(calls, any_order=True) - assert response == 0 - - -@mock.patch('vcsserver.hooks._run_command') -def test_git_post_receive_first_commit_revs(cmd_mock): - extras = { - 'hooks': ['push'], - 'hooks_uri': 'fake_hook_uri' - } - rev_lines = [ - '0000000000000000000000000000000000000000 ' - '9695eef57205c17566a3ae543be187759b310bb7 refs/heads/master\n'] - with mock_hook_response(status=0): - response = hooks.git_post_receive( - None, rev_lines, {'RC_SCM_DATA': json.dumps(extras)}) - - calls = [ - mock.call(['git', 'show', 'HEAD']), - mock.call(['git', 'for-each-ref', '--format=%(refname)', - 'refs/heads/*']), - mock.call(['git', 'log', '--reverse', '--pretty=format:%H', - '--', '9695eef57205c17566a3ae543be187759b310bb7', '--not', - '']) - ] - cmd_mock.assert_has_calls(calls, any_order=True) - - assert response == 0 - - -def test_git_pre_pull(): - extras = { - 'hooks': ['pull'], - 'hooks_uri': 'fake_hook_uri', - } - with mock_hook_response(status=1, output='foo'): - assert hooks.git_pre_pull(extras) == hooks.HookResponse(1, 'foo') - - -def test_git_pre_pull_exception_is_caught(): - extras = { - 'hooks': ['pull'], - 'hooks_uri': 'fake_hook_uri', - } - with mock_hook_response(status=2, exception=Exception('foo')): - assert hooks.git_pre_pull(extras).status == 128 - - def test_git_pre_pull_is_disabled(): assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '') -def test_git_post_pull(): - extras = { - 'hooks': ['pull'], - 'hooks_uri': 'fake_hook_uri', - } - with mock_hook_response(status=1, output='foo'): - assert hooks.git_post_pull(extras) == hooks.HookResponse(1, 'foo') - - -def test_git_post_pull_exception_is_caught(): - extras = { - 'hooks': ['pull'], - 'hooks_uri': 'fake_hook_uri', - } - with mock_hook_response(status=2, exception='Exception', - exception_args=('foo',)): - assert hooks.git_post_pull(extras).status == 128 - - def test_git_post_pull_is_disabled(): assert ( hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')) class TestGetHooksClient(object): - def test_returns_pyro_client_when_protocol_matches(self): - hooks_uri = 'localhost:8000' - result = hooks._get_hooks_client({ - 'hooks_uri': hooks_uri, - 'hooks_protocol': 'pyro4' - }) - assert isinstance(result, hooks.HooksPyro4Client) - assert result.hooks_uri == hooks_uri def test_returns_http_client_when_protocol_matches(self): hooks_uri = 'localhost:8000' @@ -405,14 +127,6 @@ class TestGetHooksClient(object): assert isinstance(result, hooks.HooksHttpClient) assert result.hooks_uri == hooks_uri - def test_returns_pyro4_client_when_no_protocol_is_specified(self): - hooks_uri = 'localhost:8000' - result = hooks._get_hooks_client({ - 'hooks_uri': hooks_uri - }) - assert isinstance(result, hooks.HooksPyro4Client) - assert result.hooks_uri == hooks_uri - def test_returns_dummy_client_when_hooks_uri_not_specified(self): fake_module = mock.Mock() import_patcher = mock.patch.object( @@ -487,30 +201,6 @@ class TestHooksDummyClient(object): assert result == hooks_module.Hooks().__enter__().post_push() -class TestHooksPyro4Client(object): - def test_init_sets_hooks_uri(self): - uri = 'localhost:3000' - client = hooks.HooksPyro4Client(uri) - assert client.hooks_uri == uri - - def test_call_returns_hook_value(self): - hooks_uri = 'localhost:3000' - client = hooks.HooksPyro4Client(hooks_uri) - hooks_module = mock.Mock() - context_manager = mock.MagicMock() - context_manager.__enter__.return_value = hooks_module - pyro4_patcher = mock.patch.object( - hooks.Pyro4, 'Proxy', return_value=context_manager) - extras = { - 'test': 'test' - } - with pyro4_patcher as pyro4_mock: - result = client('post_push', extras) - pyro4_mock.assert_called_once_with(hooks_uri) - hooks_module.post_push.assert_called_once_with(extras) - assert result == hooks_module.post_push.return_value - - @pytest.fixture def http_mirror(request): server = MirrorHttpServer() diff --git a/vcsserver/tests/test_main.py b/vcsserver/tests/test_main_http.py rename from vcsserver/tests/test_main.py rename to vcsserver/tests/test_main_http.py --- a/vcsserver/tests/test_main.py +++ b/vcsserver/tests/test_main_http.py @@ -18,24 +18,24 @@ import mock import pytest -from vcsserver import main +from vcsserver import http_main from vcsserver.base import obfuscate_qs -@mock.patch('vcsserver.main.VcsServerCommand', mock.Mock()) +@mock.patch('vcsserver.http_main.VCS', mock.Mock()) @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities') def test_applies_largefiles_patch(patch_largefiles_capabilities): - main.main([]) + http_main.main([]) patch_largefiles_capabilities.assert_called_once_with() -@mock.patch('vcsserver.main.VcsServerCommand', mock.Mock()) -@mock.patch('vcsserver.main.MercurialFactory', None) +@mock.patch('vcsserver.http_main.VCS', mock.Mock()) +@mock.patch('vcsserver.http_main.MercurialFactory', None) @mock.patch( 'vcsserver.hgpatches.patch_largefiles_capabilities', mock.Mock(side_effect=Exception("Must not be called"))) def test_applies_largefiles_patch_only_if_mercurial_is_available(): - main.main([]) + http_main.main([]) @pytest.mark.parametrize('given, expected', [ diff --git a/vcsserver/tests/test_vcsserver.py b/vcsserver/tests/test_vcsserver.py deleted file mode 100644 --- a/vcsserver/tests/test_vcsserver.py +++ /dev/null @@ -1,132 +0,0 @@ -# RhodeCode VCSServer provides access to different vcs backends via network. -# Copyright (C) 2014-2017 RodeCode GmbH -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -import subprocess -import StringIO -import time - -import pytest - -from fixture import ContextINI - - -@pytest.mark.parametrize("arguments, expected_texts", [ - (['--threadpool=192'], [ - 'threadpool_size: 192', - 'worker pool of size 192 created', - 'Threadpool size set to 192']), - (['--locale=fake'], [ - 'Cannot set locale, not configuring the locale system']), - (['--timeout=5'], [ - 'Timeout for RPC calls set to 5.0 seconds']), - (['--log-level=info'], [ - 'log_level: info']), - (['--port={port}'], [ - 'port: {port}', - 'created daemon on localhost:{port}']), - (['--host=127.0.0.1', '--port={port}'], [ - 'port: {port}', - 'host: 127.0.0.1', - 'created daemon on 127.0.0.1:{port}']), - (['--config=/bad/file'], ['OSError: File /bad/file does not exist']), -]) -def test_vcsserver_calls(arguments, expected_texts, vcsserver_port): - port_argument = '--port={port}' - if port_argument not in arguments: - arguments.append(port_argument) - arguments = _replace_port(arguments, vcsserver_port) - expected_texts = _replace_port(expected_texts, vcsserver_port) - output = call_vcs_server_with_arguments(arguments) - for text in expected_texts: - assert text in output - - -def _replace_port(values, port): - return [value.format(port=port) for value in values] - - -def test_vcsserver_with_config(vcsserver_port): - ini_def = [ - {'DEFAULT': {'host': '127.0.0.1'}}, - {'DEFAULT': {'threadpool_size': '111'}}, - {'DEFAULT': {'port': vcsserver_port}}, - ] - - with ContextINI('test.ini', ini_def) as new_test_ini_path: - output = call_vcs_server_with_arguments( - ['--config=' + new_test_ini_path]) - - expected_texts = [ - 'host: 127.0.0.1', - 'Threadpool size set to 111', - ] - for text in expected_texts: - assert text in output - - -def test_vcsserver_with_config_cli_overwrite(vcsserver_port): - ini_def = [ - {'DEFAULT': {'host': '127.0.0.1'}}, - {'DEFAULT': {'port': vcsserver_port}}, - {'DEFAULT': {'threadpool_size': '111'}}, - {'DEFAULT': {'timeout': '0'}}, - ] - with ContextINI('test.ini', ini_def) as new_test_ini_path: - output = call_vcs_server_with_arguments([ - '--config=' + new_test_ini_path, - '--host=128.0.0.1', - '--threadpool=256', - '--timeout=5']) - expected_texts = [ - 'host: 128.0.0.1', - 'Threadpool size set to 256', - 'Timeout for RPC calls set to 5.0 seconds', - ] - for text in expected_texts: - assert text in output - - -def call_vcs_server_with_arguments(args): - vcs = subprocess.Popen( - ["vcsserver"] + args, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - output = read_output_until( - "Starting vcsserver.main", vcs.stdout) - vcs.terminate() - return output - - -def call_vcs_server_with_non_existing_config_file(args): - vcs = subprocess.Popen( - ["vcsserver", "--config=/tmp/bad"] + args, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = read_output_until( - "Starting vcsserver.main", vcs.stdout) - vcs.terminate() - return output - - -def read_output_until(expected, source, timeout=5): - ts = time.time() - buf = StringIO.StringIO() - while time.time() - ts < timeout: - line = source.readline() - buf.write(line) - if expected in line: - break - return buf.getvalue()