# HG changeset patch # User Marcin Kuzminski # Date 2018-07-13 15:31:06 # Node ID 80e9ab606a0507df4754f322f908323fab4d941b # Parent 439940e997fc92525ea55173eb24974b3a25e593 caches: replaced beaker with dogpile cache. diff --git a/configs/development.ini b/configs/development.ini old mode 120000 new mode 100644 --- a/configs/development.ini +++ b/configs/development.ini @@ -1,1 +1,79 @@ -development_http.ini \ No newline at end of file +################################################################################ +# RhodeCode VCSServer with HTTP Backend - configuration # +# # +################################################################################ + + +[server:main] +## COMMON ## +host = 0.0.0.0 +port = 9900 + +use = egg:waitress#main + + +[app:main] +use = egg:rhodecode-vcsserver + +pyramid.default_locale_name = en +pyramid.includes = + +## default locale used by VCS systems +locale = en_US.UTF-8 + + +## path to binaries for vcsserver, it should be set by the installer +## at installation time, e.g /home/user/vcsserver-1/profile/bin +core.binary_dir = "" + +## cache region for storing repo_objects cache +rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru +## cache auto-expires after N seconds +rc_cache.repo_object.expiration_time = 300 +## max size of LRU, old values will be discarded if the size of cache reaches max_size +rc_cache.repo_object.max_size = 100 + + +################################ +### LOGGING CONFIGURATION #### +################################ +[loggers] +keys = root, vcsserver + +[handlers] +keys = console + +[formatters] +keys = generic + +############# +## LOGGERS ## +############# +[logger_root] +level = NOTSET +handlers = console + +[logger_vcsserver] +level = DEBUG +handlers = +qualname = vcsserver +propagate = 1 + + +############## +## HANDLERS ## +############## + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +################ +## FORMATTERS ## +################ + +[formatter_generic] +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/development_http.ini b/configs/development_http.ini deleted file mode 100644 --- a/configs/development_http.ini +++ /dev/null @@ -1,88 +0,0 @@ -################################################################################ -# RhodeCode VCSServer with HTTP Backend - configuration # -# # -################################################################################ - - -[server:main] -## COMMON ## -host = 0.0.0.0 -port = 9900 - - -use = egg:waitress#main - - -[app:main] -use = egg:rhodecode-vcsserver - -pyramid.default_locale_name = en -pyramid.includes = - -## default locale used by VCS systems -locale = en_US.UTF-8 - -## cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -## cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - -## path to binaries for vcsserver, it should be set by the installer -## at installation time, e.g /home/user/vcsserver-1/profile/bin -core.binary_dir = "" - - - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/production.ini b/configs/production.ini old mode 120000 new mode 100644 --- a/configs/production.ini +++ b/configs/production.ini @@ -1,1 +1,100 @@ -production_http.ini \ No newline at end of file +################################################################################ +# RhodeCode VCSServer with HTTP Backend - configuration # +# # +################################################################################ + + +[server:main] +## COMMON ## +host = 127.0.0.1 +port = 9900 + + +########################## +## GUNICORN WSGI SERVER ## +########################## +## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini +use = egg:gunicorn#main +## Sets the number of process workers. Recommended +## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers +workers = 2 +## process name +proc_name = rhodecode_vcsserver +## type of worker class, currently `sync` is the only option allowed. +worker_class = sync +## The maximum number of simultaneous clients. Valid only for Gevent +#worker_connections = 10 +## max number of requests that worker will handle before being gracefully +## restarted, could prevent memory leaks +max_requests = 1000 +max_requests_jitter = 30 +## amount of time a worker can spend with handling a request before it +## gets killed and restarted. Set to 6hrs +timeout = 21600 + + +[app:main] +use = egg:rhodecode-vcsserver + +pyramid.default_locale_name = en +pyramid.includes = + +## default locale used by VCS systems +locale = en_US.UTF-8 + + +## path to binaries for vcsserver, it should be set by the installer +## at installation time, e.g /home/user/vcsserver-1/profile/bin +core.binary_dir = "" + +## cache region for storing repo_objects cache +rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru +## cache auto-expires after N seconds +rc_cache.repo_object.expiration_time = 300 +## max size of LRU, old values will be discarded if the size of cache reaches max_size +rc_cache.repo_object.max_size = 100 + + +################################ +### LOGGING CONFIGURATION #### +################################ +[loggers] +keys = root, vcsserver + +[handlers] +keys = console + +[formatters] +keys = generic + +############# +## LOGGERS ## +############# +[logger_root] +level = NOTSET +handlers = console + +[logger_vcsserver] +level = DEBUG +handlers = +qualname = vcsserver +propagate = 1 + + +############## +## HANDLERS ## +############## + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +################ +## FORMATTERS ## +################ + +[formatter_generic] +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/production_http.ini b/configs/production_http.ini deleted file mode 100644 --- a/configs/production_http.ini +++ /dev/null @@ -1,106 +0,0 @@ -################################################################################ -# RhodeCode VCSServer with HTTP Backend - configuration # -# # -################################################################################ - - -[server:main] -## COMMON ## -host = 127.0.0.1 -port = 9900 - - -########################## -## GUNICORN WSGI SERVER ## -########################## -## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini -use = egg:gunicorn#main -## Sets the number of process workers. Recommended -## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers -workers = 2 -## process name -proc_name = rhodecode_vcsserver -## type of worker class, currently `sync` is the only option allowed. -worker_class = sync -## The maximum number of simultaneous clients. Valid only for Gevent -#worker_connections = 10 -## max number of requests that worker will handle before being gracefully -## restarted, could prevent memory leaks -max_requests = 1000 -max_requests_jitter = 30 -## amount of time a worker can spend with handling a request before it -## gets killed and restarted. Set to 6hrs -timeout = 21600 - - -[app:main] -use = egg:rhodecode-vcsserver - -pyramid.default_locale_name = en -pyramid.includes = - -## default locale used by VCS systems -locale = en_US.UTF-8 - -## cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -## cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - -## path to binaries for vcsserver, it should be set by the installer -## at installation time, e.g /home/user/vcsserver-1/profile/bin -core.binary_dir = "" - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/pkgs/patch_beaker/patch-beaker-lock-func-debug.diff b/pkgs/patch_beaker/patch-beaker-lock-func-debug.diff deleted file mode 100644 --- a/pkgs/patch_beaker/patch-beaker-lock-func-debug.diff +++ /dev/null @@ -1,20 +0,0 @@ -diff -rup Beaker-1.9.1-orig/beaker/container.py Beaker-1.9.1/beaker/container.py ---- Beaker-1.9.1-orig/beaker/container.py 2018-04-10 10:23:04.000000000 +0200 -+++ Beaker-1.9.1/beaker/container.py 2018-04-10 10:23:34.000000000 +0200 -@@ -353,13 +353,13 @@ class Value(object): - debug("get_value returning old value while new one is created") - return value - else: -- debug("lock_creatfunc (didnt wait)") -+ debug("lock_creatfunc `%s` (didnt wait)", self.createfunc.__name__) - has_createlock = True - - if not has_createlock: -- debug("lock_createfunc (waiting)") -+ debug("lock_createfunc `%s` (waiting)", self.createfunc.__name__) - creation_lock.acquire() -- debug("lock_createfunc (waited)") -+ debug("lock_createfunc `%s` (waited)", self.createfunc.__name__) - - try: - # see if someone created the value already diff --git a/pkgs/patch_beaker/patch-beaker-metadata-reuse.diff b/pkgs/patch_beaker/patch-beaker-metadata-reuse.diff deleted file mode 100644 --- a/pkgs/patch_beaker/patch-beaker-metadata-reuse.diff +++ /dev/null @@ -1,13 +0,0 @@ -diff -rup Beaker-1.9.1-orig/beaker/ext/database.py Beaker-1.9.1/beaker/ext/database.py ---- Beaker-1.9.1-orig/beaker/ext/database.py 2018-05-22 18:22:34.802619619 +0200 -+++ Beaker-1.9.1/beaker/ext/database.py 2018-05-22 17:07:14.048335196 +0200 -@@ -91,7 +91,8 @@ class DatabaseNamespaceManager(OpenResou - sa.Column('created', types.DateTime, nullable=False), - sa.Column('data', types.PickleType, nullable=False), - sa.UniqueConstraint('namespace'), -- schema=schema_name if schema_name else meta.schema -+ schema=schema_name if schema_name else meta.schema, -+ extend_existing=True - ) - cache.create(checkfirst=True) - return cache diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -15,13 +15,6 @@ in self: super: { - "beaker" = super."beaker".override (attrs: { - patches = [ - ./patch_beaker/patch-beaker-lock-func-debug.diff - ./patch_beaker/patch-beaker-metadata-reuse.diff - ]; - }); - "gevent" = super."gevent".override (attrs: { propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ # NOTE: (marcink) odd requirements from gevent aren't set properly, diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -37,20 +37,6 @@ self: super: { license = [ pkgs.lib.licenses.mit ]; }; }; - "beaker" = super.buildPythonPackage { - name = "beaker-1.9.1"; - doCheck = false; - propagatedBuildInputs = [ - self."funcsigs" - ]; - src = fetchurl { - url = "https://files.pythonhosted.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz"; - sha256 = "08arsn61r255lhz6hcpn2lsiqpg30clla805ysx06wmbhvb6w9rj"; - }; - meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; - }; - }; "beautifulsoup4" = super.buildPythonPackage { name = "beautifulsoup4-4.6.0"; doCheck = false; @@ -112,6 +98,28 @@ self: super: { license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ]; }; }; + "dogpile.cache" = super.buildPythonPackage { + name = "dogpile.cache-0.6.6"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/48/ca/604154d835c3668efb8a31bd979b0ea4bf39c2934a40ffecc0662296cb51/dogpile.cache-0.6.6.tar.gz"; + sha256 = "1h8n1lxd4l2qvahfkiinljkqz7pww7w3sgag0j8j9ixbl2h4wk84"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + "dogpile.core" = super.buildPythonPackage { + name = "dogpile.core-0.4.1"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz"; + sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; "dulwich" = super.buildPythonPackage { name = "dulwich-0.13.0"; doCheck = false; @@ -229,21 +237,6 @@ self: super: { license = [ pkgs.lib.licenses.mit ]; }; }; - "infrae.cache" = super.buildPythonPackage { - name = "infrae.cache-1.0.1"; - doCheck = false; - propagatedBuildInputs = [ - self."beaker" - self."repoze.lru" - ]; - src = fetchurl { - url = "https://files.pythonhosted.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz"; - sha256 = "1dvqsjn8vw253wz9d1pz17j79mf4bs53dvp2qxck2qdp1am1njw4"; - }; - meta = { - license = [ pkgs.lib.licenses.zpl21 ]; - }; - }; "ipdb" = super.buildPythonPackage { name = "ipdb-0.11"; doCheck = false; @@ -294,6 +287,17 @@ self: super: { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; + "lru-dict" = super.buildPythonPackage { + name = "lru-dict-1.1.6"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/00/a5/32ed6e10246cd341ca8cc205acea5d208e4053f48a4dced2b1b31d45ba3f/lru-dict-1.1.6.tar.gz"; + sha256 = "1k2lhd4dpl6xa6iialbwx4l6bkdzxmzhygms39pvf19x1rk5fm1n"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; "mako" = super.buildPythonPackage { name = "mako-1.0.7"; doCheck = false; @@ -680,13 +684,14 @@ self: super: { ]; doCheck = true; propagatedBuildInputs = [ - self."beaker" self."configobj" + self."dogpile.cache" + self."dogpile.core" self."decorator" self."dulwich" self."hgsubversion" self."hg-evolve" - self."infrae.cache" + self."lru-dict" self."mako" self."markupsafe" self."mercurial" diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,13 @@ ## dependencies -beaker==1.9.1 configobj==5.0.6 +dogpile.cache==0.6.6 +dogpile.core==0.4.1 decorator==4.1.2 dulwich==0.13.0 hgsubversion==1.9.2 hg-evolve==8.0.1 -infrae.cache==1.0.1 +lru-dict==1.1.6 mako==1.0.7 markupsafe==1.0.0 mercurial==4.6.1 diff --git a/vcsserver/base.py b/vcsserver/base.py --- a/vcsserver/base.py +++ b/vcsserver/base.py @@ -20,6 +20,7 @@ import traceback import logging import urlparse +from vcsserver.lib.rc_cache import region_meta log = logging.getLogger(__name__) @@ -30,9 +31,10 @@ class RepoFactory(object): It provides internal caching of the `repo` object based on the :term:`call context`. """ + repo_type = None - def __init__(self, repo_cache): - self._cache = repo_cache + def __init__(self): + self._cache_region = region_meta.dogpile_cache_regions['repo_object'] def _create_config(self, path, config): config = {} @@ -48,26 +50,19 @@ class RepoFactory(object): Uses internally the low level beaker API since the decorators introduce significant overhead. """ - def create_new_repo(): + region = self._cache_region + context = wire.get('context', None) + repo_path = wire.get('path', '') + context_uid = '{}'.format(context) + cache = wire.get('cache', True) + cache_on = context and cache + + @region.conditional_cache_on_arguments(condition=cache_on) + def create_new_repo(_repo_type, _repo_path, _context_uid): return self._create_repo(wire, create) - return self._repo(wire, create_new_repo) - - def _repo(self, wire, createfunc): - context = wire.get('context', None) - cache = wire.get('cache', True) - - if context and cache: - cache_key = (context, wire['path']) - log.debug( - 'FETCH %s@%s repo object from cache. Context: %s', - self.__class__.__name__, wire['path'], context) - return self._cache.get(key=cache_key, createfunc=createfunc) - else: - log.debug( - 'INIT %s@%s repo object based on wire %s. Context: %s', - self.__class__.__name__, wire['path'], wire, context) - return createfunc() + repo = create_new_repo(self.repo_type, repo_path, context_uid) + return repo def obfuscate_qs(query_string): diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -87,6 +87,7 @@ class Repo(DulwichRepo): class GitFactory(RepoFactory): + repo_type = 'git' def _create_repo(self, wire, create): repo_path = str_to_dulwich(wire['path']) diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -93,6 +93,7 @@ def reraise_safe_exceptions(func): class MercurialFactory(RepoFactory): + repo_type = 'hg' def _create_config(self, config, hooks=True): if not hooks: diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -26,9 +26,8 @@ from itertools import chain import simplejson as json import msgpack -from beaker.cache import CacheManager -from beaker.util import parse_cache_config_options from pyramid.config import Configurator +from pyramid.settings import asbool, aslist from pyramid.wsgi import wsgiapp from pyramid.compat import configparser @@ -65,48 +64,60 @@ def _is_request_chunked(environ): return stream +def _int_setting(settings, name, default): + settings[name] = int(settings.get(name, default)) + + +def _bool_setting(settings, name, default): + input_val = settings.get(name, default) + if isinstance(input_val, unicode): + input_val = input_val.encode('utf8') + settings[name] = asbool(input_val) + + +def _list_setting(settings, name, default): + raw_value = settings.get(name, default) + + # Otherwise we assume it uses pyramids space/newline separation. + settings[name] = aslist(raw_value) + + +def _string_setting(settings, name, default, lower=True): + value = settings.get(name, default) + if lower: + value = value.lower() + settings[name] = value + + class VCS(object): def __init__(self, locale=None, cache_config=None): self.locale = locale self.cache_config = cache_config self._configure_locale() - self._initialize_cache() if GitFactory and GitRemote: - git_repo_cache = self.cache.get_cache_region( - 'git', region='repo_object') - git_factory = GitFactory(git_repo_cache) + git_factory = GitFactory() self._git_remote = GitRemote(git_factory) else: log.info("Git client import failed") if MercurialFactory and HgRemote: - hg_repo_cache = self.cache.get_cache_region( - 'hg', region='repo_object') - hg_factory = MercurialFactory(hg_repo_cache) + hg_factory = MercurialFactory() self._hg_remote = HgRemote(hg_factory) else: log.info("Mercurial client import failed") if SubversionFactory and SvnRemote: - svn_repo_cache = self.cache.get_cache_region( - 'svn', region='repo_object') - svn_factory = SubversionFactory(svn_repo_cache) + svn_factory = SubversionFactory() + # hg factory is used for svn url validation - hg_repo_cache = self.cache.get_cache_region( - 'hg', region='repo_object') - hg_factory = MercurialFactory(hg_repo_cache) + hg_factory = MercurialFactory() self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory) else: log.info("Subversion client import failed") self._vcsserver = VcsServer() - def _initialize_cache(self): - cache_config = parse_cache_config_options(self.cache_config) - log.info('Initializing beaker cache: %s' % cache_config) - self.cache = CacheManager(**cache_config) - def _configure_locale(self): if self.locale: log.info('Settings locale: `LC_ALL` to %s' % self.locale) @@ -169,8 +180,11 @@ class HTTPApplication(object): _use_echo_app = False def __init__(self, settings=None, global_config=None): + self._sanitize_settings_and_apply_defaults(settings) + self.config = Configurator(settings=settings) self.global_config = global_config + self.config.include('vcsserver.lib.rc_cache') locale = settings.get('locale', '') or 'en_US.UTF-8' vcs = VCS(locale=locale, cache_config=settings) @@ -198,6 +212,21 @@ class HTTPApplication(object): if binary_dir: settings.BINARY_DIR = binary_dir + def _sanitize_settings_and_apply_defaults(self, settings): + # repo_object cache + _string_setting( + settings, + 'rc_cache.repo_object.backend', + 'dogpile.cache.rc.memory_lru') + _int_setting( + settings, + 'rc_cache.repo_object.expiration_time', + 300) + _int_setting( + settings, + 'rc_cache.repo_object.max_size', + 1024) + def _configure(self): self.config.add_renderer( name='msgpack', @@ -246,14 +275,17 @@ class HTTPApplication(object): wire = params.get('wire') args = params.get('args') kwargs = params.get('kwargs') + context_uid = None + if wire: try: - wire['context'] = uuid.UUID(wire['context']) + wire['context'] = context_uid = uuid.UUID(wire['context']) except KeyError: pass args.insert(0, wire) - log.debug('method called:%s with kwargs:%s', method, kwargs) + log.debug('method called:%s with kwargs:%s context_uid: %s', + method, kwargs, context_uid) try: resp = getattr(remote, method)(*args, **kwargs) except Exception as e: @@ -272,7 +304,7 @@ class HTTPApplication(object): } } try: - resp['error']['_vcs_kind'] = e._vcs_kind + resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None) except AttributeError: pass else: @@ -486,5 +518,6 @@ def main(global_config, **settings): if MercurialFactory: hgpatches.patch_largefiles_capabilities() hgpatches.patch_subrepo_type_mapping() + app = HTTPApplication(settings=settings, global_config=global_config) return app.wsgi_app() diff --git a/vcsserver/lib/__init__.py b/vcsserver/lib/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/__init__.py @@ -0,0 +1,16 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/vcsserver/lib/rc_cache/__init__.py b/vcsserver/lib/rc_cache/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/__init__.py @@ -0,0 +1,60 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import logging +from dogpile.cache import register_backend + +register_backend( + "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends", + "LRUMemoryBackend") + +log = logging.getLogger(__name__) + +from . import region_meta +from .util import key_generator, get_default_cache_settings, make_region + + +def configure_dogpile_cache(settings): + cache_dir = settings.get('cache_dir') + if cache_dir: + region_meta.dogpile_config_defaults['cache_dir'] = cache_dir + + rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.']) + + # inspect available namespaces + avail_regions = set() + for key in rc_cache_data.keys(): + namespace_name = key.split('.', 1)[0] + avail_regions.add(namespace_name) + log.debug('dogpile: found following cache regions: %s', avail_regions) + + # register them into namespace + for region_name in avail_regions: + new_region = make_region( + name=region_name, + function_key_generator=key_generator + ) + + new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name)) + + log.debug('dogpile: registering a new region %s[%s]', + region_name, new_region.__dict__) + region_meta.dogpile_cache_regions[region_name] = new_region + + +def includeme(config): + configure_dogpile_cache(config.registry.settings) diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/backends.py @@ -0,0 +1,51 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import logging + +from dogpile.cache.backends import memory as memory_backend +from lru import LRU as LRUDict + + +_default_max_size = 1024 + +log = logging.getLogger(__name__) + + +class LRUMemoryBackend(memory_backend.MemoryBackend): + pickle_values = False + + def __init__(self, arguments): + max_size = arguments.pop('max_size', _default_max_size) + callback = None + if arguments.pop('log_max_size_reached', None): + def evicted(key, value): + log.debug( + 'LRU: evicting key `%s` due to max size %s reach', key, max_size) + callback = evicted + + arguments['cache_dict'] = LRUDict(max_size, callback=callback) + super(LRUMemoryBackend, self).__init__(arguments) + + def delete(self, key): + if self._cache.has_key(key): + del self._cache[key] + + def delete_multi(self, keys): + for key in keys: + if self._cache.has_key(key): + del self._cache[key] diff --git a/vcsserver/lib/rc_cache/region_meta.py b/vcsserver/lib/rc_cache/region_meta.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/region_meta.py @@ -0,0 +1,26 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import tempfile + +dogpile_config_defaults = { + 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache') +} + +# GLOBAL TO STORE ALL REGISTERED REGIONS +dogpile_cache_regions = {} diff --git a/vcsserver/lib/rc_cache/util.py b/vcsserver/lib/rc_cache/util.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/util.py @@ -0,0 +1,136 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import logging +import functools + +from vcsserver.utils import safe_str, sha1 +from dogpile.cache import CacheRegion +from dogpile.cache.util import compat + +log = logging.getLogger(__name__) + + +class RhodeCodeCacheRegion(CacheRegion): + + def conditional_cache_on_arguments( + self, namespace=None, + expiration_time=None, + should_cache_fn=None, + to_str=compat.string_type, + function_key_generator=None, + condition=True): + """ + Custom conditional decorator, that will not touch any dogpile internals if + condition isn't meet. This works a bit different than should_cache_fn + And it's faster in cases we don't ever want to compute cached values + """ + expiration_time_is_callable = compat.callable(expiration_time) + + if function_key_generator is None: + function_key_generator = self.function_key_generator + + def decorator(fn): + if to_str is compat.string_type: + # backwards compatible + key_generator = function_key_generator(namespace, fn) + else: + key_generator = function_key_generator(namespace, fn, to_str=to_str) + + @functools.wraps(fn) + def decorate(*arg, **kw): + key = key_generator(*arg, **kw) + + @functools.wraps(fn) + def creator(): + return fn(*arg, **kw) + + if not condition: + return creator() + + timeout = expiration_time() if expiration_time_is_callable \ + else expiration_time + + return self.get_or_create(key, creator, timeout, should_cache_fn) + + def invalidate(*arg, **kw): + key = key_generator(*arg, **kw) + self.delete(key) + + def set_(value, *arg, **kw): + key = key_generator(*arg, **kw) + self.set(key, value) + + def get(*arg, **kw): + key = key_generator(*arg, **kw) + return self.get(key) + + def refresh(*arg, **kw): + key = key_generator(*arg, **kw) + value = fn(*arg, **kw) + self.set(key, value) + return value + + decorate.set = set_ + decorate.invalidate = invalidate + decorate.refresh = refresh + decorate.get = get + decorate.original = fn + decorate.key_generator = key_generator + + return decorate + + return decorator + + +def make_region(*arg, **kw): + return RhodeCodeCacheRegion(*arg, **kw) + + +def get_default_cache_settings(settings, prefixes=None): + prefixes = prefixes or [] + cache_settings = {} + for key in settings.keys(): + for prefix in prefixes: + if key.startswith(prefix): + name = key.split(prefix)[1].strip() + val = settings[key] + if isinstance(val, basestring): + val = val.strip() + cache_settings[name] = val + return cache_settings + + +def compute_key_from_params(*args): + """ + Helper to compute key from given params to be used in cache manager + """ + return sha1("_".join(map(safe_str, args))) + + +def key_generator(namespace, fn): + fname = fn.__name__ + + def generate_key(*args): + namespace_pref = namespace or 'default' + arg_key = compute_key_from_params(*args) + final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key) + + return final_key + + return generate_key diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -40,13 +40,13 @@ log = logging.getLogger(__name__) # Set of svn compatible version flags. # Compare with subversion/svnadmin/svnadmin.c -svn_compatible_versions = set([ +svn_compatible_versions = { 'pre-1.4-compatible', 'pre-1.5-compatible', 'pre-1.6-compatible', 'pre-1.8-compatible', - 'pre-1.9-compatible', -]) + 'pre-1.9-compatible' +} svn_compatible_versions_map = { 'pre-1.4-compatible': '1.3', @@ -71,6 +71,7 @@ def reraise_safe_exceptions(func): class SubversionFactory(RepoFactory): + repo_type = 'svn' def _create_repo(self, wire, create, compatible_version): path = svn.core.svn_path_canonicalize(wire['path']) @@ -92,10 +93,25 @@ class SubversionFactory(RepoFactory): return repo def repo(self, wire, create=False, compatible_version=None): - def create_new_repo(): + """ + Get a repository instance for the given path. + + Uses internally the low level beaker API since the decorators introduce + significant overhead. + """ + region = self._cache_region + context = wire.get('context', None) + repo_path = wire.get('path', '') + context_uid = '{}'.format(context) + cache = wire.get('cache', True) + cache_on = context and cache + + @region.conditional_cache_on_arguments(condition=cache_on) + def create_new_repo(_repo_type, _repo_path, _context_uid, compatible_version_id): return self._create_repo(wire, create, compatible_version) - return self._repo(wire, create_new_repo) + return create_new_repo(self.repo_type, repo_path, context_uid, + compatible_version) NODE_TYPE_MAPPING = { diff --git a/vcsserver/tests/conftest.py b/vcsserver/tests/conftest.py --- a/vcsserver/tests/conftest.py +++ b/vcsserver/tests/conftest.py @@ -40,7 +40,7 @@ def repeat(request): @pytest.fixture(scope='session') def vcsserver_port(request): port = get_available_port() - print 'Using vcsserver port %s' % (port, ) + print('Using vcsserver port %s' % (port, )) return port @@ -55,4 +55,3 @@ def get_available_port(): mysocket.close() del mysocket return port - diff --git a/vcsserver/tests/test_git.py b/vcsserver/tests/test_git.py --- a/vcsserver/tests/test_git.py +++ b/vcsserver/tests/test_git.py @@ -152,11 +152,14 @@ class TestDulwichRepoWrapper(object): class TestGitFactory(object): def test_create_repo_returns_dulwich_wrapper(self): - factory = git.GitFactory(repo_cache=Mock()) - wire = { - 'path': '/tmp/abcde' - } - isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True) - with isdir_patcher: - result = factory._create_repo(wire, True) - assert isinstance(result, git.Repo) + + with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock: + mock.side_effect = {'repo_objects': ''} + factory = git.GitFactory() + wire = { + 'path': '/tmp/abcde' + } + isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True) + with isdir_patcher: + result = factory._create_repo(wire, True) + assert isinstance(result, git.Repo) diff --git a/vcsserver/tests/test_http_performance.py b/vcsserver/tests/test_http_performance.py --- a/vcsserver/tests/test_http_performance.py +++ b/vcsserver/tests/test_http_performance.py @@ -12,11 +12,6 @@ from vcsserver.http_main import main def vcs_app(): stub_settings = { 'dev.use_echo_app': 'true', - 'beaker.cache.regions': 'repo_object', - 'beaker.cache.repo_object.type': 'memorylru', - 'beaker.cache.repo_object.max_items': '100', - 'beaker.cache.repo_object.expire': '300', - 'beaker.cache.repo_object.enabled': 'true', 'locale': 'en_US.UTF-8', } vcs_app = main({}, **stub_settings) diff --git a/vcsserver/utils.py b/vcsserver/utils.py --- a/vcsserver/utils.py +++ b/vcsserver/utils.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging +import hashlib log = logging.getLogger(__name__) @@ -80,3 +81,9 @@ class AttributeDict(dict): return self.get(attr, None) __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ + + +def sha1(val): + return hashlib.sha1(val).hexdigest() + +