##// END OF EJS Templates
caches: replaced beaker with dogpile cache.
marcink -
r483:80e9ab60 default
parent child Browse files
Show More
@@ -0,0 +1,16 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@@ -0,0 +1,60 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import logging
19 from dogpile.cache import register_backend
20
21 register_backend(
22 "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
23 "LRUMemoryBackend")
24
25 log = logging.getLogger(__name__)
26
27 from . import region_meta
28 from .util import key_generator, get_default_cache_settings, make_region
29
30
31 def configure_dogpile_cache(settings):
32 cache_dir = settings.get('cache_dir')
33 if cache_dir:
34 region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
35
36 rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
37
38 # inspect available namespaces
39 avail_regions = set()
40 for key in rc_cache_data.keys():
41 namespace_name = key.split('.', 1)[0]
42 avail_regions.add(namespace_name)
43 log.debug('dogpile: found following cache regions: %s', avail_regions)
44
45 # register them into namespace
46 for region_name in avail_regions:
47 new_region = make_region(
48 name=region_name,
49 function_key_generator=key_generator
50 )
51
52 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
53
54 log.debug('dogpile: registering a new region %s[%s]',
55 region_name, new_region.__dict__)
56 region_meta.dogpile_cache_regions[region_name] = new_region
57
58
59 def includeme(config):
60 configure_dogpile_cache(config.registry.settings)
@@ -0,0 +1,51 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import logging
19
20 from dogpile.cache.backends import memory as memory_backend
21 from lru import LRU as LRUDict
22
23
24 _default_max_size = 1024
25
26 log = logging.getLogger(__name__)
27
28
29 class LRUMemoryBackend(memory_backend.MemoryBackend):
30 pickle_values = False
31
32 def __init__(self, arguments):
33 max_size = arguments.pop('max_size', _default_max_size)
34 callback = None
35 if arguments.pop('log_max_size_reached', None):
36 def evicted(key, value):
37 log.debug(
38 'LRU: evicting key `%s` due to max size %s reach', key, max_size)
39 callback = evicted
40
41 arguments['cache_dict'] = LRUDict(max_size, callback=callback)
42 super(LRUMemoryBackend, self).__init__(arguments)
43
44 def delete(self, key):
45 if self._cache.has_key(key):
46 del self._cache[key]
47
48 def delete_multi(self, keys):
49 for key in keys:
50 if self._cache.has_key(key):
51 del self._cache[key]
@@ -0,0 +1,26 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import os
19 import tempfile
20
21 dogpile_config_defaults = {
22 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
23 }
24
25 # GLOBAL TO STORE ALL REGISTERED REGIONS
26 dogpile_cache_regions = {}
@@ -0,0 +1,136 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import os
19 import logging
20 import functools
21
22 from vcsserver.utils import safe_str, sha1
23 from dogpile.cache import CacheRegion
24 from dogpile.cache.util import compat
25
26 log = logging.getLogger(__name__)
27
28
29 class RhodeCodeCacheRegion(CacheRegion):
30
31 def conditional_cache_on_arguments(
32 self, namespace=None,
33 expiration_time=None,
34 should_cache_fn=None,
35 to_str=compat.string_type,
36 function_key_generator=None,
37 condition=True):
38 """
39 Custom conditional decorator, that will not touch any dogpile internals if
40 condition isn't meet. This works a bit different than should_cache_fn
41 And it's faster in cases we don't ever want to compute cached values
42 """
43 expiration_time_is_callable = compat.callable(expiration_time)
44
45 if function_key_generator is None:
46 function_key_generator = self.function_key_generator
47
48 def decorator(fn):
49 if to_str is compat.string_type:
50 # backwards compatible
51 key_generator = function_key_generator(namespace, fn)
52 else:
53 key_generator = function_key_generator(namespace, fn, to_str=to_str)
54
55 @functools.wraps(fn)
56 def decorate(*arg, **kw):
57 key = key_generator(*arg, **kw)
58
59 @functools.wraps(fn)
60 def creator():
61 return fn(*arg, **kw)
62
63 if not condition:
64 return creator()
65
66 timeout = expiration_time() if expiration_time_is_callable \
67 else expiration_time
68
69 return self.get_or_create(key, creator, timeout, should_cache_fn)
70
71 def invalidate(*arg, **kw):
72 key = key_generator(*arg, **kw)
73 self.delete(key)
74
75 def set_(value, *arg, **kw):
76 key = key_generator(*arg, **kw)
77 self.set(key, value)
78
79 def get(*arg, **kw):
80 key = key_generator(*arg, **kw)
81 return self.get(key)
82
83 def refresh(*arg, **kw):
84 key = key_generator(*arg, **kw)
85 value = fn(*arg, **kw)
86 self.set(key, value)
87 return value
88
89 decorate.set = set_
90 decorate.invalidate = invalidate
91 decorate.refresh = refresh
92 decorate.get = get
93 decorate.original = fn
94 decorate.key_generator = key_generator
95
96 return decorate
97
98 return decorator
99
100
101 def make_region(*arg, **kw):
102 return RhodeCodeCacheRegion(*arg, **kw)
103
104
105 def get_default_cache_settings(settings, prefixes=None):
106 prefixes = prefixes or []
107 cache_settings = {}
108 for key in settings.keys():
109 for prefix in prefixes:
110 if key.startswith(prefix):
111 name = key.split(prefix)[1].strip()
112 val = settings[key]
113 if isinstance(val, basestring):
114 val = val.strip()
115 cache_settings[name] = val
116 return cache_settings
117
118
119 def compute_key_from_params(*args):
120 """
121 Helper to compute key from given params to be used in cache manager
122 """
123 return sha1("_".join(map(safe_str, args)))
124
125
126 def key_generator(namespace, fn):
127 fname = fn.__name__
128
129 def generate_key(*args):
130 namespace_pref = namespace or 'default'
131 arg_key = compute_key_from_params(*args)
132 final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
133
134 return final_key
135
136 return generate_key
@@ -1,1 +1,79 b''
1 development_http.ini No newline at end of file
1 ################################################################################
2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 # #
4 ################################################################################
5
6
7 [server:main]
8 ## COMMON ##
9 host = 0.0.0.0
10 port = 9900
11
12 use = egg:waitress#main
13
14
15 [app:main]
16 use = egg:rhodecode-vcsserver
17
18 pyramid.default_locale_name = en
19 pyramid.includes =
20
21 ## default locale used by VCS systems
22 locale = en_US.UTF-8
23
24
25 ## path to binaries for vcsserver, it should be set by the installer
26 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
27 core.binary_dir = ""
28
29 ## cache region for storing repo_objects cache
30 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
31 ## cache auto-expires after N seconds
32 rc_cache.repo_object.expiration_time = 300
33 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
34 rc_cache.repo_object.max_size = 100
35
36
37 ################################
38 ### LOGGING CONFIGURATION ####
39 ################################
40 [loggers]
41 keys = root, vcsserver
42
43 [handlers]
44 keys = console
45
46 [formatters]
47 keys = generic
48
49 #############
50 ## LOGGERS ##
51 #############
52 [logger_root]
53 level = NOTSET
54 handlers = console
55
56 [logger_vcsserver]
57 level = DEBUG
58 handlers =
59 qualname = vcsserver
60 propagate = 1
61
62
63 ##############
64 ## HANDLERS ##
65 ##############
66
67 [handler_console]
68 class = StreamHandler
69 args = (sys.stderr,)
70 level = DEBUG
71 formatter = generic
72
73 ################
74 ## FORMATTERS ##
75 ################
76
77 [formatter_generic]
78 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
79 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,1 +1,100 b''
1 production_http.ini No newline at end of file
1 ################################################################################
2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 # #
4 ################################################################################
5
6
7 [server:main]
8 ## COMMON ##
9 host = 127.0.0.1
10 port = 9900
11
12
13 ##########################
14 ## GUNICORN WSGI SERVER ##
15 ##########################
16 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
17 use = egg:gunicorn#main
18 ## Sets the number of process workers. Recommended
19 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
20 workers = 2
21 ## process name
22 proc_name = rhodecode_vcsserver
23 ## type of worker class, currently `sync` is the only option allowed.
24 worker_class = sync
25 ## The maximum number of simultaneous clients. Valid only for Gevent
26 #worker_connections = 10
27 ## max number of requests that worker will handle before being gracefully
28 ## restarted, could prevent memory leaks
29 max_requests = 1000
30 max_requests_jitter = 30
31 ## amount of time a worker can spend with handling a request before it
32 ## gets killed and restarted. Set to 6hrs
33 timeout = 21600
34
35
36 [app:main]
37 use = egg:rhodecode-vcsserver
38
39 pyramid.default_locale_name = en
40 pyramid.includes =
41
42 ## default locale used by VCS systems
43 locale = en_US.UTF-8
44
45
46 ## path to binaries for vcsserver, it should be set by the installer
47 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
48 core.binary_dir = ""
49
50 ## cache region for storing repo_objects cache
51 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
52 ## cache auto-expires after N seconds
53 rc_cache.repo_object.expiration_time = 300
54 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
55 rc_cache.repo_object.max_size = 100
56
57
58 ################################
59 ### LOGGING CONFIGURATION ####
60 ################################
61 [loggers]
62 keys = root, vcsserver
63
64 [handlers]
65 keys = console
66
67 [formatters]
68 keys = generic
69
70 #############
71 ## LOGGERS ##
72 #############
73 [logger_root]
74 level = NOTSET
75 handlers = console
76
77 [logger_vcsserver]
78 level = DEBUG
79 handlers =
80 qualname = vcsserver
81 propagate = 1
82
83
84 ##############
85 ## HANDLERS ##
86 ##############
87
88 [handler_console]
89 class = StreamHandler
90 args = (sys.stderr,)
91 level = DEBUG
92 formatter = generic
93
94 ################
95 ## FORMATTERS ##
96 ################
97
98 [formatter_generic]
99 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
100 datefmt = %Y-%m-%d %H:%M:%S
@@ -15,13 +15,6 b' in'
15 15
16 16 self: super: {
17 17
18 "beaker" = super."beaker".override (attrs: {
19 patches = [
20 ./patch_beaker/patch-beaker-lock-func-debug.diff
21 ./patch_beaker/patch-beaker-metadata-reuse.diff
22 ];
23 });
24
25 18 "gevent" = super."gevent".override (attrs: {
26 19 propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
27 20 # NOTE: (marcink) odd requirements from gevent aren't set properly,
@@ -37,20 +37,6 b' self: super: {'
37 37 license = [ pkgs.lib.licenses.mit ];
38 38 };
39 39 };
40 "beaker" = super.buildPythonPackage {
41 name = "beaker-1.9.1";
42 doCheck = false;
43 propagatedBuildInputs = [
44 self."funcsigs"
45 ];
46 src = fetchurl {
47 url = "https://files.pythonhosted.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz";
48 sha256 = "08arsn61r255lhz6hcpn2lsiqpg30clla805ysx06wmbhvb6w9rj";
49 };
50 meta = {
51 license = [ pkgs.lib.licenses.bsdOriginal ];
52 };
53 };
54 40 "beautifulsoup4" = super.buildPythonPackage {
55 41 name = "beautifulsoup4-4.6.0";
56 42 doCheck = false;
@@ -112,6 +98,28 b' self: super: {'
112 98 license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
113 99 };
114 100 };
101 "dogpile.cache" = super.buildPythonPackage {
102 name = "dogpile.cache-0.6.6";
103 doCheck = false;
104 src = fetchurl {
105 url = "https://files.pythonhosted.org/packages/48/ca/604154d835c3668efb8a31bd979b0ea4bf39c2934a40ffecc0662296cb51/dogpile.cache-0.6.6.tar.gz";
106 sha256 = "1h8n1lxd4l2qvahfkiinljkqz7pww7w3sgag0j8j9ixbl2h4wk84";
107 };
108 meta = {
109 license = [ pkgs.lib.licenses.bsdOriginal ];
110 };
111 };
112 "dogpile.core" = super.buildPythonPackage {
113 name = "dogpile.core-0.4.1";
114 doCheck = false;
115 src = fetchurl {
116 url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
117 sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy";
118 };
119 meta = {
120 license = [ pkgs.lib.licenses.bsdOriginal ];
121 };
122 };
115 123 "dulwich" = super.buildPythonPackage {
116 124 name = "dulwich-0.13.0";
117 125 doCheck = false;
@@ -229,21 +237,6 b' self: super: {'
229 237 license = [ pkgs.lib.licenses.mit ];
230 238 };
231 239 };
232 "infrae.cache" = super.buildPythonPackage {
233 name = "infrae.cache-1.0.1";
234 doCheck = false;
235 propagatedBuildInputs = [
236 self."beaker"
237 self."repoze.lru"
238 ];
239 src = fetchurl {
240 url = "https://files.pythonhosted.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
241 sha256 = "1dvqsjn8vw253wz9d1pz17j79mf4bs53dvp2qxck2qdp1am1njw4";
242 };
243 meta = {
244 license = [ pkgs.lib.licenses.zpl21 ];
245 };
246 };
247 240 "ipdb" = super.buildPythonPackage {
248 241 name = "ipdb-0.11";
249 242 doCheck = false;
@@ -294,6 +287,17 b' self: super: {'
294 287 license = [ pkgs.lib.licenses.bsdOriginal ];
295 288 };
296 289 };
290 "lru-dict" = super.buildPythonPackage {
291 name = "lru-dict-1.1.6";
292 doCheck = false;
293 src = fetchurl {
294 url = "https://files.pythonhosted.org/packages/00/a5/32ed6e10246cd341ca8cc205acea5d208e4053f48a4dced2b1b31d45ba3f/lru-dict-1.1.6.tar.gz";
295 sha256 = "1k2lhd4dpl6xa6iialbwx4l6bkdzxmzhygms39pvf19x1rk5fm1n";
296 };
297 meta = {
298 license = [ pkgs.lib.licenses.mit ];
299 };
300 };
297 301 "mako" = super.buildPythonPackage {
298 302 name = "mako-1.0.7";
299 303 doCheck = false;
@@ -680,13 +684,14 b' self: super: {'
680 684 ];
681 685 doCheck = true;
682 686 propagatedBuildInputs = [
683 self."beaker"
684 687 self."configobj"
688 self."dogpile.cache"
689 self."dogpile.core"
685 690 self."decorator"
686 691 self."dulwich"
687 692 self."hgsubversion"
688 693 self."hg-evolve"
689 self."infrae.cache"
694 self."lru-dict"
690 695 self."mako"
691 696 self."markupsafe"
692 697 self."mercurial"
@@ -1,12 +1,13 b''
1 1 ## dependencies
2 2
3 beaker==1.9.1
4 3 configobj==5.0.6
4 dogpile.cache==0.6.6
5 dogpile.core==0.4.1
5 6 decorator==4.1.2
6 7 dulwich==0.13.0
7 8 hgsubversion==1.9.2
8 9 hg-evolve==8.0.1
9 infrae.cache==1.0.1
10 lru-dict==1.1.6
10 11 mako==1.0.7
11 12 markupsafe==1.0.0
12 13 mercurial==4.6.1
@@ -20,6 +20,7 b' import traceback'
20 20 import logging
21 21 import urlparse
22 22
23 from vcsserver.lib.rc_cache import region_meta
23 24 log = logging.getLogger(__name__)
24 25
25 26
@@ -30,9 +31,10 b' class RepoFactory(object):'
30 31 It provides internal caching of the `repo` object based on
31 32 the :term:`call context`.
32 33 """
34 repo_type = None
33 35
34 def __init__(self, repo_cache):
35 self._cache = repo_cache
36 def __init__(self):
37 self._cache_region = region_meta.dogpile_cache_regions['repo_object']
36 38
37 39 def _create_config(self, path, config):
38 40 config = {}
@@ -48,26 +50,19 b' class RepoFactory(object):'
48 50 Uses internally the low level beaker API since the decorators introduce
49 51 significant overhead.
50 52 """
51 def create_new_repo():
53 region = self._cache_region
54 context = wire.get('context', None)
55 repo_path = wire.get('path', '')
56 context_uid = '{}'.format(context)
57 cache = wire.get('cache', True)
58 cache_on = context and cache
59
60 @region.conditional_cache_on_arguments(condition=cache_on)
61 def create_new_repo(_repo_type, _repo_path, _context_uid):
52 62 return self._create_repo(wire, create)
53 63
54 return self._repo(wire, create_new_repo)
55
56 def _repo(self, wire, createfunc):
57 context = wire.get('context', None)
58 cache = wire.get('cache', True)
59
60 if context and cache:
61 cache_key = (context, wire['path'])
62 log.debug(
63 'FETCH %s@%s repo object from cache. Context: %s',
64 self.__class__.__name__, wire['path'], context)
65 return self._cache.get(key=cache_key, createfunc=createfunc)
66 else:
67 log.debug(
68 'INIT %s@%s repo object based on wire %s. Context: %s',
69 self.__class__.__name__, wire['path'], wire, context)
70 return createfunc()
64 repo = create_new_repo(self.repo_type, repo_path, context_uid)
65 return repo
71 66
72 67
73 68 def obfuscate_qs(query_string):
@@ -87,6 +87,7 b' class Repo(DulwichRepo):'
87 87
88 88
89 89 class GitFactory(RepoFactory):
90 repo_type = 'git'
90 91
91 92 def _create_repo(self, wire, create):
92 93 repo_path = str_to_dulwich(wire['path'])
@@ -93,6 +93,7 b' def reraise_safe_exceptions(func):'
93 93
94 94
95 95 class MercurialFactory(RepoFactory):
96 repo_type = 'hg'
96 97
97 98 def _create_config(self, config, hooks=True):
98 99 if not hooks:
@@ -26,9 +26,8 b' from itertools import chain'
26 26
27 27 import simplejson as json
28 28 import msgpack
29 from beaker.cache import CacheManager
30 from beaker.util import parse_cache_config_options
31 29 from pyramid.config import Configurator
30 from pyramid.settings import asbool, aslist
32 31 from pyramid.wsgi import wsgiapp
33 32 from pyramid.compat import configparser
34 33
@@ -65,48 +64,60 b' def _is_request_chunked(environ):'
65 64 return stream
66 65
67 66
67 def _int_setting(settings, name, default):
68 settings[name] = int(settings.get(name, default))
69
70
71 def _bool_setting(settings, name, default):
72 input_val = settings.get(name, default)
73 if isinstance(input_val, unicode):
74 input_val = input_val.encode('utf8')
75 settings[name] = asbool(input_val)
76
77
78 def _list_setting(settings, name, default):
79 raw_value = settings.get(name, default)
80
81 # Otherwise we assume it uses pyramids space/newline separation.
82 settings[name] = aslist(raw_value)
83
84
85 def _string_setting(settings, name, default, lower=True):
86 value = settings.get(name, default)
87 if lower:
88 value = value.lower()
89 settings[name] = value
90
91
68 92 class VCS(object):
69 93 def __init__(self, locale=None, cache_config=None):
70 94 self.locale = locale
71 95 self.cache_config = cache_config
72 96 self._configure_locale()
73 self._initialize_cache()
74 97
75 98 if GitFactory and GitRemote:
76 git_repo_cache = self.cache.get_cache_region(
77 'git', region='repo_object')
78 git_factory = GitFactory(git_repo_cache)
99 git_factory = GitFactory()
79 100 self._git_remote = GitRemote(git_factory)
80 101 else:
81 102 log.info("Git client import failed")
82 103
83 104 if MercurialFactory and HgRemote:
84 hg_repo_cache = self.cache.get_cache_region(
85 'hg', region='repo_object')
86 hg_factory = MercurialFactory(hg_repo_cache)
105 hg_factory = MercurialFactory()
87 106 self._hg_remote = HgRemote(hg_factory)
88 107 else:
89 108 log.info("Mercurial client import failed")
90 109
91 110 if SubversionFactory and SvnRemote:
92 svn_repo_cache = self.cache.get_cache_region(
93 'svn', region='repo_object')
94 svn_factory = SubversionFactory(svn_repo_cache)
111 svn_factory = SubversionFactory()
112
95 113 # hg factory is used for svn url validation
96 hg_repo_cache = self.cache.get_cache_region(
97 'hg', region='repo_object')
98 hg_factory = MercurialFactory(hg_repo_cache)
114 hg_factory = MercurialFactory()
99 115 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
100 116 else:
101 117 log.info("Subversion client import failed")
102 118
103 119 self._vcsserver = VcsServer()
104 120
105 def _initialize_cache(self):
106 cache_config = parse_cache_config_options(self.cache_config)
107 log.info('Initializing beaker cache: %s' % cache_config)
108 self.cache = CacheManager(**cache_config)
109
110 121 def _configure_locale(self):
111 122 if self.locale:
112 123 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
@@ -169,8 +180,11 b' class HTTPApplication(object):'
169 180 _use_echo_app = False
170 181
171 182 def __init__(self, settings=None, global_config=None):
183 self._sanitize_settings_and_apply_defaults(settings)
184
172 185 self.config = Configurator(settings=settings)
173 186 self.global_config = global_config
187 self.config.include('vcsserver.lib.rc_cache')
174 188
175 189 locale = settings.get('locale', '') or 'en_US.UTF-8'
176 190 vcs = VCS(locale=locale, cache_config=settings)
@@ -198,6 +212,21 b' class HTTPApplication(object):'
198 212 if binary_dir:
199 213 settings.BINARY_DIR = binary_dir
200 214
215 def _sanitize_settings_and_apply_defaults(self, settings):
216 # repo_object cache
217 _string_setting(
218 settings,
219 'rc_cache.repo_object.backend',
220 'dogpile.cache.rc.memory_lru')
221 _int_setting(
222 settings,
223 'rc_cache.repo_object.expiration_time',
224 300)
225 _int_setting(
226 settings,
227 'rc_cache.repo_object.max_size',
228 1024)
229
201 230 def _configure(self):
202 231 self.config.add_renderer(
203 232 name='msgpack',
@@ -246,14 +275,17 b' class HTTPApplication(object):'
246 275 wire = params.get('wire')
247 276 args = params.get('args')
248 277 kwargs = params.get('kwargs')
278 context_uid = None
279
249 280 if wire:
250 281 try:
251 wire['context'] = uuid.UUID(wire['context'])
282 wire['context'] = context_uid = uuid.UUID(wire['context'])
252 283 except KeyError:
253 284 pass
254 285 args.insert(0, wire)
255 286
256 log.debug('method called:%s with kwargs:%s', method, kwargs)
287 log.debug('method called:%s with kwargs:%s context_uid: %s',
288 method, kwargs, context_uid)
257 289 try:
258 290 resp = getattr(remote, method)(*args, **kwargs)
259 291 except Exception as e:
@@ -272,7 +304,7 b' class HTTPApplication(object):'
272 304 }
273 305 }
274 306 try:
275 resp['error']['_vcs_kind'] = e._vcs_kind
307 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
276 308 except AttributeError:
277 309 pass
278 310 else:
@@ -486,5 +518,6 b' def main(global_config, **settings):'
486 518 if MercurialFactory:
487 519 hgpatches.patch_largefiles_capabilities()
488 520 hgpatches.patch_subrepo_type_mapping()
521
489 522 app = HTTPApplication(settings=settings, global_config=global_config)
490 523 return app.wsgi_app()
@@ -40,13 +40,13 b' log = logging.getLogger(__name__)'
40 40
41 41 # Set of svn compatible version flags.
42 42 # Compare with subversion/svnadmin/svnadmin.c
43 svn_compatible_versions = set([
43 svn_compatible_versions = {
44 44 'pre-1.4-compatible',
45 45 'pre-1.5-compatible',
46 46 'pre-1.6-compatible',
47 47 'pre-1.8-compatible',
48 'pre-1.9-compatible',
49 ])
48 'pre-1.9-compatible'
49 }
50 50
51 51 svn_compatible_versions_map = {
52 52 'pre-1.4-compatible': '1.3',
@@ -71,6 +71,7 b' def reraise_safe_exceptions(func):'
71 71
72 72
73 73 class SubversionFactory(RepoFactory):
74 repo_type = 'svn'
74 75
75 76 def _create_repo(self, wire, create, compatible_version):
76 77 path = svn.core.svn_path_canonicalize(wire['path'])
@@ -92,10 +93,25 b' class SubversionFactory(RepoFactory):'
92 93 return repo
93 94
94 95 def repo(self, wire, create=False, compatible_version=None):
95 def create_new_repo():
96 """
97 Get a repository instance for the given path.
98
99 Uses internally the low level beaker API since the decorators introduce
100 significant overhead.
101 """
102 region = self._cache_region
103 context = wire.get('context', None)
104 repo_path = wire.get('path', '')
105 context_uid = '{}'.format(context)
106 cache = wire.get('cache', True)
107 cache_on = context and cache
108
109 @region.conditional_cache_on_arguments(condition=cache_on)
110 def create_new_repo(_repo_type, _repo_path, _context_uid, compatible_version_id):
96 111 return self._create_repo(wire, create, compatible_version)
97 112
98 return self._repo(wire, create_new_repo)
113 return create_new_repo(self.repo_type, repo_path, context_uid,
114 compatible_version)
99 115
100 116
101 117 NODE_TYPE_MAPPING = {
@@ -40,7 +40,7 b' def repeat(request):'
40 40 @pytest.fixture(scope='session')
41 41 def vcsserver_port(request):
42 42 port = get_available_port()
43 print 'Using vcsserver port %s' % (port, )
43 print('Using vcsserver port %s' % (port, ))
44 44 return port
45 45
46 46
@@ -55,4 +55,3 b' def get_available_port():'
55 55 mysocket.close()
56 56 del mysocket
57 57 return port
58
@@ -152,7 +152,10 b' class TestDulwichRepoWrapper(object):'
152 152
153 153 class TestGitFactory(object):
154 154 def test_create_repo_returns_dulwich_wrapper(self):
155 factory = git.GitFactory(repo_cache=Mock())
155
156 with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
157 mock.side_effect = {'repo_objects': ''}
158 factory = git.GitFactory()
156 159 wire = {
157 160 'path': '/tmp/abcde'
158 161 }
@@ -12,11 +12,6 b' from vcsserver.http_main import main'
12 12 def vcs_app():
13 13 stub_settings = {
14 14 'dev.use_echo_app': 'true',
15 'beaker.cache.regions': 'repo_object',
16 'beaker.cache.repo_object.type': 'memorylru',
17 'beaker.cache.repo_object.max_items': '100',
18 'beaker.cache.repo_object.expire': '300',
19 'beaker.cache.repo_object.enabled': 'true',
20 15 'locale': 'en_US.UTF-8',
21 16 }
22 17 vcs_app = main({}, **stub_settings)
@@ -15,6 +15,7 b''
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17 import logging
18 import hashlib
18 19
19 20 log = logging.getLogger(__name__)
20 21
@@ -80,3 +81,9 b' class AttributeDict(dict):'
80 81 return self.get(attr, None)
81 82 __setattr__ = dict.__setitem__
82 83 __delattr__ = dict.__delitem__
84
85
86 def sha1(val):
87 return hashlib.sha1(val).hexdigest()
88
89
1 NO CONTENT: file was removed
1 NO CONTENT: file was removed
1 NO CONTENT: file was removed
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now