##// END OF EJS Templates
exc_store: allow to specify a custom path for exception store.
marcink -
r539:de749a38 default
parent child Browse files
Show More
@@ -1,79 +1,87 b''
1 ################################################################################
1 ################################################################################
2 # RhodeCode VCSServer with HTTP Backend - configuration #
2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 # #
3 # #
4 ################################################################################
4 ################################################################################
5
5
6
6
7 [server:main]
7 [server:main]
8 ## COMMON ##
8 ## COMMON ##
9 host = 0.0.0.0
9 host = 0.0.0.0
10 port = 9900
10 port = 9900
11
11
12 use = egg:waitress#main
12 use = egg:waitress#main
13
13
14
14
15 [app:main]
15 [app:main]
16 use = egg:rhodecode-vcsserver
16 use = egg:rhodecode-vcsserver
17
17
18 pyramid.default_locale_name = en
18 pyramid.default_locale_name = en
19 pyramid.includes =
19 pyramid.includes =
20
20
21 ## default locale used by VCS systems
21 ## default locale used by VCS systems
22 locale = en_US.UTF-8
22 locale = en_US.UTF-8
23
23
24
24
25 ## path to binaries for vcsserver, it should be set by the installer
25 ## path to binaries for vcsserver, it should be set by the installer
26 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
26 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
27 core.binary_dir = ""
27 core.binary_dir = ""
28
28
29 ## custom exception store path, defaults to TMPDIR
30 exception_store_path =
31
32 ## Default cache dir for caches. Putting this into a ramdisk
33 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
34 ## large ammount of space
35 cache_dir = %(here)s/rcdev/data
36
29 ## cache region for storing repo_objects cache
37 ## cache region for storing repo_objects cache
30 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
38 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
31 ## cache auto-expires after N seconds
39 ## cache auto-expires after N seconds
32 rc_cache.repo_object.expiration_time = 300
40 rc_cache.repo_object.expiration_time = 300
33 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
41 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
34 rc_cache.repo_object.max_size = 100
42 rc_cache.repo_object.max_size = 100
35
43
36
44
37 ################################
45 ################################
38 ### LOGGING CONFIGURATION ####
46 ### LOGGING CONFIGURATION ####
39 ################################
47 ################################
40 [loggers]
48 [loggers]
41 keys = root, vcsserver
49 keys = root, vcsserver
42
50
43 [handlers]
51 [handlers]
44 keys = console
52 keys = console
45
53
46 [formatters]
54 [formatters]
47 keys = generic
55 keys = generic
48
56
49 #############
57 #############
50 ## LOGGERS ##
58 ## LOGGERS ##
51 #############
59 #############
52 [logger_root]
60 [logger_root]
53 level = NOTSET
61 level = NOTSET
54 handlers = console
62 handlers = console
55
63
56 [logger_vcsserver]
64 [logger_vcsserver]
57 level = DEBUG
65 level = DEBUG
58 handlers =
66 handlers =
59 qualname = vcsserver
67 qualname = vcsserver
60 propagate = 1
68 propagate = 1
61
69
62
70
63 ##############
71 ##############
64 ## HANDLERS ##
72 ## HANDLERS ##
65 ##############
73 ##############
66
74
67 [handler_console]
75 [handler_console]
68 class = StreamHandler
76 class = StreamHandler
69 args = (sys.stderr,)
77 args = (sys.stderr,)
70 level = DEBUG
78 level = DEBUG
71 formatter = generic
79 formatter = generic
72
80
73 ################
81 ################
74 ## FORMATTERS ##
82 ## FORMATTERS ##
75 ################
83 ################
76
84
77 [formatter_generic]
85 [formatter_generic]
78 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
86 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
79 datefmt = %Y-%m-%d %H:%M:%S
87 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,100 +1,108 b''
1 ################################################################################
1 ################################################################################
2 # RhodeCode VCSServer with HTTP Backend - configuration #
2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 # #
3 # #
4 ################################################################################
4 ################################################################################
5
5
6
6
7 [server:main]
7 [server:main]
8 ## COMMON ##
8 ## COMMON ##
9 host = 127.0.0.1
9 host = 127.0.0.1
10 port = 9900
10 port = 9900
11
11
12
12
13 ##########################
13 ##########################
14 ## GUNICORN WSGI SERVER ##
14 ## GUNICORN WSGI SERVER ##
15 ##########################
15 ##########################
16 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
16 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
17 use = egg:gunicorn#main
17 use = egg:gunicorn#main
18 ## Sets the number of process workers. Recommended
18 ## Sets the number of process workers. Recommended
19 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
19 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
20 workers = 2
20 workers = 2
21 ## process name
21 ## process name
22 proc_name = rhodecode_vcsserver
22 proc_name = rhodecode_vcsserver
23 ## type of worker class, currently `sync` is the only option allowed.
23 ## type of worker class, currently `sync` is the only option allowed.
24 worker_class = sync
24 worker_class = sync
25 ## The maximum number of simultaneous clients. Valid only for Gevent
25 ## The maximum number of simultaneous clients. Valid only for Gevent
26 #worker_connections = 10
26 #worker_connections = 10
27 ## max number of requests that worker will handle before being gracefully
27 ## max number of requests that worker will handle before being gracefully
28 ## restarted, could prevent memory leaks
28 ## restarted, could prevent memory leaks
29 max_requests = 1000
29 max_requests = 1000
30 max_requests_jitter = 30
30 max_requests_jitter = 30
31 ## amount of time a worker can spend with handling a request before it
31 ## amount of time a worker can spend with handling a request before it
32 ## gets killed and restarted. Set to 6hrs
32 ## gets killed and restarted. Set to 6hrs
33 timeout = 21600
33 timeout = 21600
34
34
35
35
36 [app:main]
36 [app:main]
37 use = egg:rhodecode-vcsserver
37 use = egg:rhodecode-vcsserver
38
38
39 pyramid.default_locale_name = en
39 pyramid.default_locale_name = en
40 pyramid.includes =
40 pyramid.includes =
41
41
42 ## default locale used by VCS systems
42 ## default locale used by VCS systems
43 locale = en_US.UTF-8
43 locale = en_US.UTF-8
44
44
45
45
46 ## path to binaries for vcsserver, it should be set by the installer
46 ## path to binaries for vcsserver, it should be set by the installer
47 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
47 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
48 core.binary_dir = ""
48 core.binary_dir = ""
49
49
50 ## custom exception store path, defaults to TMPDIR
51 exception_store_path =
52
53 ## Default cache dir for caches. Putting this into a ramdisk
54 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
55 ## large ammount of space
56 cache_dir = %(here)s/rcdev/data
57
50 ## cache region for storing repo_objects cache
58 ## cache region for storing repo_objects cache
51 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
59 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
52 ## cache auto-expires after N seconds
60 ## cache auto-expires after N seconds
53 rc_cache.repo_object.expiration_time = 300
61 rc_cache.repo_object.expiration_time = 300
54 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
62 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
55 rc_cache.repo_object.max_size = 100
63 rc_cache.repo_object.max_size = 100
56
64
57
65
58 ################################
66 ################################
59 ### LOGGING CONFIGURATION ####
67 ### LOGGING CONFIGURATION ####
60 ################################
68 ################################
61 [loggers]
69 [loggers]
62 keys = root, vcsserver
70 keys = root, vcsserver
63
71
64 [handlers]
72 [handlers]
65 keys = console
73 keys = console
66
74
67 [formatters]
75 [formatters]
68 keys = generic
76 keys = generic
69
77
70 #############
78 #############
71 ## LOGGERS ##
79 ## LOGGERS ##
72 #############
80 #############
73 [logger_root]
81 [logger_root]
74 level = NOTSET
82 level = NOTSET
75 handlers = console
83 handlers = console
76
84
77 [logger_vcsserver]
85 [logger_vcsserver]
78 level = DEBUG
86 level = DEBUG
79 handlers =
87 handlers =
80 qualname = vcsserver
88 qualname = vcsserver
81 propagate = 1
89 propagate = 1
82
90
83
91
84 ##############
92 ##############
85 ## HANDLERS ##
93 ## HANDLERS ##
86 ##############
94 ##############
87
95
88 [handler_console]
96 [handler_console]
89 class = StreamHandler
97 class = StreamHandler
90 args = (sys.stderr,)
98 args = (sys.stderr,)
91 level = DEBUG
99 level = DEBUG
92 formatter = generic
100 formatter = generic
93
101
94 ################
102 ################
95 ## FORMATTERS ##
103 ## FORMATTERS ##
96 ################
104 ################
97
105
98 [formatter_generic]
106 [formatter_generic]
99 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
107 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
100 datefmt = %Y-%m-%d %H:%M:%S
108 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,21 +1,28 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import pkgutil
18 import pkgutil
19
19
20
20
21 __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
21 __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
22
23 # link to config for pyramid
24 CONFIG = {}
25
26 # Populated with the settings dictionary from application init in
27 #
28 PYRAMID_SETTINGS = {}
@@ -1,563 +1,598 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import sys
19 import sys
20 import base64
20 import base64
21 import locale
21 import locale
22 import logging
22 import logging
23 import uuid
23 import uuid
24 import wsgiref.util
24 import wsgiref.util
25 import traceback
25 import traceback
26 import tempfile
26 from itertools import chain
27 from itertools import chain
27
28
28 import simplejson as json
29 import simplejson as json
29 import msgpack
30 import msgpack
30 from pyramid.config import Configurator
31 from pyramid.config import Configurator
31 from pyramid.settings import asbool, aslist
32 from pyramid.settings import asbool, aslist
32 from pyramid.wsgi import wsgiapp
33 from pyramid.wsgi import wsgiapp
33 from pyramid.compat import configparser
34 from pyramid.compat import configparser
34
35
35
36
36 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
37
38
38 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
39 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
39 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
40 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
40
41
41 try:
42 try:
42 locale.setlocale(locale.LC_ALL, '')
43 locale.setlocale(locale.LC_ALL, '')
43 except locale.Error as e:
44 except locale.Error as e:
44 log.error(
45 log.error(
45 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
46 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
46 os.environ['LC_ALL'] = 'C'
47 os.environ['LC_ALL'] = 'C'
47
48
48
49 import vcsserver
49 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
50 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
50 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
51 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
51 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
52 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
52 from vcsserver.echo_stub.echo_app import EchoApp
53 from vcsserver.echo_stub.echo_app import EchoApp
53 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
54 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
54 from vcsserver.lib.exc_tracking import store_exception
55 from vcsserver.lib.exc_tracking import store_exception
55 from vcsserver.server import VcsServer
56 from vcsserver.server import VcsServer
56
57
57 try:
58 try:
58 from vcsserver.git import GitFactory, GitRemote
59 from vcsserver.git import GitFactory, GitRemote
59 except ImportError:
60 except ImportError:
60 GitFactory = None
61 GitFactory = None
61 GitRemote = None
62 GitRemote = None
62
63
63 try:
64 try:
64 from vcsserver.hg import MercurialFactory, HgRemote
65 from vcsserver.hg import MercurialFactory, HgRemote
65 except ImportError:
66 except ImportError:
66 MercurialFactory = None
67 MercurialFactory = None
67 HgRemote = None
68 HgRemote = None
68
69
69 try:
70 try:
70 from vcsserver.svn import SubversionFactory, SvnRemote
71 from vcsserver.svn import SubversionFactory, SvnRemote
71 except ImportError:
72 except ImportError:
72 SubversionFactory = None
73 SubversionFactory = None
73 SvnRemote = None
74 SvnRemote = None
74
75
75
76
76
77
78 def _is_request_chunked(environ):
77 def _is_request_chunked(environ):
79 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
78 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
80 return stream
79 return stream
81
80
82
81
83 def _int_setting(settings, name, default):
82 def _int_setting(settings, name, default):
84 settings[name] = int(settings.get(name, default))
83 settings[name] = int(settings.get(name, default))
84 return settings[name]
85
85
86
86
87 def _bool_setting(settings, name, default):
87 def _bool_setting(settings, name, default):
88 input_val = settings.get(name, default)
88 input_val = settings.get(name, default)
89 if isinstance(input_val, unicode):
89 if isinstance(input_val, unicode):
90 input_val = input_val.encode('utf8')
90 input_val = input_val.encode('utf8')
91 settings[name] = asbool(input_val)
91 settings[name] = asbool(input_val)
92 return settings[name]
92
93
93
94
94 def _list_setting(settings, name, default):
95 def _list_setting(settings, name, default):
95 raw_value = settings.get(name, default)
96 raw_value = settings.get(name, default)
96
97
97 # Otherwise we assume it uses pyramids space/newline separation.
98 # Otherwise we assume it uses pyramids space/newline separation.
98 settings[name] = aslist(raw_value)
99 settings[name] = aslist(raw_value)
100 return settings[name]
99
101
100
102
101 def _string_setting(settings, name, default, lower=True):
103 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
102 value = settings.get(name, default)
104 value = settings.get(name, default)
105
106 if default_when_empty and not value:
107 # use default value when value is empty
108 value = default
109
103 if lower:
110 if lower:
104 value = value.lower()
111 value = value.lower()
105 settings[name] = value
112 settings[name] = value
113 return settings[name]
106
114
107
115
108 class VCS(object):
116 class VCS(object):
109 def __init__(self, locale=None, cache_config=None):
117 def __init__(self, locale=None, cache_config=None):
110 self.locale = locale
118 self.locale = locale
111 self.cache_config = cache_config
119 self.cache_config = cache_config
112 self._configure_locale()
120 self._configure_locale()
113
121
114 if GitFactory and GitRemote:
122 if GitFactory and GitRemote:
115 git_factory = GitFactory()
123 git_factory = GitFactory()
116 self._git_remote = GitRemote(git_factory)
124 self._git_remote = GitRemote(git_factory)
117 else:
125 else:
118 log.info("Git client import failed")
126 log.info("Git client import failed")
119
127
120 if MercurialFactory and HgRemote:
128 if MercurialFactory and HgRemote:
121 hg_factory = MercurialFactory()
129 hg_factory = MercurialFactory()
122 self._hg_remote = HgRemote(hg_factory)
130 self._hg_remote = HgRemote(hg_factory)
123 else:
131 else:
124 log.info("Mercurial client import failed")
132 log.info("Mercurial client import failed")
125
133
126 if SubversionFactory and SvnRemote:
134 if SubversionFactory and SvnRemote:
127 svn_factory = SubversionFactory()
135 svn_factory = SubversionFactory()
128
136
129 # hg factory is used for svn url validation
137 # hg factory is used for svn url validation
130 hg_factory = MercurialFactory()
138 hg_factory = MercurialFactory()
131 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
139 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
132 else:
140 else:
133 log.info("Subversion client import failed")
141 log.info("Subversion client import failed")
134
142
135 self._vcsserver = VcsServer()
143 self._vcsserver = VcsServer()
136
144
137 def _configure_locale(self):
145 def _configure_locale(self):
138 if self.locale:
146 if self.locale:
139 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
147 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
140 else:
148 else:
141 log.info(
149 log.info(
142 'Configuring locale subsystem based on environment variables')
150 'Configuring locale subsystem based on environment variables')
143 try:
151 try:
144 # If self.locale is the empty string, then the locale
152 # If self.locale is the empty string, then the locale
145 # module will use the environment variables. See the
153 # module will use the environment variables. See the
146 # documentation of the package `locale`.
154 # documentation of the package `locale`.
147 locale.setlocale(locale.LC_ALL, self.locale)
155 locale.setlocale(locale.LC_ALL, self.locale)
148
156
149 language_code, encoding = locale.getlocale()
157 language_code, encoding = locale.getlocale()
150 log.info(
158 log.info(
151 'Locale set to language code "%s" with encoding "%s".',
159 'Locale set to language code "%s" with encoding "%s".',
152 language_code, encoding)
160 language_code, encoding)
153 except locale.Error:
161 except locale.Error:
154 log.exception(
162 log.exception(
155 'Cannot set locale, not configuring the locale system')
163 'Cannot set locale, not configuring the locale system')
156
164
157
165
158 class WsgiProxy(object):
166 class WsgiProxy(object):
159 def __init__(self, wsgi):
167 def __init__(self, wsgi):
160 self.wsgi = wsgi
168 self.wsgi = wsgi
161
169
162 def __call__(self, environ, start_response):
170 def __call__(self, environ, start_response):
163 input_data = environ['wsgi.input'].read()
171 input_data = environ['wsgi.input'].read()
164 input_data = msgpack.unpackb(input_data)
172 input_data = msgpack.unpackb(input_data)
165
173
166 error = None
174 error = None
167 try:
175 try:
168 data, status, headers = self.wsgi.handle(
176 data, status, headers = self.wsgi.handle(
169 input_data['environment'], input_data['input_data'],
177 input_data['environment'], input_data['input_data'],
170 *input_data['args'], **input_data['kwargs'])
178 *input_data['args'], **input_data['kwargs'])
171 except Exception as e:
179 except Exception as e:
172 data, status, headers = [], None, None
180 data, status, headers = [], None, None
173 error = {
181 error = {
174 'message': str(e),
182 'message': str(e),
175 '_vcs_kind': getattr(e, '_vcs_kind', None)
183 '_vcs_kind': getattr(e, '_vcs_kind', None)
176 }
184 }
177
185
178 start_response(200, {})
186 start_response(200, {})
179 return self._iterator(error, status, headers, data)
187 return self._iterator(error, status, headers, data)
180
188
181 def _iterator(self, error, status, headers, data):
189 def _iterator(self, error, status, headers, data):
182 initial_data = [
190 initial_data = [
183 error,
191 error,
184 status,
192 status,
185 headers,
193 headers,
186 ]
194 ]
187
195
188 for d in chain(initial_data, data):
196 for d in chain(initial_data, data):
189 yield msgpack.packb(d)
197 yield msgpack.packb(d)
190
198
191
199
192 class HTTPApplication(object):
200 class HTTPApplication(object):
193 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
201 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
194
202
195 remote_wsgi = remote_wsgi
203 remote_wsgi = remote_wsgi
196 _use_echo_app = False
204 _use_echo_app = False
197
205
198 def __init__(self, settings=None, global_config=None):
206 def __init__(self, settings=None, global_config=None):
199 self._sanitize_settings_and_apply_defaults(settings)
207 self._sanitize_settings_and_apply_defaults(settings)
200
208
201 self.config = Configurator(settings=settings)
209 self.config = Configurator(settings=settings)
202 self.global_config = global_config
210 self.global_config = global_config
203 self.config.include('vcsserver.lib.rc_cache')
211 self.config.include('vcsserver.lib.rc_cache')
204
212
205 locale = settings.get('locale', '') or 'en_US.UTF-8'
213 locale = settings.get('locale', '') or 'en_US.UTF-8'
206 vcs = VCS(locale=locale, cache_config=settings)
214 vcs = VCS(locale=locale, cache_config=settings)
207 self._remotes = {
215 self._remotes = {
208 'hg': vcs._hg_remote,
216 'hg': vcs._hg_remote,
209 'git': vcs._git_remote,
217 'git': vcs._git_remote,
210 'svn': vcs._svn_remote,
218 'svn': vcs._svn_remote,
211 'server': vcs._vcsserver,
219 'server': vcs._vcsserver,
212 }
220 }
213 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
221 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
214 self._use_echo_app = True
222 self._use_echo_app = True
215 log.warning("Using EchoApp for VCS operations.")
223 log.warning("Using EchoApp for VCS operations.")
216 self.remote_wsgi = remote_wsgi_stub
224 self.remote_wsgi = remote_wsgi_stub
217 self._configure_settings(settings)
225
226 self._configure_settings(global_config, settings)
218 self._configure()
227 self._configure()
219
228
220 def _configure_settings(self, app_settings):
229 def _configure_settings(self, global_config, app_settings):
221 """
230 """
222 Configure the settings module.
231 Configure the settings module.
223 """
232 """
233 settings_merged = global_config.copy()
234 settings_merged.update(app_settings)
235
224 git_path = app_settings.get('git_path', None)
236 git_path = app_settings.get('git_path', None)
225 if git_path:
237 if git_path:
226 settings.GIT_EXECUTABLE = git_path
238 settings.GIT_EXECUTABLE = git_path
227 binary_dir = app_settings.get('core.binary_dir', None)
239 binary_dir = app_settings.get('core.binary_dir', None)
228 if binary_dir:
240 if binary_dir:
229 settings.BINARY_DIR = binary_dir
241 settings.BINARY_DIR = binary_dir
230
242
243 # Store the settings to make them available to other modules.
244 vcsserver.PYRAMID_SETTINGS = settings_merged
245 vcsserver.CONFIG = settings_merged
246
231 def _sanitize_settings_and_apply_defaults(self, settings):
247 def _sanitize_settings_and_apply_defaults(self, settings):
248
249 default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
250
251 # save default, cache dir, and use it for all backends later.
252 default_cache_dir = _string_setting(
253 settings,
254 'cache_dir',
255 default_cache_dir, lower=False, default_when_empty=True)
256
257 # ensure we have our dir created
258 if not os.path.isdir(default_cache_dir):
259 os.makedirs(default_cache_dir, mode=0755)
260
261 # exception store cache
262 _string_setting(
263 settings,
264 'exception_store_path',
265 default_cache_dir, lower=False)
266
232 # repo_object cache
267 # repo_object cache
233 _string_setting(
268 _string_setting(
234 settings,
269 settings,
235 'rc_cache.repo_object.backend',
270 'rc_cache.repo_object.backend',
236 'dogpile.cache.rc.memory_lru')
271 'dogpile.cache.rc.memory_lru')
237 _int_setting(
272 _int_setting(
238 settings,
273 settings,
239 'rc_cache.repo_object.expiration_time',
274 'rc_cache.repo_object.expiration_time',
240 300)
275 300)
241 _int_setting(
276 _int_setting(
242 settings,
277 settings,
243 'rc_cache.repo_object.max_size',
278 'rc_cache.repo_object.max_size',
244 1024)
279 1024)
245
280
246 def _configure(self):
281 def _configure(self):
247 self.config.add_renderer(
282 self.config.add_renderer(
248 name='msgpack',
283 name='msgpack',
249 factory=self._msgpack_renderer_factory)
284 factory=self._msgpack_renderer_factory)
250
285
251 self.config.add_route('service', '/_service')
286 self.config.add_route('service', '/_service')
252 self.config.add_route('status', '/status')
287 self.config.add_route('status', '/status')
253 self.config.add_route('hg_proxy', '/proxy/hg')
288 self.config.add_route('hg_proxy', '/proxy/hg')
254 self.config.add_route('git_proxy', '/proxy/git')
289 self.config.add_route('git_proxy', '/proxy/git')
255 self.config.add_route('vcs', '/{backend}')
290 self.config.add_route('vcs', '/{backend}')
256 self.config.add_route('stream_git', '/stream/git/*repo_name')
291 self.config.add_route('stream_git', '/stream/git/*repo_name')
257 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
292 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
258
293
259 self.config.add_view(
294 self.config.add_view(
260 self.status_view, route_name='status', renderer='json')
295 self.status_view, route_name='status', renderer='json')
261 self.config.add_view(
296 self.config.add_view(
262 self.service_view, route_name='service', renderer='msgpack')
297 self.service_view, route_name='service', renderer='msgpack')
263
298
264 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
299 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
265 self.config.add_view(self.git_proxy(), route_name='git_proxy')
300 self.config.add_view(self.git_proxy(), route_name='git_proxy')
266 self.config.add_view(
301 self.config.add_view(
267 self.vcs_view, route_name='vcs', renderer='msgpack',
302 self.vcs_view, route_name='vcs', renderer='msgpack',
268 custom_predicates=[self.is_vcs_view])
303 custom_predicates=[self.is_vcs_view])
269
304
270 self.config.add_view(self.hg_stream(), route_name='stream_hg')
305 self.config.add_view(self.hg_stream(), route_name='stream_hg')
271 self.config.add_view(self.git_stream(), route_name='stream_git')
306 self.config.add_view(self.git_stream(), route_name='stream_git')
272
307
273 def notfound(request):
308 def notfound(request):
274 return {'status': '404 NOT FOUND'}
309 return {'status': '404 NOT FOUND'}
275 self.config.add_notfound_view(notfound, renderer='json')
310 self.config.add_notfound_view(notfound, renderer='json')
276
311
277 self.config.add_view(self.handle_vcs_exception, context=Exception)
312 self.config.add_view(self.handle_vcs_exception, context=Exception)
278
313
279 self.config.add_tween(
314 self.config.add_tween(
280 'vcsserver.tweens.RequestWrapperTween',
315 'vcsserver.tweens.RequestWrapperTween',
281 )
316 )
282
317
283 def wsgi_app(self):
318 def wsgi_app(self):
284 return self.config.make_wsgi_app()
319 return self.config.make_wsgi_app()
285
320
286 def vcs_view(self, request):
321 def vcs_view(self, request):
287 remote = self._remotes[request.matchdict['backend']]
322 remote = self._remotes[request.matchdict['backend']]
288 payload = msgpack.unpackb(request.body, use_list=True)
323 payload = msgpack.unpackb(request.body, use_list=True)
289 method = payload.get('method')
324 method = payload.get('method')
290 params = payload.get('params')
325 params = payload.get('params')
291 wire = params.get('wire')
326 wire = params.get('wire')
292 args = params.get('args')
327 args = params.get('args')
293 kwargs = params.get('kwargs')
328 kwargs = params.get('kwargs')
294 context_uid = None
329 context_uid = None
295
330
296 if wire:
331 if wire:
297 try:
332 try:
298 wire['context'] = context_uid = uuid.UUID(wire['context'])
333 wire['context'] = context_uid = uuid.UUID(wire['context'])
299 except KeyError:
334 except KeyError:
300 pass
335 pass
301 args.insert(0, wire)
336 args.insert(0, wire)
302
337
303 log.debug('method called:%s with kwargs:%s context_uid: %s',
338 log.debug('method called:%s with kwargs:%s context_uid: %s',
304 method, kwargs, context_uid)
339 method, kwargs, context_uid)
305 try:
340 try:
306 resp = getattr(remote, method)(*args, **kwargs)
341 resp = getattr(remote, method)(*args, **kwargs)
307 except Exception as e:
342 except Exception as e:
308 exc_info = list(sys.exc_info())
343 exc_info = list(sys.exc_info())
309 exc_type, exc_value, exc_traceback = exc_info
344 exc_type, exc_value, exc_traceback = exc_info
310
345
311 org_exc = getattr(e, '_org_exc', None)
346 org_exc = getattr(e, '_org_exc', None)
312 org_exc_name = None
347 org_exc_name = None
313 if org_exc:
348 if org_exc:
314 org_exc_name = org_exc.__class__.__name__
349 org_exc_name = org_exc.__class__.__name__
315 # replace our "faked" exception with our org
350 # replace our "faked" exception with our org
316 exc_info[0] = org_exc.__class__
351 exc_info[0] = org_exc.__class__
317 exc_info[1] = org_exc
352 exc_info[1] = org_exc
318
353
319 store_exception(id(exc_info), exc_info)
354 store_exception(id(exc_info), exc_info)
320
355
321 tb_info = ''.join(
356 tb_info = ''.join(
322 traceback.format_exception(exc_type, exc_value, exc_traceback))
357 traceback.format_exception(exc_type, exc_value, exc_traceback))
323
358
324 type_ = e.__class__.__name__
359 type_ = e.__class__.__name__
325 if type_ not in self.ALLOWED_EXCEPTIONS:
360 if type_ not in self.ALLOWED_EXCEPTIONS:
326 type_ = None
361 type_ = None
327
362
328 resp = {
363 resp = {
329 'id': payload.get('id'),
364 'id': payload.get('id'),
330 'error': {
365 'error': {
331 'message': e.message,
366 'message': e.message,
332 'traceback': tb_info,
367 'traceback': tb_info,
333 'org_exc': org_exc_name,
368 'org_exc': org_exc_name,
334 'type': type_
369 'type': type_
335 }
370 }
336 }
371 }
337 try:
372 try:
338 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
373 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
339 except AttributeError:
374 except AttributeError:
340 pass
375 pass
341 else:
376 else:
342 resp = {
377 resp = {
343 'id': payload.get('id'),
378 'id': payload.get('id'),
344 'result': resp
379 'result': resp
345 }
380 }
346
381
347 return resp
382 return resp
348
383
349 def status_view(self, request):
384 def status_view(self, request):
350 import vcsserver
385 import vcsserver
351 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
386 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
352 'pid': os.getpid()}
387 'pid': os.getpid()}
353
388
354 def service_view(self, request):
389 def service_view(self, request):
355 import vcsserver
390 import vcsserver
356
391
357 payload = msgpack.unpackb(request.body, use_list=True)
392 payload = msgpack.unpackb(request.body, use_list=True)
358
393
359 try:
394 try:
360 path = self.global_config['__file__']
395 path = self.global_config['__file__']
361 config = configparser.ConfigParser()
396 config = configparser.ConfigParser()
362 config.read(path)
397 config.read(path)
363 parsed_ini = config
398 parsed_ini = config
364 if parsed_ini.has_section('server:main'):
399 if parsed_ini.has_section('server:main'):
365 parsed_ini = dict(parsed_ini.items('server:main'))
400 parsed_ini = dict(parsed_ini.items('server:main'))
366 except Exception:
401 except Exception:
367 log.exception('Failed to read .ini file for display')
402 log.exception('Failed to read .ini file for display')
368 parsed_ini = {}
403 parsed_ini = {}
369
404
370 resp = {
405 resp = {
371 'id': payload.get('id'),
406 'id': payload.get('id'),
372 'result': dict(
407 'result': dict(
373 version=vcsserver.__version__,
408 version=vcsserver.__version__,
374 config=parsed_ini,
409 config=parsed_ini,
375 payload=payload,
410 payload=payload,
376 )
411 )
377 }
412 }
378 return resp
413 return resp
379
414
380 def _msgpack_renderer_factory(self, info):
415 def _msgpack_renderer_factory(self, info):
381 def _render(value, system):
416 def _render(value, system):
382 value = msgpack.packb(value)
417 value = msgpack.packb(value)
383 request = system.get('request')
418 request = system.get('request')
384 if request is not None:
419 if request is not None:
385 response = request.response
420 response = request.response
386 ct = response.content_type
421 ct = response.content_type
387 if ct == response.default_content_type:
422 if ct == response.default_content_type:
388 response.content_type = 'application/x-msgpack'
423 response.content_type = 'application/x-msgpack'
389 return value
424 return value
390 return _render
425 return _render
391
426
392 def set_env_from_config(self, environ, config):
427 def set_env_from_config(self, environ, config):
393 dict_conf = {}
428 dict_conf = {}
394 try:
429 try:
395 for elem in config:
430 for elem in config:
396 if elem[0] == 'rhodecode':
431 if elem[0] == 'rhodecode':
397 dict_conf = json.loads(elem[2])
432 dict_conf = json.loads(elem[2])
398 break
433 break
399 except Exception:
434 except Exception:
400 log.exception('Failed to fetch SCM CONFIG')
435 log.exception('Failed to fetch SCM CONFIG')
401 return
436 return
402
437
403 username = dict_conf.get('username')
438 username = dict_conf.get('username')
404 if username:
439 if username:
405 environ['REMOTE_USER'] = username
440 environ['REMOTE_USER'] = username
406 # mercurial specific, some extension api rely on this
441 # mercurial specific, some extension api rely on this
407 environ['HGUSER'] = username
442 environ['HGUSER'] = username
408
443
409 ip = dict_conf.get('ip')
444 ip = dict_conf.get('ip')
410 if ip:
445 if ip:
411 environ['REMOTE_HOST'] = ip
446 environ['REMOTE_HOST'] = ip
412
447
413 if _is_request_chunked(environ):
448 if _is_request_chunked(environ):
414 # set the compatibility flag for webob
449 # set the compatibility flag for webob
415 environ['wsgi.input_terminated'] = True
450 environ['wsgi.input_terminated'] = True
416
451
417 def hg_proxy(self):
452 def hg_proxy(self):
418 @wsgiapp
453 @wsgiapp
419 def _hg_proxy(environ, start_response):
454 def _hg_proxy(environ, start_response):
420 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
455 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
421 return app(environ, start_response)
456 return app(environ, start_response)
422 return _hg_proxy
457 return _hg_proxy
423
458
424 def git_proxy(self):
459 def git_proxy(self):
425 @wsgiapp
460 @wsgiapp
426 def _git_proxy(environ, start_response):
461 def _git_proxy(environ, start_response):
427 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
462 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
428 return app(environ, start_response)
463 return app(environ, start_response)
429 return _git_proxy
464 return _git_proxy
430
465
431 def hg_stream(self):
466 def hg_stream(self):
432 if self._use_echo_app:
467 if self._use_echo_app:
433 @wsgiapp
468 @wsgiapp
434 def _hg_stream(environ, start_response):
469 def _hg_stream(environ, start_response):
435 app = EchoApp('fake_path', 'fake_name', None)
470 app = EchoApp('fake_path', 'fake_name', None)
436 return app(environ, start_response)
471 return app(environ, start_response)
437 return _hg_stream
472 return _hg_stream
438 else:
473 else:
439 @wsgiapp
474 @wsgiapp
440 def _hg_stream(environ, start_response):
475 def _hg_stream(environ, start_response):
441 log.debug('http-app: handling hg stream')
476 log.debug('http-app: handling hg stream')
442 repo_path = environ['HTTP_X_RC_REPO_PATH']
477 repo_path = environ['HTTP_X_RC_REPO_PATH']
443 repo_name = environ['HTTP_X_RC_REPO_NAME']
478 repo_name = environ['HTTP_X_RC_REPO_NAME']
444 packed_config = base64.b64decode(
479 packed_config = base64.b64decode(
445 environ['HTTP_X_RC_REPO_CONFIG'])
480 environ['HTTP_X_RC_REPO_CONFIG'])
446 config = msgpack.unpackb(packed_config)
481 config = msgpack.unpackb(packed_config)
447 app = scm_app.create_hg_wsgi_app(
482 app = scm_app.create_hg_wsgi_app(
448 repo_path, repo_name, config)
483 repo_path, repo_name, config)
449
484
450 # Consistent path information for hgweb
485 # Consistent path information for hgweb
451 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
486 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
452 environ['REPO_NAME'] = repo_name
487 environ['REPO_NAME'] = repo_name
453 self.set_env_from_config(environ, config)
488 self.set_env_from_config(environ, config)
454
489
455 log.debug('http-app: starting app handler '
490 log.debug('http-app: starting app handler '
456 'with %s and process request', app)
491 'with %s and process request', app)
457 return app(environ, ResponseFilter(start_response))
492 return app(environ, ResponseFilter(start_response))
458 return _hg_stream
493 return _hg_stream
459
494
460 def git_stream(self):
495 def git_stream(self):
461 if self._use_echo_app:
496 if self._use_echo_app:
462 @wsgiapp
497 @wsgiapp
463 def _git_stream(environ, start_response):
498 def _git_stream(environ, start_response):
464 app = EchoApp('fake_path', 'fake_name', None)
499 app = EchoApp('fake_path', 'fake_name', None)
465 return app(environ, start_response)
500 return app(environ, start_response)
466 return _git_stream
501 return _git_stream
467 else:
502 else:
468 @wsgiapp
503 @wsgiapp
469 def _git_stream(environ, start_response):
504 def _git_stream(environ, start_response):
470 log.debug('http-app: handling git stream')
505 log.debug('http-app: handling git stream')
471 repo_path = environ['HTTP_X_RC_REPO_PATH']
506 repo_path = environ['HTTP_X_RC_REPO_PATH']
472 repo_name = environ['HTTP_X_RC_REPO_NAME']
507 repo_name = environ['HTTP_X_RC_REPO_NAME']
473 packed_config = base64.b64decode(
508 packed_config = base64.b64decode(
474 environ['HTTP_X_RC_REPO_CONFIG'])
509 environ['HTTP_X_RC_REPO_CONFIG'])
475 config = msgpack.unpackb(packed_config)
510 config = msgpack.unpackb(packed_config)
476
511
477 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
512 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
478 self.set_env_from_config(environ, config)
513 self.set_env_from_config(environ, config)
479
514
480 content_type = environ.get('CONTENT_TYPE', '')
515 content_type = environ.get('CONTENT_TYPE', '')
481
516
482 path = environ['PATH_INFO']
517 path = environ['PATH_INFO']
483 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
518 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
484 log.debug(
519 log.debug(
485 'LFS: Detecting if request `%s` is LFS server path based '
520 'LFS: Detecting if request `%s` is LFS server path based '
486 'on content type:`%s`, is_lfs:%s',
521 'on content type:`%s`, is_lfs:%s',
487 path, content_type, is_lfs_request)
522 path, content_type, is_lfs_request)
488
523
489 if not is_lfs_request:
524 if not is_lfs_request:
490 # fallback detection by path
525 # fallback detection by path
491 if GIT_LFS_PROTO_PAT.match(path):
526 if GIT_LFS_PROTO_PAT.match(path):
492 is_lfs_request = True
527 is_lfs_request = True
493 log.debug(
528 log.debug(
494 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
529 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
495 path, is_lfs_request)
530 path, is_lfs_request)
496
531
497 if is_lfs_request:
532 if is_lfs_request:
498 app = scm_app.create_git_lfs_wsgi_app(
533 app = scm_app.create_git_lfs_wsgi_app(
499 repo_path, repo_name, config)
534 repo_path, repo_name, config)
500 else:
535 else:
501 app = scm_app.create_git_wsgi_app(
536 app = scm_app.create_git_wsgi_app(
502 repo_path, repo_name, config)
537 repo_path, repo_name, config)
503
538
504 log.debug('http-app: starting app handler '
539 log.debug('http-app: starting app handler '
505 'with %s and process request', app)
540 'with %s and process request', app)
506
541
507 return app(environ, start_response)
542 return app(environ, start_response)
508
543
509 return _git_stream
544 return _git_stream
510
545
511 def is_vcs_view(self, context, request):
546 def is_vcs_view(self, context, request):
512 """
547 """
513 View predicate that returns true if given backend is supported by
548 View predicate that returns true if given backend is supported by
514 defined remotes.
549 defined remotes.
515 """
550 """
516 backend = request.matchdict.get('backend')
551 backend = request.matchdict.get('backend')
517 return backend in self._remotes
552 return backend in self._remotes
518
553
519 def handle_vcs_exception(self, exception, request):
554 def handle_vcs_exception(self, exception, request):
520 _vcs_kind = getattr(exception, '_vcs_kind', '')
555 _vcs_kind = getattr(exception, '_vcs_kind', '')
521 if _vcs_kind == 'repo_locked':
556 if _vcs_kind == 'repo_locked':
522 # Get custom repo-locked status code if present.
557 # Get custom repo-locked status code if present.
523 status_code = request.headers.get('X-RC-Locked-Status-Code')
558 status_code = request.headers.get('X-RC-Locked-Status-Code')
524 return HTTPRepoLocked(
559 return HTTPRepoLocked(
525 title=exception.message, status_code=status_code)
560 title=exception.message, status_code=status_code)
526
561
527 elif _vcs_kind == 'repo_branch_protected':
562 elif _vcs_kind == 'repo_branch_protected':
528 # Get custom repo-branch-protected status code if present.
563 # Get custom repo-branch-protected status code if present.
529 return HTTPRepoBranchProtected(title=exception.message)
564 return HTTPRepoBranchProtected(title=exception.message)
530
565
531 exc_info = request.exc_info
566 exc_info = request.exc_info
532 store_exception(id(exc_info), exc_info)
567 store_exception(id(exc_info), exc_info)
533
568
534 traceback_info = 'unavailable'
569 traceback_info = 'unavailable'
535 if request.exc_info:
570 if request.exc_info:
536 exc_type, exc_value, exc_tb = request.exc_info
571 exc_type, exc_value, exc_tb = request.exc_info
537 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
572 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
538
573
539 log.error(
574 log.error(
540 'error occurred handling this request for path: %s, \n tb: %s',
575 'error occurred handling this request for path: %s, \n tb: %s',
541 request.path, traceback_info)
576 request.path, traceback_info)
542 raise exception
577 raise exception
543
578
544
579
545 class ResponseFilter(object):
580 class ResponseFilter(object):
546
581
547 def __init__(self, start_response):
582 def __init__(self, start_response):
548 self._start_response = start_response
583 self._start_response = start_response
549
584
550 def __call__(self, status, response_headers, exc_info=None):
585 def __call__(self, status, response_headers, exc_info=None):
551 headers = tuple(
586 headers = tuple(
552 (h, v) for h, v in response_headers
587 (h, v) for h, v in response_headers
553 if not wsgiref.util.is_hop_by_hop(h))
588 if not wsgiref.util.is_hop_by_hop(h))
554 return self._start_response(status, headers, exc_info)
589 return self._start_response(status, headers, exc_info)
555
590
556
591
557 def main(global_config, **settings):
592 def main(global_config, **settings):
558 if MercurialFactory:
593 if MercurialFactory:
559 hgpatches.patch_largefiles_capabilities()
594 hgpatches.patch_largefiles_capabilities()
560 hgpatches.patch_subrepo_type_mapping()
595 hgpatches.patch_subrepo_type_mapping()
561
596
562 app = HTTPApplication(settings=settings, global_config=global_config)
597 app = HTTPApplication(settings=settings, global_config=global_config)
563 return app.wsgi_app()
598 return app.wsgi_app()
@@ -1,146 +1,151 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
3 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2018 RhodeCode GmbH
4 # Copyright (C) 2014-2018 RhodeCode GmbH
5 #
5 #
6 # This program is free software; you can redistribute it and/or modify
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
9 # (at your option) any later version.
10 #
10 #
11 # This program is distributed in the hope that it will be useful,
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
14 # GNU General Public License for more details.
15 #
15 #
16 # You should have received a copy of the GNU General Public License
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
17 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
19
20
20
21 import os
21 import os
22 import time
22 import time
23 import datetime
23 import datetime
24 import msgpack
24 import msgpack
25 import logging
25 import logging
26 import traceback
26 import traceback
27 import tempfile
27 import tempfile
28
28
29
29
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
32 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
33 global_prefix = 'vcsserver'
33 global_prefix = 'vcsserver'
34 exc_store_dir_name = 'rc_exception_store_v1'
34
35
35
36
36 def exc_serialize(exc_id, tb, exc_type):
37 def exc_serialize(exc_id, tb, exc_type):
37
38
38 data = {
39 data = {
39 'version': 'v1',
40 'version': 'v1',
40 'exc_id': exc_id,
41 'exc_id': exc_id,
41 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 'exc_timestamp': repr(time.time()),
43 'exc_timestamp': repr(time.time()),
43 'exc_message': tb,
44 'exc_message': tb,
44 'exc_type': exc_type,
45 'exc_type': exc_type,
45 }
46 }
46 return msgpack.packb(data), data
47 return msgpack.packb(data), data
47
48
48
49
49 def exc_unserialize(tb):
50 def exc_unserialize(tb):
50 return msgpack.unpackb(tb)
51 return msgpack.unpackb(tb)
51
52
52
53
53 def get_exc_store():
54 def get_exc_store():
54 """
55 """
55 Get and create exception store if it's not existing
56 Get and create exception store if it's not existing
56 """
57 """
57 exc_store_dir = 'rc_exception_store_v1'
58 import vcsserver as app
58 # fallback
59 _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir)
60
59
61 exc_store_dir = '' # TODO: need a persistent cross instance store here
60 exc_store_dir = app.CONFIG.get('exception_store_path', '') or tempfile.gettempdir()
62 if exc_store_dir:
61 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
63 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir)
64
62
65 _exc_store_path = os.path.abspath(_exc_store_path)
63 _exc_store_path = os.path.abspath(_exc_store_path)
66 if not os.path.isdir(_exc_store_path):
64 if not os.path.isdir(_exc_store_path):
67 os.makedirs(_exc_store_path)
65 os.makedirs(_exc_store_path)
68 log.debug('Initializing exceptions store at %s', _exc_store_path)
66 log.debug('Initializing exceptions store at %s', _exc_store_path)
69 return _exc_store_path
67 return _exc_store_path
70
68
71
69
72 def _store_exception(exc_id, exc_info, prefix):
70 def _store_exception(exc_id, exc_info, prefix):
73 exc_type, exc_value, exc_traceback = exc_info
71 exc_type, exc_value, exc_traceback = exc_info
74 tb = ''.join(traceback.format_exception(
72 tb = ''.join(traceback.format_exception(
75 exc_type, exc_value, exc_traceback, None))
73 exc_type, exc_value, exc_traceback, None))
76
74
77 exc_type_name = exc_type.__name__
75 exc_type_name = exc_type.__name__
78 exc_store_path = get_exc_store()
76 exc_store_path = get_exc_store()
79 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
77 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
80 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
78 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
81 if not os.path.isdir(exc_store_path):
79 if not os.path.isdir(exc_store_path):
82 os.makedirs(exc_store_path)
80 os.makedirs(exc_store_path)
83 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
81 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
84 with open(stored_exc_path, 'wb') as f:
82 with open(stored_exc_path, 'wb') as f:
85 f.write(exc_data)
83 f.write(exc_data)
86 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
84 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
87
85
88
86
89 def store_exception(exc_id, exc_info, prefix=global_prefix):
87 def store_exception(exc_id, exc_info, prefix=global_prefix):
88 """
89 Example usage::
90
91 exc_info = sys.exc_info()
92 store_exception(id(exc_info), exc_info)
93 """
94
90 try:
95 try:
91 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
96 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
92 except Exception:
97 except Exception:
93 log.exception('Failed to store exception `%s` information', exc_id)
98 log.exception('Failed to store exception `%s` information', exc_id)
94 # there's no way this can fail, it will crash server badly if it does.
99 # there's no way this can fail, it will crash server badly if it does.
95 pass
100 pass
96
101
97
102
98 def _find_exc_file(exc_id, prefix=global_prefix):
103 def _find_exc_file(exc_id, prefix=global_prefix):
99 exc_store_path = get_exc_store()
104 exc_store_path = get_exc_store()
100 if prefix:
105 if prefix:
101 exc_id = '{}_{}'.format(exc_id, prefix)
106 exc_id = '{}_{}'.format(exc_id, prefix)
102 else:
107 else:
103 # search without a prefix
108 # search without a prefix
104 exc_id = '{}'.format(exc_id)
109 exc_id = '{}'.format(exc_id)
105
110
106 # we need to search the store for such start pattern as above
111 # we need to search the store for such start pattern as above
107 for fname in os.listdir(exc_store_path):
112 for fname in os.listdir(exc_store_path):
108 if fname.startswith(exc_id):
113 if fname.startswith(exc_id):
109 exc_id = os.path.join(exc_store_path, fname)
114 exc_id = os.path.join(exc_store_path, fname)
110 break
115 break
111 continue
116 continue
112 else:
117 else:
113 exc_id = None
118 exc_id = None
114
119
115 return exc_id
120 return exc_id
116
121
117
122
118 def _read_exception(exc_id, prefix):
123 def _read_exception(exc_id, prefix):
119 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
124 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
120 if exc_id_file_path:
125 if exc_id_file_path:
121 with open(exc_id_file_path, 'rb') as f:
126 with open(exc_id_file_path, 'rb') as f:
122 return exc_unserialize(f.read())
127 return exc_unserialize(f.read())
123 else:
128 else:
124 log.debug('Exception File `%s` not found', exc_id_file_path)
129 log.debug('Exception File `%s` not found', exc_id_file_path)
125 return None
130 return None
126
131
127
132
128 def read_exception(exc_id, prefix=global_prefix):
133 def read_exception(exc_id, prefix=global_prefix):
129 try:
134 try:
130 return _read_exception(exc_id=exc_id, prefix=prefix)
135 return _read_exception(exc_id=exc_id, prefix=prefix)
131 except Exception:
136 except Exception:
132 log.exception('Failed to read exception `%s` information', exc_id)
137 log.exception('Failed to read exception `%s` information', exc_id)
133 # there's no way this can fail, it will crash server badly if it does.
138 # there's no way this can fail, it will crash server badly if it does.
134 return None
139 return None
135
140
136
141
137 def delete_exception(exc_id, prefix=global_prefix):
142 def delete_exception(exc_id, prefix=global_prefix):
138 try:
143 try:
139 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
144 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
140 if exc_id_file_path:
145 if exc_id_file_path:
141 os.remove(exc_id_file_path)
146 os.remove(exc_id_file_path)
142
147
143 except Exception:
148 except Exception:
144 log.exception('Failed to remove exception `%s` information', exc_id)
149 log.exception('Failed to remove exception `%s` information', exc_id)
145 # there's no way this can fail, it will crash server badly if it does.
150 # there's no way this can fail, it will crash server badly if it does.
146 pass
151 pass
@@ -1,57 +1,57 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2018 RhodeCode GmbH
2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import mock
18 import mock
19 import pytest
19 import pytest
20
20
21 from vcsserver import http_main
21 from vcsserver import http_main
22 from vcsserver.base import obfuscate_qs
22 from vcsserver.base import obfuscate_qs
23
23
24
24
25 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
25 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
28 http_main.main([])
28 http_main.main({})
29 patch_largefiles_capabilities.assert_called_once_with()
29 patch_largefiles_capabilities.assert_called_once_with()
30
30
31
31
32 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
32 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
33 @mock.patch('vcsserver.http_main.MercurialFactory', None)
33 @mock.patch('vcsserver.http_main.MercurialFactory', None)
34 @mock.patch(
34 @mock.patch(
35 'vcsserver.hgpatches.patch_largefiles_capabilities',
35 'vcsserver.hgpatches.patch_largefiles_capabilities',
36 mock.Mock(side_effect=Exception("Must not be called")))
36 mock.Mock(side_effect=Exception("Must not be called")))
37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
38 http_main.main([])
38 http_main.main({})
39
39
40
40
41 @pytest.mark.parametrize('given, expected', [
41 @pytest.mark.parametrize('given, expected', [
42 ('bad', 'bad'),
42 ('bad', 'bad'),
43 ('query&foo=bar', 'query&foo=bar'),
43 ('query&foo=bar', 'query&foo=bar'),
44 ('equery&auth_token=bar', 'equery&auth_token=*****'),
44 ('equery&auth_token=bar', 'equery&auth_token=*****'),
45 ('a;b;c;query&foo=bar&auth_token=secret',
45 ('a;b;c;query&foo=bar&auth_token=secret',
46 'a&b&c&query&foo=bar&auth_token=*****'),
46 'a&b&c&query&foo=bar&auth_token=*****'),
47 ('', ''),
47 ('', ''),
48 (None, None),
48 (None, None),
49 ('foo=bar', 'foo=bar'),
49 ('foo=bar', 'foo=bar'),
50 ('auth_token=secret', 'auth_token=*****'),
50 ('auth_token=secret', 'auth_token=*****'),
51 ('auth_token=secret&api_key=secret2',
51 ('auth_token=secret&api_key=secret2',
52 'auth_token=*****&api_key=*****'),
52 'auth_token=*****&api_key=*****'),
53 ('auth_token=secret&api_key=secret2&param=value',
53 ('auth_token=secret&api_key=secret2&param=value',
54 'auth_token=*****&api_key=*****&param=value'),
54 'auth_token=*****&api_key=*****&param=value'),
55 ])
55 ])
56 def test_obfuscate_qs(given, expected):
56 def test_obfuscate_qs(given, expected):
57 assert expected == obfuscate_qs(given)
57 assert expected == obfuscate_qs(given)
General Comments 0
You need to be logged in to leave comments. Login now