##// END OF EJS Templates
subrepo: Apply mercurial sub repository patch.
Martin Bornhold -
r100:2582dee7 default
parent child Browse files
Show More
@@ -1,358 +1,359 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2016 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import base64
18 import base64
19 import locale
19 import locale
20 import logging
20 import logging
21 import uuid
21 import uuid
22 import wsgiref.util
22 import wsgiref.util
23 from itertools import chain
23 from itertools import chain
24
24
25 import msgpack
25 import msgpack
26 from beaker.cache import CacheManager
26 from beaker.cache import CacheManager
27 from beaker.util import parse_cache_config_options
27 from beaker.util import parse_cache_config_options
28 from pyramid.config import Configurator
28 from pyramid.config import Configurator
29 from pyramid.wsgi import wsgiapp
29 from pyramid.wsgi import wsgiapp
30
30
31 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
31 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
32 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
32 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
33 from vcsserver.echo_stub.echo_app import EchoApp
33 from vcsserver.echo_stub.echo_app import EchoApp
34 from vcsserver.exceptions import HTTPRepoLocked
34 from vcsserver.exceptions import HTTPRepoLocked
35 from vcsserver.server import VcsServer
35 from vcsserver.server import VcsServer
36
36
37 try:
37 try:
38 from vcsserver.git import GitFactory, GitRemote
38 from vcsserver.git import GitFactory, GitRemote
39 except ImportError:
39 except ImportError:
40 GitFactory = None
40 GitFactory = None
41 GitRemote = None
41 GitRemote = None
42 try:
42 try:
43 from vcsserver.hg import MercurialFactory, HgRemote
43 from vcsserver.hg import MercurialFactory, HgRemote
44 except ImportError:
44 except ImportError:
45 MercurialFactory = None
45 MercurialFactory = None
46 HgRemote = None
46 HgRemote = None
47 try:
47 try:
48 from vcsserver.svn import SubversionFactory, SvnRemote
48 from vcsserver.svn import SubversionFactory, SvnRemote
49 except ImportError:
49 except ImportError:
50 SubversionFactory = None
50 SubversionFactory = None
51 SvnRemote = None
51 SvnRemote = None
52
52
53 log = logging.getLogger(__name__)
53 log = logging.getLogger(__name__)
54
54
55
55
56 class VCS(object):
56 class VCS(object):
57 def __init__(self, locale=None, cache_config=None):
57 def __init__(self, locale=None, cache_config=None):
58 self.locale = locale
58 self.locale = locale
59 self.cache_config = cache_config
59 self.cache_config = cache_config
60 self._configure_locale()
60 self._configure_locale()
61 self._initialize_cache()
61 self._initialize_cache()
62
62
63 if GitFactory and GitRemote:
63 if GitFactory and GitRemote:
64 git_repo_cache = self.cache.get_cache_region(
64 git_repo_cache = self.cache.get_cache_region(
65 'git', region='repo_object')
65 'git', region='repo_object')
66 git_factory = GitFactory(git_repo_cache)
66 git_factory = GitFactory(git_repo_cache)
67 self._git_remote = GitRemote(git_factory)
67 self._git_remote = GitRemote(git_factory)
68 else:
68 else:
69 log.info("Git client import failed")
69 log.info("Git client import failed")
70
70
71 if MercurialFactory and HgRemote:
71 if MercurialFactory and HgRemote:
72 hg_repo_cache = self.cache.get_cache_region(
72 hg_repo_cache = self.cache.get_cache_region(
73 'hg', region='repo_object')
73 'hg', region='repo_object')
74 hg_factory = MercurialFactory(hg_repo_cache)
74 hg_factory = MercurialFactory(hg_repo_cache)
75 self._hg_remote = HgRemote(hg_factory)
75 self._hg_remote = HgRemote(hg_factory)
76 else:
76 else:
77 log.info("Mercurial client import failed")
77 log.info("Mercurial client import failed")
78
78
79 if SubversionFactory and SvnRemote:
79 if SubversionFactory and SvnRemote:
80 svn_repo_cache = self.cache.get_cache_region(
80 svn_repo_cache = self.cache.get_cache_region(
81 'svn', region='repo_object')
81 'svn', region='repo_object')
82 svn_factory = SubversionFactory(svn_repo_cache)
82 svn_factory = SubversionFactory(svn_repo_cache)
83 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
83 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
84 else:
84 else:
85 log.info("Subversion client import failed")
85 log.info("Subversion client import failed")
86
86
87 self._vcsserver = VcsServer()
87 self._vcsserver = VcsServer()
88
88
89 def _initialize_cache(self):
89 def _initialize_cache(self):
90 cache_config = parse_cache_config_options(self.cache_config)
90 cache_config = parse_cache_config_options(self.cache_config)
91 log.info('Initializing beaker cache: %s' % cache_config)
91 log.info('Initializing beaker cache: %s' % cache_config)
92 self.cache = CacheManager(**cache_config)
92 self.cache = CacheManager(**cache_config)
93
93
94 def _configure_locale(self):
94 def _configure_locale(self):
95 if self.locale:
95 if self.locale:
96 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
96 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
97 else:
97 else:
98 log.info(
98 log.info(
99 'Configuring locale subsystem based on environment variables')
99 'Configuring locale subsystem based on environment variables')
100 try:
100 try:
101 # If self.locale is the empty string, then the locale
101 # If self.locale is the empty string, then the locale
102 # module will use the environment variables. See the
102 # module will use the environment variables. See the
103 # documentation of the package `locale`.
103 # documentation of the package `locale`.
104 locale.setlocale(locale.LC_ALL, self.locale)
104 locale.setlocale(locale.LC_ALL, self.locale)
105
105
106 language_code, encoding = locale.getlocale()
106 language_code, encoding = locale.getlocale()
107 log.info(
107 log.info(
108 'Locale set to language code "%s" with encoding "%s".',
108 'Locale set to language code "%s" with encoding "%s".',
109 language_code, encoding)
109 language_code, encoding)
110 except locale.Error:
110 except locale.Error:
111 log.exception(
111 log.exception(
112 'Cannot set locale, not configuring the locale system')
112 'Cannot set locale, not configuring the locale system')
113
113
114
114
115 class WsgiProxy(object):
115 class WsgiProxy(object):
116 def __init__(self, wsgi):
116 def __init__(self, wsgi):
117 self.wsgi = wsgi
117 self.wsgi = wsgi
118
118
119 def __call__(self, environ, start_response):
119 def __call__(self, environ, start_response):
120 input_data = environ['wsgi.input'].read()
120 input_data = environ['wsgi.input'].read()
121 input_data = msgpack.unpackb(input_data)
121 input_data = msgpack.unpackb(input_data)
122
122
123 error = None
123 error = None
124 try:
124 try:
125 data, status, headers = self.wsgi.handle(
125 data, status, headers = self.wsgi.handle(
126 input_data['environment'], input_data['input_data'],
126 input_data['environment'], input_data['input_data'],
127 *input_data['args'], **input_data['kwargs'])
127 *input_data['args'], **input_data['kwargs'])
128 except Exception as e:
128 except Exception as e:
129 data, status, headers = [], None, None
129 data, status, headers = [], None, None
130 error = {
130 error = {
131 'message': str(e),
131 'message': str(e),
132 '_vcs_kind': getattr(e, '_vcs_kind', None)
132 '_vcs_kind': getattr(e, '_vcs_kind', None)
133 }
133 }
134
134
135 start_response(200, {})
135 start_response(200, {})
136 return self._iterator(error, status, headers, data)
136 return self._iterator(error, status, headers, data)
137
137
138 def _iterator(self, error, status, headers, data):
138 def _iterator(self, error, status, headers, data):
139 initial_data = [
139 initial_data = [
140 error,
140 error,
141 status,
141 status,
142 headers,
142 headers,
143 ]
143 ]
144
144
145 for d in chain(initial_data, data):
145 for d in chain(initial_data, data):
146 yield msgpack.packb(d)
146 yield msgpack.packb(d)
147
147
148
148
149 class HTTPApplication(object):
149 class HTTPApplication(object):
150 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
150 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
151
151
152 remote_wsgi = remote_wsgi
152 remote_wsgi = remote_wsgi
153 _use_echo_app = False
153 _use_echo_app = False
154
154
155 def __init__(self, settings=None):
155 def __init__(self, settings=None):
156 self.config = Configurator(settings=settings)
156 self.config = Configurator(settings=settings)
157 locale = settings.get('', 'en_US.UTF-8')
157 locale = settings.get('', 'en_US.UTF-8')
158 vcs = VCS(locale=locale, cache_config=settings)
158 vcs = VCS(locale=locale, cache_config=settings)
159 self._remotes = {
159 self._remotes = {
160 'hg': vcs._hg_remote,
160 'hg': vcs._hg_remote,
161 'git': vcs._git_remote,
161 'git': vcs._git_remote,
162 'svn': vcs._svn_remote,
162 'svn': vcs._svn_remote,
163 'server': vcs._vcsserver,
163 'server': vcs._vcsserver,
164 }
164 }
165 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
165 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
166 self._use_echo_app = True
166 self._use_echo_app = True
167 log.warning("Using EchoApp for VCS operations.")
167 log.warning("Using EchoApp for VCS operations.")
168 self.remote_wsgi = remote_wsgi_stub
168 self.remote_wsgi = remote_wsgi_stub
169 self._configure_settings(settings)
169 self._configure_settings(settings)
170 self._configure()
170 self._configure()
171
171
172 def _configure_settings(self, app_settings):
172 def _configure_settings(self, app_settings):
173 """
173 """
174 Configure the settings module.
174 Configure the settings module.
175 """
175 """
176 git_path = app_settings.get('git_path', None)
176 git_path = app_settings.get('git_path', None)
177 if git_path:
177 if git_path:
178 settings.GIT_EXECUTABLE = git_path
178 settings.GIT_EXECUTABLE = git_path
179
179
180 def _configure(self):
180 def _configure(self):
181 self.config.add_renderer(
181 self.config.add_renderer(
182 name='msgpack',
182 name='msgpack',
183 factory=self._msgpack_renderer_factory)
183 factory=self._msgpack_renderer_factory)
184
184
185 self.config.add_route('status', '/status')
185 self.config.add_route('status', '/status')
186 self.config.add_route('hg_proxy', '/proxy/hg')
186 self.config.add_route('hg_proxy', '/proxy/hg')
187 self.config.add_route('git_proxy', '/proxy/git')
187 self.config.add_route('git_proxy', '/proxy/git')
188 self.config.add_route('vcs', '/{backend}')
188 self.config.add_route('vcs', '/{backend}')
189 self.config.add_route('stream_git', '/stream/git/*repo_name')
189 self.config.add_route('stream_git', '/stream/git/*repo_name')
190 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
190 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
191
191
192 self.config.add_view(
192 self.config.add_view(
193 self.status_view, route_name='status', renderer='json')
193 self.status_view, route_name='status', renderer='json')
194 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
194 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
195 self.config.add_view(self.git_proxy(), route_name='git_proxy')
195 self.config.add_view(self.git_proxy(), route_name='git_proxy')
196 self.config.add_view(
196 self.config.add_view(
197 self.vcs_view, route_name='vcs', renderer='msgpack')
197 self.vcs_view, route_name='vcs', renderer='msgpack')
198
198
199 self.config.add_view(self.hg_stream(), route_name='stream_hg')
199 self.config.add_view(self.hg_stream(), route_name='stream_hg')
200 self.config.add_view(self.git_stream(), route_name='stream_git')
200 self.config.add_view(self.git_stream(), route_name='stream_git')
201 self.config.add_view(
201 self.config.add_view(
202 self.handle_vcs_exception, context=Exception,
202 self.handle_vcs_exception, context=Exception,
203 custom_predicates=[self.is_vcs_exception])
203 custom_predicates=[self.is_vcs_exception])
204
204
205 def wsgi_app(self):
205 def wsgi_app(self):
206 return self.config.make_wsgi_app()
206 return self.config.make_wsgi_app()
207
207
208 def vcs_view(self, request):
208 def vcs_view(self, request):
209 remote = self._remotes[request.matchdict['backend']]
209 remote = self._remotes[request.matchdict['backend']]
210 payload = msgpack.unpackb(request.body, use_list=True)
210 payload = msgpack.unpackb(request.body, use_list=True)
211 method = payload.get('method')
211 method = payload.get('method')
212 params = payload.get('params')
212 params = payload.get('params')
213 wire = params.get('wire')
213 wire = params.get('wire')
214 args = params.get('args')
214 args = params.get('args')
215 kwargs = params.get('kwargs')
215 kwargs = params.get('kwargs')
216 if wire:
216 if wire:
217 try:
217 try:
218 wire['context'] = uuid.UUID(wire['context'])
218 wire['context'] = uuid.UUID(wire['context'])
219 except KeyError:
219 except KeyError:
220 pass
220 pass
221 args.insert(0, wire)
221 args.insert(0, wire)
222
222
223 try:
223 try:
224 resp = getattr(remote, method)(*args, **kwargs)
224 resp = getattr(remote, method)(*args, **kwargs)
225 except Exception as e:
225 except Exception as e:
226 type_ = e.__class__.__name__
226 type_ = e.__class__.__name__
227 if type_ not in self.ALLOWED_EXCEPTIONS:
227 if type_ not in self.ALLOWED_EXCEPTIONS:
228 type_ = None
228 type_ = None
229
229
230 resp = {
230 resp = {
231 'id': payload.get('id'),
231 'id': payload.get('id'),
232 'error': {
232 'error': {
233 'message': e.message,
233 'message': e.message,
234 'type': type_
234 'type': type_
235 }
235 }
236 }
236 }
237 try:
237 try:
238 resp['error']['_vcs_kind'] = e._vcs_kind
238 resp['error']['_vcs_kind'] = e._vcs_kind
239 except AttributeError:
239 except AttributeError:
240 pass
240 pass
241 else:
241 else:
242 resp = {
242 resp = {
243 'id': payload.get('id'),
243 'id': payload.get('id'),
244 'result': resp
244 'result': resp
245 }
245 }
246
246
247 return resp
247 return resp
248
248
249 def status_view(self, request):
249 def status_view(self, request):
250 return {'status': 'OK'}
250 return {'status': 'OK'}
251
251
252 def _msgpack_renderer_factory(self, info):
252 def _msgpack_renderer_factory(self, info):
253 def _render(value, system):
253 def _render(value, system):
254 value = msgpack.packb(value)
254 value = msgpack.packb(value)
255 request = system.get('request')
255 request = system.get('request')
256 if request is not None:
256 if request is not None:
257 response = request.response
257 response = request.response
258 ct = response.content_type
258 ct = response.content_type
259 if ct == response.default_content_type:
259 if ct == response.default_content_type:
260 response.content_type = 'application/x-msgpack'
260 response.content_type = 'application/x-msgpack'
261 return value
261 return value
262 return _render
262 return _render
263
263
264 def hg_proxy(self):
264 def hg_proxy(self):
265 @wsgiapp
265 @wsgiapp
266 def _hg_proxy(environ, start_response):
266 def _hg_proxy(environ, start_response):
267 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
267 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
268 return app(environ, start_response)
268 return app(environ, start_response)
269 return _hg_proxy
269 return _hg_proxy
270
270
271 def git_proxy(self):
271 def git_proxy(self):
272 @wsgiapp
272 @wsgiapp
273 def _git_proxy(environ, start_response):
273 def _git_proxy(environ, start_response):
274 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
274 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
275 return app(environ, start_response)
275 return app(environ, start_response)
276 return _git_proxy
276 return _git_proxy
277
277
278 def hg_stream(self):
278 def hg_stream(self):
279 if self._use_echo_app:
279 if self._use_echo_app:
280 @wsgiapp
280 @wsgiapp
281 def _hg_stream(environ, start_response):
281 def _hg_stream(environ, start_response):
282 app = EchoApp('fake_path', 'fake_name', None)
282 app = EchoApp('fake_path', 'fake_name', None)
283 return app(environ, start_response)
283 return app(environ, start_response)
284 return _hg_stream
284 return _hg_stream
285 else:
285 else:
286 @wsgiapp
286 @wsgiapp
287 def _hg_stream(environ, start_response):
287 def _hg_stream(environ, start_response):
288 repo_path = environ['HTTP_X_RC_REPO_PATH']
288 repo_path = environ['HTTP_X_RC_REPO_PATH']
289 repo_name = environ['HTTP_X_RC_REPO_NAME']
289 repo_name = environ['HTTP_X_RC_REPO_NAME']
290 packed_config = base64.b64decode(
290 packed_config = base64.b64decode(
291 environ['HTTP_X_RC_REPO_CONFIG'])
291 environ['HTTP_X_RC_REPO_CONFIG'])
292 config = msgpack.unpackb(packed_config)
292 config = msgpack.unpackb(packed_config)
293 app = scm_app.create_hg_wsgi_app(
293 app = scm_app.create_hg_wsgi_app(
294 repo_path, repo_name, config)
294 repo_path, repo_name, config)
295
295
296 # Consitent path information for hgweb
296 # Consitent path information for hgweb
297 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
297 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
298 environ['REPO_NAME'] = repo_name
298 environ['REPO_NAME'] = repo_name
299 return app(environ, ResponseFilter(start_response))
299 return app(environ, ResponseFilter(start_response))
300 return _hg_stream
300 return _hg_stream
301
301
302 def git_stream(self):
302 def git_stream(self):
303 if self._use_echo_app:
303 if self._use_echo_app:
304 @wsgiapp
304 @wsgiapp
305 def _git_stream(environ, start_response):
305 def _git_stream(environ, start_response):
306 app = EchoApp('fake_path', 'fake_name', None)
306 app = EchoApp('fake_path', 'fake_name', None)
307 return app(environ, start_response)
307 return app(environ, start_response)
308 return _git_stream
308 return _git_stream
309 else:
309 else:
310 @wsgiapp
310 @wsgiapp
311 def _git_stream(environ, start_response):
311 def _git_stream(environ, start_response):
312 repo_path = environ['HTTP_X_RC_REPO_PATH']
312 repo_path = environ['HTTP_X_RC_REPO_PATH']
313 repo_name = environ['HTTP_X_RC_REPO_NAME']
313 repo_name = environ['HTTP_X_RC_REPO_NAME']
314 packed_config = base64.b64decode(
314 packed_config = base64.b64decode(
315 environ['HTTP_X_RC_REPO_CONFIG'])
315 environ['HTTP_X_RC_REPO_CONFIG'])
316 config = msgpack.unpackb(packed_config)
316 config = msgpack.unpackb(packed_config)
317
317
318 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
318 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
319 app = scm_app.create_git_wsgi_app(
319 app = scm_app.create_git_wsgi_app(
320 repo_path, repo_name, config)
320 repo_path, repo_name, config)
321 return app(environ, start_response)
321 return app(environ, start_response)
322 return _git_stream
322 return _git_stream
323
323
324 def is_vcs_exception(self, context, request):
324 def is_vcs_exception(self, context, request):
325 """
325 """
326 View predicate that returns true if the context object is a VCS
326 View predicate that returns true if the context object is a VCS
327 exception.
327 exception.
328 """
328 """
329 return hasattr(context, '_vcs_kind')
329 return hasattr(context, '_vcs_kind')
330
330
331 def handle_vcs_exception(self, exception, request):
331 def handle_vcs_exception(self, exception, request):
332 if exception._vcs_kind == 'repo_locked':
332 if exception._vcs_kind == 'repo_locked':
333 # Get custom repo-locked status code if present.
333 # Get custom repo-locked status code if present.
334 status_code = request.headers.get('X-RC-Locked-Status-Code')
334 status_code = request.headers.get('X-RC-Locked-Status-Code')
335 return HTTPRepoLocked(
335 return HTTPRepoLocked(
336 title=exception.message, status_code=status_code)
336 title=exception.message, status_code=status_code)
337
337
338 # Re-raise exception if we can not handle it.
338 # Re-raise exception if we can not handle it.
339 raise exception
339 raise exception
340
340
341
341
342 class ResponseFilter(object):
342 class ResponseFilter(object):
343
343
344 def __init__(self, start_response):
344 def __init__(self, start_response):
345 self._start_response = start_response
345 self._start_response = start_response
346
346
347 def __call__(self, status, response_headers, exc_info=None):
347 def __call__(self, status, response_headers, exc_info=None):
348 headers = tuple(
348 headers = tuple(
349 (h, v) for h, v in response_headers
349 (h, v) for h, v in response_headers
350 if not wsgiref.util.is_hop_by_hop(h))
350 if not wsgiref.util.is_hop_by_hop(h))
351 return self._start_response(status, headers, exc_info)
351 return self._start_response(status, headers, exc_info)
352
352
353
353
354 def main(global_config, **settings):
354 def main(global_config, **settings):
355 if MercurialFactory:
355 if MercurialFactory:
356 hgpatches.patch_largefiles_capabilities()
356 hgpatches.patch_largefiles_capabilities()
357 hgpatches.patch_subrepo_type_mapping()
357 app = HTTPApplication(settings=settings)
358 app = HTTPApplication(settings=settings)
358 return app.wsgi_app()
359 return app.wsgi_app()
@@ -1,507 +1,508 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2016 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import atexit
18 import atexit
19 import locale
19 import locale
20 import logging
20 import logging
21 import optparse
21 import optparse
22 import os
22 import os
23 import textwrap
23 import textwrap
24 import threading
24 import threading
25 import sys
25 import sys
26
26
27 import configobj
27 import configobj
28 import Pyro4
28 import Pyro4
29 from beaker.cache import CacheManager
29 from beaker.cache import CacheManager
30 from beaker.util import parse_cache_config_options
30 from beaker.util import parse_cache_config_options
31
31
32 try:
32 try:
33 from vcsserver.git import GitFactory, GitRemote
33 from vcsserver.git import GitFactory, GitRemote
34 except ImportError:
34 except ImportError:
35 GitFactory = None
35 GitFactory = None
36 GitRemote = None
36 GitRemote = None
37 try:
37 try:
38 from vcsserver.hg import MercurialFactory, HgRemote
38 from vcsserver.hg import MercurialFactory, HgRemote
39 except ImportError:
39 except ImportError:
40 MercurialFactory = None
40 MercurialFactory = None
41 HgRemote = None
41 HgRemote = None
42 try:
42 try:
43 from vcsserver.svn import SubversionFactory, SvnRemote
43 from vcsserver.svn import SubversionFactory, SvnRemote
44 except ImportError:
44 except ImportError:
45 SubversionFactory = None
45 SubversionFactory = None
46 SvnRemote = None
46 SvnRemote = None
47
47
48 from server import VcsServer
48 from server import VcsServer
49 from vcsserver import hgpatches, remote_wsgi, settings
49 from vcsserver import hgpatches, remote_wsgi, settings
50 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
50 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
51
51
52 log = logging.getLogger(__name__)
52 log = logging.getLogger(__name__)
53
53
54 HERE = os.path.dirname(os.path.abspath(__file__))
54 HERE = os.path.dirname(os.path.abspath(__file__))
55 SERVER_RUNNING_FILE = None
55 SERVER_RUNNING_FILE = None
56
56
57
57
58 # HOOKS - inspired by gunicorn #
58 # HOOKS - inspired by gunicorn #
59
59
60 def when_ready(server):
60 def when_ready(server):
61 """
61 """
62 Called just after the server is started.
62 Called just after the server is started.
63 """
63 """
64
64
65 def _remove_server_running_file():
65 def _remove_server_running_file():
66 if os.path.isfile(SERVER_RUNNING_FILE):
66 if os.path.isfile(SERVER_RUNNING_FILE):
67 os.remove(SERVER_RUNNING_FILE)
67 os.remove(SERVER_RUNNING_FILE)
68
68
69 # top up to match to level location
69 # top up to match to level location
70 if SERVER_RUNNING_FILE:
70 if SERVER_RUNNING_FILE:
71 with open(SERVER_RUNNING_FILE, 'wb') as f:
71 with open(SERVER_RUNNING_FILE, 'wb') as f:
72 f.write(str(os.getpid()))
72 f.write(str(os.getpid()))
73 # register cleanup of that file when server exits
73 # register cleanup of that file when server exits
74 atexit.register(_remove_server_running_file)
74 atexit.register(_remove_server_running_file)
75
75
76
76
77 class LazyWriter(object):
77 class LazyWriter(object):
78 """
78 """
79 File-like object that opens a file lazily when it is first written
79 File-like object that opens a file lazily when it is first written
80 to.
80 to.
81 """
81 """
82
82
83 def __init__(self, filename, mode='w'):
83 def __init__(self, filename, mode='w'):
84 self.filename = filename
84 self.filename = filename
85 self.fileobj = None
85 self.fileobj = None
86 self.lock = threading.Lock()
86 self.lock = threading.Lock()
87 self.mode = mode
87 self.mode = mode
88
88
89 def open(self):
89 def open(self):
90 if self.fileobj is None:
90 if self.fileobj is None:
91 with self.lock:
91 with self.lock:
92 self.fileobj = open(self.filename, self.mode)
92 self.fileobj = open(self.filename, self.mode)
93 return self.fileobj
93 return self.fileobj
94
94
95 def close(self):
95 def close(self):
96 fileobj = self.fileobj
96 fileobj = self.fileobj
97 if fileobj is not None:
97 if fileobj is not None:
98 fileobj.close()
98 fileobj.close()
99
99
100 def __del__(self):
100 def __del__(self):
101 self.close()
101 self.close()
102
102
103 def write(self, text):
103 def write(self, text):
104 fileobj = self.open()
104 fileobj = self.open()
105 fileobj.write(text)
105 fileobj.write(text)
106 fileobj.flush()
106 fileobj.flush()
107
107
108 def writelines(self, text):
108 def writelines(self, text):
109 fileobj = self.open()
109 fileobj = self.open()
110 fileobj.writelines(text)
110 fileobj.writelines(text)
111 fileobj.flush()
111 fileobj.flush()
112
112
113 def flush(self):
113 def flush(self):
114 self.open().flush()
114 self.open().flush()
115
115
116
116
117 class Application(object):
117 class Application(object):
118 """
118 """
119 Represents the vcs server application.
119 Represents the vcs server application.
120
120
121 This object is responsible to initialize the application and all needed
121 This object is responsible to initialize the application and all needed
122 libraries. After that it hooks together the different objects and provides
122 libraries. After that it hooks together the different objects and provides
123 them a way to access things like configuration.
123 them a way to access things like configuration.
124 """
124 """
125
125
126 def __init__(
126 def __init__(
127 self, host, port=None, locale='', threadpool_size=None,
127 self, host, port=None, locale='', threadpool_size=None,
128 timeout=None, cache_config=None, remote_wsgi_=None):
128 timeout=None, cache_config=None, remote_wsgi_=None):
129
129
130 self.host = host
130 self.host = host
131 self.port = int(port) or settings.PYRO_PORT
131 self.port = int(port) or settings.PYRO_PORT
132 self.threadpool_size = (
132 self.threadpool_size = (
133 int(threadpool_size) if threadpool_size else None)
133 int(threadpool_size) if threadpool_size else None)
134 self.locale = locale
134 self.locale = locale
135 self.timeout = timeout
135 self.timeout = timeout
136 self.cache_config = cache_config
136 self.cache_config = cache_config
137 self.remote_wsgi = remote_wsgi_ or remote_wsgi
137 self.remote_wsgi = remote_wsgi_ or remote_wsgi
138
138
139 def init(self):
139 def init(self):
140 """
140 """
141 Configure and hook together all relevant objects.
141 Configure and hook together all relevant objects.
142 """
142 """
143 self._configure_locale()
143 self._configure_locale()
144 self._configure_pyro()
144 self._configure_pyro()
145 self._initialize_cache()
145 self._initialize_cache()
146 self._create_daemon_and_remote_objects(host=self.host, port=self.port)
146 self._create_daemon_and_remote_objects(host=self.host, port=self.port)
147
147
148 def run(self):
148 def run(self):
149 """
149 """
150 Start the main loop of the application.
150 Start the main loop of the application.
151 """
151 """
152
152
153 if hasattr(os, 'getpid'):
153 if hasattr(os, 'getpid'):
154 log.info('Starting %s in PID %i.', __name__, os.getpid())
154 log.info('Starting %s in PID %i.', __name__, os.getpid())
155 else:
155 else:
156 log.info('Starting %s.', __name__)
156 log.info('Starting %s.', __name__)
157 if SERVER_RUNNING_FILE:
157 if SERVER_RUNNING_FILE:
158 log.info('PID file written as %s', SERVER_RUNNING_FILE)
158 log.info('PID file written as %s', SERVER_RUNNING_FILE)
159 else:
159 else:
160 log.info('No PID file written by default.')
160 log.info('No PID file written by default.')
161 when_ready(self)
161 when_ready(self)
162 try:
162 try:
163 self._pyrodaemon.requestLoop(
163 self._pyrodaemon.requestLoop(
164 loopCondition=lambda: not self._vcsserver._shutdown)
164 loopCondition=lambda: not self._vcsserver._shutdown)
165 finally:
165 finally:
166 self._pyrodaemon.shutdown()
166 self._pyrodaemon.shutdown()
167
167
168 def _configure_locale(self):
168 def _configure_locale(self):
169 if self.locale:
169 if self.locale:
170 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
170 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
171 else:
171 else:
172 log.info(
172 log.info(
173 'Configuring locale subsystem based on environment variables')
173 'Configuring locale subsystem based on environment variables')
174
174
175 try:
175 try:
176 # If self.locale is the empty string, then the locale
176 # If self.locale is the empty string, then the locale
177 # module will use the environment variables. See the
177 # module will use the environment variables. See the
178 # documentation of the package `locale`.
178 # documentation of the package `locale`.
179 locale.setlocale(locale.LC_ALL, self.locale)
179 locale.setlocale(locale.LC_ALL, self.locale)
180
180
181 language_code, encoding = locale.getlocale()
181 language_code, encoding = locale.getlocale()
182 log.info(
182 log.info(
183 'Locale set to language code "%s" with encoding "%s".',
183 'Locale set to language code "%s" with encoding "%s".',
184 language_code, encoding)
184 language_code, encoding)
185 except locale.Error:
185 except locale.Error:
186 log.exception(
186 log.exception(
187 'Cannot set locale, not configuring the locale system')
187 'Cannot set locale, not configuring the locale system')
188
188
189 def _configure_pyro(self):
189 def _configure_pyro(self):
190 if self.threadpool_size is not None:
190 if self.threadpool_size is not None:
191 log.info("Threadpool size set to %s", self.threadpool_size)
191 log.info("Threadpool size set to %s", self.threadpool_size)
192 Pyro4.config.THREADPOOL_SIZE = self.threadpool_size
192 Pyro4.config.THREADPOOL_SIZE = self.threadpool_size
193 if self.timeout not in (None, 0, 0.0, '0'):
193 if self.timeout not in (None, 0, 0.0, '0'):
194 log.info("Timeout for RPC calls set to %s seconds", self.timeout)
194 log.info("Timeout for RPC calls set to %s seconds", self.timeout)
195 Pyro4.config.COMMTIMEOUT = float(self.timeout)
195 Pyro4.config.COMMTIMEOUT = float(self.timeout)
196 Pyro4.config.SERIALIZER = 'pickle'
196 Pyro4.config.SERIALIZER = 'pickle'
197 Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
197 Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
198 Pyro4.config.SOCK_REUSE = True
198 Pyro4.config.SOCK_REUSE = True
199 # Uncomment the next line when you need to debug remote errors
199 # Uncomment the next line when you need to debug remote errors
200 # Pyro4.config.DETAILED_TRACEBACK = True
200 # Pyro4.config.DETAILED_TRACEBACK = True
201
201
202 def _initialize_cache(self):
202 def _initialize_cache(self):
203 cache_config = parse_cache_config_options(self.cache_config)
203 cache_config = parse_cache_config_options(self.cache_config)
204 log.info('Initializing beaker cache: %s' % cache_config)
204 log.info('Initializing beaker cache: %s' % cache_config)
205 self.cache = CacheManager(**cache_config)
205 self.cache = CacheManager(**cache_config)
206
206
207 def _create_daemon_and_remote_objects(self, host='localhost',
207 def _create_daemon_and_remote_objects(self, host='localhost',
208 port=settings.PYRO_PORT):
208 port=settings.PYRO_PORT):
209 daemon = Pyro4.Daemon(host=host, port=port)
209 daemon = Pyro4.Daemon(host=host, port=port)
210
210
211 self._vcsserver = VcsServer()
211 self._vcsserver = VcsServer()
212 uri = daemon.register(
212 uri = daemon.register(
213 self._vcsserver, objectId=settings.PYRO_VCSSERVER)
213 self._vcsserver, objectId=settings.PYRO_VCSSERVER)
214 log.info("Object registered = %s", uri)
214 log.info("Object registered = %s", uri)
215
215
216 if GitFactory and GitRemote:
216 if GitFactory and GitRemote:
217 git_repo_cache = self.cache.get_cache_region('git', region='repo_object')
217 git_repo_cache = self.cache.get_cache_region('git', region='repo_object')
218 git_factory = GitFactory(git_repo_cache)
218 git_factory = GitFactory(git_repo_cache)
219 self._git_remote = GitRemote(git_factory)
219 self._git_remote = GitRemote(git_factory)
220 uri = daemon.register(self._git_remote, objectId=settings.PYRO_GIT)
220 uri = daemon.register(self._git_remote, objectId=settings.PYRO_GIT)
221 log.info("Object registered = %s", uri)
221 log.info("Object registered = %s", uri)
222 else:
222 else:
223 log.info("Git client import failed")
223 log.info("Git client import failed")
224
224
225 if MercurialFactory and HgRemote:
225 if MercurialFactory and HgRemote:
226 hg_repo_cache = self.cache.get_cache_region('hg', region='repo_object')
226 hg_repo_cache = self.cache.get_cache_region('hg', region='repo_object')
227 hg_factory = MercurialFactory(hg_repo_cache)
227 hg_factory = MercurialFactory(hg_repo_cache)
228 self._hg_remote = HgRemote(hg_factory)
228 self._hg_remote = HgRemote(hg_factory)
229 uri = daemon.register(self._hg_remote, objectId=settings.PYRO_HG)
229 uri = daemon.register(self._hg_remote, objectId=settings.PYRO_HG)
230 log.info("Object registered = %s", uri)
230 log.info("Object registered = %s", uri)
231 else:
231 else:
232 log.info("Mercurial client import failed")
232 log.info("Mercurial client import failed")
233
233
234 if SubversionFactory and SvnRemote:
234 if SubversionFactory and SvnRemote:
235 svn_repo_cache = self.cache.get_cache_region('svn', region='repo_object')
235 svn_repo_cache = self.cache.get_cache_region('svn', region='repo_object')
236 svn_factory = SubversionFactory(svn_repo_cache)
236 svn_factory = SubversionFactory(svn_repo_cache)
237 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
237 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
238 uri = daemon.register(self._svn_remote, objectId=settings.PYRO_SVN)
238 uri = daemon.register(self._svn_remote, objectId=settings.PYRO_SVN)
239 log.info("Object registered = %s", uri)
239 log.info("Object registered = %s", uri)
240 else:
240 else:
241 log.info("Subversion client import failed")
241 log.info("Subversion client import failed")
242
242
243 self._git_remote_wsgi = self.remote_wsgi.GitRemoteWsgi()
243 self._git_remote_wsgi = self.remote_wsgi.GitRemoteWsgi()
244 uri = daemon.register(self._git_remote_wsgi,
244 uri = daemon.register(self._git_remote_wsgi,
245 objectId=settings.PYRO_GIT_REMOTE_WSGI)
245 objectId=settings.PYRO_GIT_REMOTE_WSGI)
246 log.info("Object registered = %s", uri)
246 log.info("Object registered = %s", uri)
247
247
248 self._hg_remote_wsgi = self.remote_wsgi.HgRemoteWsgi()
248 self._hg_remote_wsgi = self.remote_wsgi.HgRemoteWsgi()
249 uri = daemon.register(self._hg_remote_wsgi,
249 uri = daemon.register(self._hg_remote_wsgi,
250 objectId=settings.PYRO_HG_REMOTE_WSGI)
250 objectId=settings.PYRO_HG_REMOTE_WSGI)
251 log.info("Object registered = %s", uri)
251 log.info("Object registered = %s", uri)
252
252
253 self._pyrodaemon = daemon
253 self._pyrodaemon = daemon
254
254
255
255
256 class VcsServerCommand(object):
256 class VcsServerCommand(object):
257
257
258 usage = '%prog'
258 usage = '%prog'
259 description = """
259 description = """
260 Runs the VCS server
260 Runs the VCS server
261 """
261 """
262 default_verbosity = 1
262 default_verbosity = 1
263
263
264 parser = optparse.OptionParser(
264 parser = optparse.OptionParser(
265 usage,
265 usage,
266 description=textwrap.dedent(description)
266 description=textwrap.dedent(description)
267 )
267 )
268 parser.add_option(
268 parser.add_option(
269 '--host',
269 '--host',
270 type="str",
270 type="str",
271 dest="host",
271 dest="host",
272 )
272 )
273 parser.add_option(
273 parser.add_option(
274 '--port',
274 '--port',
275 type="int",
275 type="int",
276 dest="port"
276 dest="port"
277 )
277 )
278 parser.add_option(
278 parser.add_option(
279 '--running-file',
279 '--running-file',
280 dest='running_file',
280 dest='running_file',
281 metavar='RUNNING_FILE',
281 metavar='RUNNING_FILE',
282 help="Create a running file after the server is initalized with "
282 help="Create a running file after the server is initalized with "
283 "stored PID of process"
283 "stored PID of process"
284 )
284 )
285 parser.add_option(
285 parser.add_option(
286 '--locale',
286 '--locale',
287 dest='locale',
287 dest='locale',
288 help="Allows to set the locale, e.g. en_US.UTF-8",
288 help="Allows to set the locale, e.g. en_US.UTF-8",
289 default=""
289 default=""
290 )
290 )
291 parser.add_option(
291 parser.add_option(
292 '--log-file',
292 '--log-file',
293 dest='log_file',
293 dest='log_file',
294 metavar='LOG_FILE',
294 metavar='LOG_FILE',
295 help="Save output to the given log file (redirects stdout)"
295 help="Save output to the given log file (redirects stdout)"
296 )
296 )
297 parser.add_option(
297 parser.add_option(
298 '--log-level',
298 '--log-level',
299 dest="log_level",
299 dest="log_level",
300 metavar="LOG_LEVEL",
300 metavar="LOG_LEVEL",
301 help="use LOG_LEVEL to set log level "
301 help="use LOG_LEVEL to set log level "
302 "(debug,info,warning,error,critical)"
302 "(debug,info,warning,error,critical)"
303 )
303 )
304 parser.add_option(
304 parser.add_option(
305 '--threadpool',
305 '--threadpool',
306 dest='threadpool_size',
306 dest='threadpool_size',
307 type='int',
307 type='int',
308 help="Set the size of the threadpool used to communicate with the "
308 help="Set the size of the threadpool used to communicate with the "
309 "WSGI workers. This should be at least 6 times the number of "
309 "WSGI workers. This should be at least 6 times the number of "
310 "WSGI worker processes."
310 "WSGI worker processes."
311 )
311 )
312 parser.add_option(
312 parser.add_option(
313 '--timeout',
313 '--timeout',
314 dest='timeout',
314 dest='timeout',
315 type='float',
315 type='float',
316 help="Set the timeout for RPC communication in seconds."
316 help="Set the timeout for RPC communication in seconds."
317 )
317 )
318 parser.add_option(
318 parser.add_option(
319 '--config',
319 '--config',
320 dest='config_file',
320 dest='config_file',
321 type='string',
321 type='string',
322 help="Configuration file for vcsserver."
322 help="Configuration file for vcsserver."
323 )
323 )
324
324
325 def __init__(self, argv, quiet=False):
325 def __init__(self, argv, quiet=False):
326 self.options, self.args = self.parser.parse_args(argv[1:])
326 self.options, self.args = self.parser.parse_args(argv[1:])
327 if quiet:
327 if quiet:
328 self.options.verbose = 0
328 self.options.verbose = 0
329
329
330 def _get_file_config(self):
330 def _get_file_config(self):
331 ini_conf = {}
331 ini_conf = {}
332 conf = configobj.ConfigObj(self.options.config_file)
332 conf = configobj.ConfigObj(self.options.config_file)
333 if 'DEFAULT' in conf:
333 if 'DEFAULT' in conf:
334 ini_conf = conf['DEFAULT']
334 ini_conf = conf['DEFAULT']
335
335
336 return ini_conf
336 return ini_conf
337
337
338 def _show_config(self, vcsserver_config):
338 def _show_config(self, vcsserver_config):
339 order = [
339 order = [
340 'config_file',
340 'config_file',
341 'host',
341 'host',
342 'port',
342 'port',
343 'log_file',
343 'log_file',
344 'log_level',
344 'log_level',
345 'locale',
345 'locale',
346 'threadpool_size',
346 'threadpool_size',
347 'timeout',
347 'timeout',
348 'cache_config',
348 'cache_config',
349 ]
349 ]
350
350
351 def sorter(k):
351 def sorter(k):
352 return dict([(y, x) for x, y in enumerate(order)]).get(k)
352 return dict([(y, x) for x, y in enumerate(order)]).get(k)
353
353
354 _config = []
354 _config = []
355 for k in sorted(vcsserver_config.keys(), key=sorter):
355 for k in sorted(vcsserver_config.keys(), key=sorter):
356 v = vcsserver_config[k]
356 v = vcsserver_config[k]
357 # construct padded key for display eg %-20s % = key: val
357 # construct padded key for display eg %-20s % = key: val
358 k_formatted = ('%-'+str(len(max(order, key=len))+1)+'s') % (k+':')
358 k_formatted = ('%-'+str(len(max(order, key=len))+1)+'s') % (k+':')
359 _config.append(' * %s %s' % (k_formatted, v))
359 _config.append(' * %s %s' % (k_formatted, v))
360 log.info('\n[vcsserver configuration]:\n'+'\n'.join(_config))
360 log.info('\n[vcsserver configuration]:\n'+'\n'.join(_config))
361
361
362 def _get_vcsserver_configuration(self):
362 def _get_vcsserver_configuration(self):
363 _defaults = {
363 _defaults = {
364 'config_file': None,
364 'config_file': None,
365 'git_path': 'git',
365 'git_path': 'git',
366 'host': 'localhost',
366 'host': 'localhost',
367 'port': settings.PYRO_PORT,
367 'port': settings.PYRO_PORT,
368 'log_file': None,
368 'log_file': None,
369 'log_level': 'debug',
369 'log_level': 'debug',
370 'locale': None,
370 'locale': None,
371 'threadpool_size': 16,
371 'threadpool_size': 16,
372 'timeout': None,
372 'timeout': None,
373
373
374 # Development support
374 # Development support
375 'dev.use_echo_app': False,
375 'dev.use_echo_app': False,
376
376
377 # caches, baker style config
377 # caches, baker style config
378 'beaker.cache.regions': 'repo_object',
378 'beaker.cache.regions': 'repo_object',
379 'beaker.cache.repo_object.expire': '10',
379 'beaker.cache.repo_object.expire': '10',
380 'beaker.cache.repo_object.type': 'memory',
380 'beaker.cache.repo_object.type': 'memory',
381 }
381 }
382 config = {}
382 config = {}
383 config.update(_defaults)
383 config.update(_defaults)
384 # overwrite defaults with one loaded from file
384 # overwrite defaults with one loaded from file
385 config.update(self._get_file_config())
385 config.update(self._get_file_config())
386
386
387 # overwrite with self.option which has the top priority
387 # overwrite with self.option which has the top priority
388 for k, v in self.options.__dict__.items():
388 for k, v in self.options.__dict__.items():
389 if v or v == 0:
389 if v or v == 0:
390 config[k] = v
390 config[k] = v
391
391
392 # clear all "extra" keys if they are somehow passed,
392 # clear all "extra" keys if they are somehow passed,
393 # we only want defaults, so any extra stuff from self.options is cleared
393 # we only want defaults, so any extra stuff from self.options is cleared
394 # except beaker stuff which needs to be dynamic
394 # except beaker stuff which needs to be dynamic
395 for k in [k for k in config.copy().keys() if not k.startswith('beaker.cache.')]:
395 for k in [k for k in config.copy().keys() if not k.startswith('beaker.cache.')]:
396 if k not in _defaults:
396 if k not in _defaults:
397 del config[k]
397 del config[k]
398
398
399 # group together the cache into one key.
399 # group together the cache into one key.
400 # Needed further for beaker lib configuration
400 # Needed further for beaker lib configuration
401 _k = {}
401 _k = {}
402 for k in [k for k in config.copy() if k.startswith('beaker.cache.')]:
402 for k in [k for k in config.copy() if k.startswith('beaker.cache.')]:
403 _k[k] = config.pop(k)
403 _k[k] = config.pop(k)
404 config['cache_config'] = _k
404 config['cache_config'] = _k
405
405
406 return config
406 return config
407
407
408 def out(self, msg): # pragma: no cover
408 def out(self, msg): # pragma: no cover
409 if self.options.verbose > 0:
409 if self.options.verbose > 0:
410 print(msg)
410 print(msg)
411
411
412 def run(self): # pragma: no cover
412 def run(self): # pragma: no cover
413 vcsserver_config = self._get_vcsserver_configuration()
413 vcsserver_config = self._get_vcsserver_configuration()
414
414
415 # Ensure the log file is writeable
415 # Ensure the log file is writeable
416 if vcsserver_config['log_file']:
416 if vcsserver_config['log_file']:
417 stdout_log = self._configure_logfile()
417 stdout_log = self._configure_logfile()
418 else:
418 else:
419 stdout_log = None
419 stdout_log = None
420
420
421 # set PID file with running lock
421 # set PID file with running lock
422 if self.options.running_file:
422 if self.options.running_file:
423 global SERVER_RUNNING_FILE
423 global SERVER_RUNNING_FILE
424 SERVER_RUNNING_FILE = self.options.running_file
424 SERVER_RUNNING_FILE = self.options.running_file
425
425
426 # configure logging, and logging based on configuration file
426 # configure logging, and logging based on configuration file
427 self._configure_logging(level=vcsserver_config['log_level'],
427 self._configure_logging(level=vcsserver_config['log_level'],
428 stream=stdout_log)
428 stream=stdout_log)
429 if self.options.config_file:
429 if self.options.config_file:
430 if not os.path.isfile(self.options.config_file):
430 if not os.path.isfile(self.options.config_file):
431 raise OSError('File %s does not exist' %
431 raise OSError('File %s does not exist' %
432 self.options.config_file)
432 self.options.config_file)
433
433
434 self._configure_file_logging(self.options.config_file)
434 self._configure_file_logging(self.options.config_file)
435
435
436 self._configure_settings(vcsserver_config)
436 self._configure_settings(vcsserver_config)
437
437
438 # display current configuration of vcsserver
438 # display current configuration of vcsserver
439 self._show_config(vcsserver_config)
439 self._show_config(vcsserver_config)
440
440
441 if not vcsserver_config['dev.use_echo_app']:
441 if not vcsserver_config['dev.use_echo_app']:
442 remote_wsgi_mod = remote_wsgi
442 remote_wsgi_mod = remote_wsgi
443 else:
443 else:
444 log.warning("Using EchoApp for VCS endpoints.")
444 log.warning("Using EchoApp for VCS endpoints.")
445 remote_wsgi_mod = remote_wsgi_stub
445 remote_wsgi_mod = remote_wsgi_stub
446
446
447 app = Application(
447 app = Application(
448 host=vcsserver_config['host'],
448 host=vcsserver_config['host'],
449 port=vcsserver_config['port'],
449 port=vcsserver_config['port'],
450 locale=vcsserver_config['locale'],
450 locale=vcsserver_config['locale'],
451 threadpool_size=vcsserver_config['threadpool_size'],
451 threadpool_size=vcsserver_config['threadpool_size'],
452 timeout=vcsserver_config['timeout'],
452 timeout=vcsserver_config['timeout'],
453 cache_config=vcsserver_config['cache_config'],
453 cache_config=vcsserver_config['cache_config'],
454 remote_wsgi_=remote_wsgi_mod)
454 remote_wsgi_=remote_wsgi_mod)
455 app.init()
455 app.init()
456 app.run()
456 app.run()
457
457
458 def _configure_logging(self, level, stream=None):
458 def _configure_logging(self, level, stream=None):
459 _format = (
459 _format = (
460 '%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s')
460 '%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s')
461 levels = {
461 levels = {
462 'debug': logging.DEBUG,
462 'debug': logging.DEBUG,
463 'info': logging.INFO,
463 'info': logging.INFO,
464 'warning': logging.WARNING,
464 'warning': logging.WARNING,
465 'error': logging.ERROR,
465 'error': logging.ERROR,
466 'critical': logging.CRITICAL,
466 'critical': logging.CRITICAL,
467 }
467 }
468 try:
468 try:
469 level = levels[level]
469 level = levels[level]
470 except KeyError:
470 except KeyError:
471 raise AttributeError(
471 raise AttributeError(
472 'Invalid log level please use one of %s' % (levels.keys(),))
472 'Invalid log level please use one of %s' % (levels.keys(),))
473 logging.basicConfig(format=_format, stream=stream, level=level)
473 logging.basicConfig(format=_format, stream=stream, level=level)
474 logging.getLogger('Pyro4').setLevel(level)
474 logging.getLogger('Pyro4').setLevel(level)
475
475
476 def _configure_file_logging(self, config):
476 def _configure_file_logging(self, config):
477 import logging.config
477 import logging.config
478 try:
478 try:
479 logging.config.fileConfig(config)
479 logging.config.fileConfig(config)
480 except Exception as e:
480 except Exception as e:
481 log.warning('Failed to configure logging based on given '
481 log.warning('Failed to configure logging based on given '
482 'config file. Error: %s' % e)
482 'config file. Error: %s' % e)
483
483
484 def _configure_logfile(self):
484 def _configure_logfile(self):
485 try:
485 try:
486 writeable_log_file = open(self.options.log_file, 'a')
486 writeable_log_file = open(self.options.log_file, 'a')
487 except IOError as ioe:
487 except IOError as ioe:
488 msg = 'Error: Unable to write to log file: %s' % ioe
488 msg = 'Error: Unable to write to log file: %s' % ioe
489 raise ValueError(msg)
489 raise ValueError(msg)
490 writeable_log_file.close()
490 writeable_log_file.close()
491 stdout_log = LazyWriter(self.options.log_file, 'a')
491 stdout_log = LazyWriter(self.options.log_file, 'a')
492 sys.stdout = stdout_log
492 sys.stdout = stdout_log
493 sys.stderr = stdout_log
493 sys.stderr = stdout_log
494 return stdout_log
494 return stdout_log
495
495
496 def _configure_settings(self, config):
496 def _configure_settings(self, config):
497 """
497 """
498 Configure the settings module based on the given `config`.
498 Configure the settings module based on the given `config`.
499 """
499 """
500 settings.GIT_EXECUTABLE = config['git_path']
500 settings.GIT_EXECUTABLE = config['git_path']
501
501
502
502
503 def main(argv=sys.argv, quiet=False):
503 def main(argv=sys.argv, quiet=False):
504 if MercurialFactory:
504 if MercurialFactory:
505 hgpatches.patch_largefiles_capabilities()
505 hgpatches.patch_largefiles_capabilities()
506 hgpatches.patch_subrepo_type_mapping()
506 command = VcsServerCommand(argv, quiet=quiet)
507 command = VcsServerCommand(argv, quiet=quiet)
507 return command.run()
508 return command.run()
General Comments 0
You need to be logged in to leave comments. Login now