##// END OF EJS Templates
celery: Reorder celery imports to fix case where celery tasks were being...
dan -
r628:4d980275 default
parent child Browse files
Show More
@@ -1,183 +1,188 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Pylons environment configuration
22 Pylons environment configuration
23 """
23 """
24
24
25 import os
25 import os
26 import logging
26 import logging
27 import rhodecode
27 import rhodecode
28 import platform
28 import platform
29 import re
29 import re
30 import io
30 import io
31
31
32 from mako.lookup import TemplateLookup
32 from mako.lookup import TemplateLookup
33 from pylons.configuration import PylonsConfig
33 from pylons.configuration import PylonsConfig
34 from pylons.error import handle_mako_error
34 from pylons.error import handle_mako_error
35 from pyramid.settings import asbool
35 from pyramid.settings import asbool
36
36
37 # don't remove this import it does magic for celery
37 # ------------------------------------------------------------------------------
38 from rhodecode.lib import celerypylons # noqa
38 # CELERY magic until refactor - issue #4163 - import order matters here:
39 from rhodecode.lib import celerypylons # this must be first, celerypylons
40 # sets config settings upon import
39
41
40 import rhodecode.lib.app_globals as app_globals
42 import rhodecode.integrations # any modules using celery task
43 # decorators should be added afterwards:
44 # ------------------------------------------------------------------------------
41
45
46 from rhodecode.lib import app_globals
42 from rhodecode.config import utils
47 from rhodecode.config import utils
43 from rhodecode.config.routing import make_map
48 from rhodecode.config.routing import make_map
44 from rhodecode.config.jsroutes import generate_jsroutes_content
49 from rhodecode.config.jsroutes import generate_jsroutes_content
45
50
46 from rhodecode.lib import helpers
51 from rhodecode.lib import helpers
47 from rhodecode.lib.auth import set_available_permissions
52 from rhodecode.lib.auth import set_available_permissions
48 from rhodecode.lib.utils import (
53 from rhodecode.lib.utils import (
49 repo2db_mapper, make_db_config, set_rhodecode_config,
54 repo2db_mapper, make_db_config, set_rhodecode_config,
50 load_rcextensions)
55 load_rcextensions)
51 from rhodecode.lib.utils2 import str2bool, aslist
56 from rhodecode.lib.utils2 import str2bool, aslist
52 from rhodecode.lib.vcs import connect_vcs, start_vcs_server
57 from rhodecode.lib.vcs import connect_vcs, start_vcs_server
53 from rhodecode.model.scm import ScmModel
58 from rhodecode.model.scm import ScmModel
54
59
55 log = logging.getLogger(__name__)
60 log = logging.getLogger(__name__)
56
61
57 def load_environment(global_conf, app_conf, initial=False,
62 def load_environment(global_conf, app_conf, initial=False,
58 test_env=None, test_index=None):
63 test_env=None, test_index=None):
59 """
64 """
60 Configure the Pylons environment via the ``pylons.config``
65 Configure the Pylons environment via the ``pylons.config``
61 object
66 object
62 """
67 """
63 config = PylonsConfig()
68 config = PylonsConfig()
64
69
65
70
66 # Pylons paths
71 # Pylons paths
67 root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
72 root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
68 paths = {
73 paths = {
69 'root': root,
74 'root': root,
70 'controllers': os.path.join(root, 'controllers'),
75 'controllers': os.path.join(root, 'controllers'),
71 'static_files': os.path.join(root, 'public'),
76 'static_files': os.path.join(root, 'public'),
72 'templates': [os.path.join(root, 'templates')],
77 'templates': [os.path.join(root, 'templates')],
73 }
78 }
74
79
75 # Initialize config with the basic options
80 # Initialize config with the basic options
76 config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
81 config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
77
82
78 # store some globals into rhodecode
83 # store some globals into rhodecode
79 rhodecode.CELERY_ENABLED = str2bool(config['app_conf'].get('use_celery'))
84 rhodecode.CELERY_ENABLED = str2bool(config['app_conf'].get('use_celery'))
80 rhodecode.CELERY_EAGER = str2bool(
85 rhodecode.CELERY_EAGER = str2bool(
81 config['app_conf'].get('celery.always.eager'))
86 config['app_conf'].get('celery.always.eager'))
82
87
83 config['routes.map'] = make_map(config)
88 config['routes.map'] = make_map(config)
84
89
85 if asbool(config.get('generate_js_files', 'false')):
90 if asbool(config.get('generate_js_files', 'false')):
86 jsroutes = config['routes.map'].jsroutes()
91 jsroutes = config['routes.map'].jsroutes()
87 jsroutes_file_content = generate_jsroutes_content(jsroutes)
92 jsroutes_file_content = generate_jsroutes_content(jsroutes)
88 jsroutes_file_path = os.path.join(
93 jsroutes_file_path = os.path.join(
89 paths['static_files'], 'js', 'rhodecode', 'routes.js')
94 paths['static_files'], 'js', 'rhodecode', 'routes.js')
90
95
91 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
96 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
92 f.write(jsroutes_file_content)
97 f.write(jsroutes_file_content)
93
98
94 config['pylons.app_globals'] = app_globals.Globals(config)
99 config['pylons.app_globals'] = app_globals.Globals(config)
95 config['pylons.h'] = helpers
100 config['pylons.h'] = helpers
96 rhodecode.CONFIG = config
101 rhodecode.CONFIG = config
97
102
98 load_rcextensions(root_path=config['here'])
103 load_rcextensions(root_path=config['here'])
99
104
100 # Setup cache object as early as possible
105 # Setup cache object as early as possible
101 import pylons
106 import pylons
102 pylons.cache._push_object(config['pylons.app_globals'].cache)
107 pylons.cache._push_object(config['pylons.app_globals'].cache)
103
108
104 # Create the Mako TemplateLookup, with the default auto-escaping
109 # Create the Mako TemplateLookup, with the default auto-escaping
105 config['pylons.app_globals'].mako_lookup = TemplateLookup(
110 config['pylons.app_globals'].mako_lookup = TemplateLookup(
106 directories=paths['templates'],
111 directories=paths['templates'],
107 error_handler=handle_mako_error,
112 error_handler=handle_mako_error,
108 module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
113 module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
109 input_encoding='utf-8', default_filters=['escape'],
114 input_encoding='utf-8', default_filters=['escape'],
110 imports=['from webhelpers.html import escape'])
115 imports=['from webhelpers.html import escape'])
111
116
112 # sets the c attribute access when don't existing attribute are accessed
117 # sets the c attribute access when don't existing attribute are accessed
113 config['pylons.strict_tmpl_context'] = True
118 config['pylons.strict_tmpl_context'] = True
114
119
115 # configure channelstream
120 # configure channelstream
116 config['channelstream_config'] = {
121 config['channelstream_config'] = {
117 'enabled': asbool(config.get('channelstream.enabled', False)),
122 'enabled': asbool(config.get('channelstream.enabled', False)),
118 'server': config.get('channelstream.server'),
123 'server': config.get('channelstream.server'),
119 'secret': config.get('channelstream.secret')
124 'secret': config.get('channelstream.secret')
120 }
125 }
121
126
122 set_available_permissions(config)
127 set_available_permissions(config)
123 db_cfg = make_db_config(clear_session=True)
128 db_cfg = make_db_config(clear_session=True)
124
129
125 repos_path = list(db_cfg.items('paths'))[0][1]
130 repos_path = list(db_cfg.items('paths'))[0][1]
126 config['base_path'] = repos_path
131 config['base_path'] = repos_path
127
132
128 # store db config also in main global CONFIG
133 # store db config also in main global CONFIG
129 set_rhodecode_config(config)
134 set_rhodecode_config(config)
130
135
131 # configure instance id
136 # configure instance id
132 utils.set_instance_id(config)
137 utils.set_instance_id(config)
133
138
134 # CONFIGURATION OPTIONS HERE (note: all config options will override
139 # CONFIGURATION OPTIONS HERE (note: all config options will override
135 # any Pylons config options)
140 # any Pylons config options)
136
141
137 # store config reference into our module to skip import magic of pylons
142 # store config reference into our module to skip import magic of pylons
138 rhodecode.CONFIG.update(config)
143 rhodecode.CONFIG.update(config)
139
144
140 return config
145 return config
141
146
142
147
143 def load_pyramid_environment(global_config, settings):
148 def load_pyramid_environment(global_config, settings):
144 # Some parts of the code expect a merge of global and app settings.
149 # Some parts of the code expect a merge of global and app settings.
145 settings_merged = global_config.copy()
150 settings_merged = global_config.copy()
146 settings_merged.update(settings)
151 settings_merged.update(settings)
147
152
148 # Store the settings to make them available to other modules.
153 # Store the settings to make them available to other modules.
149 rhodecode.PYRAMID_SETTINGS = settings_merged
154 rhodecode.PYRAMID_SETTINGS = settings_merged
150
155
151 # If this is a test run we prepare the test environment like
156 # If this is a test run we prepare the test environment like
152 # creating a test database, test search index and test repositories.
157 # creating a test database, test search index and test repositories.
153 # This has to be done before the database connection is initialized.
158 # This has to be done before the database connection is initialized.
154 if settings['is_test']:
159 if settings['is_test']:
155 rhodecode.is_test = True
160 rhodecode.is_test = True
156 utils.initialize_test_environment(settings_merged)
161 utils.initialize_test_environment(settings_merged)
157
162
158 # Initialize the database connection.
163 # Initialize the database connection.
159 utils.initialize_database(settings_merged)
164 utils.initialize_database(settings_merged)
160
165
161 # Limit backends to `vcs.backends` from configuration
166 # Limit backends to `vcs.backends` from configuration
162 for alias in rhodecode.BACKENDS.keys():
167 for alias in rhodecode.BACKENDS.keys():
163 if alias not in settings['vcs.backends']:
168 if alias not in settings['vcs.backends']:
164 del rhodecode.BACKENDS[alias]
169 del rhodecode.BACKENDS[alias]
165 log.info('Enabled VCS backends: %s', rhodecode.BACKENDS.keys())
170 log.info('Enabled VCS backends: %s', rhodecode.BACKENDS.keys())
166
171
167 # initialize vcs client and optionally run the server if enabled
172 # initialize vcs client and optionally run the server if enabled
168 vcs_server_uri = settings['vcs.server']
173 vcs_server_uri = settings['vcs.server']
169 vcs_server_enabled = settings['vcs.server.enable']
174 vcs_server_enabled = settings['vcs.server.enable']
170 start_server = (
175 start_server = (
171 settings['vcs.start_server'] and
176 settings['vcs.start_server'] and
172 not int(os.environ.get('RC_VCSSERVER_TEST_DISABLE', '0')))
177 not int(os.environ.get('RC_VCSSERVER_TEST_DISABLE', '0')))
173
178
174 if vcs_server_enabled and start_server:
179 if vcs_server_enabled and start_server:
175 log.info("Starting vcsserver")
180 log.info("Starting vcsserver")
176 start_vcs_server(server_and_port=vcs_server_uri,
181 start_vcs_server(server_and_port=vcs_server_uri,
177 protocol=utils.get_vcs_server_protocol(settings),
182 protocol=utils.get_vcs_server_protocol(settings),
178 log_level=settings['vcs.server.log_level'])
183 log_level=settings['vcs.server.log_level'])
179
184
180 utils.configure_pyro4(settings)
185 utils.configure_pyro4(settings)
181 utils.configure_vcs(settings)
186 utils.configure_vcs(settings)
182 if vcs_server_enabled:
187 if vcs_server_enabled:
183 connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
188 connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
@@ -1,475 +1,474 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Pylons middleware initialization
22 Pylons middleware initialization
23 """
23 """
24 import logging
24 import logging
25 from collections import OrderedDict
25 from collections import OrderedDict
26
26
27 from paste.registry import RegistryManager
27 from paste.registry import RegistryManager
28 from paste.gzipper import make_gzip_middleware
28 from paste.gzipper import make_gzip_middleware
29 from pylons.wsgiapp import PylonsApp
29 from pylons.wsgiapp import PylonsApp
30 from pyramid.authorization import ACLAuthorizationPolicy
30 from pyramid.authorization import ACLAuthorizationPolicy
31 from pyramid.config import Configurator
31 from pyramid.config import Configurator
32 from pyramid.settings import asbool, aslist
32 from pyramid.settings import asbool, aslist
33 from pyramid.wsgi import wsgiapp
33 from pyramid.wsgi import wsgiapp
34 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
34 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
35 from pyramid.events import ApplicationCreated
35 from pyramid.events import ApplicationCreated
36 import pyramid.httpexceptions as httpexceptions
36 import pyramid.httpexceptions as httpexceptions
37 from pyramid.renderers import render_to_response
37 from pyramid.renderers import render_to_response
38 from routes.middleware import RoutesMiddleware
38 from routes.middleware import RoutesMiddleware
39 import routes.util
39 import routes.util
40
40
41 import rhodecode
41 import rhodecode
42 import rhodecode.integrations # do not remove this as it registers celery tasks
43 from rhodecode.config import patches
42 from rhodecode.config import patches
44 from rhodecode.config.routing import STATIC_FILE_PREFIX
43 from rhodecode.config.routing import STATIC_FILE_PREFIX
45 from rhodecode.config.environment import (
44 from rhodecode.config.environment import (
46 load_environment, load_pyramid_environment)
45 load_environment, load_pyramid_environment)
47 from rhodecode.lib.middleware import csrf
46 from rhodecode.lib.middleware import csrf
48 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
47 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
49 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
48 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
50 from rhodecode.lib.middleware.https_fixup import HttpsFixup
49 from rhodecode.lib.middleware.https_fixup import HttpsFixup
51 from rhodecode.lib.middleware.vcs import VCSMiddleware
50 from rhodecode.lib.middleware.vcs import VCSMiddleware
52 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
51 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
53 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
52 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
54 from rhodecode.subscribers import scan_repositories_if_enabled
53 from rhodecode.subscribers import scan_repositories_if_enabled
55
54
56
55
57 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
58
57
59
58
60 # this is used to avoid avoid the route lookup overhead in routesmiddleware
59 # this is used to avoid avoid the route lookup overhead in routesmiddleware
61 # for certain routes which won't go to pylons to - eg. static files, debugger
60 # for certain routes which won't go to pylons to - eg. static files, debugger
62 # it is only needed for the pylons migration and can be removed once complete
61 # it is only needed for the pylons migration and can be removed once complete
63 class SkippableRoutesMiddleware(RoutesMiddleware):
62 class SkippableRoutesMiddleware(RoutesMiddleware):
64 """ Routes middleware that allows you to skip prefixes """
63 """ Routes middleware that allows you to skip prefixes """
65
64
66 def __init__(self, *args, **kw):
65 def __init__(self, *args, **kw):
67 self.skip_prefixes = kw.pop('skip_prefixes', [])
66 self.skip_prefixes = kw.pop('skip_prefixes', [])
68 super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
67 super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
69
68
70 def __call__(self, environ, start_response):
69 def __call__(self, environ, start_response):
71 for prefix in self.skip_prefixes:
70 for prefix in self.skip_prefixes:
72 if environ['PATH_INFO'].startswith(prefix):
71 if environ['PATH_INFO'].startswith(prefix):
73 # added to avoid the case when a missing /_static route falls
72 # added to avoid the case when a missing /_static route falls
74 # through to pylons and causes an exception as pylons is
73 # through to pylons and causes an exception as pylons is
75 # expecting wsgiorg.routingargs to be set in the environ
74 # expecting wsgiorg.routingargs to be set in the environ
76 # by RoutesMiddleware.
75 # by RoutesMiddleware.
77 if 'wsgiorg.routing_args' not in environ:
76 if 'wsgiorg.routing_args' not in environ:
78 environ['wsgiorg.routing_args'] = (None, {})
77 environ['wsgiorg.routing_args'] = (None, {})
79 return self.app(environ, start_response)
78 return self.app(environ, start_response)
80
79
81 return super(SkippableRoutesMiddleware, self).__call__(
80 return super(SkippableRoutesMiddleware, self).__call__(
82 environ, start_response)
81 environ, start_response)
83
82
84
83
85 def make_app(global_conf, static_files=True, **app_conf):
84 def make_app(global_conf, static_files=True, **app_conf):
86 """Create a Pylons WSGI application and return it
85 """Create a Pylons WSGI application and return it
87
86
88 ``global_conf``
87 ``global_conf``
89 The inherited configuration for this application. Normally from
88 The inherited configuration for this application. Normally from
90 the [DEFAULT] section of the Paste ini file.
89 the [DEFAULT] section of the Paste ini file.
91
90
92 ``app_conf``
91 ``app_conf``
93 The application's local configuration. Normally specified in
92 The application's local configuration. Normally specified in
94 the [app:<name>] section of the Paste ini file (where <name>
93 the [app:<name>] section of the Paste ini file (where <name>
95 defaults to main).
94 defaults to main).
96
95
97 """
96 """
98 # Apply compatibility patches
97 # Apply compatibility patches
99 patches.kombu_1_5_1_python_2_7_11()
98 patches.kombu_1_5_1_python_2_7_11()
100 patches.inspect_getargspec()
99 patches.inspect_getargspec()
101
100
102 # Configure the Pylons environment
101 # Configure the Pylons environment
103 config = load_environment(global_conf, app_conf)
102 config = load_environment(global_conf, app_conf)
104
103
105 # The Pylons WSGI app
104 # The Pylons WSGI app
106 app = PylonsApp(config=config)
105 app = PylonsApp(config=config)
107 if rhodecode.is_test:
106 if rhodecode.is_test:
108 app = csrf.CSRFDetector(app)
107 app = csrf.CSRFDetector(app)
109
108
110 expected_origin = config.get('expected_origin')
109 expected_origin = config.get('expected_origin')
111 if expected_origin:
110 if expected_origin:
112 # The API can be accessed from other Origins.
111 # The API can be accessed from other Origins.
113 app = csrf.OriginChecker(app, expected_origin,
112 app = csrf.OriginChecker(app, expected_origin,
114 skip_urls=[routes.util.url_for('api')])
113 skip_urls=[routes.util.url_for('api')])
115
114
116 # Establish the Registry for this application
115 # Establish the Registry for this application
117 app = RegistryManager(app)
116 app = RegistryManager(app)
118
117
119 app.config = config
118 app.config = config
120
119
121 return app
120 return app
122
121
123
122
124 def make_pyramid_app(global_config, **settings):
123 def make_pyramid_app(global_config, **settings):
125 """
124 """
126 Constructs the WSGI application based on Pyramid and wraps the Pylons based
125 Constructs the WSGI application based on Pyramid and wraps the Pylons based
127 application.
126 application.
128
127
129 Specials:
128 Specials:
130
129
131 * We migrate from Pylons to Pyramid. While doing this, we keep both
130 * We migrate from Pylons to Pyramid. While doing this, we keep both
132 frameworks functional. This involves moving some WSGI middlewares around
131 frameworks functional. This involves moving some WSGI middlewares around
133 and providing access to some data internals, so that the old code is
132 and providing access to some data internals, so that the old code is
134 still functional.
133 still functional.
135
134
136 * The application can also be integrated like a plugin via the call to
135 * The application can also be integrated like a plugin via the call to
137 `includeme`. This is accompanied with the other utility functions which
136 `includeme`. This is accompanied with the other utility functions which
138 are called. Changing this should be done with great care to not break
137 are called. Changing this should be done with great care to not break
139 cases when these fragments are assembled from another place.
138 cases when these fragments are assembled from another place.
140
139
141 """
140 """
142 # The edition string should be available in pylons too, so we add it here
141 # The edition string should be available in pylons too, so we add it here
143 # before copying the settings.
142 # before copying the settings.
144 settings.setdefault('rhodecode.edition', 'Community Edition')
143 settings.setdefault('rhodecode.edition', 'Community Edition')
145
144
146 # As long as our Pylons application does expect "unprepared" settings, make
145 # As long as our Pylons application does expect "unprepared" settings, make
147 # sure that we keep an unmodified copy. This avoids unintentional change of
146 # sure that we keep an unmodified copy. This avoids unintentional change of
148 # behavior in the old application.
147 # behavior in the old application.
149 settings_pylons = settings.copy()
148 settings_pylons = settings.copy()
150
149
151 sanitize_settings_and_apply_defaults(settings)
150 sanitize_settings_and_apply_defaults(settings)
152 config = Configurator(settings=settings)
151 config = Configurator(settings=settings)
153 add_pylons_compat_data(config.registry, global_config, settings_pylons)
152 add_pylons_compat_data(config.registry, global_config, settings_pylons)
154
153
155 load_pyramid_environment(global_config, settings)
154 load_pyramid_environment(global_config, settings)
156
155
157 includeme_first(config)
156 includeme_first(config)
158 includeme(config)
157 includeme(config)
159 pyramid_app = config.make_wsgi_app()
158 pyramid_app = config.make_wsgi_app()
160 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
159 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
161 pyramid_app.config = config
160 pyramid_app.config = config
162 return pyramid_app
161 return pyramid_app
163
162
164
163
165 def make_not_found_view(config):
164 def make_not_found_view(config):
166 """
165 """
167 This creates the view which should be registered as not-found-view to
166 This creates the view which should be registered as not-found-view to
168 pyramid. Basically it contains of the old pylons app, converted to a view.
167 pyramid. Basically it contains of the old pylons app, converted to a view.
169 Additionally it is wrapped by some other middlewares.
168 Additionally it is wrapped by some other middlewares.
170 """
169 """
171 settings = config.registry.settings
170 settings = config.registry.settings
172 vcs_server_enabled = settings['vcs.server.enable']
171 vcs_server_enabled = settings['vcs.server.enable']
173
172
174 # Make pylons app from unprepared settings.
173 # Make pylons app from unprepared settings.
175 pylons_app = make_app(
174 pylons_app = make_app(
176 config.registry._pylons_compat_global_config,
175 config.registry._pylons_compat_global_config,
177 **config.registry._pylons_compat_settings)
176 **config.registry._pylons_compat_settings)
178 config.registry._pylons_compat_config = pylons_app.config
177 config.registry._pylons_compat_config = pylons_app.config
179
178
180 # Appenlight monitoring.
179 # Appenlight monitoring.
181 pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
180 pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
182 pylons_app, settings)
181 pylons_app, settings)
183
182
184 # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
183 # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
185 # a view to handle the request. Therefore we wrap it around the pylons app.
184 # a view to handle the request. Therefore we wrap it around the pylons app.
186 if vcs_server_enabled:
185 if vcs_server_enabled:
187 pylons_app = VCSMiddleware(
186 pylons_app = VCSMiddleware(
188 pylons_app, settings, appenlight_client, registry=config.registry)
187 pylons_app, settings, appenlight_client, registry=config.registry)
189
188
190 pylons_app_as_view = wsgiapp(pylons_app)
189 pylons_app_as_view = wsgiapp(pylons_app)
191
190
192 # Protect from VCS Server error related pages when server is not available
191 # Protect from VCS Server error related pages when server is not available
193 if not vcs_server_enabled:
192 if not vcs_server_enabled:
194 pylons_app_as_view = DisableVCSPagesWrapper(pylons_app_as_view)
193 pylons_app_as_view = DisableVCSPagesWrapper(pylons_app_as_view)
195
194
196 def pylons_app_with_error_handler(context, request):
195 def pylons_app_with_error_handler(context, request):
197 """
196 """
198 Handle exceptions from rc pylons app:
197 Handle exceptions from rc pylons app:
199
198
200 - old webob type exceptions get converted to pyramid exceptions
199 - old webob type exceptions get converted to pyramid exceptions
201 - pyramid exceptions are passed to the error handler view
200 - pyramid exceptions are passed to the error handler view
202 """
201 """
203 def is_vcs_response(response):
202 def is_vcs_response(response):
204 return 'X-RhodeCode-Backend' in response.headers
203 return 'X-RhodeCode-Backend' in response.headers
205
204
206 def is_http_error(response):
205 def is_http_error(response):
207 # webob type error responses
206 # webob type error responses
208 return (400 <= response.status_int <= 599)
207 return (400 <= response.status_int <= 599)
209
208
210 def is_error_handling_needed(response):
209 def is_error_handling_needed(response):
211 return is_http_error(response) and not is_vcs_response(response)
210 return is_http_error(response) and not is_vcs_response(response)
212
211
213 try:
212 try:
214 response = pylons_app_as_view(context, request)
213 response = pylons_app_as_view(context, request)
215 if is_error_handling_needed(response):
214 if is_error_handling_needed(response):
216 response = webob_to_pyramid_http_response(response)
215 response = webob_to_pyramid_http_response(response)
217 return error_handler(response, request)
216 return error_handler(response, request)
218 except HTTPError as e: # pyramid type exceptions
217 except HTTPError as e: # pyramid type exceptions
219 return error_handler(e, request)
218 return error_handler(e, request)
220 except Exception:
219 except Exception:
221 if settings.get('debugtoolbar.enabled', False):
220 if settings.get('debugtoolbar.enabled', False):
222 raise
221 raise
223 return error_handler(HTTPInternalServerError(), request)
222 return error_handler(HTTPInternalServerError(), request)
224 return response
223 return response
225
224
226 return pylons_app_with_error_handler
225 return pylons_app_with_error_handler
227
226
228
227
229 def add_pylons_compat_data(registry, global_config, settings):
228 def add_pylons_compat_data(registry, global_config, settings):
230 """
229 """
231 Attach data to the registry to support the Pylons integration.
230 Attach data to the registry to support the Pylons integration.
232 """
231 """
233 registry._pylons_compat_global_config = global_config
232 registry._pylons_compat_global_config = global_config
234 registry._pylons_compat_settings = settings
233 registry._pylons_compat_settings = settings
235
234
236
235
237 def webob_to_pyramid_http_response(webob_response):
236 def webob_to_pyramid_http_response(webob_response):
238 ResponseClass = httpexceptions.status_map[webob_response.status_int]
237 ResponseClass = httpexceptions.status_map[webob_response.status_int]
239 pyramid_response = ResponseClass(webob_response.status)
238 pyramid_response = ResponseClass(webob_response.status)
240 pyramid_response.status = webob_response.status
239 pyramid_response.status = webob_response.status
241 pyramid_response.headers.update(webob_response.headers)
240 pyramid_response.headers.update(webob_response.headers)
242 if pyramid_response.headers['content-type'] == 'text/html':
241 if pyramid_response.headers['content-type'] == 'text/html':
243 pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
242 pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
244 return pyramid_response
243 return pyramid_response
245
244
246
245
247 def error_handler(exception, request):
246 def error_handler(exception, request):
248 # TODO: dan: replace the old pylons error controller with this
247 # TODO: dan: replace the old pylons error controller with this
249 from rhodecode.model.settings import SettingsModel
248 from rhodecode.model.settings import SettingsModel
250 from rhodecode.lib.utils2 import AttributeDict
249 from rhodecode.lib.utils2 import AttributeDict
251
250
252 try:
251 try:
253 rc_config = SettingsModel().get_all_settings()
252 rc_config = SettingsModel().get_all_settings()
254 except Exception:
253 except Exception:
255 log.exception('failed to fetch settings')
254 log.exception('failed to fetch settings')
256 rc_config = {}
255 rc_config = {}
257
256
258 base_response = HTTPInternalServerError()
257 base_response = HTTPInternalServerError()
259 # prefer original exception for the response since it may have headers set
258 # prefer original exception for the response since it may have headers set
260 if isinstance(exception, HTTPError):
259 if isinstance(exception, HTTPError):
261 base_response = exception
260 base_response = exception
262
261
263 c = AttributeDict()
262 c = AttributeDict()
264 c.error_message = base_response.status
263 c.error_message = base_response.status
265 c.error_explanation = base_response.explanation or str(base_response)
264 c.error_explanation = base_response.explanation or str(base_response)
266 c.visual = AttributeDict()
265 c.visual = AttributeDict()
267
266
268 c.visual.rhodecode_support_url = (
267 c.visual.rhodecode_support_url = (
269 request.registry.settings.get('rhodecode_support_url') or
268 request.registry.settings.get('rhodecode_support_url') or
270 request.route_url('rhodecode_support')
269 request.route_url('rhodecode_support')
271 )
270 )
272 c.redirect_time = 0
271 c.redirect_time = 0
273 c.rhodecode_name = rc_config.get('rhodecode_title', '')
272 c.rhodecode_name = rc_config.get('rhodecode_title', '')
274 if not c.rhodecode_name:
273 if not c.rhodecode_name:
275 c.rhodecode_name = 'Rhodecode'
274 c.rhodecode_name = 'Rhodecode'
276
275
277 response = render_to_response(
276 response = render_to_response(
278 '/errors/error_document.html', {'c': c}, request=request,
277 '/errors/error_document.html', {'c': c}, request=request,
279 response=base_response)
278 response=base_response)
280
279
281 return response
280 return response
282
281
283
282
284 def includeme(config):
283 def includeme(config):
285 settings = config.registry.settings
284 settings = config.registry.settings
286
285
287 # plugin information
286 # plugin information
288 config.registry.rhodecode_plugins = OrderedDict()
287 config.registry.rhodecode_plugins = OrderedDict()
289
288
290 config.add_directive(
289 config.add_directive(
291 'register_rhodecode_plugin', register_rhodecode_plugin)
290 'register_rhodecode_plugin', register_rhodecode_plugin)
292
291
293 if asbool(settings.get('appenlight', 'false')):
292 if asbool(settings.get('appenlight', 'false')):
294 config.include('appenlight_client.ext.pyramid_tween')
293 config.include('appenlight_client.ext.pyramid_tween')
295
294
296 # Includes which are required. The application would fail without them.
295 # Includes which are required. The application would fail without them.
297 config.include('pyramid_mako')
296 config.include('pyramid_mako')
298 config.include('pyramid_beaker')
297 config.include('pyramid_beaker')
299 config.include('rhodecode.channelstream')
298 config.include('rhodecode.channelstream')
300 config.include('rhodecode.admin')
299 config.include('rhodecode.admin')
301 config.include('rhodecode.authentication')
300 config.include('rhodecode.authentication')
302 config.include('rhodecode.integrations')
301 config.include('rhodecode.integrations')
303 config.include('rhodecode.login')
302 config.include('rhodecode.login')
304 config.include('rhodecode.tweens')
303 config.include('rhodecode.tweens')
305 config.include('rhodecode.api')
304 config.include('rhodecode.api')
306 config.include('rhodecode.svn_support')
305 config.include('rhodecode.svn_support')
307 config.add_route(
306 config.add_route(
308 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
307 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
309
308
310 # Add subscribers.
309 # Add subscribers.
311 config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
310 config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
312
311
313 # Set the authorization policy.
312 # Set the authorization policy.
314 authz_policy = ACLAuthorizationPolicy()
313 authz_policy = ACLAuthorizationPolicy()
315 config.set_authorization_policy(authz_policy)
314 config.set_authorization_policy(authz_policy)
316
315
317 # Set the default renderer for HTML templates to mako.
316 # Set the default renderer for HTML templates to mako.
318 config.add_mako_renderer('.html')
317 config.add_mako_renderer('.html')
319
318
320 # include RhodeCode plugins
319 # include RhodeCode plugins
321 includes = aslist(settings.get('rhodecode.includes', []))
320 includes = aslist(settings.get('rhodecode.includes', []))
322 for inc in includes:
321 for inc in includes:
323 config.include(inc)
322 config.include(inc)
324
323
325 # This is the glue which allows us to migrate in chunks. By registering the
324 # This is the glue which allows us to migrate in chunks. By registering the
326 # pylons based application as the "Not Found" view in Pyramid, we will
325 # pylons based application as the "Not Found" view in Pyramid, we will
327 # fallback to the old application each time the new one does not yet know
326 # fallback to the old application each time the new one does not yet know
328 # how to handle a request.
327 # how to handle a request.
329 config.add_notfound_view(make_not_found_view(config))
328 config.add_notfound_view(make_not_found_view(config))
330
329
331 if not settings.get('debugtoolbar.enabled', False):
330 if not settings.get('debugtoolbar.enabled', False):
332 # if no toolbar, then any exception gets caught and rendered
331 # if no toolbar, then any exception gets caught and rendered
333 config.add_view(error_handler, context=Exception)
332 config.add_view(error_handler, context=Exception)
334
333
335 config.add_view(error_handler, context=HTTPError)
334 config.add_view(error_handler, context=HTTPError)
336
335
337
336
338 def includeme_first(config):
337 def includeme_first(config):
339 # redirect automatic browser favicon.ico requests to correct place
338 # redirect automatic browser favicon.ico requests to correct place
340 def favicon_redirect(context, request):
339 def favicon_redirect(context, request):
341 return HTTPFound(
340 return HTTPFound(
342 request.static_path('rhodecode:public/images/favicon.ico'))
341 request.static_path('rhodecode:public/images/favicon.ico'))
343
342
344 config.add_view(favicon_redirect, route_name='favicon')
343 config.add_view(favicon_redirect, route_name='favicon')
345 config.add_route('favicon', '/favicon.ico')
344 config.add_route('favicon', '/favicon.ico')
346
345
347 config.add_static_view(
346 config.add_static_view(
348 '_static/deform', 'deform:static')
347 '_static/deform', 'deform:static')
349 config.add_static_view(
348 config.add_static_view(
350 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
349 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
351
350
352
351
353 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
352 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
354 """
353 """
355 Apply outer WSGI middlewares around the application.
354 Apply outer WSGI middlewares around the application.
356
355
357 Part of this has been moved up from the Pylons layer, so that the
356 Part of this has been moved up from the Pylons layer, so that the
358 data is also available if old Pylons code is hit through an already ported
357 data is also available if old Pylons code is hit through an already ported
359 view.
358 view.
360 """
359 """
361 settings = config.registry.settings
360 settings = config.registry.settings
362
361
363 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
362 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
364 pyramid_app = HttpsFixup(pyramid_app, settings)
363 pyramid_app = HttpsFixup(pyramid_app, settings)
365
364
366 # Add RoutesMiddleware to support the pylons compatibility tween during
365 # Add RoutesMiddleware to support the pylons compatibility tween during
367 # migration to pyramid.
366 # migration to pyramid.
368 pyramid_app = SkippableRoutesMiddleware(
367 pyramid_app = SkippableRoutesMiddleware(
369 pyramid_app, config.registry._pylons_compat_config['routes.map'],
368 pyramid_app, config.registry._pylons_compat_config['routes.map'],
370 skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
369 skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
371
370
372 pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
371 pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
373
372
374 if settings['gzip_responses']:
373 if settings['gzip_responses']:
375 pyramid_app = make_gzip_middleware(
374 pyramid_app = make_gzip_middleware(
376 pyramid_app, settings, compress_level=1)
375 pyramid_app, settings, compress_level=1)
377
376
378 return pyramid_app
377 return pyramid_app
379
378
380
379
381 def sanitize_settings_and_apply_defaults(settings):
380 def sanitize_settings_and_apply_defaults(settings):
382 """
381 """
383 Applies settings defaults and does all type conversion.
382 Applies settings defaults and does all type conversion.
384
383
385 We would move all settings parsing and preparation into this place, so that
384 We would move all settings parsing and preparation into this place, so that
386 we have only one place left which deals with this part. The remaining parts
385 we have only one place left which deals with this part. The remaining parts
387 of the application would start to rely fully on well prepared settings.
386 of the application would start to rely fully on well prepared settings.
388
387
389 This piece would later be split up per topic to avoid a big fat monster
388 This piece would later be split up per topic to avoid a big fat monster
390 function.
389 function.
391 """
390 """
392
391
393 # Pyramid's mako renderer has to search in the templates folder so that the
392 # Pyramid's mako renderer has to search in the templates folder so that the
394 # old templates still work. Ported and new templates are expected to use
393 # old templates still work. Ported and new templates are expected to use
395 # real asset specifications for the includes.
394 # real asset specifications for the includes.
396 mako_directories = settings.setdefault('mako.directories', [
395 mako_directories = settings.setdefault('mako.directories', [
397 # Base templates of the original Pylons application
396 # Base templates of the original Pylons application
398 'rhodecode:templates',
397 'rhodecode:templates',
399 ])
398 ])
400 log.debug(
399 log.debug(
401 "Using the following Mako template directories: %s",
400 "Using the following Mako template directories: %s",
402 mako_directories)
401 mako_directories)
403
402
404 # Default includes, possible to change as a user
403 # Default includes, possible to change as a user
405 pyramid_includes = settings.setdefault('pyramid.includes', [
404 pyramid_includes = settings.setdefault('pyramid.includes', [
406 'rhodecode.lib.middleware.request_wrapper',
405 'rhodecode.lib.middleware.request_wrapper',
407 ])
406 ])
408 log.debug(
407 log.debug(
409 "Using the following pyramid.includes: %s",
408 "Using the following pyramid.includes: %s",
410 pyramid_includes)
409 pyramid_includes)
411
410
412 # TODO: johbo: Re-think this, usually the call to config.include
411 # TODO: johbo: Re-think this, usually the call to config.include
413 # should allow to pass in a prefix.
412 # should allow to pass in a prefix.
414 settings.setdefault('rhodecode.api.url', '/_admin/api')
413 settings.setdefault('rhodecode.api.url', '/_admin/api')
415
414
416 # Sanitize generic settings.
415 # Sanitize generic settings.
417 _list_setting(settings, 'default_encoding', 'UTF-8')
416 _list_setting(settings, 'default_encoding', 'UTF-8')
418 _bool_setting(settings, 'is_test', 'false')
417 _bool_setting(settings, 'is_test', 'false')
419 _bool_setting(settings, 'gzip_responses', 'false')
418 _bool_setting(settings, 'gzip_responses', 'false')
420
419
421 # Call split out functions that sanitize settings for each topic.
420 # Call split out functions that sanitize settings for each topic.
422 _sanitize_appenlight_settings(settings)
421 _sanitize_appenlight_settings(settings)
423 _sanitize_vcs_settings(settings)
422 _sanitize_vcs_settings(settings)
424
423
425 return settings
424 return settings
426
425
427
426
428 def _sanitize_appenlight_settings(settings):
427 def _sanitize_appenlight_settings(settings):
429 _bool_setting(settings, 'appenlight', 'false')
428 _bool_setting(settings, 'appenlight', 'false')
430
429
431
430
432 def _sanitize_vcs_settings(settings):
431 def _sanitize_vcs_settings(settings):
433 """
432 """
434 Applies settings defaults and does type conversion for all VCS related
433 Applies settings defaults and does type conversion for all VCS related
435 settings.
434 settings.
436 """
435 """
437 _string_setting(settings, 'vcs.svn.compatible_version', '')
436 _string_setting(settings, 'vcs.svn.compatible_version', '')
438 _string_setting(settings, 'git_rev_filter', '--all')
437 _string_setting(settings, 'git_rev_filter', '--all')
439 _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
438 _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
440 _string_setting(settings, 'vcs.server', '')
439 _string_setting(settings, 'vcs.server', '')
441 _string_setting(settings, 'vcs.server.log_level', 'debug')
440 _string_setting(settings, 'vcs.server.log_level', 'debug')
442 _string_setting(settings, 'vcs.server.protocol', 'pyro4')
441 _string_setting(settings, 'vcs.server.protocol', 'pyro4')
443 _bool_setting(settings, 'startup.import_repos', 'false')
442 _bool_setting(settings, 'startup.import_repos', 'false')
444 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
443 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
445 _bool_setting(settings, 'vcs.server.enable', 'true')
444 _bool_setting(settings, 'vcs.server.enable', 'true')
446 _bool_setting(settings, 'vcs.start_server', 'false')
445 _bool_setting(settings, 'vcs.start_server', 'false')
447 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
446 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
448 _int_setting(settings, 'vcs.connection_timeout', 3600)
447 _int_setting(settings, 'vcs.connection_timeout', 3600)
449
448
450
449
451 def _int_setting(settings, name, default):
450 def _int_setting(settings, name, default):
452 settings[name] = int(settings.get(name, default))
451 settings[name] = int(settings.get(name, default))
453
452
454
453
455 def _bool_setting(settings, name, default):
454 def _bool_setting(settings, name, default):
456 input = settings.get(name, default)
455 input = settings.get(name, default)
457 if isinstance(input, unicode):
456 if isinstance(input, unicode):
458 input = input.encode('utf8')
457 input = input.encode('utf8')
459 settings[name] = asbool(input)
458 settings[name] = asbool(input)
460
459
461
460
462 def _list_setting(settings, name, default):
461 def _list_setting(settings, name, default):
463 raw_value = settings.get(name, default)
462 raw_value = settings.get(name, default)
464
463
465 old_separator = ','
464 old_separator = ','
466 if old_separator in raw_value:
465 if old_separator in raw_value:
467 # If we get a comma separated list, pass it to our own function.
466 # If we get a comma separated list, pass it to our own function.
468 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
467 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
469 else:
468 else:
470 # Otherwise we assume it uses pyramids space/newline separation.
469 # Otherwise we assume it uses pyramids space/newline separation.
471 settings[name] = aslist(raw_value)
470 settings[name] = aslist(raw_value)
472
471
473
472
474 def _string_setting(settings, name, default):
473 def _string_setting(settings, name, default):
475 settings[name] = settings.get(name, default).lower()
474 settings[name] = settings.get(name, default).lower()
General Comments 0
You need to be logged in to leave comments. Login now