config: Move initial repo scan up to the pyramid layer.

It is now subscribed to the ApplicationCreated event from pyramid.

rhodecode/config/environment.py

@@ -1,199 +1,196 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Pylons environment configuration
 """
 
 import os
 import logging
 import rhodecode
 import platform
 import re
 import io
 
 from mako.lookup import TemplateLookup
 from pylons.configuration import PylonsConfig
 from pylons.error import handle_mako_error
 from pyramid.settings import asbool
 
 # don't remove this import it does magic for celery
 from rhodecode.lib import celerypylons  # noqa
 
 import rhodecode.lib.app_globals as app_globals
 
 from rhodecode.config import utils
 from rhodecode.config.routing import make_map
 from rhodecode.config.jsroutes import generate_jsroutes_content
 
 from rhodecode.lib import helpers
 from rhodecode.lib.auth import set_available_permissions
 from rhodecode.lib.utils import (
     repo2db_mapper, make_db_config, set_rhodecode_config,
     load_rcextensions)
 from rhodecode.lib.utils2 import str2bool, aslist
 from rhodecode.lib.vcs import connect_vcs, start_vcs_server
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 def load_environment(global_conf, app_conf, initial=False,
                      test_env=None, test_index=None):
     """
     Configure the Pylons environment via the ``pylons.config``
     object
     """
     config = PylonsConfig()
 
 
     # Pylons paths
     root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     paths = {
         'root': root,
         'controllers': os.path.join(root, 'controllers'),
         'static_files': os.path.join(root, 'public'),
         'templates': [os.path.join(root, 'templates')],
     }
 
     # Initialize config with the basic options
     config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
 
     # store some globals into rhodecode
     rhodecode.CELERY_ENABLED = str2bool(config['app_conf'].get('use_celery'))
     rhodecode.CELERY_EAGER = str2bool(
         config['app_conf'].get('celery.always.eager'))
 
     config['routes.map'] = make_map(config)
 
     if asbool(config.get('generate_js_files', 'false')):
         jsroutes = config['routes.map'].jsroutes()
         jsroutes_file_content = generate_jsroutes_content(jsroutes)
         jsroutes_file_path = os.path.join(
             paths['static_files'], 'js', 'rhodecode', 'routes.js')
 
         with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
             f.write(jsroutes_file_content)
 
     config['pylons.app_globals'] = app_globals.Globals(config)
     config['pylons.h'] = helpers
     rhodecode.CONFIG = config
 
     load_rcextensions(root_path=config['here'])
 
     # Setup cache object as early as possible
     import pylons
     pylons.cache._push_object(config['pylons.app_globals'].cache)
 
     # Create the Mako TemplateLookup, with the default auto-escaping
     config['pylons.app_globals'].mako_lookup = TemplateLookup(
         directories=paths['templates'],
         error_handler=handle_mako_error,
         module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
         input_encoding='utf-8', default_filters=['escape'],
         imports=['from webhelpers.html import escape'])
 
     # sets the c attribute access when don't existing attribute are accessed
     config['pylons.strict_tmpl_context'] = True
 
     # configure channelstream
     config['channelstream_config'] = {
         'enabled': asbool(config.get('channelstream.enabled', False)),
         'server': config.get('channelstream.server'),
         'secret': config.get('channelstream.secret')
     }
 
     # Limit backends to "vcs.backends" from configuration
     backends = config['vcs.backends'] = aslist(
         config.get('vcs.backends', 'hg,git'), sep=',')
     for alias in rhodecode.BACKENDS.keys():
         if alias not in backends:
             del rhodecode.BACKENDS[alias]
     log.info("Enabled backends: %s", backends)
 
     # initialize vcs client and optionally run the server if enabled
     vcs_server_uri = config.get('vcs.server', '')
     vcs_server_enabled = str2bool(config.get('vcs.server.enable', 'true'))
     start_server = (
         str2bool(config.get('vcs.start_server', 'false')) and
         not int(os.environ.get('RC_VCSSERVER_TEST_DISABLE', '0')))
     if vcs_server_enabled and start_server:
         log.info("Starting vcsserver")
         start_vcs_server(server_and_port=vcs_server_uri,
                          protocol=utils.get_vcs_server_protocol(config),
                          log_level=config['vcs.server.log_level'])
 
     set_available_permissions(config)
     db_cfg = make_db_config(clear_session=True)
 
     repos_path = list(db_cfg.items('paths'))[0][1]
     config['base_path'] = repos_path
 
     config['vcs.hooks.direct_calls'] = _use_direct_hook_calls(config)
     config['vcs.hooks.protocol'] = _get_vcs_hooks_protocol(config)
 
     # store db config also in main global CONFIG
     set_rhodecode_config(config)
 
     # configure instance id
     utils.set_instance_id(config)
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)
 
     # store config reference into our module to skip import magic of pylons
     rhodecode.CONFIG.update(config)
 
     utils.configure_pyro4(config)
     utils.configure_vcs(config)
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(config))
 
-    import_on_startup = str2bool(config.get('startup.import_repos', False))
-    if vcs_server_enabled and import_on_startup:
-        repo2db_mapper(ScmModel().repo_scan(repos_path), remove_obsolete=False)
     return config
 
 
 def _use_direct_hook_calls(config):
     default_direct_hook_calls = 'false'
     direct_hook_calls = str2bool(
         config.get('vcs.hooks.direct_calls', default_direct_hook_calls))
     return direct_hook_calls
 
 
 def _get_vcs_hooks_protocol(config):
     protocol = config.get('vcs.hooks.protocol', 'pyro4').lower()
     return protocol
 
 
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
     settings_merged.update(settings)
 
     # If this is a test run we prepare the test environment like
     # creating a test database, test search index and test repositories.
     # This has to be done before the database connection is initialized.
     if settings['is_test']:
         rhodecode.is_test = True
         utils.initialize_test_environment(settings_merged)
 
     # Initialize the database connection.
     utils.initialize_database(settings_merged)

rhodecode/config/middleware.py

@@ -1,404 +1,409 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Pylons middleware initialization
 """
 import logging
 from collections import OrderedDict
 
 from paste.registry import RegistryManager
 from paste.gzipper import make_gzip_middleware
 from pylons.wsgiapp import PylonsApp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.static import static_view
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError
 from pylons.controllers.util import abort, redirect
+from pyramid.events import ApplicationCreated
 import pyramid.httpexceptions as httpexceptions
 from pyramid.renderers import render_to_response, render
 from routes.middleware import RoutesMiddleware
 import routes.util
 
 import rhodecode
 import rhodecode.integrations # do not remove this as it registers celery tasks
 from rhodecode.config import patches
 from rhodecode.config.routing import STATIC_FILE_PREFIX
 from rhodecode.config.environment import (
     load_environment, load_pyramid_environment)
 from rhodecode.lib.middleware import csrf
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
+from rhodecode.subscribers import scan_repositories_if_enabled
 
 
 log = logging.getLogger(__name__)
 
 
 # this is used to avoid avoid the route lookup overhead in routesmiddleware
 # for certain routes which won't go to pylons to - eg. static files, debugger
 # it is only needed for the pylons migration and can be removed once complete
 class SkippableRoutesMiddleware(RoutesMiddleware):
     """ Routes middleware that allows you to skip prefixes """
 
     def __init__(self, *args, **kw):
         self.skip_prefixes = kw.pop('skip_prefixes', [])
         super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
 
     def __call__(self, environ, start_response):
         for prefix in self.skip_prefixes:
             if environ['PATH_INFO'].startswith(prefix):
                 # added to avoid the case when a missing /_static route falls
                 # through to pylons and causes an exception as pylons is
                 # expecting wsgiorg.routingargs to be set in the environ
                 # by RoutesMiddleware.
                 if 'wsgiorg.routing_args' not in environ:
                     environ['wsgiorg.routing_args'] = (None, {})
                 return self.app(environ, start_response)
 
         return super(SkippableRoutesMiddleware, self).__call__(
             environ, start_response)
 
 
 def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``global_conf``
         The inherited configuration for this application. Normally from
         the [DEFAULT] section of the Paste ini file.
 
     ``full_stack``
         Whether or not this application provides a full WSGI stack (by
         default, meaning it handles its own exceptions and errors).
         Disable full_stack when this application is "managed" by
         another WSGI middleware.
 
     ``app_conf``
         The application's local configuration. Normally specified in
         the [app:<name>] section of the Paste ini file (where <name>
         defaults to main).
 
     """
     # Apply compatibility patches
     patches.kombu_1_5_1_python_2_7_11()
     patches.inspect_getargspec()
 
     # Configure the Pylons environment
     config = load_environment(global_conf, app_conf)
 
     # The Pylons WSGI app
     app = PylonsApp(config=config)
     if rhodecode.is_test:
         app = csrf.CSRFDetector(app)
 
     expected_origin = config.get('expected_origin')
     if expected_origin:
         # The API can be accessed from other Origins.
         app = csrf.OriginChecker(app, expected_origin,
                                  skip_urls=[routes.util.url_for('api')])
 
 
     if asbool(full_stack):
 
         # Appenlight monitoring and error handler
         app, appenlight_client = wrap_in_appenlight_if_enabled(app, config)
 
         # we want our low level middleware to get to the request ASAP. We don't
         # need any pylons stack middleware in them
         app = VCSMiddleware(app, config, appenlight_client)
 
     # Establish the Registry for this application
     app = RegistryManager(app)
 
     app.config = config
 
     return app
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid and wraps the Pylons based
     application.
 
     Specials:
 
     * We migrate from Pylons to Pyramid. While doing this, we keep both
       frameworks functional. This involves moving some WSGI middlewares around
       and providing access to some data internals, so that the old code is
       still functional.
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     # The edition string should be available in pylons too, so we add it here
     # before copying the settings.
     settings.setdefault('rhodecode.edition', 'Community Edition')
 
     # As long as our Pylons application does expect "unprepared" settings, make
     # sure that we keep an unmodified copy. This avoids unintentional change of
     # behavior in the old application.
     settings_pylons = settings.copy()
 
     sanitize_settings_and_apply_defaults(settings)
     config = Configurator(settings=settings)
     add_pylons_compat_data(config.registry, global_config, settings_pylons)
 
     load_pyramid_environment(global_config, settings)
 
     includeme_first(config)
     includeme(config)
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     return pyramid_app
 
 
 def add_pylons_compat_data(registry, global_config, settings):
     """
     Attach data to the registry to support the Pylons integration.
     """
     registry._pylons_compat_global_config = global_config
     registry._pylons_compat_settings = settings
 
 
 def webob_to_pyramid_http_response(webob_response):
     ResponseClass = httpexceptions.status_map[webob_response.status_int]
     pyramid_response = ResponseClass(webob_response.status)
     pyramid_response.status = webob_response.status
     pyramid_response.headers.update(webob_response.headers)
     if pyramid_response.headers['content-type'] == 'text/html':
         pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
     return pyramid_response
 
 
 def error_handler(exception, request):
     # TODO: dan: replace the old pylons error controller with this
     from rhodecode.model.settings import SettingsModel
     from rhodecode.lib.utils2 import AttributeDict
 
     try:
         rc_config = SettingsModel().get_all_settings()
     except Exception:
         log.exception('failed to fetch settings')
         rc_config = {}
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPError):
         base_response = exception
 
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = base_response.explanation or str(base_response)
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rc_config.get('rhodecode_title', '')
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     response = render_to_response(
         '/errors/error_document.html', {'c': c}, request=request,
         response=base_response)
 
     return response
 
 
 def includeme(config):
     settings = config.registry.settings
 
     # plugin information
     config.registry.rhodecode_plugins = OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     if asbool(settings.get('appenlight', 'false')):
         config.include('appenlight_client.ext.pyramid_tween')
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('pyramid_beaker')
     config.include('rhodecode.channelstream')
     config.include('rhodecode.admin')
     config.include('rhodecode.authentication')
     config.include('rhodecode.integrations')
     config.include('rhodecode.login')
     config.include('rhodecode.tweens')
     config.include('rhodecode.api')
     config.include('rhodecode.svn_support')
     config.add_route(
         'rhodecode_support', 'https://rhodecode.com/help/', static=True)
 
+    # Add subscribers.
+    config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
+
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     pylons_app = make_app(
         config.registry._pylons_compat_global_config,
         **config.registry._pylons_compat_settings)
     config.registry._pylons_compat_config = pylons_app.config
 
     pylons_app_as_view = wsgiapp(pylons_app)
 
     # Protect from VCS Server error related pages when server is not available
     vcs_server_enabled = asbool(settings.get('vcs.server.enable', 'true'))
     if not vcs_server_enabled:
         pylons_app_as_view = DisableVCSPagesWrapper(pylons_app_as_view)
 
 
     def pylons_app_with_error_handler(context, request):
         """
         Handle exceptions from rc pylons app:
 
         - old webob type exceptions get converted to pyramid exceptions
         - pyramid exceptions are passed to the error handler view
         """
         try:
             response = pylons_app_as_view(context, request)
             if 400 <= response.status_int <= 599: # webob type error responses
                 return error_handler(
                     webob_to_pyramid_http_response(response), request)
         except HTTPError as e: # pyramid type exceptions
             return error_handler(e, request)
         except Exception:
             if settings.get('debugtoolbar.enabled', False):
                 raise
             return error_handler(HTTPInternalServerError(), request)
         return response
 
     # This is the glue which allows us to migrate in chunks. By registering the
     # pylons based application as the "Not Found" view in Pyramid, we will
     # fallback to the old application each time the new one does not yet know
     # how to handle a request.
     config.add_notfound_view(pylons_app_with_error_handler)
 
     if not settings.get('debugtoolbar.enabled', False):
         # if no toolbar, then any exception gets caught and rendered
         config.add_view(error_handler, context=Exception)
 
     config.add_view(error_handler, context=HTTPError)
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return redirect(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
 
     Part of this has been moved up from the Pylons layer, so that the
     data is also available if old Pylons code is hit through an already ported
     view.
     """
     settings = config.registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     # Add RoutesMiddleware to support the pylons compatibility tween during
     # migration to pyramid.
     pyramid_app = SkippableRoutesMiddleware(
         pyramid_app, config.registry._pylons_compat_config['routes.map'],
         skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
 
     if asbool(settings.get('appenlight', 'false')):
         pyramid_app, _ = wrap_in_appenlight_if_enabled(
             pyramid_app, config.registry._pylons_compat_config)
 
     if asbool(settings.get('gzip_responses', 'true')):
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     return pyramid_app
 
 
 def sanitize_settings_and_apply_defaults(settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     # Pyramid's mako renderer has to search in the templates folder so that the
     # old templates still work. Ported and new templates are expected to use
     # real asset specifications for the includes.
     mako_directories = settings.setdefault('mako.directories', [
         # Base templates of the original Pylons application
         'rhodecode:templates',
     ])
     log.debug(
         "Using the following Mako template directories: %s",
         mako_directories)
 
     # Default includes, possible to change as a user
     pyramid_includes = settings.setdefault('pyramid.includes', [
         'rhodecode.lib.middleware.request_wrapper',
     ])
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings.setdefault('rhodecode.api.url', '/_admin/api')
 
     _bool_setting(settings, 'vcs.server.enable', 'true')
     _bool_setting(settings, 'is_test', 'false')
 
     return settings
 
 
 def _bool_setting(settings, name, default):
     settings[name] = asbool(settings.get(name, default))

rhodecode/subscribers.py

@@ -1,55 +1,70 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import pylons
 
 from pyramid.i18n import get_localizer
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.translation import _ as tsf
 
 
 def add_renderer_globals(event):
     # Put pylons stuff into the context. This will be removed as soon as
     # migration to pyramid is finished.
     conf = pylons.config._current_obj()
     event['h'] = conf.get('pylons.h')
     event['c'] = pylons.tmpl_context
     event['url'] = pylons.url
 
     # TODO: When executed in pyramid view context the request is not available
     # in the event. Find a better solution to get the request.
     request = event['request'] or get_current_request()
 
     # Add Pyramid translation as '_' to context
     event['_'] = request.translate
     event['localizer'] = request.localizer
 
 
 def add_localizer(event):
     request = event.request
     localizer = get_localizer(request)
 
     def auto_translate(*args, **kwargs):
         return localizer.translate(tsf(*args, **kwargs))
 
     request.localizer = localizer
     request.translate = auto_translate
+
+
+def scan_repositories_if_enabled(event):
+    """
+    This is subscribed to the `pyramid.events.ApplicationCreated` event. It
+    does a repository scan if enabled in the settings.
+    """
+    from rhodecode.model.scm import ScmModel
+    from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
+    settings = event.app.registry.settings
+    vcs_server_enabled = settings['vcs.server.enable']
+    import_on_startup = settings['startup.import_repos']
+    if vcs_server_enabled and import_on_startup:
+        repositories = ScmModel().repo_scan(get_rhodecode_base_path())
+        repo2db_mapper(repositories, remove_obsolete=False)