wsgi: Refactor to introduce make_app_raw function with the essentials of make_app

Command line commands and WSGI applications have different environments and
runtime models. They deserve different handling.

Keep make_app doing as before so WSGI scripts keep working as before.

All "app" usage from command line is changed to use make_app_raw directly.

kallithea/bin/kallithea_cli_base.py

@@ -1,86 +1,86
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import configparser
 import functools
 import logging.config
 import os
 import re
 import sys
 
 import click
 import paste.deploy
 
 import kallithea
 import kallithea.config.application
 
 
 # kallithea_cli is usually invoked through the 'kallithea-cli' wrapper script
 # that is installed by setuptools, as specified in setup.py console_scripts
 # entry_points. The script will be using the right virtualenv (if any), and for
 # Unix, it will contain #! pointing at the right python executable. The script
 # also makes sure sys.argv[0] points back at the script path, and that is what
 # can be used to invoke 'kallithea-cli' later.
 kallithea_cli_path = sys.argv[0]
 
 
 def read_config(ini_file_name, strip_section_prefix):
     """Read ini_file_name content, and for all sections like '[X:Y]' where X is
     strip_section_prefix, replace the section name with '[Y]'."""
 
     def repl(m):
         if m.group(1) == strip_section_prefix:
             return '[%s]' % m.group(2)
         return m.group(0)
 
     with open(ini_file_name) as f:
         return re.sub(r'^\[([^:]+):(.*)]', repl, f.read(), flags=re.MULTILINE)
 
 
 # This placeholder is the main entry point for the kallithea-cli command
 @click.group(context_settings=dict(help_option_names=['-h', '--help']))
 def cli():
     """Various commands to manage a Kallithea instance."""
 
 def register_command(needs_config_file=False, config_file_initialize_app=False, hidden=False):
     """Register a kallithea-cli subcommand.
 
     If one of the needs_config_file flags are true, a config file must be specified
     with -c and it is read and logging is configured. The configuration is
     available in the kallithea.CONFIG dict.
 
     If config_file_initialize_app is true, Kallithea, TurboGears global state
     (including tg.config), and database access will also be fully initialized.
     """
     cli_command = cli.command(hidden=hidden)
     if needs_config_file or config_file_initialize_app:
         def annotator(annotated):
             @click.option('--config_file', '-c', help="Path to .ini file with app configuration.",
                 type=click.Path(dir_okay=False, exists=True, readable=True), required=True)
             @functools.wraps(annotated) # reuse meta data from the wrapped function so click can see other options
             def runtime_wrapper(config_file, *args, **kwargs):
                 path_to_ini_file = os.path.realpath(config_file)
                 config = paste.deploy.appconfig('config:' + path_to_ini_file)
                 cp = configparser.ConfigParser(strict=False)
                 cp.read_string(read_config(path_to_ini_file, strip_section_prefix=annotated.__name__))
                 logging.config.fileConfig(cp,
                     {'__file__': path_to_ini_file, 'here': os.path.dirname(path_to_ini_file)})
                 if needs_config_file:
                     annotated(*args, config=config, **kwargs)
                 if config_file_initialize_app:
-                    kallithea.config.application.make_app(config.global_conf, **config.local_conf)
+                    kallithea.config.application.make_app_raw(config.global_conf, **config.local_conf)
                     annotated(*args, **kwargs)
             return cli_command(runtime_wrapper)
         return annotator
     return cli_command

kallithea/bin/kallithea_cli_celery.py

@@ -1,53 +1,53
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import click
 from celery.bin.celery import celery as celery_command
 
 import kallithea
 import kallithea.bin.kallithea_cli_base as cli_base
 from kallithea.lib import celery_app
 from kallithea.lib.utils2 import asbool
 
 
 @cli_base.register_command(needs_config_file=True)
 @click.argument('celery_args', nargs=-1)
 def celery_run(celery_args, config):
     """Start Celery worker(s) for asynchronous tasks.
 
     This commands starts the Celery daemon which will spawn workers to handle
     certain asynchronous tasks for Kallithea.
 
     Any extra arguments you pass to this command will be passed through to
     Celery. Use '--' before such extra arguments to avoid options to be parsed
     by this CLI command.
     """
 
     if not asbool(config.get('use_celery')):
         raise Exception('Please set use_celery = true in .ini config '
                         'file before running this command')
 
     kallithea.CELERY_APP.config_from_object(celery_app.make_celery_config(config))
 
-    kallithea.CELERY_APP.loader.on_worker_process_init = lambda: kallithea.config.application.make_app(config.global_conf, **config.local_conf)
+    kallithea.CELERY_APP.loader.on_worker_process_init = lambda: kallithea.config.application.make_app_raw(config.global_conf, **config.local_conf)
 
     args = list(celery_args)
     # args[0] is generally ignored when prog_name is specified, but -h *needs* it to be 'worker' ... but will also suggest that users specify 'worker' explicitly
     if not args or args[0] != 'worker':
         args.insert(0, 'worker')
 
     # inline kallithea.CELERY_APP.start in order to allow specifying prog_name
     assert celery_command.params[0].name == 'app'
     celery_command.params[0].default = kallithea.CELERY_APP
     celery_command.main(args=args, prog_name='kallithea-cli celery-run -c CONFIG_FILE --')

kallithea/bin/vcs_hooks.py

@@ -1,203 +1,203
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.bin.vcs_hooks
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 Entry points for Kallithea hooking into Mercurial and Git.
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import os
 import sys
 
 import mercurial.hg
 import mercurial.scmutil
 import paste.deploy
 
 import kallithea
 import kallithea.config.application
 from kallithea.lib import hooks, webutils
 from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes, safe_str
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.utils.helpers import get_scm_size
 from kallithea.model import db
 
 
 log = logging.getLogger(__name__)
 
 
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """Show size of Mercurial repository.
 
     Called as Mercurial hook changegroup.kallithea_repo_size after push.
     """
     size_hg, size_root = get_scm_size('.hg', safe_str(repo.root))
 
     last_cs = repo[len(repo) - 1]
 
     msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'
            'Last revision is now r%s:%s\n') % (
         webutils.format_byte_size(size_hg),
         webutils.format_byte_size(size_root),
         webutils.format_byte_size(size_hg + size_root),
         last_cs.rev(),
         ascii_str(last_cs.hex())[:12],
     )
     ui.status(safe_bytes(msg))
 
 
 def update(ui, repo, hooktype=None, **kwargs):
     """Update repo after push. The equivalent to 'hg update' but using the same
     Mercurial as everything else.
 
     Called as Mercurial hook changegroup.kallithea_update after push.
     """
     try:
         ui.pushbuffer(error=True, subproc=True)
         rev = brev = None
         mercurial.hg.updatetotally(ui, repo, rev, brev)
     finally:
         s = ui.popbuffer()  # usually just "x files updated, x files merged, x files removed, x files unresolved"
         log.info('%s update hook output: %s', safe_str(repo.root), safe_str(s).rstrip())
 
 
 def pull_action(ui, repo, **kwargs):
     """Logs user pull action
 
     Called as Mercurial hook outgoing.kallithea_pull_action.
     """
     hooks.log_pull_action()
 
 
 def push_action(ui, repo, node, node_last, **kwargs):
     """
     Register that changes have been added to the repo - log the action *and* invalidate caches.
     Note: This hook is not only logging, but also the side effect invalidating
     caches! The function should perhaps be renamed.
 
     Called as Mercurial hook changegroup.kallithea_push_action .
 
     The pushed changesets is given by the revset 'node:node_last'.
     """
     revs = [ascii_str(repo[r].hex()) for r in mercurial.scmutil.revrange(repo, [b'%s:%s' % (node, node_last)])]
     hooks.process_pushed_raw_ids(revs)
 
 
 def _git_hook_environment(repo_path):
     """
     Create a light-weight environment for stand-alone scripts and return an UI and the
     db repository.
 
     Git hooks are executed as subprocess of Git while Kallithea is waiting, and
     they thus need enough info to be able to create an app environment and
     connect to the database.
     """
     extras = get_hook_environment()
 
     path_to_ini_file = extras['config']
     config = paste.deploy.appconfig('config:' + path_to_ini_file)
     #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
-    kallithea.config.application.make_app(config.global_conf, **config.local_conf)
+    kallithea.config.application.make_app_raw(config.global_conf, **config.local_conf)
 
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
 
     repo = db.Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database' % repo_path)
 
     return repo
 
 
 def post_receive(repo_path, git_stdin_lines):
     """Called from Git post-receive hook.
     The returned value is used as hook exit code and must be 0.
     """
     try:
         repo = _git_hook_environment(repo_path)
     except HookEnvironmentError as e:
         sys.stderr.write("Skipping Kallithea Git post-receive hook %r.\nGit was apparently not invoked by Kallithea: %s\n" % (sys.argv[0], e))
         return 0
 
     # the post push hook should never use the cached instance
     scm_repo = repo.scm_instance_no_cache()
 
     rev_data = []
     for l in git_stdin_lines:
         old_rev, new_rev, ref = l.strip().split(' ')
         _ref_data = ref.split('/')
         if _ref_data[1] in ['tags', 'heads']:
             rev_data.append({'old_rev': old_rev,
                              'new_rev': new_rev,
                              'ref': ref,
                              'type': _ref_data[1],
                              'name': '/'.join(_ref_data[2:])})
 
     git_revs = []
     for push_ref in rev_data:
         _type = push_ref['type']
         if _type == 'heads':
             if push_ref['old_rev'] == EmptyChangeset().raw_id:
                 # update the symbolic ref if we push new repo
                 if scm_repo.is_empty():
                     scm_repo._repo.refs.set_symbolic_ref(
                         b'HEAD',
                         b'refs/heads/%s' % safe_bytes(push_ref['name']))
 
                 # build exclude list without the ref
                 cmd = ['for-each-ref', '--format=%(refname)', 'refs/heads/*']
                 stdout = scm_repo.run_git_command(cmd)
                 ref = push_ref['ref']
                 heads = [head for head in stdout.splitlines() if head != ref]
                 # now list the git revs while excluding from the list
                 cmd = ['log', push_ref['new_rev'], '--reverse', '--pretty=format:%H']
                 cmd.append('--not')
                 cmd.extend(heads) # empty list is ok
                 stdout = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
 
             elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                 # delete branch case
                 git_revs += ['delete_branch=>%s' % push_ref['name']]
             else:
                 cmd = ['log', '%(old_rev)s..%(new_rev)s' % push_ref,
                        '--reverse', '--pretty=format:%H']
                 stdout = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
 
         elif _type == 'tags':
             git_revs += ['tag=>%s' % push_ref['name']]
 
     hooks.process_pushed_raw_ids(git_revs)
 
     return 0
 
 
 # Almost exactly like Mercurial contrib/hg-ssh:
 def rejectpush(ui, **kwargs):
     """Mercurial hook to be installed as pretxnopen and prepushkey for read-only repos.
     Return value 1 will make the hook fail and reject the push.
     """
     ex = get_hook_environment()
     ui.warn(safe_bytes("Push access to %r denied\n" % ex.repository))
     return 1

kallithea/config/application.py

@@ -1,68 +1,74
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """WSGI middleware initialization for the Kallithea application."""
 
 from kallithea.config.app_cfg import base_config
 from kallithea.config.middleware.https_fixup import HttpsFixup
 from kallithea.config.middleware.permanent_repo_url import PermanentRepoUrl
 from kallithea.config.middleware.simplegit import SimpleGit
 from kallithea.config.middleware.simplehg import SimpleHg
 from kallithea.config.middleware.wrapper import RequestWrapper
 from kallithea.lib.utils2 import asbool
 
 
 __all__ = ['make_app']
 
 
 def wrap_app(app):
     """Wrap the TG WSGI application in Kallithea middleware"""
     config = app.config
 
     # we want our low level middleware to get to the request ASAP. We don't
     # need any stack middleware in them - especially no StatusCodeRedirect buffering
     app = SimpleHg(app, config)
     app = SimpleGit(app, config)
 
     # Enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     if config.get('url_scheme_variable') or asbool(config.get('force_https')) or asbool(config.get('use_htsts')):
         app = HttpsFixup(app, config)
 
     app = PermanentRepoUrl(app, config)
 
     # Optional and undocumented wrapper - gives more verbose request/response logging, but has a slight overhead
     if asbool(config.get('use_wsgi_wrapper')):
         app = RequestWrapper(app, config)
 
     return app
 
 
 def make_app(global_conf, **app_conf):
+    """Return WSGI app with logging Mercurial stdout/stderr - to be used as
+    Paste or mod_wsgi entry point"""
+    return make_app_raw(global_conf, **app_conf)
+
+
+def make_app_raw(global_conf, **app_conf):
     """
     Set up Kallithea with the settings found in the PasteDeploy configuration
     file used.
 
     :param global_conf: The global settings for Kallithea (those
         defined under the ``[DEFAULT]`` section).
     :return: The Kallithea application with all the relevant middleware
         loaded.
 
     This is the PasteDeploy factory for the Kallithea application.
 
     ``app_conf`` contains all the application-specific settings (those defined
     under ``[app:main]``.
     """
     assert app_conf.get('sqlalchemy.url')  # must be called with a Kallithea .ini file, which for example must have this config option
     assert global_conf.get('here') and global_conf.get('__file__')  # app config should be initialized the paste way ...
 
     return base_config.make_wsgi_app(global_conf, app_conf, wrap_app=wrap_app)

kallithea/tests/scripts/manual_test_concurrency.py

@@ -1,212 +1,212
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_concurrency
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Test suite for making push/pull operations
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 30, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 
 """
 
 import logging
 import os
 import shutil
 import sys
 import tempfile
 from os.path import dirname
 from subprocess import PIPE, Popen
 
 from paste.deploy import appconfig
 from sqlalchemy import engine_from_config
 
 import kallithea.config.application
 from kallithea.lib.utils2 import get_crypt_password
 from kallithea.model import db, meta
 from kallithea.model.base import init_model
 from kallithea.model.repo import RepoModel
 from kallithea.tests.base import HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS
 
 
 rel_path = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
 conf = appconfig('config:development.ini', relative_to=rel_path)
-kallithea.config.application.make_app(conf.global_conf, **conf.local_conf)
+kallithea.config.application.make_app_raw(conf.global_conf, **conf.local_conf)
 
 USER = TEST_USER_ADMIN_LOGIN
 PASS = TEST_USER_ADMIN_PASS
 HOST = 'server.local'
 METHOD = 'pull'
 DEBUG = True
 log = logging.getLogger(__name__)
 
 
 class Command(object):
 
     def __init__(self, cwd):
         self.cwd = cwd
 
     def execute(self, cmd, *args):
         """Runs command on the system with given ``args``.
         """
 
         command = cmd + ' ' + ' '.join(args)
         log.debug('Executing %s', command)
         if DEBUG:
             print(command)
         p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
         stdout, stderr = p.communicate()
         if DEBUG:
             print(stdout, stderr)
         return stdout, stderr
 
 
 def get_session():
     engine = engine_from_config(conf, 'sqlalchemy.')
     init_model(engine)
     sa = meta.Session
     return sa
 
 
 def create_test_user(force=True):
     print('creating test user')
     sa = get_session()
 
     user = sa.query(db.User).filter(db.User.username == USER).scalar()
 
     if force and user is not None:
         print('removing current user')
         for repo in sa.query(db.Repository).filter(db.Repository.user == user).all():
             sa.delete(repo)
         sa.delete(user)
         sa.commit()
 
     if user is None or force:
         print('creating new one')
         new_usr = db.User()
         new_usr.username = USER
         new_usr.password = get_crypt_password(PASS)
         new_usr.email = '[e-mail]'
         new_usr.name = 'test'
         new_usr.lastname = 'lasttestname'
         new_usr.active = True
         new_usr.admin = True
         sa.add(new_usr)
         sa.commit()
 
     print('done')
 
 
 def create_test_repo(force=True):
     print('creating test repo')
     sa = get_session()
 
     user = sa.query(db.User).filter(db.User.username == USER).scalar()
     if user is None:
         raise Exception('user not found')
 
     repo = sa.query(db.Repository).filter(db.Repository.repo_name == HG_REPO).scalar()
 
     if repo is None:
         print('repo not found creating')
 
         form_data = {'repo_name': HG_REPO,
                      'repo_type': 'hg',
                      'private': False,
                      'clone_uri': ''}
         rm = RepoModel()
         rm.base_path = '/home/hg'
         rm.create(form_data, user)
 
     print('done')
 
 
 def set_anonymous_access(enable=True):
     sa = get_session()
     user = sa.query(db.User).filter(db.User.username == 'default').one()
     user.active = enable
     sa.add(user)
     sa.commit()
 
 
 def get_anonymous_access():
     sa = get_session()
     return sa.query(db.User).filter(db.User.username == 'default').one().active
 
 
 #==============================================================================
 # TESTS
 #==============================================================================
 def test_clone_with_credentials(no_errors=False, repo=HG_REPO, method=METHOD,
                                 backend='hg'):
     cwd = path = os.path.join(db.Ui.get_by_key('paths', '/').ui_value, repo)
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % os.path.join(path)
     except OSError:
         raise
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user': USER,
                    'pass': PASS,
                    'host': HOST,
                    'cloned_repo': repo, }
 
     dest = tempfile.mktemp(dir=path, prefix='dest-')
     if method == 'pull':
         stdout, stderr = Command(cwd).execute(backend, method, '--cwd', dest, clone_url)
     else:
         stdout, stderr = Command(cwd).execute(backend, method, clone_url, dest)
         if not no_errors:
             if backend == 'hg':
                 assert """adding file changes""" in stdout, 'no messages about cloning'
                 assert """abort""" not in stderr, 'got error from clone'
             elif backend == 'git':
                 assert """Cloning into""" in stdout, 'no messages about cloning'
 
 
 if __name__ == '__main__':
     try:
         create_test_user(force=False)
         import time
 
         try:
             METHOD = sys.argv[3]
         except IndexError:
             pass
 
         try:
             backend = sys.argv[4]
         except IndexError:
             backend = 'hg'
 
         if METHOD == 'pull':
             seq = next(tempfile._RandomNameSequence())  # pytype: disable=module-attr
             test_clone_with_credentials(repo=sys.argv[1], method='clone',
                                         backend=backend)
         s = time.time()
         for i in range(1, int(sys.argv[2]) + 1):
             print('take', i)
             test_clone_with_credentials(repo=sys.argv[1], method=METHOD,
                                         backend=backend)
         print('time taken %.3f' % (time.time() - s))
     except Exception as e:
         sys.exit('stop on %s' % e)