rhodecode/subscribers.py

# -*- coding: utf-8 -*-

# Copyright (C) 2010-2020 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import io
import shlex

import math
import re
import os
import datetime
import logging
import Queue
import subprocess


from dateutil.parser import parse
from pyramid.threadlocal import get_current_request
from pyramid.interfaces import IRoutesMapper
from pyramid.settings import asbool
from pyramid.path import AssetResolver
from threading import Thread

from rhodecode.config.jsroutes import generate_jsroutes_content
from rhodecode.lib.base import get_auth_user

import rhodecode


log = logging.getLogger(__name__)


def add_renderer_globals(event):
    from rhodecode.lib import helpers

    # TODO: When executed in pyramid view context the request is not available
    # in the event. Find a better solution to get the request.
    request = event['request'] or get_current_request()

    # Add Pyramid translation as '_' to context
    event['_'] = request.translate
    event['_ungettext'] = request.plularize
    event['h'] = helpers


def set_user_lang(event):
    request = event.request
    cur_user = getattr(request, 'user', None)

    if cur_user:
        user_lang = cur_user.get_instance().user_data.get('language')
        if user_lang:
            log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
            event.request._LOCALE_ = user_lang


def update_celery_conf(event):
    from rhodecode.lib.celerylib.loader import set_celery_conf
    log.debug('Setting celery config from new request')
    set_celery_conf(request=event.request, registry=event.request.registry)


def add_request_user_context(event):
    """
    Adds auth user into request context
    """
    request = event.request
    # access req_id as soon as possible
    req_id = request.req_id

    if hasattr(request, 'vcs_call'):
        # skip vcs calls
        return

    if hasattr(request, 'rpc_method'):
        # skip api calls
        return

    auth_user, auth_token = get_auth_user(request)
    request.user = auth_user
    request.user_auth_token = auth_token
    request.environ['rc_auth_user'] = auth_user
    request.environ['rc_auth_user_id'] = auth_user.user_id
    request.environ['rc_req_id'] = req_id


def reset_log_bucket(event):
    """
    reset the log bucket on new request
    """
    request = event.request
    request.req_id_records_init()


def scan_repositories_if_enabled(event):
    """
    This is subscribed to the `pyramid.events.ApplicationCreated` event. It
    does a repository scan if enabled in the settings.
    """
    settings = event.app.registry.settings
    vcs_server_enabled = settings['vcs.server.enable']
    import_on_startup = settings['startup.import_repos']
    if vcs_server_enabled and import_on_startup:
        from rhodecode.model.scm import ScmModel
        from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
        repositories = ScmModel().repo_scan(get_rhodecode_base_path())
        repo2db_mapper(repositories, remove_obsolete=False)


def write_metadata_if_needed(event):
    """
    Writes upgrade metadata
    """
    import rhodecode
    from rhodecode.lib import system_info
    from rhodecode.lib import ext_json

    fname = '.rcmetadata.json'
    ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
    metadata_destination = os.path.join(ini_loc, fname)

    def get_update_age():
        now = datetime.datetime.utcnow()

        with open(metadata_destination, 'rb') as f:
            data = ext_json.json.loads(f.read())
            if 'created_on' in data:
                update_date = parse(data['created_on'])
                diff = now - update_date
                return diff.total_seconds() / 60.0

        return 0

    def write():
        configuration = system_info.SysInfo(
            system_info.rhodecode_config)()['value']
        license_token = configuration['config']['license_token']

        setup = dict(
            workers=configuration['config']['server:main'].get(
                'workers', '?'),
            worker_type=configuration['config']['server:main'].get(
                'worker_class', 'sync'),
        )
        dbinfo = system_info.SysInfo(system_info.database_info)()['value']
        del dbinfo['url']

        metadata = dict(
            desc='upgrade metadata info',
            license_token=license_token,
            created_on=datetime.datetime.utcnow().isoformat(),
            usage=system_info.SysInfo(system_info.usage_info)()['value'],
            platform=system_info.SysInfo(system_info.platform_type)()['value'],
            database=dbinfo,
            cpu=system_info.SysInfo(system_info.cpu)()['value'],
            memory=system_info.SysInfo(system_info.memory)()['value'],
            setup=setup
        )

        with open(metadata_destination, 'wb') as f:
            f.write(ext_json.json.dumps(metadata))

    settings = event.app.registry.settings
    if settings.get('metadata.skip'):
        return

    # only write this every 24h, workers restart caused unwanted delays
    try:
        age_in_min = get_update_age()
    except Exception:
        age_in_min = 0

    if age_in_min > 60 * 60 * 24:
        return

    try:
        write()
    except Exception:
        pass


def write_usage_data(event):
    import rhodecode
    from rhodecode.lib import system_info
    from rhodecode.lib import ext_json

    settings = event.app.registry.settings
    instance_tag = settings.get('metadata.write_usage_tag')
    if not settings.get('metadata.write_usage'):
        return

    def get_update_age(dest_file):
        now = datetime.datetime.utcnow()

        with open(dest_file, 'rb') as f:
            data = ext_json.json.loads(f.read())
            if 'created_on' in data:
                update_date = parse(data['created_on'])
                diff = now - update_date
                return math.ceil(diff.total_seconds() / 60.0)

        return 0

    utc_date = datetime.datetime.utcnow()
    hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.))
    fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format(
        date=utc_date, hour=hour_quarter)
    ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))

    usage_dir = os.path.join(ini_loc, '.rcusage')
    if not os.path.isdir(usage_dir):
        os.makedirs(usage_dir)
    usage_metadata_destination = os.path.join(usage_dir, fname)

    try:
        age_in_min = get_update_age(usage_metadata_destination)
    except Exception:
        age_in_min = 0

    # write every 6th hour
    if age_in_min and age_in_min < 60 * 6:
        log.debug('Usage file created %s minutes ago, skipping (threshold: %s minutes)...',
                  age_in_min, 60 * 6)
        return

    def write(dest_file):
        configuration = system_info.SysInfo(system_info.rhodecode_config)()['value']
        license_token = configuration['config']['license_token']

        metadata = dict(
            desc='Usage data',
            instance_tag=instance_tag,
            license_token=license_token,
            created_on=datetime.datetime.utcnow().isoformat(),
            usage=system_info.SysInfo(system_info.usage_info)()['value'],
        )

        with open(dest_file, 'wb') as f:
            f.write(ext_json.json.dumps(metadata, indent=2, sort_keys=True))

    try:
        log.debug('Writing usage file at: %s', usage_metadata_destination)
        write(usage_metadata_destination)
    except Exception:
        pass


def write_js_routes_if_enabled(event):
    registry = event.app.registry

    mapper = registry.queryUtility(IRoutesMapper)
    _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')

    def _extract_route_information(route):
        """
        Convert a route into tuple(name, path, args), eg:
            ('show_user', '/profile/%(username)s', ['username'])
        """

        routepath = route.pattern
        pattern = route.pattern

        def replace(matchobj):
            if matchobj.group(1):
                return "%%(%s)s" % matchobj.group(1).split(':')[0]
            else:
                return "%%(%s)s" % matchobj.group(2)

        routepath = _argument_prog.sub(replace, routepath)

        if not routepath.startswith('/'):
            routepath = '/'+routepath

        return (
            route.name,
            routepath,
            [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
              for arg in _argument_prog.findall(pattern)]
        )

    def get_routes():
        # pyramid routes
        for route in mapper.get_routes():
            if not route.name.startswith('__'):
                yield _extract_route_information(route)

    if asbool(registry.settings.get('generate_js_files', 'false')):
        static_path = AssetResolver().resolve('rhodecode:public').abspath()
        jsroutes = get_routes()
        jsroutes_file_content = generate_jsroutes_content(jsroutes)
        jsroutes_file_path = os.path.join(
            static_path, 'js', 'rhodecode', 'routes.js')

        try:
            with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
                f.write(jsroutes_file_content)
        except Exception:
            log.exception('Failed to write routes.js into %s', jsroutes_file_path)


class Subscriber(object):
    """
    Base class for subscribers to the pyramid event system.
    """
    def __call__(self, event):
        self.run(event)

    def run(self, event):
        raise NotImplementedError('Subclass has to implement this.')


class AsyncSubscriber(Subscriber):
    """
    Subscriber that handles the execution of events in a separate task to not
    block the execution of the code which triggers the event. It puts the
    received events into a queue from which the worker process takes them in
    order.
    """
    def __init__(self):
        self._stop = False
        self._eventq = Queue.Queue()
        self._worker = self.create_worker()
        self._worker.start()

    def __call__(self, event):
        self._eventq.put(event)

    def create_worker(self):
        worker = Thread(target=self.do_work)
        worker.daemon = True
        return worker

    def stop_worker(self):
        self._stop = False
        self._eventq.put(None)
        self._worker.join()

    def do_work(self):
        while not self._stop:
            event = self._eventq.get()
            if event is not None:
                self.run(event)


class AsyncSubprocessSubscriber(AsyncSubscriber):
    """
    Subscriber that uses the subprocess module to execute a command if an
    event is received. Events are handled asynchronously::

        subscriber = AsyncSubprocessSubscriber('ls -la', timeout=10)
        subscriber(dummyEvent) # running __call__(event)

    """

    def __init__(self, cmd, timeout=None):
        if not isinstance(cmd, (list, tuple)):
            cmd = shlex.split(cmd)
        super(AsyncSubprocessSubscriber, self).__init__()
        self._cmd = cmd
        self._timeout = timeout

    def run(self, event):
        cmd = self._cmd
        timeout = self._timeout
        log.debug('Executing command %s.', cmd)

        try:
            output = subprocess.check_output(
                cmd, timeout=timeout, stderr=subprocess.STDOUT)
            log.debug('Command finished %s', cmd)
            if output:
                log.debug('Command output: %s', output)
        except subprocess.TimeoutExpired as e:
            log.exception('Timeout while executing command.')
            if e.output:
                log.error('Command output: %s', e.output)
        except subprocess.CalledProcessError as e:
            log.exception('Error while executing command.')
            if e.output:
                log.error('Command output: %s', e.output)
        except Exception:
            log.exception(
                'Exception while executing command %s.', cmd)