# Copyright (C) 2010-2023 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 # (only), as published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ import io import shlex import math import re import os import datetime import logging import queue import subprocess from dateutil.parser import parse from pyramid.interfaces import IRoutesMapper from pyramid.settings import asbool from pyramid.path import AssetResolver from threading import Thread from rhodecode.config.jsroutes import generate_jsroutes_content from rhodecode.lib.base import get_auth_user from rhodecode.lib.celerylib.loader import set_celery_conf import rhodecode log = logging.getLogger(__name__) def add_renderer_globals(event): from rhodecode.lib import helpers # TODO: When executed in pyramid view context the request is not available # in the event. Find a better solution to get the request. from pyramid.threadlocal import get_current_request request = event['request'] or get_current_request() # Add Pyramid translation as '_' to context event['_'] = request.translate event['_ungettext'] = request.plularize event['h'] = helpers def set_user_lang(event): request = event.request cur_user = getattr(request, 'user', None) if cur_user: user_lang = cur_user.get_instance().user_data.get('language') if user_lang: log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang) event.request._LOCALE_ = user_lang def update_celery_conf(event): log.debug('Setting celery config from new request') set_celery_conf(request=event.request, registry=event.request.registry) def add_request_user_context(event): """ Adds auth user into request context """ request = event.request # access req_id as soon as possible req_id = request.req_id if hasattr(request, 'vcs_call'): # skip vcs calls return if hasattr(request, 'rpc_method'): # skip api calls return auth_user, auth_token = get_auth_user(request) request.user = auth_user request.user_auth_token = auth_token request.environ['rc_auth_user'] = auth_user request.environ['rc_auth_user_id'] = str(auth_user.user_id) request.environ['rc_req_id'] = req_id def reset_log_bucket(event): """ reset the log bucket on new request """ request = event.request request.req_id_records_init() def scan_repositories_if_enabled(event): """ This is subscribed to the `pyramid.events.ApplicationCreated` event. It does a repository scan if enabled in the settings. """ settings = event.app.registry.settings vcs_server_enabled = settings['vcs.server.enable'] import_on_startup = settings['startup.import_repos'] if vcs_server_enabled and import_on_startup: from rhodecode.model.scm import ScmModel from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path repositories = ScmModel().repo_scan(get_rhodecode_base_path()) repo2db_mapper(repositories, remove_obsolete=False) def write_metadata_if_needed(event): """ Writes upgrade metadata """ import rhodecode from rhodecode.lib import system_info from rhodecode.lib import ext_json fname = '.rcmetadata.json' ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__')) metadata_destination = os.path.join(ini_loc, fname) def get_update_age(): now = datetime.datetime.utcnow() with open(metadata_destination, 'rb') as f: data = ext_json.json.loads(f.read()) if 'created_on' in data: update_date = parse(data['created_on']) diff = now - update_date return diff.total_seconds() / 60.0 return 0 def write(): configuration = system_info.SysInfo( system_info.rhodecode_config)()['value'] license_token = configuration['config']['license_token'] setup = dict( workers=configuration['config']['server:main'].get( 'workers', '?'), worker_type=configuration['config']['server:main'].get( 'worker_class', 'sync'), ) dbinfo = system_info.SysInfo(system_info.database_info)()['value'] del dbinfo['url'] metadata = dict( desc='upgrade metadata info', license_token=license_token, created_on=datetime.datetime.utcnow().isoformat(), usage=system_info.SysInfo(system_info.usage_info)()['value'], platform=system_info.SysInfo(system_info.platform_type)()['value'], database=dbinfo, cpu=system_info.SysInfo(system_info.cpu)()['value'], memory=system_info.SysInfo(system_info.memory)()['value'], setup=setup ) with open(metadata_destination, 'wb') as f: f.write(ext_json.json.dumps(metadata)) settings = event.app.registry.settings if settings.get('metadata.skip'): return # only write this every 24h, workers restart caused unwanted delays try: age_in_min = get_update_age() except Exception: age_in_min = 0 if age_in_min > 60 * 60 * 24: return try: write() except Exception: pass def write_usage_data(event): import rhodecode from rhodecode.lib import system_info from rhodecode.lib import ext_json settings = event.app.registry.settings instance_tag = settings.get('metadata.write_usage_tag') if not settings.get('metadata.write_usage'): return def get_update_age(dest_file): now = datetime.datetime.utcnow() with open(dest_file, 'rb') as f: data = ext_json.json.loads(f.read()) if 'created_on' in data: update_date = parse(data['created_on']) diff = now - update_date return math.ceil(diff.total_seconds() / 60.0) return 0 utc_date = datetime.datetime.utcnow() hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.)) fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format( date=utc_date, hour=hour_quarter) ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__')) usage_dir = os.path.join(ini_loc, '.rcusage') if not os.path.isdir(usage_dir): os.makedirs(usage_dir) usage_metadata_destination = os.path.join(usage_dir, fname) try: age_in_min = get_update_age(usage_metadata_destination) except Exception: age_in_min = 0 # write every 6th hour if age_in_min and age_in_min < 60 * 6: log.debug('Usage file created %s minutes ago, skipping (threshold: %s minutes)...', age_in_min, 60 * 6) return def write(dest_file): configuration = system_info.SysInfo(system_info.rhodecode_config)()['value'] license_token = configuration['config']['license_token'] metadata = dict( desc='Usage data', instance_tag=instance_tag, license_token=license_token, created_on=datetime.datetime.utcnow().isoformat(), usage=system_info.SysInfo(system_info.usage_info)()['value'], ) with open(dest_file, 'wb') as f: f.write(ext_json.formatted_json(metadata)) try: log.debug('Writing usage file at: %s', usage_metadata_destination) write(usage_metadata_destination) except Exception: pass def write_js_routes_if_enabled(event): registry = event.app.registry mapper = registry.queryUtility(IRoutesMapper) _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)') def _extract_route_information(route): """ Convert a route into tuple(name, path, args), eg: ('show_user', '/profile/%(username)s', ['username']) """ route_path = route.pattern pattern = route.pattern def replace(matchobj): if matchobj.group(1): return "%%(%s)s" % matchobj.group(1).split(':')[0] else: return "%%(%s)s" % matchobj.group(2) route_path = _argument_prog.sub(replace, route_path) if not route_path.startswith('/'): route_path = f'/{route_path}' return ( route.name, route_path, [(arg[0].split(':')[0] if arg[0] != '' else arg[1]) for arg in _argument_prog.findall(pattern)] ) def get_routes(): # pyramid routes for route in mapper.get_routes(): if not route.name.startswith('__'): yield _extract_route_information(route) if asbool(registry.settings.get('generate_js_files', 'false')): static_path = AssetResolver().resolve('rhodecode:public').abspath() jsroutes = get_routes() jsroutes_file_content = generate_jsroutes_content(jsroutes) jsroutes_file_path = os.path.join( static_path, 'js', 'rhodecode', 'routes.js') try: with open(jsroutes_file_path, 'w', encoding='utf-8') as f: f.write(jsroutes_file_content) log.debug('generated JS files in %s', jsroutes_file_path) except Exception: log.exception('Failed to write routes.js into %s', jsroutes_file_path) class Subscriber(object): """ Base class for subscribers to the pyramid event system. """ def __call__(self, event): self.run(event) def run(self, event): raise NotImplementedError('Subclass has to implement this.') class AsyncSubscriber(Subscriber): """ Subscriber that handles the execution of events in a separate task to not block the execution of the code which triggers the event. It puts the received events into a queue from which the worker process takes them in order. """ def __init__(self): self._stop = False self._eventq = queue.Queue() self._worker = self.create_worker() self._worker.start() def __call__(self, event): self._eventq.put(event) def create_worker(self): worker = Thread(target=self.do_work) worker.daemon = True return worker def stop_worker(self): self._stop = False self._eventq.put(None) self._worker.join() def do_work(self): while not self._stop: event = self._eventq.get() if event is not None: self.run(event) class AsyncSubprocessSubscriber(AsyncSubscriber): """ Subscriber that uses the subprocess module to execute a command if an event is received. Events are handled asynchronously:: subscriber = AsyncSubprocessSubscriber('ls -la', timeout=10) subscriber(dummyEvent) # running __call__(event) """ def __init__(self, cmd, timeout=None): if not isinstance(cmd, (list, tuple)): cmd = shlex.split(cmd) super().__init__() self._cmd = cmd self._timeout = timeout def run(self, event): cmd = self._cmd timeout = self._timeout log.debug('Executing command %s.', cmd) try: output = subprocess.check_output( cmd, timeout=timeout, stderr=subprocess.STDOUT) log.debug('Command finished %s', cmd) if output: log.debug('Command output: %s', output) except subprocess.TimeoutExpired as e: log.exception('Timeout while executing command.') if e.output: log.error('Command output: %s', e.output) except subprocess.CalledProcessError as e: log.exception('Error while executing command.') if e.output: log.error('Command output: %s', e.output) except Exception: log.exception( 'Exception while executing command %s.', cmd)