webhook: quote URL variables to prevent url errors with special chars like # in pr title.
webhook: quote URL variables to prevent url errors with special chars like # in pr title.

File last commit:

r3363:f08e98b1 default
r3477:976a0af2 default
Show More
subscribers.py
327 lines | 10.1 KiB | text/x-python | PythonLexer
project: added all source files and assets
r1 # -*- coding: utf-8 -*-
docs: updated copyrights to 2019
r3363 # Copyright (C) 2010-2019 RhodeCode GmbH
project: added all source files and assets
r1 #
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
application: moved JS routes generation into pyramid....
r1538 import io
import re
core: handle bogus GET params so we don't crash on translate subscriber.
r3132 import os
core: added metadata for control upgrades.
r1392 import datetime
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019 import logging
import Queue
import subprocess32
i18n: Use new translation string factory....
r51
metadata: store workers and prevent to excesive metadata writes.
r2488
from dateutil.parser import parse
pyramid: Fix problem with adding '_' to renderer globals....
r21 from pyramid.threadlocal import get_current_request
application: moved JS routes generation into pyramid....
r1538 from pyramid.interfaces import IRoutesMapper
from pyramid.settings import asbool
from pyramid.path import AssetResolver
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019 from threading import Thread
project: added all source files and assets
r1
i18n: Use new translation string factory....
r51 from rhodecode.translation import _ as tsf
application: moved JS routes generation into pyramid....
r1538 from rhodecode.config.jsroutes import generate_jsroutes_content
users: ported controllers from pylons into pyramid views.
r2114 from rhodecode.lib import auth
pylons: remove pylons as dependency...
r2351 from rhodecode.lib.base import get_auth_user
core: use proper event to bootstrap pylons env....
r1309 import rhodecode
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019 log = logging.getLogger(__name__)
project: added all source files and assets
r1 def add_renderer_globals(event):
dependencies: bumped pyramid to 1.9 webob to 1.7.3 and webtest to 2.0.27...
r1906 from rhodecode.lib import helpers
pyramid: Fix problem with adding '_' to renderer globals....
r21 # TODO: When executed in pyramid view context the request is not available
# in the event. Find a better solution to get the request.
request = event['request'] or get_current_request()
project: added all source files and assets
r1 # Add Pyramid translation as '_' to context
event['_'] = request.translate
i18n: added pluralize to renderer globals.
r1304 event['_ungettext'] = request.plularize
dependencies: bumped pyramid to 1.9 webob to 1.7.3 and webtest to 2.0.27...
r1906 event['h'] = helpers
project: added all source files and assets
r1
def add_localizer(event):
request = event.request
core: use application wide detection of invalid bytes sent via URL/GET/POST data.
r3145 localizer = request.localizer
project: added all source files and assets
r1
def auto_translate(*args, **kwargs):
return localizer.translate(tsf(*args, **kwargs))
request.translate = auto_translate
i18n: added pluralize to renderer globals.
r1304 request.plularize = localizer.pluralize
Martin Bornhold
config: Move initial repo scan up to the pyramid layer....
r580
i18n: use consistent way of setting user language.
r1307 def set_user_lang(event):
core: use proper event to bootstrap pylons env....
r1309 request = event.request
cur_user = getattr(request, 'user', None)
i18n: use consistent way of setting user language.
r1307
if cur_user:
user_lang = cur_user.get_instance().user_data.get('language')
if user_lang:
system-info: fix unicode problem on translation
r1308 log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
i18n: use consistent way of setting user language.
r1307 event.request._LOCALE_ = user_lang
pyramid: moved extraction of user into a seperate subscriber.
r1903 def add_request_user_context(event):
"""
Adds auth user into request context
"""
request = event.request
debug: add new custom logging to track unique requests across systems.
r2794 # access req_id as soon as possible
req_id = request.req_id
pyramid: moved extraction of user into a seperate subscriber.
r1903
if hasattr(request, 'vcs_call'):
# skip vcs calls
return
if hasattr(request, 'rpc_method'):
# skip api calls
return
auth_user = get_auth_user(request)
request.user = auth_user
request.environ['rc_auth_user'] = auth_user
security: update lastactivity when on audit logs....
r2930 request.environ['rc_auth_user_id'] = auth_user.user_id
debug: add new custom logging to track unique requests across systems.
r2794 request.environ['rc_req_id'] = req_id
pyramid: moved extraction of user into a seperate subscriber.
r1903
events: add event to catch permission changed so we can flush affected users permission caches
r2849
users: ported controllers from pylons into pyramid views.
r2114 def inject_app_settings(event):
settings = event.app.registry.settings
# inject info about available permissions
auth.set_available_permissions(settings)
Martin Bornhold
config: Move initial repo scan up to the pyramid layer....
r580 def scan_repositories_if_enabled(event):
"""
This is subscribed to the `pyramid.events.ApplicationCreated` event. It
does a repository scan if enabled in the settings.
"""
settings = event.app.registry.settings
vcs_server_enabled = settings['vcs.server.enable']
import_on_startup = settings['startup.import_repos']
if vcs_server_enabled and import_on_startup:
core: remove writing of largeobject dirs on AppStartup....
r1680 from rhodecode.model.scm import ScmModel
from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
Martin Bornhold
config: Move initial repo scan up to the pyramid layer....
r580 repositories = ScmModel().repo_scan(get_rhodecode_base_path())
repo2db_mapper(repositories, remove_obsolete=False)
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019
core: added metadata for control upgrades.
r1392 def write_metadata_if_needed(event):
"""
Writes upgrade metadata
"""
import rhodecode
from rhodecode.lib import system_info
from rhodecode.lib import ext_json
metadata: store workers and prevent to excesive metadata writes.
r2488 fname = '.rcmetadata.json'
ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
metadata_destination = os.path.join(ini_loc, fname)
def get_update_age():
now = datetime.datetime.utcnow()
with open(metadata_destination, 'rb') as f:
data = ext_json.json.loads(f.read())
if 'created_on' in data:
update_date = parse(data['created_on'])
diff = now - update_date
return diff.total_seconds() / 60.0
return 0
core: added metadata for control upgrades.
r1392 def write():
metadata: write license token for license migration for installer.
r1524 configuration = system_info.SysInfo(
system_info.rhodecode_config)()['value']
license_token = configuration['config']['license_token']
metadata: store workers and prevent to excesive metadata writes.
r2488
setup = dict(
workers=configuration['config']['server:main'].get(
'workers', '?'),
worker_type=configuration['config']['server:main'].get(
'worker_class', 'sync'),
)
core: added metadata for control upgrades.
r1392 dbinfo = system_info.SysInfo(system_info.database_info)()['value']
del dbinfo['url']
metadata: store workers and prevent to excesive metadata writes.
r2488
core: added metadata for control upgrades.
r1392 metadata = dict(
desc='upgrade metadata info',
metadata: write license token for license migration for installer.
r1524 license_token=license_token,
core: added metadata for control upgrades.
r1392 created_on=datetime.datetime.utcnow().isoformat(),
usage=system_info.SysInfo(system_info.usage_info)()['value'],
platform=system_info.SysInfo(system_info.platform_type)()['value'],
database=dbinfo,
cpu=system_info.SysInfo(system_info.cpu)()['value'],
memory=system_info.SysInfo(system_info.memory)()['value'],
metadata: store workers and prevent to excesive metadata writes.
r2488 setup=setup
core: added metadata for control upgrades.
r1392 )
with open(metadata_destination, 'wb') as f:
f.write(ext_json.json.dumps(metadata))
core: add possibility to skip write of metadata by defining metadata.skip key.
r1681 settings = event.app.registry.settings
if settings.get('metadata.skip'):
return
metadata: store workers and prevent to excesive metadata writes.
r2488 # only write this every 24h, workers restart caused unwanted delays
try:
age_in_min = get_update_age()
except Exception:
age_in_min = 0
meta: fix write condition.
r2539 if age_in_min > 60 * 60 * 24:
metadata: store workers and prevent to excesive metadata writes.
r2488 return
core: added metadata for control upgrades.
r1392 try:
write()
except Exception:
pass
application: moved JS routes generation into pyramid....
r1538 def write_js_routes_if_enabled(event):
registry = event.app.registry
mapper = registry.queryUtility(IRoutesMapper)
_argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
def _extract_route_information(route):
"""
Convert a route into tuple(name, path, args), eg:
('show_user', '/profile/%(username)s', ['username'])
"""
routepath = route.pattern
pattern = route.pattern
def replace(matchobj):
if matchobj.group(1):
return "%%(%s)s" % matchobj.group(1).split(':')[0]
else:
return "%%(%s)s" % matchobj.group(2)
routepath = _argument_prog.sub(replace, routepath)
routing: fixed pyramid routing generation....
r1884 if not routepath.startswith('/'):
routepath = '/'+routepath
application: moved JS routes generation into pyramid....
r1538 return (
route.name,
routepath,
[(arg[0].split(':')[0] if arg[0] != '' else arg[1])
for arg in _argument_prog.findall(pattern)]
)
def get_routes():
# pyramid routes
for route in mapper.get_routes():
if not route.name.startswith('__'):
yield _extract_route_information(route)
if asbool(registry.settings.get('generate_js_files', 'false')):
static_path = AssetResolver().resolve('rhodecode:public').abspath()
jsroutes = get_routes()
jsroutes_file_content = generate_jsroutes_content(jsroutes)
jsroutes_file_path = os.path.join(
static_path, 'js', 'rhodecode', 'routes.js')
subscribers: make creation of routes.js safer....
r2823 try:
with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
f.write(jsroutes_file_content)
except Exception:
log.exception('Failed to write routes.js into %s', jsroutes_file_path)
application: moved JS routes generation into pyramid....
r1538
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019 class Subscriber(object):
Martin Bornhold
events: Add comments to the subcriber classes.
r1020 """
Base class for subscribers to the pyramid event system.
"""
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019 def __call__(self, event):
self.run(event)
def run(self, event):
raise NotImplementedError('Subclass has to implement this.')
class AsyncSubscriber(Subscriber):
Martin Bornhold
events: Add comments to the subcriber classes.
r1020 """
Subscriber that handles the execution of events in a separate task to not
block the execution of the code which triggers the event. It puts the
received events into a queue from which the worker process takes them in
order.
"""
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019 def __init__(self):
self._stop = False
self._eventq = Queue.Queue()
self._worker = self.create_worker()
self._worker.start()
def __call__(self, event):
self._eventq.put(event)
def create_worker(self):
worker = Thread(target=self.do_work)
worker.daemon = True
return worker
def stop_worker(self):
self._stop = False
self._eventq.put(None)
self._worker.join()
def do_work(self):
while not self._stop:
event = self._eventq.get()
if event is not None:
self.run(event)
class AsyncSubprocessSubscriber(AsyncSubscriber):
Martin Bornhold
events: Add comments to the subcriber classes.
r1020 """
Subscriber that uses the subprocess32 module to execute a command if an
event is received. Events are handled asynchronously.
"""
Martin Bornhold
events: Move subscriber classes from svn_support to rhodecode.subscribers....
r1019
def __init__(self, cmd, timeout=None):
super(AsyncSubprocessSubscriber, self).__init__()
self._cmd = cmd
self._timeout = timeout
def run(self, event):
cmd = self._cmd
timeout = self._timeout
log.debug('Executing command %s.', cmd)
try:
output = subprocess32.check_output(
cmd, timeout=timeout, stderr=subprocess32.STDOUT)
log.debug('Command finished %s', cmd)
if output:
log.debug('Command output: %s', output)
except subprocess32.TimeoutExpired as e:
log.exception('Timeout while executing command.')
if e.output:
log.error('Command output: %s', e.output)
except subprocess32.CalledProcessError as e:
log.exception('Error while executing command.')
if e.output:
log.error('Command output: %s', e.output)
except:
log.exception(
'Exception while executing command %s.', cmd)